#!/usr/bin/python
#
# Copyright 2010 Google Inc. All Rights Reserved.
"""Tool script for auto dejagnu."""
__author__ = 'shenhan@google.com (Han Shen)'
import getpass
import optparse
import os
from os import path
import re
import shutil
import stat
import sys
import tempfile
import time
import lock_machine
import tc_enter_chroot
from cros_utils import command_executer
from cros_utils import constants
from cros_utils import logger
from cros_utils import misc
def ProcessArguments(argv):
"""Processing/validating script arguments."""
parser = optparse.OptionParser(description=(
'Launches gcc dejagnu test in chroot for chromeos toolchain, compares '
'the test result with a repository baseline and prints out the result.'),
usage='run_dejagnu options')
parser.add_option('-c',
'--chromeos_root',
dest='chromeos_root',
help='Required. Specify chromeos root')
parser.add_option('-m',
'--mount',
dest='mount',
help=('Specify gcc source to mount instead of "auto". '
'Under "auto" mode, which is the default - gcc is '
'checked out and built automatically at default '
'directories. Under "mount" mode '
'- the gcc_source is set to "$chromeos_'
'root/chroot/usr/local/toolchain_root/gcc", which is '
'the mount point for this option value, the '
'gcc-build-dir then is computed as '
'"${gcc_source_dir}-build-${ctarget}". In this mode, '
'a complete gcc build must be performed in the '
'computed gcc-build-dir beforehand.'))
parser.add_option('-b',
'--board',
dest='board',
help=('Required. Specify board.'))
parser.add_option('-r',
'--remote',
dest='remote',
help=('Required. Specify addresses/names of the board, '
'seperate each address/name using comma(\',\').'))
parser.add_option('-f',
'--flags',
dest='flags',
help='Optional. Extra run test flags to pass to dejagnu.')
parser.add_option('-k',
'--keep',
dest='keep_intermediate_files',
action='store_true',
default=False,
help=('Optional. Default to false. Do not remove dejagnu '
'intermediate files after test run.'))
parser.add_option('--cleanup',
dest='cleanup',
default=None,
help=('Optional. Values to this option could be '
'\'mount\' (unmount gcc source and '
'directory directory, '
'only valid when --mount is given), '
'\'chroot\' (delete chroot) and '
'\'chromeos\' (delete the whole chromeos tree).'))
parser.add_option('-t',
'--tools',
dest='tools',
default='gcc,g++',
help=('Optional. Specify which tools to check, using '
'","(comma) as separator. A typical value would be '
'"g++" so that only g++ tests are performed. '
'Defaults to "gcc,g++".'))
options, args = parser.parse_args(argv)
if not options.chromeos_root:
raise SyntaxError('Missing argument for --chromeos_root.')
if not options.remote:
raise SyntaxError('Missing argument for --remote.')
if not options.board:
raise SyntaxError('Missing argument for --board.')
if options.cleanup == 'mount' and not options.mount:
raise SyntaxError('--cleanup=\'mount\' not valid unless --mount is given.')
if options.cleanup and not (
options.cleanup == 'mount' or \
options.cleanup == 'chroot' or options.cleanup == 'chromeos'):
raise ValueError('Invalid option value for --cleanup')
if options.cleanup and options.keep_intermediate_files:
raise SyntaxError('Only one of --keep and --cleanup could be given.')
return options
class DejagnuExecuter(object):
"""The class wrapper for dejagnu test executer."""
def __init__(self, base_dir, mount, chromeos_root, remote, board, flags,
keep_intermediate_files, tools, cleanup):
self._l = logger.GetLogger()
self._chromeos_root = chromeos_root
self._chromeos_chroot = path.join(chromeos_root, 'chroot')
if mount:
self._gcc_source_dir_to_mount = mount
self._gcc_source_dir = path.join(constants.MOUNTED_TOOLCHAIN_ROOT, 'gcc')
else:
self._gcc_source_dir = None
self._remote = remote
self._board = board
## Compute target from board
self._target = misc.GetCtargetFromBoard(board, chromeos_root)
if not self._target:
raise ValueError('Unsupported board "%s"' % board)
self._executer = command_executer.GetCommandExecuter()
self._flags = flags or ''
self._base_dir = base_dir
self._tmp_abs = None
self._keep_intermediate_files = keep_intermediate_files
self._tools = tools.split(',')
self._cleanup = cleanup
def SetupTestingDir(self):
self._tmp_abs = tempfile.mkdtemp(
prefix='dejagnu_',
dir=path.join(self._chromeos_chroot, 'tmp'))
self._tmp = self._tmp_abs[len(self._chromeos_chroot):]
self._tmp_testing_rsa = path.join(self._tmp, 'testing_rsa')
self._tmp_testing_rsa_abs = path.join(self._tmp_abs, 'testing_rsa')
def MakeCheckString(self):
return ' '.join(['check-{0}'.format(t) for t in self._tools if t])
def CleanupIntermediateFiles(self):
if self._tmp_abs and path.isdir(self._tmp_abs):
if self._keep_intermediate_files:
self._l.LogOutput(
'Your intermediate dejagnu files are kept, you can re-run '
'inside chroot the command:')
self._l.LogOutput(
' DEJAGNU={0} make -C {1} {2} RUNTESTFLAGS="--target_board={3} {4}"' \
.format(path.join(self._tmp, 'site.exp'), self._gcc_build_dir,
self.MakeCheckString(), self._board, self._flags))
else:
self._l.LogOutput('[Cleanup] - Removing temp dir - {0}'.format(
self._tmp_abs))
shutil.rmtree(self._tmp_abs)
def Cleanup(self):
if not self._cleanup:
return
# Optionally cleanup mounted diretory, chroot and chromeos tree.
if self._cleanup == 'mount' or self._cleanup == 'chroot' or \
self._cleanup == 'chromeos':
# No exceptions are allowed from this method.
try:
self._l.LogOutput('[Cleanup] - Unmounting directories ...')
self.MountGccSourceAndBuildDir(unmount=True)
except:
print 'Warning: failed to unmount gcc source/build directory.'
if self._cleanup == 'chroot' or self._cleanup == 'chromeos':
self._l.LogOutput('[Cleanup]: Deleting chroot inside \'{0}\''.format(
self._chromeos_root))
command = 'cd %s; cros_sdk --delete' % self._chromeos_root
rv = self._executer.RunCommand(command)
if rv:
self._l.LogWarning('Warning - failed to delete chroot.')
# Delete .cache - crosbug.com/34956
command = 'sudo rm -fr %s' % os.path.join(self._chromeos_root, '.cache')
rv = self._executer.RunCommand(command)
if rv:
self._l.LogWarning('Warning - failed to delete \'.cache\'.')
if self._cleanup == 'chromeos':
self._l.LogOutput('[Cleanup]: Deleting chromeos tree \'{0}\' ...'.format(
self._chromeos_root))
command = 'rm -fr {0}'.format(self._chromeos_root)
rv = self._executer.RunCommand(command)
if rv:
self._l.LogWarning('Warning - failed to remove chromeos tree.')
def PrepareTestingRsaKeys(self):
if not path.isfile(self._tmp_testing_rsa_abs):
shutil.copy(
path.join(self._chromeos_root,
'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa'),
self._tmp_testing_rsa_abs)
os.chmod(self._tmp_testing_rsa_abs, stat.S_IRUSR)
def PrepareTestFiles(self):
"""Prepare site.exp and board exp files."""
# Create the boards directory.
os.mkdir('%s/boards' % self._tmp_abs)
# Generate the chromeos.exp file.
with open('%s/chromeos.exp.in' % self._base_dir, 'r') as template_file:
content = template_file.read()
substitutions = dict({
'__boardname__': self._board,
'__board_hostname__': self._remote,
'__tmp_testing_rsa__': self._tmp_testing_rsa,
'__tmp_dir__': self._tmp
})
for pat, sub in substitutions.items():
content = content.replace(pat, sub)
board_file_name = '%s/boards/%s.exp' % (self._tmp_abs, self._board)
with open(board_file_name, 'w') as board_file:
board_file.write(content)
# Generate the site file
with open('%s/site.exp' % self._tmp_abs, 'w') as site_file:
site_file.write('set target_list "%s"\n' % self._board)
def PrepareGcc(self):
if self._gcc_source_dir:
self.PrepareGccFromCustomizedPath()
else:
self.PrepareGccDefault()
self._l.LogOutput('Gcc source dir - {0}'.format(self._gcc_source_dir))
self._l.LogOutput('Gcc build dir - {0}'.format(self._gcc_top_build_dir))
def PrepareGccFromCustomizedPath(self):
"""Prepare gcc source, build directory from mounted source."""
# We have these source directories -
# _gcc_source_dir
# e.g. '/usr/local/toolchain_root/gcc'
# _gcc_source_dir_abs
# e.g. '/somewhere/chromeos.live/chroot/usr/local/toolchain_root/gcc'
# _gcc_source_dir_to_mount
# e.g. '/somewhere/gcc'
self._gcc_source_dir_abs = path.join(self._chromeos_chroot,
self._gcc_source_dir.lstrip('/'))
if not path.isdir(self._gcc_source_dir_abs) and \
self._executer.RunCommand(
'sudo mkdir -p {0}'.format(self._gcc_source_dir_abs)):
raise RuntimeError("Failed to create \'{0}\' inside chroot.".format(
self._gcc_source_dir))
if not (path.isdir(self._gcc_source_dir_to_mount) and
path.isdir(path.join(self._gcc_source_dir_to_mount, 'gcc'))):
raise RuntimeError('{0} is not a valid gcc source tree.'.format(
self._gcc_source_dir_to_mount))
# We have these build directories -
# _gcc_top_build_dir
# e.g. '/usr/local/toolchain_root/gcc-build-x86_64-cros-linux-gnu'
# _gcc_top_build_dir_abs
# e.g. '/somewhere/chromeos.live/chroo/tusr/local/toolchain_root/
# gcc-build-x86_64-cros-linux-gnu'
# _gcc_build_dir
# e.g. '/usr/local/toolchain_root/gcc-build-x86_64-cros-linux-gnu/gcc'
# _gcc_build_dir_to_mount
# e.g. '/somewhere/gcc-build-x86_64-cros-linux-gnu'
self._gcc_top_build_dir = '{0}-build-{1}'.format(
self._gcc_source_dir.rstrip('/'), self._target)
self._gcc_build_dir = path.join(self._gcc_top_build_dir, 'gcc')
self._gcc_build_dir_to_mount = '{0}-build-{1}'.format(
self._gcc_source_dir_to_mount, self._target)
self._gcc_top_build_dir_abs = path.join(self._chromeos_chroot,
self._gcc_top_build_dir.lstrip('/'))
if not path.isdir(self._gcc_top_build_dir_abs) and \
self._executer.RunCommand(
'sudo mkdir -p {0}'.format(self._gcc_top_build_dir_abs)):
raise RuntimeError('Failed to create \'{0}\' inside chroot.'.format(
self._gcc_top_build_dir))
if not (path.isdir(self._gcc_build_dir_to_mount) and path.join(
self._gcc_build_dir_to_mount, 'gcc')):
raise RuntimeError('{0} is not a valid gcc build tree.'.format(
self._gcc_build_dir_to_mount))
# All check passed. Now mount gcc source and build directories.
self.MountGccSourceAndBuildDir()
def PrepareGccDefault(self):
"""Auto emerging gcc for building purpose only."""
ret = self._executer.ChrootRunCommandWOutput(
self._chromeos_root, 'equery w cross-%s/gcc' % self._target)[1]
ret = path.basename(ret.strip())
# ret is expected to be something like 'gcc-4.6.2-r11.ebuild' or
# 'gcc-9999.ebuild' parse it.
matcher = re.match('((.*)-r\d+).ebuild', ret)
if matcher:
gccrevision, gccversion = matcher.group(1, 2)
elif ret == 'gcc-9999.ebuild':
gccrevision = 'gcc-9999'
gccversion = 'gcc-9999'
else:
raise RuntimeError('Failed to get gcc version.')
gcc_portage_dir = '/var/tmp/portage/cross-%s/%s/work' % (self._target,
gccrevision)
self._gcc_source_dir = path.join(gcc_portage_dir, gccversion)
self._gcc_top_build_dir = (gcc_portage_dir + '/%s-build-%s') % (
gccversion, self._target)
self._gcc_build_dir = path.join(self._gcc_top_build_dir, 'gcc')
gcc_build_dir_abs = path.join(self._chromeos_root, 'chroot',
self._gcc_build_dir.lstrip('/'))
if not path.isdir(gcc_build_dir_abs):
ret = self._executer.ChrootRunCommand(self._chromeos_root, (
'ebuild $(equery w cross-%s/gcc) clean prepare compile' % (
self._target)))
if ret:
raise RuntimeError('ebuild gcc failed.')
def MakeCheck(self):
self.MountGccSourceAndBuildDir()
cmd = ('cd %s ; '
'DEJAGNU=%s make %s RUNTESTFLAGS="--target_board=%s %s"' %
(self._gcc_build_dir, path.join(self._tmp, 'site.exp'),
self.MakeCheckString(), self._board, self._flags))
self._executer.ChrootRunCommand(self._chromeos_root, cmd)
def ValidateFailures(self):
validate_failures_py = path.join(
self._gcc_source_dir,
'contrib/testsuite-management/validate_failures.py')
cmd = 'cd {0} ; {1} --build_dir={0}'.format(self._gcc_top_build_dir,
validate_failures_py)
self.MountGccSourceAndBuildDir()
ret = self._executer.ChrootRunCommandWOutput(self._chromeos_root, cmd)
if ret[0] != 0:
self._l.LogWarning('*** validate_failures.py exited with non-zero code,'
'please run it manually inside chroot - \n'
' ' + cmd)
return ret
# This method ensures necessary mount points before executing chroot comamnd.
def MountGccSourceAndBuildDir(self, unmount=False):
mount_points = [tc_enter_chroot.MountPoint(self._gcc_source_dir_to_mount,
self._gcc_source_dir_abs,
getpass.getuser(), 'ro'),
tc_enter_chroot.MountPoint(self._gcc_build_dir_to_mount,
self._gcc_top_build_dir_abs,
getpass.getuser(), 'rw')]
for mp in mount_points:
if unmount:
if mp.UnMount():
raise RuntimeError('Failed to unmount {0}'.format(mp.mount_dir))
else:
self._l.LogOutput('{0} unmounted successfully.'.format(mp.mount_dir))
elif mp.DoMount():
raise RuntimeError(
'Failed to mount {0} onto {1}'.format(mp.external_dir,
mp.mount_dir))
else:
self._l.LogOutput('{0} mounted successfully.'.format(mp.mount_dir))
# The end of class DejagnuExecuter
def TryAcquireMachine(remotes):
available_machine = None
for r in remotes.split(','):
machine = lock_machine.Machine(r)
if machine.TryLock(timeout=300, exclusive=True):
available_machine = machine
break
else:
logger.GetLogger().LogWarning(
'*** Failed to lock machine \'{0}\'.'.format(r))
if not available_machine:
raise RuntimeError("Failed to acquire one machine from \'{0}\'.".format(
remotes))
return available_machine
def Main(argv):
opts = ProcessArguments(argv)
available_machine = TryAcquireMachine(opts.remote)
executer = DejagnuExecuter(
misc.GetRoot(argv[0])[0], opts.mount, opts.chromeos_root,
available_machine._name, opts.board, opts.flags,
opts.keep_intermediate_files, opts.tools, opts.cleanup)
# Return value is a 3- or 4-element tuple
# element#1 - exit code
# element#2 - stdout
# element#3 - stderr
# element#4 - exception infor
# Some other scripts need these detailed information.
ret = (1, '', '')
try:
executer.SetupTestingDir()
executer.PrepareTestingRsaKeys()
executer.PrepareTestFiles()
executer.PrepareGcc()
executer.MakeCheck()
ret = executer.ValidateFailures()
except Exception as e:
# At least log the exception on console.
print e
# The #4 element encodes the runtime exception.
ret = (1, '', '', 'Exception happened during execution: \n' + str(e))
finally:
available_machine.Unlock(exclusive=True)
executer.CleanupIntermediateFiles()
executer.Cleanup()
return ret
if __name__ == '__main__':
retval = Main(sys.argv)[0]
sys.exit(retval)