# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Utilities for launching and accessing ChromeOS buildbots."""
from __future__ import print_function
import ast
import json
import os
import re
import time
from cros_utils import command_executer
from cros_utils import logger
INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot.
SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot.
# Some of our slower builders (llmv-next) are taking more
# than 8 hours. So, increase this TIME_OUT to 9 hours.
TIME_OUT = 32400 # Decide the build is dead or will never finish
class BuildbotTimeout(Exception):
"""Exception to throw when a buildbot operation timesout."""
pass
def RunCommandInPath(path, cmd):
ce = command_executer.GetCommandExecuter()
cwd = os.getcwd()
os.chdir(path)
status, stdout, stderr = ce.RunCommandWOutput(cmd, print_to_console=False)
os.chdir(cwd)
return status, stdout, stderr
def PeekTrybotImage(chromeos_root, buildbucket_id):
"""Get the artifact URL of a given tryjob.
Args:
buildbucket_id: buildbucket-id
chromeos_root: root dir of chrome os checkout
Returns:
(status, url) where status can be 'pass', 'fail', 'running',
and url looks like:
gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789
"""
command = (
'cros buildresult --report json --buildbucket-id %s' % buildbucket_id)
rc, out, _ = RunCommandInPath(chromeos_root, command)
# Current implementation of cros buildresult returns fail when a job is still
# running.
if rc != 0:
return ('running', None)
results = json.loads(out)[buildbucket_id]
return (results['status'], results['artifacts_url'].rstrip('/'))
def ParseTryjobBuildbucketId(msg):
"""Find the buildbucket-id in the messages from `cros tryjob`.
Args:
msg: messages from `cros tryjob`
Returns:
buildbucket-id, which will be passed to `cros buildresult`
"""
output_list = ast.literal_eval(msg)
output_dict = output_list[0]
if 'buildbucket_id' in output_dict:
return output_dict['buildbucket_id']
return None
def SubmitTryjob(chromeos_root,
buildbot_name,
patch_list,
tryjob_flags=None,
build_toolchain=False):
"""Calls `cros tryjob ...`
Args:
chromeos_root: the path to the ChromeOS root, needed for finding chromite
and launching the buildbot.
buildbot_name: the name of the buildbot queue, such as lumpy-release or
daisy-paladin.
patch_list: a python list of the patches, if any, for the buildbot to use.
tryjob_flags: See cros tryjob --help for available options.
build_toolchain: builds and uses the latest toolchain, rather than the
prebuilt one in SDK.
Returns:
buildbucket id
"""
patch_arg = ''
if patch_list:
for p in patch_list:
patch_arg = patch_arg + ' -g ' + repr(p)
if not tryjob_flags:
tryjob_flags = []
if build_toolchain:
tryjob_flags.append('--latest-toolchain')
tryjob_flags = ' '.join(tryjob_flags)
# Launch buildbot with appropriate flags.
build = buildbot_name
command = ('cros tryjob --yes --json --nochromesdk %s %s %s' %
(tryjob_flags, patch_arg, build))
print('CMD: %s' % command)
_, out, _ = RunCommandInPath(chromeos_root, command)
buildbucket_id = ParseTryjobBuildbucketId(out)
print('buildbucket_id: %s' % repr(buildbucket_id))
if not buildbucket_id:
logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
'%s' % command)
return buildbucket_id
def GetTrybotImage(chromeos_root,
buildbot_name,
patch_list,
tryjob_flags=None,
build_toolchain=False,
async=False):
"""Launch buildbot and get resulting trybot artifact name.
This function launches a buildbot with the appropriate flags to
build the test ChromeOS image, with the current ToT mobile compiler. It
checks every 10 minutes to see if the trybot has finished. When the trybot
has finished, it parses the resulting report logs to find the trybot
artifact (if one was created), and returns that artifact name.
Args:
chromeos_root: the path to the ChromeOS root, needed for finding chromite
and launching the buildbot.
buildbot_name: the name of the buildbot queue, such as lumpy-release or
daisy-paladin.
patch_list: a python list of the patches, if any, for the buildbot to use.
tryjob_flags: See cros tryjob --help for available options.
build_toolchain: builds and uses the latest toolchain, rather than the
prebuilt one in SDK.
async: don't wait for artifacts; just return the buildbucket id
Returns:
(buildbucket id, partial image url) e.g.
(8952271933586980528, trybot-elm-release-tryjob/R67-10480.0.0-b2373596)
"""
buildbucket_id = SubmitTryjob(chromeos_root, buildbot_name, patch_list,
tryjob_flags, build_toolchain)
if async:
return buildbucket_id, ' '
# The trybot generally takes more than 2 hours to finish.
# Wait two hours before polling the status.
time.sleep(INITIAL_SLEEP_TIME)
elapsed = INITIAL_SLEEP_TIME
status = 'running'
image = ''
while True:
status, image = PeekTrybotImage(chromeos_root, buildbucket_id)
if status == 'running':
if elapsed > TIME_OUT:
logger.GetLogger().LogFatal(
'Unable to get build result for target %s.' % buildbot_name)
else:
wait_msg = 'Unable to find build result; job may be running.'
logger.GetLogger().LogOutput(wait_msg)
logger.GetLogger().LogOutput('{0} minutes elapsed.'.format(elapsed / 60))
logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
time.sleep(SLEEP_TIME)
elapsed += SLEEP_TIME
else:
break
if not buildbot_name.endswith('-toolchain') and status == 'fail':
# For rotating testers, we don't care about their status
# result, because if any HWTest failed it will be non-zero.
#
# The nightly performance tests do not run HWTests, so if
# their status is non-zero, we do care. In this case
# non-zero means the image itself probably did not build.
image = ''
if not image:
logger.GetLogger().LogError(
'Trybot job (buildbucket id: %s) failed with'
'status %s; no trybot image generated. ' % (buildbucket_id, status))
else:
# Convert full gs path to what crosperf expects. For example, convert
# gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789
# to
# trybot-elm-release-tryjob/R67-10468.0.0-b20789
image = '/'.join(image.split('/')[-2:])
logger.GetLogger().LogOutput("image is '%s'" % image)
logger.GetLogger().LogOutput('status is %s' % status)
return buildbucket_id, image
def DoesImageExist(chromeos_root, build):
"""Check if the image for the given build exists."""
ce = command_executer.GetCommandExecuter()
command = ('gsutil ls gs://chromeos-image-archive/%s'
'/chromiumos_test_image.tar.xz' % (build))
ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False)
return not ret
def WaitForImage(chromeos_root, build):
"""Wait for an image to be ready."""
elapsed_time = 0
while elapsed_time < TIME_OUT:
if DoesImageExist(chromeos_root, build):
return
logger.GetLogger().LogOutput(
'Image %s not ready, waiting for 10 minutes' % build)
time.sleep(SLEEP_TIME)
elapsed_time += SLEEP_TIME
logger.GetLogger().LogOutput(
'Image %s not found, waited for %d hours' % (build, (TIME_OUT / 3600)))
raise BuildbotTimeout('Timeout while waiting for image %s' % build)
def GetLatestImage(chromeos_root, path):
"""Get latest image"""
fmt = re.compile(r'R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)')
ce = command_executer.GetCommandExecuter()
command = ('gsutil ls gs://chromeos-image-archive/%s' % path)
_, out, _ = ce.ChrootRunCommandWOutput(
chromeos_root, command, print_to_console=False)
candidates = [l.split('/')[-2] for l in out.split()]
candidates = map(fmt.match, candidates)
candidates = [[int(r) for r in m.group(1, 2, 3, 4)] for m in candidates if m]
candidates.sort(reverse=True)
for c in candidates:
build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3])
if DoesImageExist(chromeos_root, build):
return build