# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import glob
import logging
import os
import re
import time
from distutils import version
import common
from autotest_lib.client.common_lib import autotemp
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import utils
class ManifestVersionsException(Exception):
"""Base class for exceptions from this package."""
pass
class QueryException(ManifestVersionsException):
"""Raised to indicate a failure while searching for manifests."""
pass
class CloneException(ManifestVersionsException):
"""Raised when `git clone` fails to create the repository."""
pass
def _SystemOutput(command, timeout=None, args=()):
"""Shell out to run a command, expecting data on stderr. Return stdout.
Shells out to run |command|, optionally passing escaped |args|.
Instead of logging stderr at ERROR level, will log at default
stdout log level. Normal stdout is returned.
@param command: command string to execute.
@param timeout: time limit in seconds before attempting to kill the
running process. The function will take a few seconds longer
than 'timeout' to complete if it has to kill the process.
@param args: sequence of strings of arguments to be given to the command
inside " quotes after they have been escaped for that; each
element in the sequence will be given as a separate command
argument.
@return a string with the stdout output of the command.
"""
out = utils.run(command, timeout=timeout, ignore_status=False,
stderr_is_expected=True, args=args).stdout
return out.rstrip('\n')
def _System(command, timeout=None):
"""Run a command, expecting data on stderr.
@param command: command string to execute.
@param timeout: timeout in seconds
"""
utils.run(command, timeout=timeout, ignore_status=False,
stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS,
stderr_is_expected=True)
class ManifestVersions(object):
"""Class to allow discovery of manifests for new successful CrOS builds.
Manifest versions is a repository that contains information on which
builds passed/failed. This class is responsible for keeping a temp
copy of the repository up to date.
@var _CLONE_RETRY_SECONDS: Number of seconds to wait before retrying
a failed `git clone` operation.
@var _CLONE_MAX_RETRIES: Maximum number of times to retry a failed
a failed `git clone` operation.
@var _MANIFEST_VERSIONS_URL: URL of the internal manifest-versions git repo.
@var _BOARD_MANIFEST_GLOB_PATTERN: pattern for shell glob for passed-build
manifests for a given board.
@var _BOARD_MANIFEST_RE_PATTERN: pattern for regex that parses paths to
manifests for a given board.
@var _git: absolute path of git binary.
@var _tempdir: a scoped tempdir. Will be destroyed on instance deletion.
"""
_CLONE_RETRY_SECONDS = 5 * 60
_CLONE_MAX_RETRIES = 60 * 60 / _CLONE_RETRY_SECONDS
_MANIFEST_VERSIONS_URL = ('https://chrome-internal-review.googlesource.com/'
'chromeos/manifest-versions.git')
_ANY_MANIFEST_GLOB_PATTERN = 'build-name/*/pass/'
_BOARD_MANIFEST_GLOB_PATTERN = 'build-name/%s-*/pass/'
_BOARD_MANIFEST_RE_PATTERN = (r'build-name/%s-([^-]+)'
r'(?:-group)?/pass/(\d+)/([0-9.]+)\.xml')
_BOARD_BRANCH_MANIFEST_GLOB_PATTERN = 'build-name/%s-%s/pass/'
def __init__(self, tmp_repo_dir=None):
"""Create a manifest versions manager.
@param tmp_repo_dir: For use in testing, if one does not wish to repeatedly
clone the manifest versions repo that is currently a few GB in size.
"""
self._git = _SystemOutput('which git')
if tmp_repo_dir:
self._tempdir = autotemp.dummy_dir(tmp_repo_dir)
else:
self._tempdir = autotemp.tempdir(unique_id='_suite_scheduler')
def AnyManifestsSinceRev(self, revision):
"""Determine if any builds passed since git |revision|.
@param revision: the git revision to look back to.
@return True if any builds have passed; False otherwise.
"""
manifest_paths = self._ExpandGlobMinusPrefix(
self._tempdir.name, self._ANY_MANIFEST_GLOB_PATTERN)
if not manifest_paths:
logging.error('No paths to check for manifests???')
return False
logging.info('Checking if any manifests landed since %s', revision)
log_cmd = self._BuildCommand('log',
revision + '..HEAD',
'--pretty="format:%H"',
'--',
' '.join(manifest_paths))
return _SystemOutput(log_cmd).strip() != ''
def Initialize(self):
"""Set up internal state. Must be called before other methods.
Clone manifest-versions.git into tempdir managed by this instance.
"""
# If gerrit goes down during suite_scheduler operation,
# we'll enter a loop like the following:
# 1. suite_scheduler fails executing some `git` command.
# 2. The failure is logged at ERROR level, causing an
# e-mail notification of the failure.
# 3. suite_scheduler terminates.
# 4. Upstart respawns suite_scheduler.
# 5. suite_scheduler comes here to restart with a new
# manifest-versions repo.
# 6. `git clone` fails, and we go back to step 2.
#
# We want to rate limit the e-mail notifications, so we
# retry failed `git clone` operations for a time before we
# finally give up.
retry_count = 0
msg = None
while retry_count <= self._CLONE_MAX_RETRIES:
if retry_count:
time.sleep(self._CLONE_RETRY_SECONDS)
retry_count += 1
try:
logging.debug('Cloning manifest-versions.git,'
' attempt %d.', retry_count)
self._Clone()
logging.debug('manifest-versions.git cloned.')
return
except error.CmdError as e:
msg = str(e)
logging.debug('Clone failed: %s', msg)
raise CloneException('Failed to clone %s after %d attempts: %s' %
(self._MANIFEST_VERSIONS_URL, retry_count, msg))
def ManifestsSinceDate(self, since_date, board):
"""Return map of branch:manifests for |board| since |since_date|.
To fully specify a 'branch', one needs both the type and the numeric
milestone the branch was cut for, e.g. ('release', '19') or
('factory', '17').
@param since_date: a datetime object, return all manifest files
since |since_date|
@param board: the board whose manifests we want to check for.
@return {(branch_type, milestone): [manifests, oldest, to, newest]}
"""
return self._GetManifests(
re.compile(self._BOARD_MANIFEST_RE_PATTERN % board),
self._QueryManifestsSinceDate(since_date, board))
def ManifestsSinceRev(self, rev, board):
"""Return map of branch:manifests for |board| since git |rev|.
To fully specify a 'branch', one needs both the type and the numeric
milestone the branch was cut for, e.g. ('release', '19') or
('factory', '17').
@param rev: return all manifest files from |rev| up to HEAD.
@param board: the board whose manifests we want to check for.
@return {(branch_type, milestone): [manifests, oldest, to, newest]}
"""
return self._GetManifests(
re.compile(self._BOARD_MANIFEST_RE_PATTERN % board),
self._QueryManifestsSinceRev(rev, board))
def GetLatestManifest(self, board, build_type, milestone=None):
"""Get the latest manifest of a given board and type.
@param board: the board whose manifests we want to check for.
@param build_type: Type of a build, e.g., release, factory or firmware.
@param milestone: Milestone to look for the latest manifest. Default to
None, i.e., use the latest milestone.
@return: (milestone, manifest), e.g., (46, '7268.0.0')
"""
milestones_folder = os.path.join(
self._tempdir.name,
self._BOARD_BRANCH_MANIFEST_GLOB_PATTERN % (board, build_type))
if not milestone:
try:
milestone_names = os.listdir(milestones_folder)
except OSError:
milestone_names = None
if not milestone_names:
raise QueryException('There is no milestone existed in %s.' %
milestones_folder)
milestone = max([m for m in milestone_names if m.isdigit()])
manifests_folder = os.path.join(milestones_folder, str(milestone))
manifests = [m.strip('.xml') for m in os.listdir(manifests_folder)
if m.endswith('.xml')]
if not manifests:
raise QueryException('There is no build existed in %s.' %
manifests_folder)
manifests.sort(key=version.LooseVersion)
return milestone, manifests[-1]
def _GetManifests(self, matcher, manifest_paths):
"""Parse a list of manifest_paths into a map of branch:manifests.
Given a regexp |matcher| and a list of paths to manifest files,
parse the paths and build up a map of branches to manifests of
builds on those branches.
To fully specify a 'branch', one needs both the type and the numeric
milestone the branch was cut for, e.g. ('release', '19') or
('factory', '17').
@param matcher: a compiled regexp that can be used to parse info
out of the path to a manifest file.
@param manifest_paths: an iterable of paths to manifest files.
@return {(branch_type, milestone): [manifests, oldest, to, newest]}
"""
branch_manifests = {}
for manifest_path in manifest_paths:
logging.debug('parsing manifest path %s', manifest_path)
match = matcher.match(manifest_path)
if not match:
logging.warning('Failed to parse path %s, regex: %s',
manifest_path, matcher.pattern)
continue
groups = match.groups()
config_type, milestone, manifest = groups
branch = branch_manifests.setdefault((config_type, milestone), [])
branch.append(manifest)
for manifest_list in branch_manifests.itervalues():
manifest_list.sort(key=version.LooseVersion)
return branch_manifests
def GetCheckpoint(self):
"""Return the latest checked-out git revision in manifest-versions.git.
@return the git hash of the latest git revision.
"""
return _SystemOutput(self._BuildCommand('log',
'--pretty="format:%H"',
'--max-count=1')).strip()
def Update(self):
"""Get latest manifest information."""
return _System(self._BuildCommand('pull'))
def _BuildCommand(self, command, *args):
"""Build a git CLI |command|, passing space-delineated |args|.
@param command: the git sub-command to use.
@param args: args for the git sub-command. Will be space-delineated.
@return a string with the above formatted into it.
"""
return '%s --git-dir=%s --work-tree=%s %s %s' % (
self._git, os.path.join(self._tempdir.name, '.git'),
self._tempdir.name, command, ' '.join(args))
def _Clone(self):
"""Clone self._MANIFEST_VERSIONS_URL into a local temp dir."""
# Can't use --depth here because the internal gerrit server doesn't
# support it. Wish we could. http://crosbug.com/29047
# Also, note that --work-tree and --git-dir interact oddly with
# 'git clone', so we don't use them.
_System('%s clone %s %s' % (self._git,
self._MANIFEST_VERSIONS_URL,
self._tempdir.name))
def _ShowCmd(self):
"""Return a git command that shows file names added by commits."""
return self._BuildCommand('show',
'--pretty="format:"',
'--name-only',
'--diff-filter=A')
def _QueryManifestsSinceRev(self, git_rev, board):
"""Get manifest filenames for |board|, since |git_rev|.
@param git_rev: check for manifests newer than this git commit.
@param board: the board whose manifests we want to check for.
@return whitespace-delineated
@raise QueryException if errors occur.
"""
return self._QueryManifestsSince(git_rev + '..HEAD', board)
def _QueryManifestsSinceDate(self, since_date, board):
"""Return list of manifest files for |board| since |since_date|.
@param sync_date: a datetime object, return all manifest files
since |since_date|.
@param board: the board whose manifests we want to check for.
@raise QueryException if errors occur.
"""
return self._QueryManifestsSince('--since="%s"' % since_date, board)
def _ExpandGlobMinusPrefix(self, prefix, path_glob):
"""Expand |path_glob| under dir |prefix|, then remove |prefix|.
Path-concatenate prefix and path_glob, then expand the resulting glob.
Take the results and remove |prefix| (and path separator) from each.
Return the resulting list.
Assuming /tmp/foo/baz and /tmp/bar/baz both exist,
_ExpandGlobMinusPrefix('/tmp', '*/baz') # ['bar/baz', 'foo/baz']
@param prefix: a path under which to expand |path_glob|.
@param path_glob: the glob to expand.
@return a list of paths relative to |prefix|, based on |path_glob|.
"""
full_glob = os.path.join(prefix, path_glob)
return [p[len(prefix)+1:] for p in glob.iglob(full_glob)]
def _QueryManifestsSince(self, since_spec, board):
"""Return list of manifest files for |board|, since |since_spec|.
@param since_spec: a formatted arg to git log that specifies a starting
point to list commits from, e.g.
'--since="2 days ago"' or 'd34db33f..'
@param board: the board whose manifests we want to check for.
@raise QueryException if git log or git show errors occur.
"""
manifest_paths = self._ExpandGlobMinusPrefix(
self._tempdir.name, self._BOARD_MANIFEST_GLOB_PATTERN % board)
log_cmd = self._BuildCommand('log',
since_spec,
'--pretty="format:%H"',
'--',
' '.join(manifest_paths))
try:
# If we pass nothing to git show, we get unexpected results.
# So, return early if git log is going to give us nothing.
if not manifest_paths or not _SystemOutput(log_cmd):
return []
manifests = _SystemOutput('%s|xargs %s' % (log_cmd,
self._ShowCmd()))
except (IOError, OSError) as e:
raise QueryException(e)
logging.debug('found %s', manifests)
return [m for m in re.split('\s+', manifests) if m]