#!/usr/bin/python
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Automatically update the afe_stable_versions table.
This command updates the stable repair version for selected boards
in the lab. For each board, if the version that Omaha is serving
on the Beta channel for the board is more recent than the current
stable version in the AFE database, then the AFE is updated to use
the version on Omaha.
The upgrade process is applied to every "managed board" in the test
lab. Generally, a managed board is a board with both spare and
critical scheduling pools.
See `autotest_lib.site_utils.lab_inventory` for the full definition
of "managed board".
The command supports a `--dry-run` option that reports changes that
would be made, without making the actual RPC calls to change the
database.
"""
import argparse
import logging
import common
from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
from autotest_lib.site_utils import lab_inventory
from autotest_lib.site_utils import loglib
from autotest_lib.site_utils.stable_images import build_data
from chromite.lib import ts_mon_config
from chromite.lib import metrics
# _DEFAULT_BOARD - The distinguished board name used to identify a
# stable version mapping that is used for any board without an explicit
# mapping of its own.
#
# _DEFAULT_VERSION_TAG - A string used to signify that there is no
# mapping for a board, in other words, the board is mapped to the
# default version.
#
_DEFAULT_BOARD = 'DEFAULT'
_DEFAULT_VERSION_TAG = '(default)'
_METRICS_PREFIX = 'chromeos/autotest/assign_stable_images'
class _VersionUpdater(object):
"""
Class to report and apply version changes.
This class is responsible for the low-level logic of applying
version upgrades and reporting them as command output.
This class exists to solve two problems:
1. To distinguish "normal" vs. "dry-run" modes. Each mode has a
subclass; methods that perform actual AFE updates are
implemented for the normal mode subclass only.
2. To provide hooks for unit tests. The unit tests override both
the reporting and modification behaviors, in order to test the
higher level logic that decides what changes are needed.
Methods meant merely to report changes to command output have names
starting with "report" or "_report". Methods that are meant to
change the AFE in normal mode have names starting with "_do"
"""
def __init__(self, afe, dry_run):
"""Initialize us.
@param afe: A frontend.AFE object.
@param dry_run: A boolean indicating whether to execute in dry run mode.
No updates are persisted to the afe in dry run.
"""
self._dry_run = dry_run
image_types = [afe.CROS_IMAGE_TYPE, afe.FIRMWARE_IMAGE_TYPE]
self._version_maps = {
image_type: afe.get_stable_version_map(image_type)
for image_type in image_types
}
self._cros_map = self._version_maps[afe.CROS_IMAGE_TYPE]
self._selected_map = None
def select_version_map(self, image_type):
"""
Select an AFE version map object based on `image_type`.
This creates and remembers an AFE version mapper object to be
used for making changes in normal mode.
@param image_type Image type parameter for the version mapper
object.
"""
self._selected_map = self._version_maps[image_type]
return self._selected_map.get_all_versions()
def report_default_changed(self, old_default, new_default):
"""
Report that the default version mapping is changing.
This merely reports a text description of the pending change
without executing it.
@param old_default The original default version.
@param new_default The new default version to be applied.
"""
logging.debug('Default %s -> %s', old_default, new_default)
def _report_board_changed(self, board, old_version, new_version):
"""
Report a change in one board's assigned version mapping.
This merely reports a text description of the pending change
without executing it.
@param board The board with the changing version.
@param old_version The original version mapped to the board.
@param new_version The new version to be applied to the board.
"""
logging.debug(' %-22s %s -> %s', board, old_version, new_version)
def report_board_unchanged(self, board, old_version):
"""
Report that a board's version mapping is unchanged.
This reports that a board has a non-default mapping that will be
unchanged.
@param board The board that is not changing.
@param old_version The board's version mapping.
"""
self._report_board_changed(board, '(no change)', old_version)
def _do_set_mapping(self, board, new_version):
"""
Change one board's assigned version mapping.
@param board The board with the changing version.
@param new_version The new version to be applied to the board.
"""
if self._dry_run:
logging.info('DRYRUN: Would have set %s version to %s',
board, new_version)
else:
self._selected_map.set_version(board, new_version)
def _do_delete_mapping(self, board):
"""
Delete one board's assigned version mapping.
@param board The board with the version to be deleted.
"""
if self._dry_run:
logging.info('DRYRUN: Would have deleted version for %s', board)
else:
self._selected_map.delete_version(board)
def set_mapping(self, board, old_version, new_version):
"""
Change and report a board version mapping.
@param board The board with the changing version.
@param old_version The original version mapped to the board.
@param new_version The new version to be applied to the board.
"""
self._report_board_changed(board, old_version, new_version)
self._do_set_mapping(board, new_version)
def upgrade_default(self, new_default):
"""
Apply a default version change.
@param new_default The new default version to be applied.
"""
self._do_set_mapping(_DEFAULT_BOARD, new_default)
def delete_mapping(self, board, old_version):
"""
Delete a board version mapping, and report the change.
@param board The board with the version to be deleted.
@param old_version The board's verson prior to deletion.
"""
assert board != _DEFAULT_BOARD
self._report_board_changed(board,
old_version,
_DEFAULT_VERSION_TAG)
self._do_delete_mapping(board)
def _get_upgrade_versions(cros_versions, omaha_versions, boards):
"""
Get the new stable versions to which we should update.
The new versions are returned as a tuple of a dictionary mapping
board names to versions, plus a new default board setting. The
new default is determined as the most commonly used version
across the given boards.
The new dictionary will have a mapping for every board in `boards`.
That mapping will be taken from `cros_versions`, unless the board has
a mapping in `omaha_versions` _and_ the omaha version is more recent
than the AFE version.
@param cros_versions The current board->version mappings in the
AFE.
@param omaha_versions The current board->version mappings from
Omaha for the Beta channel.
@param boards Set of boards to be upgraded.
@return Tuple of (mapping, default) where mapping is a dictionary
mapping boards to versions, and default is a version string.
"""
upgrade_versions = {}
version_counts = {}
afe_default = cros_versions[_DEFAULT_BOARD]
for board in boards:
version = build_data.get_omaha_upgrade(
omaha_versions, board,
cros_versions.get(board, afe_default))
upgrade_versions[board] = version
version_counts.setdefault(version, 0)
version_counts[version] += 1
return (upgrade_versions,
max(version_counts.items(), key=lambda x: x[1])[0])
def _get_firmware_upgrades(cros_versions):
"""
Get the new firmware versions to which we should update.
@param cros_versions Current board->cros version mappings in the
AFE.
@return A dictionary mapping boards/models to firmware upgrade versions.
If the build is unibuild, the key is a model name; else, the key
is a board name.
"""
firmware_upgrades = {}
for board, version in cros_versions.iteritems():
firmware_upgrades.update(
build_data.get_firmware_versions(board, version))
return firmware_upgrades
def _apply_cros_upgrades(updater, old_versions, new_versions,
new_default):
"""
Change CrOS stable version mappings in the AFE.
The input `old_versions` dictionary represents the content of the
`afe_stable_versions` database table; it contains mappings for a
default version, plus exceptions for boards with non-default
mappings.
The `new_versions` dictionary contains a mapping for every board,
including boards that will be mapped to the new default version.
This function applies the AFE changes necessary to produce the new
AFE mappings indicated by `new_versions` and `new_default`. The
changes are ordered so that at any moment, every board is mapped
either according to the old or the new mapping.
@param updater Instance of _VersionUpdater responsible for
making the actual database changes.
@param old_versions The current board->version mappings in the
AFE.
@param new_versions New board->version mappings obtained by
applying Beta channel upgrades from Omaha.
@param new_default The new default build for the AFE.
"""
old_default = old_versions[_DEFAULT_BOARD]
if old_default != new_default:
updater.report_default_changed(old_default, new_default)
logging.info('Applying stable version changes:')
default_count = 0
for board, new_build in new_versions.items():
if new_build == new_default:
default_count += 1
elif board in old_versions and new_build == old_versions[board]:
updater.report_board_unchanged(board, new_build)
else:
old_build = old_versions.get(board)
if old_build is None:
old_build = _DEFAULT_VERSION_TAG
updater.set_mapping(board, old_build, new_build)
if old_default != new_default:
updater.upgrade_default(new_default)
for board, new_build in new_versions.items():
if new_build == new_default and board in old_versions:
updater.delete_mapping(board, old_versions[board])
logging.info('%d boards now use the default mapping', default_count)
def _apply_firmware_upgrades(updater, old_versions, new_versions):
"""
Change firmware version mappings in the AFE.
The input `old_versions` dictionary represents the content of the
firmware mappings in the `afe_stable_versions` database table.
There is no default version; missing boards simply have no current
version.
This function applies the AFE changes necessary to produce the new
AFE mappings indicated by `new_versions`.
TODO(jrbarnette) This function ought to remove any mapping not found
in `new_versions`. However, in theory, that's only needed to
account for boards that are removed from the lab, and that hasn't
happened yet.
@param updater Instance of _VersionUpdater responsible for
making the actual database changes.
@param old_versions The current board->version mappings in the
AFE.
@param new_versions New board->version mappings obtained by
applying Beta channel upgrades from Omaha.
"""
unchanged = 0
no_version = 0
for board, new_firmware in new_versions.items():
if new_firmware is None:
no_version += 1
elif board not in old_versions:
updater.set_mapping(board, '(nothing)', new_firmware)
else:
old_firmware = old_versions[board]
if new_firmware != old_firmware:
updater.set_mapping(board, old_firmware, new_firmware)
else:
unchanged += 1
logging.info('%d boards have no firmware mapping', no_version)
logging.info('%d boards are unchanged', unchanged)
def _assign_stable_images(arguments):
afe = frontend_wrappers.RetryingAFE(server=arguments.web)
updater = _VersionUpdater(afe, dry_run=arguments.dry_run)
cros_versions = updater.select_version_map(afe.CROS_IMAGE_TYPE)
omaha_versions = build_data.get_omaha_version_map()
upgrade_versions, new_default = (
_get_upgrade_versions(cros_versions, omaha_versions,
lab_inventory.get_managed_boards(afe)))
_apply_cros_upgrades(updater, cros_versions,
upgrade_versions, new_default)
logging.info('Applying firmware updates.')
fw_versions = updater.select_version_map(afe.FIRMWARE_IMAGE_TYPE)
firmware_upgrades = _get_firmware_upgrades(upgrade_versions)
_apply_firmware_upgrades(updater, fw_versions, firmware_upgrades)
def main():
"""Standard main routine."""
parser = argparse.ArgumentParser(
description='Update the stable repair version for all '
'boards')
parser.add_argument('-n', '--dry-run',
action='store_true',
help='print changes without executing them')
loglib.add_logging_options(parser)
# TODO(crbug/888046) Make these arguments required once puppet is updated to
# pass them in.
parser.add_argument('--web',
default='cautotest',
help='URL to the AFE to update.')
arguments = parser.parse_args()
loglib.configure_logging_with_args(parser, arguments)
tsmon_args = {
'service_name': parser.prog,
'indirect': False,
'auto_flush': False,
}
if arguments.dry_run:
logging.info('DRYRUN: No changes will be made.')
# metrics will be logged to logging stream anyway.
tsmon_args['debug_file'] = '/dev/null'
try:
with ts_mon_config.SetupTsMonGlobalState(**tsmon_args):
with metrics.SuccessCounter(_METRICS_PREFIX + '/tick',
fields={'afe': arguments.web}):
_assign_stable_images(arguments)
finally:
metrics.Flush()
if __name__ == '__main__':
main()