#!/usr/bin/env python2
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# NOTE: This file is NOT under GPL. See above.
"""Queries buildbot through the json interface.
"""
from __future__ import print_function
__author__ = 'maruel@chromium.org'
__version__ = '1.2'
import code
import datetime
import functools
import json
# Pylint recommends we use "from chromite.lib import cros_logging as logging".
# Chromite specific policy message, we want to keep using the standard logging.
# pylint: disable=cros-logging-import
import logging
# pylint: disable=deprecated-module
import optparse
import time
import urllib
import urllib2
import sys
try:
from natsort import natsorted
except ImportError:
# natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted
# after "vm7". Defaults to normal sorting.
natsorted = sorted
# These values are buildbot constants used for Build and BuildStep.
# This line was copied from master/buildbot/status/builder.py.
SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6)
## Generic node caching code.
class Node(object):
"""Root class for all nodes in the graph.
Provides base functionality for any node in the graph, independent if it has
children or not or if its content can be addressed through an url or needs to
be fetched as part of another node.
self.printable_attributes is only used for self documentation and for str()
implementation.
"""
printable_attributes = []
def __init__(self, parent, url):
self.printable_attributes = self.printable_attributes[:]
if url:
self.printable_attributes.append('url')
url = url.rstrip('/')
if parent is not None:
self.printable_attributes.append('parent')
self.url = url
self.parent = parent
def __str__(self):
return self.to_string()
def __repr__(self):
"""Embeds key if present."""
key = getattr(self, 'key', None)
if key is not None:
return '<%s key=%s>' % (self.__class__.__name__, key)
cached_keys = getattr(self, 'cached_keys', None)
if cached_keys is not None:
return '<%s keys=%s>' % (self.__class__.__name__, cached_keys)
return super(Node, self).__repr__()
def to_string(self, maximum=100):
out = ['%s:' % self.__class__.__name__]
assert not 'printable_attributes' in self.printable_attributes
def limit(txt):
txt = str(txt)
if maximum > 0:
if len(txt) > maximum + 2:
txt = txt[:maximum] + '...'
return txt
for k in sorted(self.printable_attributes):
if k == 'parent':
# Avoid infinite recursion.
continue
out.append(limit(' %s: %r' % (k, getattr(self, k))))
return '\n'.join(out)
def refresh(self):
"""Refreshes the data."""
self.discard()
return self.cache()
def cache(self): # pragma: no cover
"""Caches the data."""
raise NotImplementedError()
def discard(self): # pragma: no cover
"""Discards cached data.
Pretty much everything is temporary except completed Build.
"""
raise NotImplementedError()
class AddressableBaseDataNode(Node): # pylint: disable=W0223
"""A node that contains a dictionary of data that can be fetched with an url.
The node is directly addressable. It also often can be fetched by the parent.
"""
printable_attributes = Node.printable_attributes + ['data']
def __init__(self, parent, url, data):
super(AddressableBaseDataNode, self).__init__(parent, url)
self._data = data
@property
def cached_data(self):
return self._data
@property
def data(self):
self.cache()
return self._data
def cache(self):
if self._data is None:
self._data = self._readall()
return True
return False
def discard(self):
self._data = None
def read(self, suburl):
assert self.url, self.__class__.__name__
url = self.url
if suburl:
url = '%s/%s' % (self.url, suburl)
return self.parent.read(url)
def _readall(self):
return self.read('')
class AddressableDataNode(AddressableBaseDataNode): # pylint: disable=W0223
"""Automatically encodes the url."""
def __init__(self, parent, url, data):
super(AddressableDataNode, self).__init__(parent, urllib.quote(url), data)
class NonAddressableDataNode(Node): # pylint: disable=W0223
"""A node that cannot be addressed by an unique url.
The data comes directly from the parent.
"""
def __init__(self, parent, subkey):
super(NonAddressableDataNode, self).__init__(parent, None)
self.subkey = subkey
@property
def cached_data(self):
if self.parent.cached_data is None:
return None
return self.parent.cached_data[self.subkey]
@property
def data(self):
return self.parent.data[self.subkey]
def cache(self):
self.parent.cache()
def discard(self): # pragma: no cover
"""Avoid invalid state when parent recreate the object."""
raise AttributeError('Call parent discard() instead')
class VirtualNodeList(Node):
"""Base class for every node that has children.
Adds partial supports for keys and iterator functionality. 'key' can be a
string or a int. Not to be used directly.
"""
printable_attributes = Node.printable_attributes + ['keys']
def __init__(self, parent, url):
super(VirtualNodeList, self).__init__(parent, url)
# Keeps the keys independently when ordering is needed.
self._is_cached = False
self._has_keys_cached = False
def __contains__(self, key):
"""Enables 'if i in obj:'."""
return key in self.keys
def __iter__(self):
"""Enables 'for i in obj:'. It returns children."""
self.cache_keys()
for key in self.keys:
yield self[key]
def __len__(self):
"""Enables 'len(obj)' to get the number of childs."""
return len(self.keys)
def discard(self):
"""Discards data.
The default behavior is to not invalidate cached keys. The only place where
keys need to be invalidated is with Builds.
"""
self._is_cached = False
self._has_keys_cached = False
@property
def cached_children(self): # pragma: no cover
"""Returns an iterator over the children that are cached."""
raise NotImplementedError()
@property
def cached_keys(self): # pragma: no cover
raise NotImplementedError()
@property
def keys(self): # pragma: no cover
"""Returns the keys for every children."""
raise NotImplementedError()
def __getitem__(self, key): # pragma: no cover
"""Returns a child, without fetching its data.
The children could be invalid since no verification is done.
"""
raise NotImplementedError()
def cache(self): # pragma: no cover
"""Cache all the children."""
raise NotImplementedError()
def cache_keys(self): # pragma: no cover
"""Cache all children's keys."""
raise NotImplementedError()
class NodeList(VirtualNodeList): # pylint: disable=W0223
"""Adds a cache of the keys."""
def __init__(self, parent, url):
super(NodeList, self).__init__(parent, url)
self._keys = []
@property
def cached_keys(self):
return self._keys
@property
def keys(self):
self.cache_keys()
return self._keys
class NonAddressableNodeList(VirtualNodeList): # pylint: disable=W0223
"""A node that contains children but retrieves all its data from its parent.
I.e. there's no url to get directly this data.
"""
# Child class object for children of this instance. For example, BuildSteps
# has BuildStep children.
_child_cls = None
def __init__(self, parent, subkey):
super(NonAddressableNodeList, self).__init__(parent, None)
self.subkey = subkey
assert (not isinstance(self._child_cls, NonAddressableDataNode) and
issubclass(self._child_cls, NonAddressableDataNode)), (
self._child_cls.__name__)
@property
def cached_children(self):
if self.parent.cached_data is not None:
for i in xrange(len(self.parent.cached_data[self.subkey])):
yield self[i]
@property
def cached_data(self):
if self.parent.cached_data is None:
return None
return self.parent.data.get(self.subkey, None)
@property
def cached_keys(self):
if self.parent.cached_data is None:
return None
return range(len(self.parent.data.get(self.subkey, [])))
@property
def data(self):
return self.parent.data[self.subkey]
def cache(self):
self.parent.cache()
def cache_keys(self):
self.parent.cache()
def discard(self): # pragma: no cover
"""Do not call.
Avoid infinite recursion by having the caller calls the parent's
discard() explicitely.
"""
raise AttributeError('Call parent discard() instead')
def __iter__(self):
"""Enables 'for i in obj:'. It returns children."""
if self.data:
for i in xrange(len(self.data)):
yield self[i]
def __getitem__(self, key):
"""Doesn't cache the value, it's not needed.
TODO(maruel): Cache?
"""
if isinstance(key, int) and key < 0:
key = len(self.data) + key
# pylint: disable=E1102
return self._child_cls(self, key)
class AddressableNodeList(NodeList):
"""A node that has children that can be addressed with an url."""
# Child class object for children of this instance. For example, Builders has
# Builder children and Builds has Build children.
_child_cls = None
def __init__(self, parent, url):
super(AddressableNodeList, self).__init__(parent, url)
self._cache = {}
assert (not isinstance(self._child_cls, AddressableDataNode) and
issubclass(self._child_cls, AddressableDataNode)), (
self._child_cls.__name__)
@property
def cached_children(self):
for item in self._cache.itervalues():
if item.cached_data is not None:
yield item
@property
def cached_keys(self):
return self._cache.keys()
def __getitem__(self, key):
"""Enables 'obj[i]'."""
if self._has_keys_cached and not key in self._keys:
raise KeyError(key)
if not key in self._cache:
# Create an empty object.
self._create_obj(key, None)
return self._cache[key]
def cache(self):
if not self._is_cached:
data = self._readall()
for key in sorted(data):
self._create_obj(key, data[key])
self._is_cached = True
self._has_keys_cached = True
def cache_partial(self, children):
"""Caches a partial number of children.
This method is more efficient since it does a single request for all the
children instead of one request per children.
It only grab objects not already cached.
"""
# pylint: disable=W0212
if not self._is_cached:
to_fetch = [
child
for child in children
if not (child in self._cache and self._cache[child].cached_data)
]
if to_fetch:
# Similar to cache(). The only reason to sort is to simplify testing.
params = '&'.join('select=%s' % urllib.quote(str(v))
for v in sorted(to_fetch))
data = self.read('?' + params)
for key in sorted(data):
self._create_obj(key, data[key])
def cache_keys(self):
"""Implement to speed up enumeration. Defaults to call cache()."""
if not self._has_keys_cached:
self.cache()
assert self._has_keys_cached
def discard(self):
"""Discards temporary children."""
super(AddressableNodeList, self).discard()
for v in self._cache.itervalues():
v.discard()
def read(self, suburl):
assert self.url, self.__class__.__name__
url = self.url
if suburl:
url = '%s/%s' % (self.url, suburl)
return self.parent.read(url)
def _create_obj(self, key, data):
"""Creates an object of type self._child_cls."""
# pylint: disable=E1102
obj = self._child_cls(self, key, data)
# obj.key and key may be different.
# No need to overide cached data with None.
if data is not None or obj.key not in self._cache:
self._cache[obj.key] = obj
if obj.key not in self._keys:
self._keys.append(obj.key)
def _readall(self):
return self.read('')
class SubViewNodeList(VirtualNodeList): # pylint: disable=W0223
"""A node that shows a subset of children that comes from another structure.
The node is not addressable.
E.g. the keys are retrieved from parent but the actual data comes from
virtual_parent.
"""
def __init__(self, parent, virtual_parent, subkey):
super(SubViewNodeList, self).__init__(parent, None)
self.subkey = subkey
self.virtual_parent = virtual_parent
assert isinstance(self.parent, AddressableDataNode)
assert isinstance(self.virtual_parent, NodeList)
@property
def cached_children(self):
if self.parent.cached_data is not None:
for item in self.keys:
if item in self.virtual_parent.keys:
child = self[item]
if child.cached_data is not None:
yield child
@property
def cached_keys(self):
return (self.parent.cached_data or {}).get(self.subkey, [])
@property
def keys(self):
self.cache_keys()
return self.parent.data.get(self.subkey, [])
def cache(self):
"""Batch request for each child in a single read request."""
if not self._is_cached:
self.virtual_parent.cache_partial(self.keys)
self._is_cached = True
def cache_keys(self):
if not self._has_keys_cached:
self.parent.cache()
self._has_keys_cached = True
def discard(self):
if self.parent.cached_data is not None:
for child in self.virtual_parent.cached_children:
if child.key in self.keys:
child.discard()
self.parent.discard()
super(SubViewNodeList, self).discard()
def __getitem__(self, key):
"""Makes sure the key is in our key but grab it from the virtual parent."""
return self.virtual_parent[key]
def __iter__(self):
self.cache()
return super(SubViewNodeList, self).__iter__()
###############################################################################
## Buildbot-specific code
class Slave(AddressableDataNode):
"""Buildbot slave class."""
printable_attributes = AddressableDataNode.printable_attributes + [
'name',
'key',
'connected',
'version',
]
def __init__(self, parent, name, data):
super(Slave, self).__init__(parent, name, data)
self.name = name
self.key = self.name
# TODO(maruel): Add SlaveBuilders and a 'builders' property.
# TODO(maruel): Add a 'running_builds' property.
@property
def connected(self):
return self.data.get('connected', False)
@property
def version(self):
return self.data.get('version')
class Slaves(AddressableNodeList):
"""Buildbot slaves."""
_child_cls = Slave
printable_attributes = AddressableNodeList.printable_attributes + ['names']
def __init__(self, parent):
super(Slaves, self).__init__(parent, 'slaves')
@property
def names(self):
return self.keys
class BuilderSlaves(SubViewNodeList):
"""Similar to Slaves but only list slaves connected to a specific builder."""
printable_attributes = SubViewNodeList.printable_attributes + ['names']
def __init__(self, parent):
super(BuilderSlaves, self).__init__(parent, parent.parent.parent.slaves,
'slaves')
@property
def names(self):
return self.keys
class BuildStep(NonAddressableDataNode):
"""Class for a buildbot build step."""
printable_attributes = NonAddressableDataNode.printable_attributes + [
'name',
'number',
'start_time',
'end_time',
'duration',
'is_started',
'is_finished',
'is_running',
'result',
'simplified_result',
]
def __init__(self, parent, number):
"""Pre-loaded, since the data is retrieved via the Build object."""
assert isinstance(number, int)
super(BuildStep, self).__init__(parent, number)
self.number = number
@property
def start_time(self):
if self.data.get('times'):
return int(round(self.data['times'][0]))
@property
def end_time(self):
times = self.data.get('times')
if times and len(times) == 2 and times[1]:
return int(round(times[1]))
@property
def duration(self):
if self.start_time:
return (self.end_time or int(round(time.time()))) - self.start_time
@property
def name(self):
return self.data['name']
@property
def is_started(self):
return self.data.get('isStarted', False)
@property
def is_finished(self):
return self.data.get('isFinished', False)
@property
def is_running(self):
return self.is_started and not self.is_finished
@property
def result(self):
result = self.data.get('results')
if result is None:
# results may be 0, in that case with filter=1, the value won't be
# present.
if self.data.get('isFinished'):
result = self.data.get('results', 0)
while isinstance(result, list):
result = result[0]
return result
@property
def simplified_result(self):
"""Returns a simplified 3 state value, True, False or None."""
result = self.result
if result in (SUCCESS, WARNINGS):
return True
elif result in (FAILURE, EXCEPTION, RETRY):
return False
assert result in (None, SKIPPED), (result, self.data)
return None
class BuildSteps(NonAddressableNodeList):
"""Duplicates keys to support lookup by both step number and step name."""
printable_attributes = NonAddressableNodeList.printable_attributes + [
'failed',
]
_child_cls = BuildStep
def __init__(self, parent):
"""Pre-loaded, since the data is retrieved via the Build object."""
super(BuildSteps, self).__init__(parent, 'steps')
@property
def keys(self):
"""Returns the steps name in order."""
return [i['name'] for i in self.data or []]
@property
def failed(self):
"""Shortcuts that lists the step names of steps that failed."""
return [step.name for step in self if step.simplified_result is False]
def __getitem__(self, key):
"""Accept step name in addition to index number."""
if isinstance(key, basestring):
# It's a string, try to find the corresponding index.
for i, step in enumerate(self.data):
if step['name'] == key:
key = i
break
else:
raise KeyError(key)
return super(BuildSteps, self).__getitem__(key)
class Build(AddressableDataNode):
"""Buildbot build info."""
printable_attributes = AddressableDataNode.printable_attributes + [
'key',
'number',
'steps',
'blame',
'reason',
'revision',
'result',
'simplified_result',
'start_time',
'end_time',
'duration',
'slave',
'properties',
'completed',
]
def __init__(self, parent, key, data):
super(Build, self).__init__(parent, str(key), data)
self.number = int(key)
self.key = self.number
self.steps = BuildSteps(self)
@property
def blame(self):
return self.data.get('blame', [])
@property
def builder(self):
"""Returns the Builder object.
Goes up the hierarchy to find the Buildbot.builders[builder] instance.
"""
return self.parent.parent.parent.parent.builders[self.data['builderName']]
@property
def start_time(self):
if self.data.get('times'):
return int(round(self.data['times'][0]))
@property
def end_time(self):
times = self.data.get('times')
if times and len(times) == 2 and times[1]:
return int(round(times[1]))
@property
def duration(self):
if self.start_time:
return (self.end_time or int(round(time.time()))) - self.start_time
@property
def eta(self):
return self.data.get('eta', 0)
@property
def completed(self):
return self.data.get('currentStep') is None
@property
def properties(self):
return self.data.get('properties', [])
@property
def reason(self):
return self.data.get('reason')
@property
def result(self):
result = self.data.get('results')
while isinstance(result, list):
result = result[0]
if result is None and self.steps:
# results may be 0, in that case with filter=1, the value won't be
# present.
result = self.steps[-1].result
return result
@property
def revision(self):
return self.data.get('sourceStamp', {}).get('revision')
@property
def simplified_result(self):
"""Returns a simplified 3 state value, True, False or None."""
result = self.result
if result in (SUCCESS, WARNINGS, SKIPPED):
return True
elif result in (FAILURE, EXCEPTION, RETRY):
return False
assert result is None, (result, self.data)
return None
@property
def slave(self):
"""Returns the Slave object.
Goes up the hierarchy to find the Buildbot.slaves[slave] instance.
"""
return self.parent.parent.parent.parent.slaves[self.data['slave']]
def discard(self):
"""Completed Build isn't discarded."""
if self._data and self.result is None:
assert not self.steps or not self.steps[-1].data.get('isFinished')
self._data = None
class CurrentBuilds(SubViewNodeList):
"""Lists of the current builds."""
def __init__(self, parent):
super(CurrentBuilds, self).__init__(parent, parent.builds, 'currentBuilds')
class PendingBuilds(AddressableDataNode):
"""List of the pending builds."""
def __init__(self, parent):
super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None)
class Builds(AddressableNodeList):
"""Supports iteration.
Recommends using .cache() to speed up if a significant number of builds are
iterated over.
"""
_child_cls = Build
def __init__(self, parent):
super(Builds, self).__init__(parent, 'builds')
def __getitem__(self, key):
"""Support for negative reference and enable retrieving non-cached builds.
e.g. -1 is the last build, -2 is the previous build before the last one.
"""
key = int(key)
if key < 0:
# Convert negative to positive build number.
self.cache_keys()
# Since the negative value can be outside of the cache keys range, use the
# highest key value and calculate from it.
key = max(self._keys) + key + 1
if not key in self._cache:
# Create an empty object.
self._create_obj(key, None)
return self._cache[key]
def __iter__(self):
"""Returns cached Build objects in reversed order.
The most recent build is returned first and then in reverse chronological
order, up to the oldest cached build by the server. Older builds can be
accessed but will trigger significantly more I/O so they are not included by
default in the iteration.
To access the older builds, use self.iterall() instead.
"""
self.cache()
return reversed(self._cache.values())
def iterall(self):
"""Returns Build objects in decreasing order unbounded up to build 0.
The most recent build is returned first and then in reverse chronological
order. Older builds can be accessed and will trigger significantly more I/O
so use this carefully.
"""
# Only cache keys here.
self.cache_keys()
if self._keys:
for i in xrange(max(self._keys), -1, -1):
yield self[i]
def cache_keys(self):
"""Grabs the keys (build numbers) from the builder."""
if not self._has_keys_cached:
for i in self.parent.data.get('cachedBuilds', []):
i = int(i)
self._cache.setdefault(i, Build(self, i, None))
if i not in self._keys:
self._keys.append(i)
self._has_keys_cached = True
def discard(self):
super(Builds, self).discard()
# Can't keep keys.
self._has_keys_cached = False
def _readall(self):
return self.read('_all')
class Builder(AddressableDataNode):
"""Builder status."""
printable_attributes = AddressableDataNode.printable_attributes + [
'name',
'key',
'builds',
'slaves',
'pending_builds',
'current_builds',
]
def __init__(self, parent, name, data):
super(Builder, self).__init__(parent, name, data)
self.name = name
self.key = name
self.builds = Builds(self)
self.slaves = BuilderSlaves(self)
self.current_builds = CurrentBuilds(self)
self.pending_builds = PendingBuilds(self)
def discard(self):
super(Builder, self).discard()
self.builds.discard()
self.slaves.discard()
self.current_builds.discard()
class Builders(AddressableNodeList):
"""Root list of builders."""
_child_cls = Builder
def __init__(self, parent):
super(Builders, self).__init__(parent, 'builders')
class Buildbot(AddressableBaseDataNode):
"""This object should be recreated on a master restart as it caches data."""
# Throttle fetches to not kill the server.
auto_throttle = None
printable_attributes = AddressableDataNode.printable_attributes + [
'slaves',
'builders',
'last_fetch',
]
def __init__(self, url):
super(Buildbot, self).__init__(None, url.rstrip('/') + '/json', None)
self._builders = Builders(self)
self._slaves = Slaves(self)
self.last_fetch = None
@property
def builders(self):
return self._builders
@property
def slaves(self):
return self._slaves
def discard(self):
"""Discards information about Builders and Slaves."""
super(Buildbot, self).discard()
self._builders.discard()
self._slaves.discard()
def read(self, suburl):
if self.auto_throttle:
if self.last_fetch:
delta = datetime.datetime.utcnow() - self.last_fetch
remaining = (datetime.timedelta(seconds=self.auto_throttle) - delta)
if remaining > datetime.timedelta(seconds=0):
logging.debug('Sleeping for %ss', remaining)
time.sleep(remaining.seconds)
self.last_fetch = datetime.datetime.utcnow()
url = '%s/%s' % (self.url, suburl)
if '?' in url:
url += '&filter=1'
else:
url += '?filter=1'
logging.info('read(%s)', suburl)
channel = urllib.urlopen(url)
data = channel.read()
try:
return json.loads(data)
except ValueError:
if channel.getcode() >= 400:
# Convert it into an HTTPError for easier processing.
raise urllib2.HTTPError(url, channel.getcode(), '%s:\n%s' % (url, data),
channel.headers, None)
raise
def _readall(self):
return self.read('project')
###############################################################################
## Controller code
def usage(more):
def hook(fn):
fn.func_usage_more = more
return fn
return hook
def need_buildbot(fn):
"""Post-parse args to create a buildbot object."""
@functools.wraps(fn)
def hook(parser, args, *extra_args, **kwargs):
old_parse_args = parser.parse_args
def new_parse_args(args):
options, args = old_parse_args(args)
if len(args) < 1:
parser.error('Need to pass the root url of the buildbot')
url = args.pop(0)
if not url.startswith('http'):
url = 'http://' + url
buildbot = Buildbot(url)
buildbot.auto_throttle = options.throttle
return options, args, buildbot
parser.parse_args = new_parse_args
# Call the original function with the modified parser.
return fn(parser, args, *extra_args, **kwargs)
hook.func_usage_more = '[options] <url>'
return hook
@need_buildbot
def CMDpending(parser, args):
"""Lists pending jobs."""
parser.add_option('-b',
'--builder',
dest='builders',
action='append',
default=[],
help='Builders to filter on')
options, args, buildbot = parser.parse_args(args)
if args:
parser.error('Unrecognized parameters: %s' % ' '.join(args))
if not options.builders:
options.builders = buildbot.builders.keys
for builder in options.builders:
builder = buildbot.builders[builder]
pending_builds = builder.data.get('pendingBuilds', 0)
if not pending_builds:
continue
print('Builder %s: %d' % (builder.name, pending_builds))
if not options.quiet:
for pending in builder.pending_builds.data:
if 'revision' in pending['source']:
print(' revision: %s' % pending['source']['revision'])
for change in pending['source']['changes']:
print(' change:')
print(' comment: %r' % unicode(change['comments'][:50]))
print(' who: %s' % change['who'])
return 0
@usage('[options] <url> [commands] ...')
@need_buildbot
def CMDrun(parser, args):
"""Runs commands passed as parameters.
When passing commands on the command line, each command will be run as if it
was on its own line.
"""
parser.add_option('-f', '--file', help='Read script from file')
parser.add_option('-i',
dest='use_stdin',
action='store_true',
help='Read script on stdin')
# Variable 'buildbot' is not used directly.
# pylint: disable=W0612
options, args, buildbot = parser.parse_args(args)
if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1:
parser.error('Need to pass only one of: <commands>, -f <file> or -i')
if options.use_stdin:
cmds = sys.stdin.read()
elif options.file:
cmds = open(options.file).read()
else:
cmds = '\n'.join(args)
compiled = compile(cmds, '<cmd line>', 'exec')
# pylint: disable=eval-used
eval(compiled, globals(), locals())
return 0
@need_buildbot
def CMDinteractive(parser, args):
"""Runs an interactive shell to run queries."""
_, args, buildbot = parser.parse_args(args)
if args:
parser.error('Unrecognized parameters: %s' % ' '.join(args))
prompt = (
'Buildbot interactive console for "%s".\n'
'Hint: Start with typing: \'buildbot.printable_attributes\' or '
'\'print str(buildbot)\' to explore.') % buildbot.url[:-len('/json')]
local_vars = {'buildbot': buildbot, 'b': buildbot}
code.interact(prompt, None, local_vars)
@need_buildbot
def CMDidle(parser, args):
"""Lists idle slaves."""
return find_idle_busy_slaves(parser, args, True)
@need_buildbot
def CMDbusy(parser, args):
"""Lists idle slaves."""
return find_idle_busy_slaves(parser, args, False)
@need_buildbot
def CMDdisconnected(parser, args):
"""Lists disconnected slaves."""
_, args, buildbot = parser.parse_args(args)
if args:
parser.error('Unrecognized parameters: %s' % ' '.join(args))
for slave in buildbot.slaves:
if not slave.connected:
print(slave.name)
return 0
def find_idle_busy_slaves(parser, args, show_idle):
parser.add_option('-b',
'--builder',
dest='builders',
action='append',
default=[],
help='Builders to filter on')
parser.add_option('-s',
'--slave',
dest='slaves',
action='append',
default=[],
help='Slaves to filter on')
options, args, buildbot = parser.parse_args(args)
if args:
parser.error('Unrecognized parameters: %s' % ' '.join(args))
if not options.builders:
options.builders = buildbot.builders.keys
for builder in options.builders:
builder = buildbot.builders[builder]
if options.slaves:
# Only the subset of slaves connected to the builder.
slaves = list(set(options.slaves).intersection(set(builder.slaves.names)))
if not slaves:
continue
else:
slaves = builder.slaves.names
busy_slaves = [build.slave.name for build in builder.current_builds]
if show_idle:
slaves = natsorted(set(slaves) - set(busy_slaves))
else:
slaves = natsorted(set(slaves) & set(busy_slaves))
if options.quiet:
for slave in slaves:
print(slave)
else:
if slaves:
print('Builder %s: %s' % (builder.name, ', '.join(slaves)))
return 0
def last_failure(buildbot,
builders=None,
slaves=None,
steps=None,
no_cache=False):
"""Returns Build object with last failure with the specific filters."""
builders = builders or buildbot.builders.keys
for builder in builders:
builder = buildbot.builders[builder]
if slaves:
# Only the subset of slaves connected to the builder.
builder_slaves = list(set(slaves).intersection(set(builder.slaves.names)))
if not builder_slaves:
continue
else:
builder_slaves = builder.slaves.names
if not no_cache and len(builder.slaves) > 2:
# Unless you just want the last few builds, it's often faster to
# fetch the whole thing at once, at the cost of a small hickup on
# the buildbot.
# TODO(maruel): Cache only N last builds or all builds since
# datetime.
builder.builds.cache()
found = []
for build in builder.builds:
if build.slave.name not in builder_slaves or build.slave.name in found:
continue
# Only add the slave for the first completed build but still look for
# incomplete builds.
if build.completed:
found.append(build.slave.name)
if steps:
if any(build.steps[step].simplified_result is False for step in steps):
yield build
elif build.simplified_result is False:
yield build
if len(found) == len(builder_slaves):
# Found all the slaves, quit.
break
@need_buildbot
def CMDlast_failure(parser, args):
"""Lists all slaves that failed on that step on their last build.
Example: to find all slaves where their last build was a compile failure,
run with --step compile
"""
parser.add_option(
'-S',
'--step',
dest='steps',
action='append',
default=[],
help='List all slaves that failed on that step on their last build')
parser.add_option('-b',
'--builder',
dest='builders',
action='append',
default=[],
help='Builders to filter on')
parser.add_option('-s',
'--slave',
dest='slaves',
action='append',
default=[],
help='Slaves to filter on')
parser.add_option('-n',
'--no_cache',
action='store_true',
help='Don\'t load all builds at once')
options, args, buildbot = parser.parse_args(args)
if args:
parser.error('Unrecognized parameters: %s' % ' '.join(args))
print_builders = not options.quiet and len(options.builders) != 1
last_builder = None
for build in last_failure(buildbot,
builders=options.builders,
slaves=options.slaves,
steps=options.steps,
no_cache=options.no_cache):
if print_builders and last_builder != build.builder:
print(build.builder.name)
last_builder = build.builder
if options.quiet:
if options.slaves:
print('%s: %s' % (build.builder.name, build.slave.name))
else:
print(build.slave.name)
else:
out = '%d on %s: blame:%s' % (build.number, build.slave.name,
', '.join(build.blame))
if print_builders:
out = ' ' + out
print(out)
if len(options.steps) != 1:
for step in build.steps:
if step.simplified_result is False:
# Assume the first line is the text name anyway.
summary = ', '.join(step.data['text'][1:])[:40]
out = ' %s: "%s"' % (step.data['name'], summary)
if print_builders:
out = ' ' + out
print(out)
return 0
@need_buildbot
def CMDcurrent(parser, args):
"""Lists current jobs."""
parser.add_option('-b',
'--builder',
dest='builders',
action='append',
default=[],
help='Builders to filter on')
parser.add_option('--blame',
action='store_true',
help='Only print the blame list')
options, args, buildbot = parser.parse_args(args)
if args:
parser.error('Unrecognized parameters: %s' % ' '.join(args))
if not options.builders:
options.builders = buildbot.builders.keys
if options.blame:
blame = set()
for builder in options.builders:
for build in buildbot.builders[builder].current_builds:
if build.blame:
for blamed in build.blame:
blame.add(blamed)
print('\n'.join(blame))
return 0
for builder in options.builders:
builder = buildbot.builders[builder]
if not options.quiet and builder.current_builds:
print(builder.name)
for build in builder.current_builds:
if options.quiet:
print(build.slave.name)
else:
out = '%4d: slave=%10s' % (build.number, build.slave.name)
out += ' duration=%5d' % (build.duration or 0)
if build.eta:
out += ' eta=%5.0f' % build.eta
else:
out += ' '
if build.blame:
out += ' blame=' + ', '.join(build.blame)
print(out)
return 0
@need_buildbot
def CMDbuilds(parser, args):
"""Lists all builds.
Example: to find all builds on a single slave, run with -b bar -s foo
"""
parser.add_option('-r',
'--result',
type='int',
help='Build result to filter on')
parser.add_option('-b',
'--builder',
dest='builders',
action='append',
default=[],
help='Builders to filter on')
parser.add_option('-s',
'--slave',
dest='slaves',
action='append',
default=[],
help='Slaves to filter on')
parser.add_option('-n',
'--no_cache',
action='store_true',
help='Don\'t load all builds at once')
options, args, buildbot = parser.parse_args(args)
if args:
parser.error('Unrecognized parameters: %s' % ' '.join(args))
builders = options.builders or buildbot.builders.keys
for builder in builders:
builder = buildbot.builders[builder]
for build in builder.builds:
if not options.slaves or build.slave.name in options.slaves:
if options.quiet:
out = ''
if options.builders:
out += '%s/' % builder.name
if len(options.slaves) != 1:
out += '%s/' % build.slave.name
out += '%d revision:%s result:%s blame:%s' % (
build.number, build.revision, build.result, ','.join(build.blame))
print(out)
else:
print(build)
return 0
@need_buildbot
def CMDcount(parser, args):
"""Count the number of builds that occured during a specific period."""
parser.add_option('-o',
'--over',
type='int',
help='Number of seconds to look for')
parser.add_option('-b',
'--builder',
dest='builders',
action='append',
default=[],
help='Builders to filter on')
options, args, buildbot = parser.parse_args(args)
if args:
parser.error('Unrecognized parameters: %s' % ' '.join(args))
if not options.over:
parser.error(
'Specify the number of seconds, e.g. --over 86400 for the last 24 '
'hours')
builders = options.builders or buildbot.builders.keys
counts = {}
since = time.time() - options.over
for builder in builders:
builder = buildbot.builders[builder]
counts[builder.name] = 0
if not options.quiet:
print(builder.name)
for build in builder.builds.iterall():
try:
start_time = build.start_time
except urllib2.HTTPError:
# The build was probably trimmed.
print('Failed to fetch build %s/%d' % (builder.name, build.number),
file=sys.stderr)
continue
if start_time >= since:
counts[builder.name] += 1
else:
break
if not options.quiet:
print('.. %d' % counts[builder.name])
align_name = max(len(b) for b in counts)
align_number = max(len(str(c)) for c in counts.itervalues())
for builder in sorted(counts):
print('%*s: %*d' % (align_name, builder, align_number, counts[builder]))
print('Total: %d' % sum(counts.itervalues()))
return 0
def gen_parser():
"""Returns an OptionParser instance with default options.
It should be then processed with gen_usage() before being used.
"""
parser = optparse.OptionParser(version=__version__)
# Remove description formatting
parser.format_description = lambda x: parser.description
# Add common parsing.
old_parser_args = parser.parse_args
def Parse(*args, **kwargs):
options, args = old_parser_args(*args, **kwargs)
if options.verbose >= 2:
logging.basicConfig(level=logging.DEBUG)
elif options.verbose:
logging.basicConfig(level=logging.INFO)
else:
logging.basicConfig(level=logging.WARNING)
return options, args
parser.parse_args = Parse
parser.add_option('-v',
'--verbose',
action='count',
help='Use multiple times to increase logging leve')
parser.add_option(
'-q',
'--quiet',
action='store_true',
help='Reduces the output to be parsed by scripts, independent of -v')
parser.add_option('--throttle',
type='float',
help='Minimum delay to sleep between requests')
return parser
###############################################################################
## Generic subcommand handling code
def Command(name):
return getattr(sys.modules[__name__], 'CMD' + name, None)
@usage('<command>')
def CMDhelp(parser, args):
"""Print list of commands or use 'help <command>'."""
_, args = parser.parse_args(args)
if len(args) == 1:
return main(args + ['--help'])
parser.print_help()
return 0
def gen_usage(parser, command):
"""Modifies an OptionParser object with the command's documentation.
The documentation is taken from the function's docstring.
"""
obj = Command(command)
more = getattr(obj, 'func_usage_more')
# OptParser.description prefer nicely non-formatted strings.
parser.description = obj.__doc__ + '\n'
parser.set_usage('usage: %%prog %s %s' % (command, more))
def main(args=None):
# Do it late so all commands are listed.
# pylint: disable=E1101
CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join(
' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0])
for fn in dir(sys.modules[__name__]) if fn.startswith('CMD'))
parser = gen_parser()
if args is None:
args = sys.argv[1:]
if args:
command = Command(args[0])
if command:
# "fix" the usage and the description now that we know the subcommand.
gen_usage(parser, args[0])
return command(parser, args[1:])
# Not a known command. Default to help.
gen_usage(parser, 'help')
return CMDhelp(parser, args)
if __name__ == '__main__':
sys.exit(main())