#!/usr/bin/python
# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
This script crawls crbug. Sort-of.
Invocation:
Get all bugs with labels, strings (in summary and/or comments):
crbug_crawler.py --labels 'one two three'
--queries '"first query" "second query"'
Get baddest open bugs of all time:
crbug_crawler.py --reap
Tips:
- Label based queries will return faster than text queries.
- contrib/crbug_shell.py is a wrapper that allows you to incrementally
filter search results using this script.
"""
import argparse
import cmd
import logging
import sys
import shlex
import common
from autotest_lib.client.common_lib import global_config
from autotest_lib.server.cros.dynamic_suite import reporting
def _parse_args(args):
if not args:
import crbug_crawler
logging.error('Improper usage of crbug_crawler: %s',
crbug_crawler.__doc__)
sys.exit(1)
description = ('Usage: crbug_crawler.py --reap')
parser = argparse.ArgumentParser(description=description)
parser.add_argument('--quiet', help=('Turn off logging noise.'),
action='store_true', default=False)
parser.add_argument('--num', help='Number of issues to output.', default=10,
type=int)
parser.add_argument('--queries',
help=('Search query. Eg: --queries "%s %s"' %
('build_Root', 'login')),
default='')
parser.add_argument('--labels',
help=('Search labels. Eg: --labels "%s %s"' %
('autofiled', 'Pri-1')), default=None)
parser.add_argument('--reap', help=('Top autofiled bugs ordered by count.'),
action='store_true', default=False)
return parser.parse_args(args)
class Update(object):
"""Class encapsulating fields of an update to a bug.
"""
open_statuses = ['Unconfirmed', 'Untriaged', 'Available', 'Assigned',
'Started', 'ExternalDependency']
closed_statuses = ['Fixed', 'Verified', 'Duplicate', 'WontFix', 'Archived']
def __init__(self, comment='', labels='', status=''):
self.comment = comment
self.labels = labels if labels else []
self.status = status
def __str__(self):
msg = 'status: %s' % self.status
if self.labels:
msg = '%s labels: %s' % (msg, self.labels)
if self.comment:
msg = '%s comment: %s' % (msg, self.comment)
return msg
class UpdateManager(object):
"""Update manager that allows you to revert status updates.
This class keeps track of the last update applied and is capable
of reverting it.
"""
def __init__(self, autocommit=False):
"""Initialize update manager.
@param autocommit: If False just print out the update instead
of committing it.
"""
self.history = {}
self.present = {}
self.reporter = reporting.Reporter()
self.phapi_lib = self.reporter.get_bug_tracker_client()
self.autocommit = autocommit
def revert(self):
"""Only manages status reverts as of now.
"""
for issue_id, update in self.history.iteritems():
logging.warning('You will have to manually update %s and %s on %s',
self.present[issue_id].labels,
self.present[issue_id].comment, issue_id)
# Create a new update with just the status.
self.update(issue_id, Update(status=update.status))
def update(self, old_issue, update):
"""Record the state of an issue before updating it.
@param old_issue: The issue to update. If an id is specified an
issue is constructed. If an issue object (as defined in phapi_lib
Issue)is passed in, it is used directly.
@param update: The Update object to apply to the issue.
"""
if type(old_issue) == int:
old_issue = self.phapi_lib.get_tracker_issue_by_id(old_issue)
old_update = Update(
labels=old_issue.labels, status=old_issue.status)
if not update.status:
update.status = old_update.status
elif (update.status not in Update.open_statuses and
update.status not in Update.closed_statuses):
raise ValueError('Unknown status %s' % update.status)
if not self.autocommit:
logging.warning('Would have applied the following update: '
'%s -> %s', old_update, update)
return
self.history[old_issue.id] = old_update
self.reporter.modify_bug_report(
issue_id=old_issue.id, comment=update.comment,
label_update=update.labels,
status=update.status)
self.present[old_issue.id] = update
class Crawler(object):
"""Class capable of crawling crbug.
This class applies filters to issues it crawls and caches them locally.
"""
# The limit at which we ask for confirmation to proceed with the crawl.
PROMPT_LIMIT = 2000
def __init__(self):
self.reporter = reporting.Reporter()
self.phapi_client = self.reporter.get_bug_tracker_client()
self.issues = None
self.all_autofiled_query = 'ANCHOR TestFailure'
self.all_autofiled_label = 'autofiled'
self.prompted = False
def fuzzy_search(self, query='', label='', fast=True):
"""Returns all issues using one query and/or one label.
@param query: A string representing the query.
@param label: A string representing the label.
@param fast: If true, don't bother fetching comments.
@return: A list of issues matching the query. If fast is
specified the issues won't have comments.
"""
if not query and not label:
raise ValueError('Require query or labels to make a tracker query, '
'try query = "%s" or one of the predefined labels %s' %
(self.fuzzy_search_anchor(),
self.reporter._PREDEFINED_LABELS))
if type(label) != str:
raise ValueError('The crawler only supports one label per query, '
'and it must be a string. you supplied %s' % label)
return self.phapi_client.get_tracker_issues_by_text(
query, label=label, full_text=not fast)
@staticmethod
def _get_autofiled_count(issue):
"""Return the autofiled count.
@param issue: An issue object that has labels.
@return: An integer representing the autofiled count.
"""
for label in issue.labels:
if 'autofiled-count-' in label:
return int(label.replace('autofiled-count-', ''))
# Force bugs without autofiled-count to sink
return 0
def _prompt_crawl(self, new_issues, start_index):
"""Warn the user that a crawl is getting large.
This method prompts for a y/n answer in case the user wants to abort the
crawl and specify another set of labels/queries.
@param new_issues: A list of issues used with the start_index to
determine the number of issues already processed.
@param start_index: The start index of the next crawl iteration.
"""
logging.warning('Found %s issues, Crawling issues starting from %s',
len(new_issues), start_index)
if start_index > self.PROMPT_LIMIT and not self.prompted:
logging.warning('Already crawled %s issues, it is possible that'
'you\'ve specified a very general label. If this is the '
'case consider re-rodering the labels so they start with '
'the rarest. Continue crawling [y/n]?',
start_index + len(new_issues))
self.prompted = raw_input() == 'y'
if not self.prompted:
sys.exit(0)
def exhaustive_crawl(self, query='', label='', fast=True):
"""Perform an exhaustive crawl using one label and query string.
@param query: A string representing one query.
@param lable: A string representing one label.
@return A list of issues sorted by descending autofiled count.
"""
start_index = 0
self.phapi_client.set_max_results(200)
logging.warning('Performing an exhaustive crawl with label %s query %s',
label, query)
vague_issues = []
new_issues = self.fuzzy_search(query=query, label=label, fast=fast)
while new_issues:
vague_issues += new_issues
start_index += len(new_issues) + 1
self.phapi_client.set_start_index(start_index)
new_issues = self.fuzzy_search(query=query, label=label,
fast=fast)
self._prompt_crawl(new_issues, start_index)
# Subsequent calls will clear the issues cache with new results.
self.phapi_client.set_start_index(1)
return sorted(vague_issues, reverse=True,
key=lambda issue: self._get_autofiled_count(issue))
@staticmethod
def filter_labels(issues, labels):
"""Takes a list of labels and returns matching issues.
@param issues: A list of issues to parse for labels.
@param labels: A list of labels to match.
@return: A list of matching issues. The issues must contain
all the labels specified.
"""
if not labels:
return issues
matching_issues = set([])
labels = set(labels)
for issue in issues:
issue_labels = set(issue.labels)
if issue_labels.issuperset(labels):
matching_issues.add(issue)
return matching_issues
@classmethod
def does_query_match(cls, issue, query):
"""Check if a query matches the given issue.
@param issue: The issue to check.
@param query: The query to check against.
@return: True if the query matches, false otherwise.
"""
if query in issue.title or query in issue.summary:
return True
# We can only search comments if the issue is a complete issue
# i.e as defined in phapi_lib.Issue.
try:
if any(query in comment for comment in issue.comments):
return True
except (AttributeError, TypeError):
pass
return False
@classmethod
def filter_queries(cls, issues, queries):
"""Take a list of queries and returns matching issues.
@param issues: A list of issues to parse. If the issues contain
comments and a query is not in the issues title or summmary,
the comments are parsed for a substring match.
@param queries: A list of queries to parse the issues for.
This method looks for an exact substring match within each issue.
@return: A list of matching issues.
"""
if not queries:
return issues
matching_issues = set([])
for issue in issues:
# For each query, check if it's in the title, description or
# comments. If a query isn't in any of these, discard the issue.
for query in queries:
if cls.does_query_match(issue, query):
matching_issues.add(issue)
else:
if issue in matching_issues:
logging.warning('%s: %s\n \tPassed a subset of the '
'queries but failed query %s',
issue.id, issue.title, query)
matching_issues.remove(issue)
break
return matching_issues
def filter_issues(self, queries='', labels=None, fast=True):
"""Run the queries, labels filters by crawling crbug.
@param queries: A space seperated string of queries, usually passed
through the command line.
@param labels: A space seperated string of labels, usually passed
through the command line.
@param fast: If specified, skip creating comments for issues since this
can be a slow process. This value is only a suggestion, since it is
ignored if multiple queries are specified.
"""
queries = shlex.split(queries)
labels = shlex.split(labels) if labels else None
# We'll need comments to filter multiple queries.
if len(queries) > 1:
fast = False
matching_issues = self.exhaustive_crawl(
query=queries.pop(0) if queries else '',
label=labels.pop(0) if labels else '', fast=fast)
matching_issues = self.filter_labels(matching_issues, labels)
matching_issues = self.filter_queries(matching_issues, queries)
self.issues = list(matching_issues)
def dump_issues(self, limit=None):
"""Print issues.
"""
if limit and limit < len(self.issues):
issues = self.issues[:limit]
else:
issues = self.issues
#TODO: Modify formatting, include some paging etc.
for issue in issues:
try:
print ('[%s] %s crbug.com/%s %s' %
(self._get_autofiled_count(issue),
issue.status, issue.id, issue.title))
except UnicodeEncodeError as e:
print "Unicdoe error decoding issue id %s" % issue.id
continue
def _update_test(args):
"""A simple update test, to record usage.
"""
updater = UpdateManager(autocommit=True)
for issue in issues:
updater.update(issue,
Update(comment='this is bogus', labels=['bogus'],
status='Assigned'))
updater.revert()
def configure_logging(quiet=False):
"""Configure logging.
@param quiet: True to turn off warning messages.
"""
logging.basicConfig()
logger = logging.getLogger()
level = logging.WARNING
if quiet:
level = logging.ERROR
logger.setLevel(level)
def main(args):
crawler = Crawler()
if args.reap:
if args.queries or args.labels:
logging.error('Query based ranking of bugs not supported yet.')
return
queries = ''
labels = crawler.all_autofiled_label
else:
queries = args.queries
labels = args.labels
crawler.filter_issues(queries=queries, labels=labels,
fast=False if queries else True)
crawler.dump_issues(int(args.num))
logging.warning('\nThis is a truncated list of %s results, use --num %s '
'to get them all. If you want more informative results/better '
'querying capabilities try crbug_shell.py.',
args.num, len(crawler.issues))
if __name__ == '__main__':
args = _parse_args(sys.argv[1:])
configure_logging(args.quiet)
main(args)