#!/usr/bin/python
# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Kill slow queries in local autotest database."""
import logging
import MySQLdb
import optparse
import sys
import common
from autotest_lib.client.common_lib import global_config
from autotest_lib.site_utils import gmail_lib
AT_DIR='/usr/local/autotest'
DEFAULT_USER = global_config.global_config.get_config_value(
'CROS', 'db_backup_user', type=str, default='')
DEFAULT_PASSWD = global_config.global_config.get_config_value(
'CROS', 'db_backup_password', type=str, default='')
DEFAULT_MAIL = global_config.global_config.get_config_value(
'SCHEDULER', 'notify_email', type=str, default='')
def parse_options():
"""Parse the command line arguments."""
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage=usage)
parser.add_option('-u', '--user', default=DEFAULT_USER,
help='User to login to the Autotest DB. Default is the '
'one defined in config file.')
parser.add_option('-p', '--password', default=DEFAULT_PASSWD,
help='Password to login to the Autotest DB. Default is '
'the one defined in config file.')
parser.add_option('-t', '--timeout', type=int, default=300,
help='Timeout boundry of the slow database query. '
'Default is 300s')
parser.add_option('-m', '--mail', default=DEFAULT_MAIL,
help='Mail address to send the summary to. Default is '
'ChromeOS infra Deputy')
options, args = parser.parse_args()
return parser, options, args
def verify_options_and_args(options, args):
"""Verify the validity of options and args.
@param options: The parsed options to verify.
@param args: The parsed args to verify.
@returns: True if verification passes, False otherwise.
"""
if args:
logging.error('Unknown arguments: ' + str(args))
return False
if not (options.user and options.password):
logging.error('Failed to get the default user of password for Autotest'
' DB. Please specify them through the command line.')
return False
return True
def format_the_output(slow_queries):
"""Convert a list of slow queries into a readable string format.
e.g. [(a, b, c...)] -->
"Id: a
Host: b
User: c
...
"
@param slow_queries: A list of tuples, one tuple contains all the info about
one single slow query.
@returns: one clean string representation of all the slow queries.
"""
query_str_list = [('Id: %s\nUser: %s\nHost: %s\ndb: %s\nCommand: %s\n'
'Time: %s\nState: %s\nInfo: %s\n') %
q for q in slow_queries]
return '\n'.join(query_str_list)
def kill_slow_queries(user, password, timeout):
"""Kill the slow database queries running beyond the timeout limit.
@param user: User to login to the Autotest DB.
@param password: Password to login to the Autotest DB.
@param timeout: Timeout limit to kill the slow queries.
@returns: string representation of all the killed slow queries.
"""
db = MySQLdb.connect('localhost', user, password)
cursor = db.cursor()
# Get the processlist.
cursor.execute('SHOW FULL PROCESSLIST')
processlist = cursor.fetchall()
# Filter out the slow queries and kill them.
slow_queries = [p for p in processlist if p[4]=='Query' and p[5]>=timeout]
queries_str = ''
if slow_queries:
queries_str = format_the_output(slow_queries)
queries_ids = [q[0] for q in slow_queries]
logging.info('Start killing following slow queries\n%s', queries_str)
for query_id in queries_ids:
logging.info('Killing %s...', query_id)
cursor.execute('KILL %d' % query_id)
logging.info('Done!')
else:
logging.info('No slow queries over %ds!', timeout)
return queries_str
def main():
"""Main entry."""
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S', level=logging.DEBUG)
parser, options, args = parse_options()
if not verify_options_and_args(options, args):
parser.print_help()
return 1
try:
result_log_strs = kill_slow_queries(options.user, options.password,
options.timeout)
if result_log_strs:
gmail_lib.send_email(
options.mail,
'Successfully killed slow autotest db queries',
'Below are killed queries:\n%s' % result_log_strs)
except Exception as e:
logging.error('Failed to kill slow db queries.\n%s', e)
gmail_lib.send_email(
options.mail,
'Failed to kill slow autotest db queries.',
('Error occurred during killing slow db queries:\n%s\n'
'Detailed logs can be found in /var/log/slow_queries.log on db '
'backup server.\nTo avoid db crash, please check ASAP.') % e)
raise
if __name__ == '__main__':
sys.exit(main())