# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Scheduler helper libraries. """ import logging import os import common from autotest_lib.client.common_lib import global_config from autotest_lib.client.common_lib import logging_config from autotest_lib.client.common_lib import logging_manager from autotest_lib.client.common_lib import utils from autotest_lib.database import database_connection from autotest_lib.frontend import setup_django_environment from autotest_lib.frontend.afe import readonly_connection from autotest_lib.server import utils as server_utils DB_CONFIG_SECTION = 'AUTOTEST_WEB' # Translations necessary for scheduler queries to work with SQLite. # Though this is only used for testing it is included in this module to avoid # circular imports. _re_translator = database_connection.TranslatingDatabase.make_regexp_translator _DB_TRANSLATORS = ( _re_translator(r'NOW\(\)', 'time("now")'), _re_translator(r'LAST_INSERT_ID\(\)', 'LAST_INSERT_ROWID()'), # older SQLite doesn't support group_concat, so just don't bother until # it arises in an important query _re_translator(r'GROUP_CONCAT\((.*?)\)', r'\1'), _re_translator(r'TRUNCATE TABLE', 'DELETE FROM'), _re_translator(r'ISNULL\(([a-z,_]+)\)', r'ifnull(nullif(\1, NULL), \1) DESC'), ) class SchedulerError(Exception): """General parent class for exceptions raised by scheduler code.""" class MalformedRecordError(SchedulerError): """Exception raised when an individual job or record is malformed. Code that handles individual records (e.g. afe jobs, hqe entries, special tasks) should treat such an exception as a signal to skip or permanently discard this record.""" class NoHostIdError(MalformedRecordError): """Raised by the scheduler when a non-hostless job's host is None.""" class ConnectionManager(object): """Manager for the django database connections. The connection is used through scheduler_models and monitor_db. """ __metaclass__ = server_utils.Singleton def __init__(self, readonly=True, autocommit=True): """Set global django database options for correct connection handling. @param readonly: Globally disable readonly connections. @param autocommit: Initialize django autocommit options. """ self.db_connection = None # bypass the readonly connection readonly_connection.set_globally_disabled(readonly) if autocommit: # ensure Django connection is in autocommit setup_django_environment.enable_autocommit() @classmethod def open_connection(cls): """Open a new database connection. @return: An instance of the newly opened connection. """ db = database_connection.DatabaseConnection(DB_CONFIG_SECTION) db.connect(db_type='django') return db def get_connection(self): """Get a connection. @return: A database connection. """ if self.db_connection is None: self.db_connection = self.open_connection() return self.db_connection def disconnect(self): """Close the database connection.""" try: self.db_connection.disconnect() except Exception as e: logging.debug('Could not close the db connection. %s', e) def __del__(self): self.disconnect() class SchedulerLoggingConfig(logging_config.LoggingConfig): """Configure timestamped logging for a scheduler.""" GLOBAL_LEVEL = logging.INFO @classmethod def get_log_name(cls, timestamped_logfile_prefix): """Get the name of a logfile. @param timestamped_logfile_prefix: The prefix to apply to the a timestamped log. Eg: 'scheduler' will create a logfile named scheduler.log.2014-05-12-17.24.02. @return: The timestamped log name. """ return cls.get_timestamped_log_name(timestamped_logfile_prefix) def configure_logging(self, log_dir=None, logfile_name=None, timestamped_logfile_prefix='scheduler'): """Configure logging to a specified logfile. @param log_dir: The directory to log into. @param logfile_name: The name of the log file. @timestamped_logfile_prefix: The prefix to apply to the name of the logfile, if a log file name isn't specified. """ super(SchedulerLoggingConfig, self).configure_logging(use_console=True) if log_dir is None: log_dir = self.get_server_log_dir() if not logfile_name: logfile_name = self.get_log_name(timestamped_logfile_prefix) self.add_file_handler(logfile_name, logging.DEBUG, log_dir=log_dir) symlink_path = os.path.join( log_dir, '%s.latest' % timestamped_logfile_prefix) try: os.unlink(symlink_path) except OSError: pass os.symlink(os.path.join(log_dir, logfile_name), symlink_path) def setup_logging(log_dir, log_name, timestamped_logfile_prefix='scheduler'): """Setup logging to a given log directory and log file. @param log_dir: The directory to log into. @param log_name: Name of the log file. @param timestamped_logfile_prefix: The prefix to apply to the logfile. """ logging_manager.configure_logging( SchedulerLoggingConfig(), log_dir=log_dir, logfile_name=log_name, timestamped_logfile_prefix=timestamped_logfile_prefix) def check_production_settings(scheduler_options): """Check the scheduler option's production settings. @param scheduler_options: Settings for scheduler. @raises SchedulerError: If a loclhost scheduler is started with production settings. """ db_server = global_config.global_config.get_config_value('AUTOTEST_WEB', 'host') if (not scheduler_options.production and not utils.is_localhost(db_server)): raise SchedulerError('Scheduler is not running in production mode, you ' 'should not set database to hosts other than ' 'localhost. It\'s currently set to %s.\nAdd option' ' --production if you want to skip this check.' % db_server)