#pylint: disable-msg=C0111
"""
Pidfile monitor.
"""
import logging
import time
import traceback
import common
from autotest_lib.client.common_lib import utils
from autotest_lib.client.common_lib import global_config
from autotest_lib.scheduler import drone_manager
from autotest_lib.scheduler import scheduler_config
try:
from chromite.lib import metrics
except ImportError:
metrics = utils.metrics_mock
def _get_pidfile_timeout_secs():
"""@returns How long to wait for autoserv to write pidfile."""
pidfile_timeout_mins = global_config.global_config.get_config_value(
scheduler_config.CONFIG_SECTION, 'pidfile_timeout_mins', type=int)
return pidfile_timeout_mins * 60
class PidfileRunMonitor(object):
"""
Client must call either run() to start a new process or
attach_to_existing_process().
"""
class _PidfileException(Exception):
"""
Raised when there's some unexpected behavior with the pid file, but only
used internally (never allowed to escape this class).
"""
def __init__(self):
self._drone_manager = drone_manager.instance()
self.lost_process = False
self._start_time = None
self.pidfile_id = None
self._killed = False
self._state = drone_manager.PidfileContents()
def _add_nice_command(self, command, nice_level):
if not nice_level:
return command
return ['nice', '-n', str(nice_level)] + command
def _set_start_time(self):
self._start_time = time.time()
def run(self, command, working_directory, num_processes, nice_level=None,
log_file=None, pidfile_name=None, paired_with_pidfile=None,
username=None, drone_hostnames_allowed=None):
assert command is not None
if nice_level is not None:
command = ['nice', '-n', str(nice_level)] + command
self._set_start_time()
self.pidfile_id = self._drone_manager.execute_command(
command, working_directory, pidfile_name=pidfile_name,
num_processes=num_processes, log_file=log_file,
paired_with_pidfile=paired_with_pidfile, username=username,
drone_hostnames_allowed=drone_hostnames_allowed)
def attach_to_existing_process(self, execution_path,
pidfile_name=drone_manager.AUTOSERV_PID_FILE,
num_processes=None):
self._set_start_time()
self.pidfile_id = self._drone_manager.get_pidfile_id_from(
execution_path, pidfile_name=pidfile_name)
if num_processes is not None:
self._drone_manager.declare_process_count(self.pidfile_id, num_processes)
def kill(self):
if self.has_process():
self._drone_manager.kill_process(self.get_process())
self._killed = True
def has_process(self):
self._get_pidfile_info()
return self._state.process is not None
def get_process(self):
self._get_pidfile_info()
assert self._state.process is not None
return self._state.process
def _read_pidfile(self, use_second_read=False):
assert self.pidfile_id is not None, (
'You must call run() or attach_to_existing_process()')
contents = self._drone_manager.get_pidfile_contents(
self.pidfile_id, use_second_read=use_second_read)
if contents.is_invalid():
self._state = drone_manager.PidfileContents()
raise self._PidfileException(contents)
self._state = contents
def _handle_pidfile_error(self, error, message=''):
self.on_lost_process(self._state.process)
def _get_pidfile_info_helper(self):
if self.lost_process:
return
self._read_pidfile()
if self._state.process is None:
self._handle_no_process()
return
if self._state.exit_status is None:
# double check whether or not autoserv is running
if self._drone_manager.is_process_running(self._state.process):
return
# pid but no running process - maybe process *just* exited
self._read_pidfile(use_second_read=True)
if self._state.exit_status is None:
# autoserv exited without writing an exit code
# to the pidfile
self._handle_pidfile_error(
'autoserv died without writing exit code')
def _get_pidfile_info(self):
"""\
After completion, self._state will contain:
pid=None, exit_status=None if autoserv has not yet run
pid!=None, exit_status=None if autoserv is running
pid!=None, exit_status!=None if autoserv has completed
"""
try:
self._get_pidfile_info_helper()
except self._PidfileException, exc:
self._handle_pidfile_error('Pidfile error', traceback.format_exc())
def _handle_no_process(self):
"""\
Called when no pidfile is found or no pid is in the pidfile.
"""
if time.time() - self._start_time > _get_pidfile_timeout_secs():
# If we aborted the process, and we find that it has exited without
# writing a pidfile, then it's because we killed it, and thus this
# isn't a surprising situation.
if not self._killed:
metrics.Counter('chromeos/autotest/errors/scheduler/no_pidfile'
).increment()
else:
logging.warning("%s didn't exit after SIGTERM", self.pidfile_id)
self.on_lost_process()
def on_lost_process(self, process=None):
"""\
Called when autoserv has exited without writing an exit status,
or we've timed out waiting for autoserv to write a pid to the
pidfile. In either case, we just return failure and the caller
should signal some kind of warning.
process is unimportant here, as it shouldn't be used by anyone.
"""
self.lost_process = True
self._state.process = process
self._state.exit_status = 1
self._state.num_tests_failed = 0
def exit_code(self):
self._get_pidfile_info()
return self._state.exit_status
def num_tests_failed(self):
"""@returns The number of tests that failed or -1 if unknown."""
self._get_pidfile_info()
if self._state.num_tests_failed is None:
return -1
return self._state.num_tests_failed
def try_copy_results_on_drone(self, **kwargs):
if self.has_process():
# copy results logs into the normal place for job results
self._drone_manager.copy_results_on_drone(self.get_process(), **kwargs)
def try_copy_to_results_repository(self, source, **kwargs):
if self.has_process():
self._drone_manager.copy_to_results_repository(self.get_process(),
source, **kwargs)