# Copyright 2011 Google Inc. All Rights Reserved.
#
__author__ = 'kbaclawski@google.com (Krystian Baclawski)'
from collections import namedtuple
import glob
import gzip
import os.path
import pickle
import time
import xmlrpclib
from django import forms
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.template import Context
from django.views import static
Link = namedtuple('Link', 'href name')
def GetServerConnection():
return xmlrpclib.Server('http://localhost:8000')
def MakeDefaultContext(*args):
context = Context({'links': [
Link('/job-group', 'Job Groups'), Link('/machine', 'Machines')
]})
for arg in args:
context.update(arg)
return context
class JobInfo(object):
def __init__(self, job_id):
self._job = pickle.loads(GetServerConnection().GetJob(job_id))
def GetAttributes(self):
job = self._job
group = [Link('/job-group/%d' % job.group.id, job.group.label)]
predecessors = [Link('/job/%d' % pred.id, pred.label)
for pred in job.predecessors]
successors = [Link('/job/%d' % succ.id, succ.label)
for succ in job.successors]
machines = [Link('/machine/%s' % mach.hostname, mach.hostname)
for mach in job.machines]
logs = [Link('/job/%d/log' % job.id, 'Log')]
commands = enumerate(job.PrettyFormatCommand().split('\n'), start=1)
return {'text': [('Label', job.label), ('Directory', job.work_dir)],
'link': [('Group', group), ('Predecessors', predecessors),
('Successors', successors), ('Machines', machines),
('Logs', logs)],
'code': [('Command', commands)]}
def GetTimeline(self):
return [{'started': evlog.GetTimeStartedFormatted(),
'state_from': evlog.event.from_,
'state_to': evlog.event.to_,
'elapsed': evlog.GetTimeElapsedRounded()}
for evlog in self._job.timeline.GetTransitionEventHistory()]
def GetLog(self):
log_path = os.path.join(self._job.logs_dir,
'%s.gz' % self._job.log_filename_prefix)
try:
log = gzip.open(log_path, 'r')
except IOError:
content = []
else:
# There's a good chance that file is not closed yet, so EOF handling
# function and CRC calculation will fail, thus we need to monkey patch the
# _read_eof method.
log._read_eof = lambda: None
def SplitLine(line):
prefix, msg = line.split(': ', 1)
datetime, stream = prefix.rsplit(' ', 1)
return datetime, stream, msg
content = map(SplitLine, log.readlines())
finally:
log.close()
return content
class JobGroupInfo(object):
def __init__(self, job_group_id):
self._job_group = pickle.loads(GetServerConnection().GetJobGroup(
job_group_id))
def GetAttributes(self):
group = self._job_group
home_dir = [Link('/job-group/%d/files/' % group.id, group.home_dir)]
return {'text': [('Label', group.label),
('Time submitted', time.ctime(group.time_submitted)),
('State', group.status),
('Cleanup on completion', group.cleanup_on_completion),
('Cleanup on failure', group.cleanup_on_failure)],
'link': [('Directory', home_dir)]}
def _GetJobStatus(self, job):
status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'}
return status_map.get(str(job.status), None)
def GetJobList(self):
return [{'id': job.id,
'label': job.label,
'state': job.status,
'status': self._GetJobStatus(job),
'elapsed': job.timeline.GetTotalTime()}
for job in self._job_group.jobs]
def GetHomeDirectory(self):
return self._job_group.home_dir
def GetReportList(self):
job_dir_pattern = os.path.join(self._job_group.home_dir, 'job-*')
filenames = []
for job_dir in glob.glob(job_dir_pattern):
filename = os.path.join(job_dir, 'report.html')
if os.access(filename, os.F_OK):
filenames.append(filename)
reports = []
for filename in sorted(filenames, key=lambda f: os.stat(f).st_ctime):
try:
with open(filename, 'r') as report:
reports.append(report.read())
except IOError:
pass
return reports
class JobGroupListInfo(object):
def __init__(self):
self._all_job_groups = pickle.loads(GetServerConnection().GetAllJobGroups())
def _GetJobGroupState(self, group):
return str(group.status)
def _GetJobGroupStatus(self, group):
status_map = {'SUCCEEDED': 'success', 'FAILED': 'failure'}
return status_map.get(self._GetJobGroupState(group), None)
def GetList(self):
return [{'id': group.id,
'label': group.label,
'submitted': time.ctime(group.time_submitted),
'state': self._GetJobGroupState(group),
'status': self._GetJobGroupStatus(group)}
for group in self._all_job_groups]
def GetLabelList(self):
return sorted(set(group.label for group in self._all_job_groups))
def JobPageHandler(request, job_id):
job = JobInfo(int(job_id))
ctx = MakeDefaultContext({
'job_id': job_id,
'attributes': job.GetAttributes(),
'timeline': job.GetTimeline()
})
return render_to_response('job.html', ctx)
def LogPageHandler(request, job_id):
job = JobInfo(int(job_id))
ctx = MakeDefaultContext({'job_id': job_id, 'log_lines': job.GetLog()})
return render_to_response('job_log.html', ctx)
def JobGroupPageHandler(request, job_group_id):
group = JobGroupInfo(int(job_group_id))
ctx = MakeDefaultContext({
'group_id': job_group_id,
'attributes': group.GetAttributes(),
'job_list': group.GetJobList(),
'reports': group.GetReportList()
})
return render_to_response('job_group.html', ctx)
def JobGroupFilesPageHandler(request, job_group_id, path):
group = JobGroupInfo(int(job_group_id))
return static.serve(request,
path,
document_root=group.GetHomeDirectory(),
show_indexes=True)
class FilterJobGroupsForm(forms.Form):
label = forms.ChoiceField(label='Filter by label:', required=False)
def JobGroupListPageHandler(request):
groups = JobGroupListInfo()
group_list = groups.GetList()
field = FilterJobGroupsForm.base_fields['label']
field.choices = [('*', '--- no filtering ---')]
field.choices.extend([(label, label) for label in groups.GetLabelList()])
if request.method == 'POST':
form = FilterJobGroupsForm(request.POST)
if form.is_valid():
label = form.cleaned_data['label']
if label != '*':
group_list = [group for group in group_list if group['label'] == label]
else:
form = FilterJobGroupsForm({'initial': '*'})
ctx = MakeDefaultContext({'filter': form, 'groups': group_list})
return render_to_response('job_group_list.html', ctx)
def MachineListPageHandler(request):
machine_list = pickle.loads(GetServerConnection().GetMachineList())
return render_to_response('machine_list.html',
MakeDefaultContext({'machines': machine_list}))
def DefaultPageHandler(request):
return HttpResponseRedirect('/job-group')