import BaseHTTPServer
import SimpleHTTPServer
import os
import sys
import urllib, urlparse
import posixpath
import StringIO
import re
import shutil
import threading
import time
import socket
import itertools
import Reporter
import ConfigParser
###
# Various patterns matched or replaced by server.
kReportFileRE = re.compile('(.*/)?report-(.*)\\.html')
kBugKeyValueRE = re.compile('<!-- BUG([^ ]*) (.*) -->')
# <!-- REPORTPROBLEM file="crashes/clang_crash_ndSGF9.mi" stderr="crashes/clang_crash_ndSGF9.mi.stderr.txt" info="crashes/clang_crash_ndSGF9.mi.info" -->
kReportCrashEntryRE = re.compile('<!-- REPORTPROBLEM (.*?)-->')
kReportCrashEntryKeyValueRE = re.compile(' ?([^=]+)="(.*?)"')
kReportReplacements = []
# Add custom javascript.
kReportReplacements.append((re.compile('<!-- SUMMARYENDHEAD -->'), """\
<script language="javascript" type="text/javascript">
function load(url) {
if (window.XMLHttpRequest) {
req = new XMLHttpRequest();
} else if (window.ActiveXObject) {
req = new ActiveXObject("Microsoft.XMLHTTP");
}
if (req != undefined) {
req.open("GET", url, true);
req.send("");
}
}
</script>"""))
# Insert additional columns.
kReportReplacements.append((re.compile('<!-- REPORTBUGCOL -->'),
'<td></td><td></td>'))
# Insert report bug and open file links.
kReportReplacements.append((re.compile('<!-- REPORTBUG id="report-(.*)\\.html" -->'),
('<td class="Button"><a href="report/\\1">Report Bug</a></td>' +
'<td class="Button"><a href="javascript:load(\'open/\\1\')">Open File</a></td>')))
kReportReplacements.append((re.compile('<!-- REPORTHEADER -->'),
'<h3><a href="/">Summary</a> > Report %(report)s</h3>'))
kReportReplacements.append((re.compile('<!-- REPORTSUMMARYEXTRA -->'),
'<td class="Button"><a href="report/%(report)s">Report Bug</a></td>'))
# Insert report crashes link.
# Disabled for the time being until we decide exactly when this should
# be enabled. Also the radar reporter needs to be fixed to report
# multiple files.
#kReportReplacements.append((re.compile('<!-- REPORTCRASHES -->'),
# '<br>These files will automatically be attached to ' +
# 'reports filed here: <a href="report_crashes">Report Crashes</a>.'))
###
# Other simple parameters
kResources = posixpath.join(posixpath.dirname(__file__), 'Resources')
kConfigPath = os.path.expanduser('~/.scanview.cfg')
###
__version__ = "0.1"
__all__ = ["create_server"]
class ReporterThread(threading.Thread):
def __init__(self, report, reporter, parameters, server):
threading.Thread.__init__(self)
self.report = report
self.server = server
self.reporter = reporter
self.parameters = parameters
self.success = False
self.status = None
def run(self):
result = None
try:
if self.server.options.debug:
print >>sys.stderr, "%s: SERVER: submitting bug."%(sys.argv[0],)
self.status = self.reporter.fileReport(self.report, self.parameters)
self.success = True
time.sleep(3)
if self.server.options.debug:
print >>sys.stderr, "%s: SERVER: submission complete."%(sys.argv[0],)
except Reporter.ReportFailure,e:
self.status = e.value
except Exception,e:
s = StringIO.StringIO()
import traceback
print >>s,'<b>Unhandled Exception</b><br><pre>'
traceback.print_exc(e,file=s)
print >>s,'</pre>'
self.status = s.getvalue()
class ScanViewServer(BaseHTTPServer.HTTPServer):
def __init__(self, address, handler, root, reporters, options):
BaseHTTPServer.HTTPServer.__init__(self, address, handler)
self.root = root
self.reporters = reporters
self.options = options
self.halted = False
self.config = None
self.load_config()
def load_config(self):
self.config = ConfigParser.RawConfigParser()
# Add defaults
self.config.add_section('ScanView')
for r in self.reporters:
self.config.add_section(r.getName())
for p in r.getParameters():
if p.saveConfigValue():
self.config.set(r.getName(), p.getName(), '')
# Ignore parse errors
try:
self.config.read([kConfigPath])
except:
pass
# Save on exit
import atexit
atexit.register(lambda: self.save_config())
def save_config(self):
# Ignore errors (only called on exit).
try:
f = open(kConfigPath,'w')
self.config.write(f)
f.close()
except:
pass
def halt(self):
self.halted = True
if self.options.debug:
print >>sys.stderr, "%s: SERVER: halting." % (sys.argv[0],)
def serve_forever(self):
while not self.halted:
if self.options.debug > 1:
print >>sys.stderr, "%s: SERVER: waiting..." % (sys.argv[0],)
try:
self.handle_request()
except OSError,e:
print 'OSError',e.errno
def finish_request(self, request, client_address):
if self.options.autoReload:
import ScanView
self.RequestHandlerClass = reload(ScanView).ScanViewRequestHandler
BaseHTTPServer.HTTPServer.finish_request(self, request, client_address)
def handle_error(self, request, client_address):
# Ignore socket errors
info = sys.exc_info()
if info and isinstance(info[1], socket.error):
if self.options.debug > 1:
print >>sys.stderr, "%s: SERVER: ignored socket error." % (sys.argv[0],)
return
BaseHTTPServer.HTTPServer.handle_error(self, request, client_address)
# Borrowed from Quixote, with simplifications.
def parse_query(qs, fields=None):
if fields is None:
fields = {}
for chunk in filter(None, qs.split('&')):
if '=' not in chunk:
name = chunk
value = ''
else:
name, value = chunk.split('=', 1)
name = urllib.unquote(name.replace('+', ' '))
value = urllib.unquote(value.replace('+', ' '))
item = fields.get(name)
if item is None:
fields[name] = [value]
else:
item.append(value)
return fields
class ScanViewRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
server_version = "ScanViewServer/" + __version__
dynamic_mtime = time.time()
def do_HEAD(self):
try:
SimpleHTTPServer.SimpleHTTPRequestHandler.do_HEAD(self)
except Exception,e:
self.handle_exception(e)
def do_GET(self):
try:
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
except Exception,e:
self.handle_exception(e)
def do_POST(self):
"""Serve a POST request."""
try:
length = self.headers.getheader('content-length') or "0"
try:
length = int(length)
except:
length = 0
content = self.rfile.read(length)
fields = parse_query(content)
f = self.send_head(fields)
if f:
self.copyfile(f, self.wfile)
f.close()
except Exception,e:
self.handle_exception(e)
def log_message(self, format, *args):
if self.server.options.debug:
sys.stderr.write("%s: SERVER: %s - - [%s] %s\n" %
(sys.argv[0],
self.address_string(),
self.log_date_time_string(),
format%args))
def load_report(self, report):
path = os.path.join(self.server.root, 'report-%s.html'%report)
data = open(path).read()
keys = {}
for item in kBugKeyValueRE.finditer(data):
k,v = item.groups()
keys[k] = v
return keys
def load_crashes(self):
path = posixpath.join(self.server.root, 'index.html')
data = open(path).read()
problems = []
for item in kReportCrashEntryRE.finditer(data):
fieldData = item.group(1)
fields = dict([i.groups() for i in
kReportCrashEntryKeyValueRE.finditer(fieldData)])
problems.append(fields)
return problems
def handle_exception(self, exc):
import traceback
s = StringIO.StringIO()
print >>s, "INTERNAL ERROR\n"
traceback.print_exc(exc, s)
f = self.send_string(s.getvalue(), 'text/plain')
if f:
self.copyfile(f, self.wfile)
f.close()
def get_scalar_field(self, name):
if name in self.fields:
return self.fields[name][0]
else:
return None
def submit_bug(self, c):
title = self.get_scalar_field('title')
description = self.get_scalar_field('description')
report = self.get_scalar_field('report')
reporterIndex = self.get_scalar_field('reporter')
files = []
for fileID in self.fields.get('files',[]):
try:
i = int(fileID)
except:
i = None
if i is None or i<0 or i>=len(c.files):
return (False, 'Invalid file ID')
files.append(c.files[i])
if not title:
return (False, "Missing title.")
if not description:
return (False, "Missing description.")
try:
reporterIndex = int(reporterIndex)
except:
return (False, "Invalid report method.")
# Get the reporter and parameters.
reporter = self.server.reporters[reporterIndex]
parameters = {}
for o in reporter.getParameters():
name = '%s_%s'%(reporter.getName(),o.getName())
if name not in self.fields:
return (False,
'Missing field "%s" for %s report method.'%(name,
reporter.getName()))
parameters[o.getName()] = self.get_scalar_field(name)
# Update config defaults.
if report != 'None':
self.server.config.set('ScanView', 'reporter', reporterIndex)
for o in reporter.getParameters():
if o.saveConfigValue():
name = o.getName()
self.server.config.set(reporter.getName(), name, parameters[name])
# Create the report.
bug = Reporter.BugReport(title, description, files)
# Kick off a reporting thread.
t = ReporterThread(bug, reporter, parameters, self.server)
t.start()
# Wait for thread to die...
while t.isAlive():
time.sleep(.25)
submitStatus = t.status
return (t.success, t.status)
def send_report_submit(self):
report = self.get_scalar_field('report')
c = self.get_report_context(report)
if c.reportSource is None:
reportingFor = "Report Crashes > "
fileBug = """\
<a href="/report_crashes">File Bug</a> > """%locals()
else:
reportingFor = '<a href="/%s">Report %s</a> > ' % (c.reportSource,
report)
fileBug = '<a href="/report/%s">File Bug</a> > ' % report
title = self.get_scalar_field('title')
description = self.get_scalar_field('description')
res,message = self.submit_bug(c)
if res:
statusClass = 'SubmitOk'
statusName = 'Succeeded'
else:
statusClass = 'SubmitFail'
statusName = 'Failed'
result = """
<head>
<title>Bug Submission</title>
<link rel="stylesheet" type="text/css" href="/scanview.css" />
</head>
<body>
<h3>
<a href="/">Summary</a> >
%(reportingFor)s
%(fileBug)s
Submit</h3>
<form name="form" action="">
<table class="form">
<tr><td>
<table class="form_group">
<tr>
<td class="form_clabel">Title:</td>
<td class="form_value">
<input type="text" name="title" size="50" value="%(title)s" disabled>
</td>
</tr>
<tr>
<td class="form_label">Description:</td>
<td class="form_value">
<textarea rows="10" cols="80" name="description" disabled>
%(description)s
</textarea>
</td>
</table>
</td></tr>
</table>
</form>
<h1 class="%(statusClass)s">Submission %(statusName)s</h1>
%(message)s
<p>
<hr>
<a href="/">Return to Summary</a>
</body>
</html>"""%locals()
return self.send_string(result)
def send_open_report(self, report):
try:
keys = self.load_report(report)
except IOError:
return self.send_error(400, 'Invalid report.')
file = keys.get('FILE')
if not file or not posixpath.exists(file):
return self.send_error(400, 'File does not exist: "%s"' % file)
import startfile
if self.server.options.debug:
print >>sys.stderr, '%s: SERVER: opening "%s"'%(sys.argv[0],
file)
status = startfile.open(file)
if status:
res = 'Opened: "%s"' % file
else:
res = 'Open failed: "%s"' % file
return self.send_string(res, 'text/plain')
def get_report_context(self, report):
class Context:
pass
if report is None or report == 'None':
data = self.load_crashes()
# Don't allow empty reports.
if not data:
raise ValueError, 'No crashes detected!'
c = Context()
c.title = 'clang static analyzer failures'
stderrSummary = ""
for item in data:
if 'stderr' in item:
path = posixpath.join(self.server.root, item['stderr'])
if os.path.exists(path):
lns = itertools.islice(open(path), 0, 10)
stderrSummary += '%s\n--\n%s' % (item.get('src',
'<unknown>'),
''.join(lns))
c.description = """\
The clang static analyzer failed on these inputs:
%s
STDERR Summary
--------------
%s
""" % ('\n'.join([item.get('src','<unknown>') for item in data]),
stderrSummary)
c.reportSource = None
c.navMarkup = "Report Crashes > "
c.files = []
for item in data:
c.files.append(item.get('src',''))
c.files.append(posixpath.join(self.server.root,
item.get('file','')))
c.files.append(posixpath.join(self.server.root,
item.get('clangfile','')))
c.files.append(posixpath.join(self.server.root,
item.get('stderr','')))
c.files.append(posixpath.join(self.server.root,
item.get('info','')))
# Just in case something failed, ignore files which don't
# exist.
c.files = [f for f in c.files
if os.path.exists(f) and os.path.isfile(f)]
else:
# Check that this is a valid report.
path = posixpath.join(self.server.root, 'report-%s.html' % report)
if not posixpath.exists(path):
raise ValueError, 'Invalid report ID'
keys = self.load_report(report)
c = Context()
c.title = keys.get('DESC','clang error (unrecognized')
c.description = """\
Bug reported by the clang static analyzer.
Description: %s
File: %s
Line: %s
"""%(c.title, keys.get('FILE','<unknown>'), keys.get('LINE', '<unknown>'))
c.reportSource = 'report-%s.html' % report
c.navMarkup = """<a href="/%s">Report %s</a> > """ % (c.reportSource,
report)
c.files = [path]
return c
def send_report(self, report, configOverrides=None):
def getConfigOption(section, field):
if (configOverrides is not None and
section in configOverrides and
field in configOverrides[section]):
return configOverrides[section][field]
return self.server.config.get(section, field)
# report is None is used for crashes
try:
c = self.get_report_context(report)
except ValueError, e:
return self.send_error(400, e.message)
title = c.title
description= c.description
reportingFor = c.navMarkup
if c.reportSource is None:
extraIFrame = ""
else:
extraIFrame = """\
<iframe src="/%s" width="100%%" height="40%%"
scrolling="auto" frameborder="1">
<a href="/%s">View Bug Report</a>
</iframe>""" % (c.reportSource, c.reportSource)
reporterSelections = []
reporterOptions = []
try:
active = int(getConfigOption('ScanView','reporter'))
except:
active = 0
for i,r in enumerate(self.server.reporters):
selected = (i == active)
if selected:
selectedStr = ' selected'
else:
selectedStr = ''
reporterSelections.append('<option value="%d"%s>%s</option>'%(i,selectedStr,r.getName()))
options = '\n'.join([ o.getHTML(r,title,getConfigOption) for o in r.getParameters()])
display = ('none','')[selected]
reporterOptions.append("""\
<tr id="%sReporterOptions" style="display:%s">
<td class="form_label">%s Options</td>
<td class="form_value">
<table class="form_inner_group">
%s
</table>
</td>
</tr>
"""%(r.getName(),display,r.getName(),options))
reporterSelections = '\n'.join(reporterSelections)
reporterOptionsDivs = '\n'.join(reporterOptions)
reportersArray = '[%s]'%(','.join([`r.getName()` for r in self.server.reporters]))
if c.files:
fieldSize = min(5, len(c.files))
attachFileOptions = '\n'.join(["""\
<option value="%d" selected>%s</option>""" % (i,v) for i,v in enumerate(c.files)])
attachFileRow = """\
<tr>
<td class="form_label">Attach:</td>
<td class="form_value">
<select style="width:100%%" name="files" multiple size=%d>
%s
</select>
</td>
</tr>
""" % (min(5, len(c.files)), attachFileOptions)
else:
attachFileRow = ""
result = """<html>
<head>
<title>File Bug</title>
<link rel="stylesheet" type="text/css" href="/scanview.css" />
</head>
<script language="javascript" type="text/javascript">
var reporters = %(reportersArray)s;
function updateReporterOptions() {
index = document.getElementById('reporter').selectedIndex;
for (var i=0; i < reporters.length; ++i) {
o = document.getElementById(reporters[i] + "ReporterOptions");
if (i == index) {
o.style.display = "";
} else {
o.style.display = "none";
}
}
}
</script>
<body onLoad="updateReporterOptions()">
<h3>
<a href="/">Summary</a> >
%(reportingFor)s
File Bug</h3>
<form name="form" action="/report_submit" method="post">
<input type="hidden" name="report" value="%(report)s">
<table class="form">
<tr><td>
<table class="form_group">
<tr>
<td class="form_clabel">Title:</td>
<td class="form_value">
<input type="text" name="title" size="50" value="%(title)s">
</td>
</tr>
<tr>
<td class="form_label">Description:</td>
<td class="form_value">
<textarea rows="10" cols="80" name="description">
%(description)s
</textarea>
</td>
</tr>
%(attachFileRow)s
</table>
<br>
<table class="form_group">
<tr>
<td class="form_clabel">Method:</td>
<td class="form_value">
<select id="reporter" name="reporter" onChange="updateReporterOptions()">
%(reporterSelections)s
</select>
</td>
</tr>
%(reporterOptionsDivs)s
</table>
<br>
</td></tr>
<tr><td class="form_submit">
<input align="right" type="submit" name="Submit" value="Submit">
</td></tr>
</table>
</form>
%(extraIFrame)s
</body>
</html>"""%locals()
return self.send_string(result)
def send_head(self, fields=None):
if (self.server.options.onlyServeLocal and
self.client_address[0] != '127.0.0.1'):
return self.send_error(401, 'Unauthorized host.')
if fields is None:
fields = {}
self.fields = fields
o = urlparse.urlparse(self.path)
self.fields = parse_query(o.query, fields)
path = posixpath.normpath(urllib.unquote(o.path))
# Split the components and strip the root prefix.
components = path.split('/')[1:]
# Special case some top-level entries.
if components:
name = components[0]
if len(components)==2:
if name=='report':
return self.send_report(components[1])
elif name=='open':
return self.send_open_report(components[1])
elif len(components)==1:
if name=='quit':
self.server.halt()
return self.send_string('Goodbye.', 'text/plain')
elif name=='report_submit':
return self.send_report_submit()
elif name=='report_crashes':
overrides = { 'ScanView' : {},
'Radar' : {},
'Email' : {} }
for i,r in enumerate(self.server.reporters):
if r.getName() == 'Radar':
overrides['ScanView']['reporter'] = i
break
overrides['Radar']['Component'] = 'llvm - checker'
overrides['Radar']['Component Version'] = 'X'
return self.send_report(None, overrides)
elif name=='favicon.ico':
return self.send_path(posixpath.join(kResources,'bugcatcher.ico'))
# Match directory entries.
if components[-1] == '':
components[-1] = 'index.html'
suffix = '/'.join(components)
# The summary may reference source files on disk using rooted
# paths. Make sure these resolve correctly for now.
# FIXME: This isn't a very good idea... we should probably
# mark rooted paths somehow.
if os.path.exists(posixpath.join('/', suffix)):
path = posixpath.join('/', suffix)
else:
path = posixpath.join(self.server.root, suffix)
if self.server.options.debug > 1:
print >>sys.stderr, '%s: SERVER: sending path "%s"'%(sys.argv[0],
path)
return self.send_path(path)
def send_404(self):
self.send_error(404, "File not found")
return None
def send_path(self, path):
ctype = self.guess_type(path)
if ctype.startswith('text/'):
# Patch file instead
return self.send_patched_file(path, ctype)
else:
mode = 'rb'
try:
f = open(path, mode)
except IOError:
return self.send_404()
return self.send_file(f, ctype)
def send_file(self, f, ctype):
# Patch files to add links, but skip binary files.
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f
def send_string(self, s, ctype='text/html', headers=True, mtime=None):
if headers:
self.send_response(200)
self.send_header("Content-type", ctype)
self.send_header("Content-Length", str(len(s)))
if mtime is None:
mtime = self.dynamic_mtime
self.send_header("Last-Modified", self.date_time_string(mtime))
self.end_headers()
return StringIO.StringIO(s)
def send_patched_file(self, path, ctype):
# Allow a very limited set of variables. This is pretty gross.
variables = {}
variables['report'] = ''
m = kReportFileRE.match(path)
if m:
variables['report'] = m.group(2)
try:
f = open(path,'r')
except IOError:
return self.send_404()
fs = os.fstat(f.fileno())
data = f.read()
for a,b in kReportReplacements:
data = a.sub(b % variables, data)
return self.send_string(data, ctype, mtime=fs.st_mtime)
def create_server(address, options, root):
import Reporter
reporters = Reporter.getReporters()
return ScanViewServer(address, ScanViewRequestHandler,
root,
reporters,
options)