#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import print_function
import argparse
import BaseHTTPServer
import json
import os
import os.path
import re
import subprocess
import sys
import tempfile
import urllib2
# Grab the script path because that is where all the static assets are
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
# Find the tools directory for python imports
TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
# Find the root of the skia trunk for finding skpdiff binary
SKIA_ROOT_DIR = os.path.dirname(TOOLS_DIR)
# Find the default location of gm expectations
DEFAULT_GM_EXPECTATIONS_DIR = os.path.join(SKIA_ROOT_DIR, 'expectations', 'gm')
# Imports from within Skia
if TOOLS_DIR not in sys.path:
sys.path.append(TOOLS_DIR)
GM_DIR = os.path.join(SKIA_ROOT_DIR, 'gm')
if GM_DIR not in sys.path:
sys.path.append(GM_DIR)
import gm_json
import jsondiff
# A simple dictionary of file name extensions to MIME types. The empty string
# entry is used as the default when no extension was given or if the extension
# has no entry in this dictionary.
MIME_TYPE_MAP = {'': 'application/octet-stream',
'html': 'text/html',
'css': 'text/css',
'png': 'image/png',
'js': 'application/javascript',
'json': 'application/json'
}
IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
SKPDIFF_INVOKE_FORMAT = '{} --jsonp=false -o {} -f {} {}'
def get_skpdiff_path(user_path=None):
"""Find the skpdiff binary.
@param user_path If none, searches in Release and Debug out directories of
the skia root. If set, checks that the path is a real file and
returns it.
"""
skpdiff_path = None
possible_paths = []
# Use the user given path, or try out some good default paths.
if user_path:
possible_paths.append(user_path)
else:
possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out',
'Release', 'skpdiff'))
possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out',
'Release', 'skpdiff.exe'))
possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out',
'Debug', 'skpdiff'))
possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out',
'Debug', 'skpdiff.exe'))
# Use the first path that actually points to the binary
for possible_path in possible_paths:
if os.path.isfile(possible_path):
skpdiff_path = possible_path
break
# If skpdiff was not found, print out diagnostic info for the user.
if skpdiff_path is None:
print('Could not find skpdiff binary. Either build it into the ' +
'default directory, or specify the path on the command line.')
print('skpdiff paths tried:')
for possible_path in possible_paths:
print(' ', possible_path)
return skpdiff_path
def download_file(url, output_path):
"""Download the file at url and place it in output_path"""
reader = urllib2.urlopen(url)
with open(output_path, 'wb') as writer:
writer.write(reader.read())
def download_gm_image(image_name, image_path, hash_val):
"""Download the gm result into the given path.
@param image_name The GM file name, for example imageblur_gpu.png.
@param image_path Path to place the image.
@param hash_val The hash value of the image.
"""
if hash_val is None:
return
# Separate the test name from a image name
image_match = IMAGE_FILENAME_RE.match(image_name)
test_name = image_match.group(1)
# Calculate the URL of the requested image
image_url = gm_json.CreateGmActualUrl(
test_name, gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, hash_val)
# Download the image as requested
download_file(image_url, image_path)
def get_image_set_from_skpdiff(skpdiff_records):
"""Get the set of all images references in the given records.
@param skpdiff_records An array of records, which are dictionary objects.
"""
expected_set = frozenset([r['baselinePath'] for r in skpdiff_records])
actual_set = frozenset([r['testPath'] for r in skpdiff_records])
return expected_set | actual_set
def set_expected_hash_in_json(expected_results_json, image_name, hash_value):
"""Set the expected hash for the object extracted from
expected-results.json. Note that this only work with bitmap-64bitMD5 hash
types.
@param expected_results_json The Python dictionary with the results to
modify.
@param image_name The name of the image to set the hash of.
@param hash_value The hash to set for the image.
"""
expected_results = expected_results_json[gm_json.JSONKEY_EXPECTEDRESULTS]
if image_name in expected_results:
expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0][1] = hash_value
else:
expected_results[image_name] = {
gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS:
[
[
gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
hash_value
]
]
}
def get_head_version(path):
"""Get the version of the file at the given path stored inside the HEAD of
the git repository. It is returned as a string.
@param path The path of the file whose HEAD is returned. It is assumed the
path is inside a git repo rooted at SKIA_ROOT_DIR.
"""
# git-show will not work with absolute paths. This ensures we give it a path
# relative to the skia root. This path also has to use forward slashes, even
# on windows.
git_path = os.path.relpath(path, SKIA_ROOT_DIR).replace('\\', '/')
git_show_proc = subprocess.Popen(['git', 'show', 'HEAD:' + git_path],
stdout=subprocess.PIPE)
# When invoked outside a shell, git will output the last committed version
# of the file directly to stdout.
git_version_content, _ = git_show_proc.communicate()
return git_version_content
class GMInstance:
"""Information about a GM test result on a specific device:
- device_name = the name of the device that rendered it
- image_name = the GM test name and config
- expected_hash = the current expected hash value
- actual_hash = the actual hash value
- is_rebaselined = True if actual_hash is what is currently in the expected
results file, False otherwise.
"""
def __init__(self,
device_name, image_name,
expected_hash, actual_hash,
is_rebaselined):
self.device_name = device_name
self.image_name = image_name
self.expected_hash = expected_hash
self.actual_hash = actual_hash
self.is_rebaselined = is_rebaselined
class ExpectationsManager:
def __init__(self, expectations_dir, expected_name, updated_name,
skpdiff_path):
"""
@param expectations_dir The directory to traverse for results files.
This should resemble expectations/gm in the Skia trunk.
@param expected_name The name of the expected result files. These
are in the format of expected-results.json.
@param updated_name The name of the updated expected result files.
Normally this matches --expectations-filename-output for the
rebaseline.py tool.
@param skpdiff_path The path used to execute the skpdiff command.
"""
self._expectations_dir = expectations_dir
self._expected_name = expected_name
self._updated_name = updated_name
self._skpdiff_path = skpdiff_path
self._generate_gm_comparison()
def _generate_gm_comparison(self):
"""Generate all the data needed to compare GMs:
- determine which GMs changed
- download the changed images
- compare them with skpdiff
"""
# Get the expectations and compare them with actual hashes
self._get_expectations()
# Create a temporary file tree that makes sense for skpdiff to operate
# on. We take the realpath of the new temp directory because some OSs
# (*cough* osx) put the temp directory behind a symlink that gets
# resolved later down the pipeline and breaks the image map.
image_output_dir = os.path.realpath(tempfile.mkdtemp('skpdiff'))
expected_image_dir = os.path.join(image_output_dir, 'expected')
actual_image_dir = os.path.join(image_output_dir, 'actual')
os.mkdir(expected_image_dir)
os.mkdir(actual_image_dir)
# Download expected and actual images that differed into the temporary
# file tree.
self._download_expectation_images(expected_image_dir, actual_image_dir)
# Invoke skpdiff with our downloaded images and place its results in the
# temporary directory.
self._skpdiff_output_path = os.path.join(image_output_dir,
'skpdiff_output.json')
skpdiff_cmd = SKPDIFF_INVOKE_FORMAT.format(self._skpdiff_path,
self._skpdiff_output_path,
expected_image_dir,
actual_image_dir)
os.system(skpdiff_cmd)
self._load_skpdiff_output()
def _get_expectations(self):
"""Fills self._expectations with GMInstance objects for each test whose
expectation is different between the following two files:
- the local filesystem's updated results file
- git's head version of the expected results file
"""
differ = jsondiff.GMDiffer()
self._expectations = []
for root, dirs, files in os.walk(self._expectations_dir):
for expectation_file in files:
# There are many files in the expectations directory. We only
# care about expected results.
if expectation_file != self._expected_name:
continue
# Get the name of the results file, and be sure there is an
# updated result to compare against. If there is not, there is
# no point in diffing this device.
expected_file_path = os.path.join(root, self._expected_name)
updated_file_path = os.path.join(root, self._updated_name)
if not os.path.isfile(updated_file_path):
continue
# Always get the expected results from git because we may have
# changed them in a previous instance of the server.
expected_contents = get_head_version(expected_file_path)
updated_contents = None
with open(updated_file_path, 'rb') as updated_file:
updated_contents = updated_file.read()
# Read the expected results on disk to determine what we've
# already rebaselined.
commited_contents = None
with open(expected_file_path, 'rb') as expected_file:
commited_contents = expected_file.read()
# Find all expectations that did not match.
expected_diff = differ.GenerateDiffDictFromStrings(
expected_contents,
updated_contents)
# Generate a set of images that have already been rebaselined
# onto disk.
rebaselined_diff = differ.GenerateDiffDictFromStrings(
expected_contents,
commited_contents)
rebaselined_set = set(rebaselined_diff.keys())
# The name of the device corresponds to the name of the folder
# we are in.
device_name = os.path.basename(root)
# Store old and new versions of the expectation for each GM
for image_name, hashes in expected_diff.iteritems():
self._expectations.append(
GMInstance(device_name, image_name,
hashes['old'], hashes['new'],
image_name in rebaselined_set))
def _load_skpdiff_output(self):
"""Loads the results of skpdiff and annotates them with whether they
have already been rebaselined or not. The resulting data is store in
self.skpdiff_records."""
self.skpdiff_records = None
with open(self._skpdiff_output_path, 'rb') as skpdiff_output_file:
self.skpdiff_records = json.load(skpdiff_output_file)['records']
for record in self.skpdiff_records:
record['isRebaselined'] = self.image_map[record['baselinePath']][1].is_rebaselined
def _download_expectation_images(self, expected_image_dir, actual_image_dir):
"""Download the expected and actual images for the _expectations array.
@param expected_image_dir The directory to download expected images
into.
@param actual_image_dir The directory to download actual images into.
"""
image_map = {}
# Look through expectations and download their images.
for expectation in self._expectations:
# Build appropriate paths to download the images into.
expected_image_path = os.path.join(expected_image_dir,
expectation.device_name + '-' +
expectation.image_name)
actual_image_path = os.path.join(actual_image_dir,
expectation.device_name + '-' +
expectation.image_name)
print('Downloading %s for device %s' % (
expectation.image_name, expectation.device_name))
# Download images
download_gm_image(expectation.image_name,
expected_image_path,
expectation.expected_hash)
download_gm_image(expectation.image_name,
actual_image_path,
expectation.actual_hash)
# Annotate the expectations with where the images were downloaded
# to.
expectation.expected_image_path = expected_image_path
expectation.actual_image_path = actual_image_path
# Map the image paths back to the expectations.
image_map[expected_image_path] = (False, expectation)
image_map[actual_image_path] = (True, expectation)
self.image_map = image_map
def _set_expected_hash(self, device_name, image_name, hash_value):
"""Set the expected hash for the image of the given device. This always
writes directly to the expected results file of the given device
@param device_name The name of the device to write the hash to.
@param image_name The name of the image whose hash to set.
@param hash_value The value of the hash to set.
"""
# Retrieve the expected results file as it is in the working tree
json_path = os.path.join(self._expectations_dir, device_name,
self._expected_name)
expectations = gm_json.LoadFromFile(json_path)
# Set the specified hash.
set_expected_hash_in_json(expectations, image_name, hash_value)
# Write it out to disk using gm_json to keep the formatting consistent.
gm_json.WriteToFile(expectations, json_path)
def commit_rebaselines(self, rebaselines):
"""Sets the expected results file to use the hashes of the images in
the rebaselines list. If a expected result image is not in rebaselines
at all, the old hash will be used.
@param rebaselines A list of image paths to use the hash of.
"""
# Reset all expectations to their old hashes because some of them may
# have been set to the new hash by a previous call to this function.
for expectation in self._expectations:
expectation.is_rebaselined = False
self._set_expected_hash(expectation.device_name,
expectation.image_name,
expectation.expected_hash)
# Take all the images to rebaseline
for image_path in rebaselines:
# Get the metadata about the image at the path.
is_actual, expectation = self.image_map[image_path]
expectation.is_rebaselined = is_actual
expectation_hash = expectation.actual_hash if is_actual else\
expectation.expected_hash
# Write out that image's hash directly to the expected results file.
self._set_expected_hash(expectation.device_name,
expectation.image_name,
expectation_hash)
self._load_skpdiff_output()
class SkPDiffHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def send_file(self, file_path):
# Grab the extension if there is one
extension = os.path.splitext(file_path)[1]
if len(extension) >= 1:
extension = extension[1:]
# Determine the MIME type of the file from its extension
mime_type = MIME_TYPE_MAP.get(extension, MIME_TYPE_MAP[''])
# Open the file and send it over HTTP
if os.path.isfile(file_path):
with open(file_path, 'rb') as sending_file:
self.send_response(200)
self.send_header('Content-type', mime_type)
self.end_headers()
self.wfile.write(sending_file.read())
else:
self.send_error(404)
def serve_if_in_dir(self, dir_path, file_path):
# Determine if the file exists relative to the given dir_path AND exists
# under the dir_path. This is to prevent accidentally serving files
# outside the directory intended using symlinks, or '../'.
real_path = os.path.normpath(os.path.join(dir_path, file_path))
if os.path.commonprefix([real_path, dir_path]) == dir_path:
if os.path.isfile(real_path):
self.send_file(real_path)
return True
return False
def do_GET(self):
# Simple rewrite rule of the root path to 'viewer.html'
if self.path == '' or self.path == '/':
self.path = '/viewer.html'
# The [1:] chops off the leading '/'
file_path = self.path[1:]
# Handle skpdiff_output.json manually because it is was processed by the
# server when it was started and does not exist as a file.
if file_path == 'skpdiff_output.json':
self.send_response(200)
self.send_header('Content-type', MIME_TYPE_MAP['json'])
self.end_headers()
# Add JSONP padding to the JSON because the web page expects it. It
# expects it because it was designed to run with or without a web
# server. Without a web server, the only way to load JSON is with
# JSONP.
skpdiff_records = self.server.expectations_manager.skpdiff_records
self.wfile.write('var SkPDiffRecords = ')
json.dump({'records': skpdiff_records}, self.wfile)
self.wfile.write(';')
return
# Attempt to send static asset files first.
if self.serve_if_in_dir(SCRIPT_DIR, file_path):
return
# WARNING: Serving any file the user wants is incredibly insecure. Its
# redeeming quality is that we only serve gm files on a white list.
if self.path in self.server.image_set:
self.send_file(self.path)
return
# If no file to send was found, just give the standard 404
self.send_error(404)
def do_POST(self):
if self.path == '/commit_rebaselines':
content_length = int(self.headers['Content-length'])
request_data = json.loads(self.rfile.read(content_length))
rebaselines = request_data['rebaselines']
self.server.expectations_manager.commit_rebaselines(rebaselines)
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write('{"success":true}')
return
# If the we have no handler for this path, give em' the 404
self.send_error(404)
def run_server(expectations_manager, port=8080):
# It's important to parse the results file so that we can make a set of
# images that the web page might request.
skpdiff_records = expectations_manager.skpdiff_records
image_set = get_image_set_from_skpdiff(skpdiff_records)
# Do not bind to interfaces other than localhost because the server will
# attempt to serve files relative to the root directory as a last resort
# before 404ing. This means all of your files can be accessed from this
# server, so DO NOT let this server listen to anything but localhost.
server_address = ('127.0.0.1', port)
http_server = BaseHTTPServer.HTTPServer(server_address, SkPDiffHandler)
http_server.image_set = image_set
http_server.expectations_manager = expectations_manager
print('Navigate thine browser to: http://{}:{}/'.format(*server_address))
http_server.serve_forever()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--port', '-p', metavar='PORT',
type=int,
default=8080,
help='port to bind the server to; ' +
'defaults to %(default)s',
)
parser.add_argument('--expectations-dir', metavar='EXPECTATIONS_DIR',
default=DEFAULT_GM_EXPECTATIONS_DIR,
help='path to the gm expectations; ' +
'defaults to %(default)s'
)
parser.add_argument('--expected',
metavar='EXPECTATIONS_FILE_NAME',
default='expected-results.json',
help='the file name of the expectations JSON; ' +
'defaults to %(default)s'
)
parser.add_argument('--updated',
metavar='UPDATED_FILE_NAME',
default='updated-results.json',
help='the file name of the updated expectations JSON;' +
' defaults to %(default)s'
)
parser.add_argument('--skpdiff-path', metavar='SKPDIFF_PATH',
default=None,
help='the path to the skpdiff binary to use; ' +
'defaults to out/Release/skpdiff or out/Default/skpdiff'
)
args = vars(parser.parse_args()) # Convert args into a python dict
# Make sure we have access to an skpdiff binary
skpdiff_path = get_skpdiff_path(args['skpdiff_path'])
if skpdiff_path is None:
sys.exit(1)
# Print out the paths of things for easier debugging
print('script dir :', SCRIPT_DIR)
print('tools dir :', TOOLS_DIR)
print('root dir :', SKIA_ROOT_DIR)
print('expectations dir :', args['expectations_dir'])
print('skpdiff path :', skpdiff_path)
expectations_manager = ExpectationsManager(args['expectations_dir'],
args['expected'],
args['updated'],
skpdiff_path)
run_server(expectations_manager, port=args['port'])
if __name__ == '__main__':
main()