#!/usr/bin/env ruby
# Copyright (C) 2011 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
require 'rubygems'
require 'getoptlong'
require 'pathname'
require 'tempfile'
require 'socket'
begin
require 'json'
rescue LoadError => e
$stderr.puts "It does not appear that you have the 'json' package installed. Try running 'sudo gem install json'."
exit 1
end
# Configuration
CONFIGURATION_FLNM = ENV["HOME"]+"/.bencher"
unless FileTest.exist? CONFIGURATION_FLNM
$stderr.puts "Error: no configuration file at ~/.bencher."
$stderr.puts "This file should contain paths to SunSpider, V8, and Kraken, as well as a"
$stderr.puts "temporary directory that bencher can use for its remote mode. It should be"
$stderr.puts "formatted in JSON. For example:"
$stderr.puts "{"
$stderr.puts " \"sunSpiderPath\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/sunspider-1.0\","
$stderr.puts " \"v8Path\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/v8-v6\","
$stderr.puts " \"krakenPath\": \"/Volumes/Data/pizlo/kraken/kraken-e119421cb325/tests/kraken-1.1\","
$stderr.puts " \"tempPath\": \"/Volumes/Data/pizlo/bencher/temp\""
$stderr.puts "}"
exit 1
end
CONFIGURATION = JSON.parse(File::read(CONFIGURATION_FLNM))
SUNSPIDER_PATH = CONFIGURATION["sunSpiderPath"]
V8_PATH = CONFIGURATION["v8Path"]
KRAKEN_PATH = CONFIGURATION["krakenPath"]
TEMP_PATH = CONFIGURATION["tempPath"]
BENCH_DATA_PATH = TEMP_PATH + "/benchdata"
IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933,
0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684,
0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227,
0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674,
0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652,
0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604,
0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338,
0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538,
0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185,
0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694,
0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244,
0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067,
0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323,
0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129,
0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574,
0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722,
0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624,
0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321,
0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843,
0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217,
0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464,
0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599,
0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638,
0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592,
0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471,
0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283,
0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037,
0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738,
0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481,
0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085,
0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651,
0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183,
0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683,
0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154,
0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599,
0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019,
0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417,
0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795,
0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153,
0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494,
0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818,
0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128,
0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423,
0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704,
0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974,
0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232,
0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479,
0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716,
0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943,
0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192,
0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401,
0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963,
0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822,
0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007,
0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186,
0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358,
0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525,
0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686,
0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841,
0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991,
0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137,
0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278,
0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414,
0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547,
0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675,
0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799,
0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192,
0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037,
0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151,
0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262,
0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369,
0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489,
0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259,
0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689,
0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785,
0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879,
0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297,
0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059,
0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145,
0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242,
0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324,
0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404,
0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483,
0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559,
0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634,
0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707,
0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778,
0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848,
0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916,
0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983,
0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057,
0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121,
0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183,
0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244,
0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304,
0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363,
0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442,
0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476,
0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539,
0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946,
0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653,
0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712,
0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762,
0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812,
0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867,
0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915,
0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962,
0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007,
0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059,
0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103,
0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146,
0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189,
0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231,
0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272,
0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312,
0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352,
0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391,
0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429,
0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467,
0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504,
0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546,
0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582,
0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617,
0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652,
0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686,
0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724,
0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757,
0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579,
0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822,
0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853,
0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885,
0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915,
0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595,
0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979,
0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013,
0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041,
0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074,
0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102,
0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129,
0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156,
0.99616, 0.996164]
# Run-time configuration parameters (can be set with command-line options)
$rerun=1
$inner=3
$warmup=1
$outer=4
$includeSunSpider=true
$includeV8=true
$includeKraken=true
$measureGC=false
$benchmarkPattern=nil
$verbosity=0
$timeMode=:preciseTime
$forceVMKind=nil
$brief=false
$silent=false
$remoteHosts=[]
$alsoLocal=false
$sshOptions=[]
$vms = []
$needToCopyVMs = false
$dontCopyVMs = false
$prepare = true
$run = true
$analyze = []
# Helpful functions and classes
def smallUsage
puts "Use the --help option to get basic usage information."
exit 1
end
def usage
puts "bencher [options] <vm1> [<vm2> ...]"
puts
puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
puts "benchmarks, and reports detailed statistics. What makes bencher special is"
puts "that each benchmark/VM configuration is run in a single VM invocation, and"
puts "the invocations are run in random order. This minimizes systematics due to"
puts "one benchmark polluting the running time of another. The fine-grained"
puts "interleaving of VM invocations further minimizes systematics due to changes in"
puts "the performance or behavior of your machine."
puts
puts "Bencher is highly configurable. You can compare as many VMs as you like. You"
puts "can change the amount of warm-up iterations, number of iterations executed per"
puts "VM invocation, and the number of VM invocations per benchmark. By default,"
puts "SunSpider, VM, and Kraken are all run; but you can run any combination of these"
puts "suites."
puts
puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
puts "the executable and <name> is the name that you would like to give the"
puts "configuration for the purposeof reporting. If no name is given, a generic name"
puts "of the form Conf#<n> will be ascribed to the configuration automatically."
puts
puts "Options:"
puts "--rerun <n> Set the number of iterations of the benchmark that"
puts " contribute to the measured run time. Default is #{$rerun}."
puts "--inner <n> Set the number of inner (per-runtime-invocation)"
puts " iterations. Default is #{$inner}."
puts "--outer <n> Set the number of runtime invocations for each benchmark."
puts " Default is #{$outer}."
puts "--warmup <n> Set the number of warm-up runs per invocation. Default"
puts " is #{$warmup}."
puts "--timing-mode Set the way that bencher measures time. Possible values"
puts " are 'preciseTime' and 'date'. Default is 'preciseTime'."
puts "--force-vm-kind Turn off auto-detection of VM kind, and assume that it is"
puts " the one specified. Valid arguments are 'jsc' or"
puts " 'DumpRenderTree'."
puts "--force-vm-copy Force VM builds to be copied to bencher's working directory."
puts " This may reduce pathologies resulting from path names."
puts "--dont-copy-vms Don't copy VMs even when doing a remote benchmarking run;"
puts " instead assume that they are already there."
puts "--v8-only Only run V8."
puts "--sunspider-only Only run SunSpider."
puts "--kraken-only Only run Kraken."
puts "--exclude-v8 Exclude V8 (only run SunSpider and Kraken)."
puts "--exclude-sunspider Exclude SunSpider (only run V8 and Kraken)."
puts "--exclude-kraken Exclude Kraken (only run SunSpider and V8)."
puts "--benchmarks Only run benchmarks matching the given regular expression."
puts "--measure-gc Turn off manual calls to gc(), so that GC time is measured."
puts " Works best with large values of --inner. You can also say"
puts " --measure-gc <conf>, which turns this on for one"
puts " configuration only."
puts "--verbose or -v Print more stuff."
puts "--brief Print only the final result for each VM."
puts "--silent Don't print progress. This might slightly reduce some"
puts " performance perturbation."
puts "--remote <sshhosts> Performance performance measurements remotely, on the given"
puts " SSH host(s). Easiest way to use this is to specify the SSH"
puts " user@host string. However, you can also supply a comma-"
puts " separated list of SSH hosts. Alternatively, you can use this"
puts " option multiple times to specify multiple hosts. This"
puts " automatically copies the WebKit release builds of the VMs"
puts " you specified to all of the hosts."
puts "--ssh-options Pass additional options to SSH."
puts "--local Also do a local benchmark run even when doing --remote."
puts "--prepare-only Only prepare the bencher runscript (a shell script that"
puts " invokes the VMs to run benchmarks) but don't run it."
puts "--analyze Only read the output of the runscript but don't do anything"
puts " else. This requires passing the same arguments to bencher"
puts " that you passed when running --prepare-only."
puts "--help or -h Display this message."
puts
puts "Example:"
puts "bencher TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
exit 1
end
def fail(reason)
if reason.respond_to? :backtrace
puts "FAILED: #{reason}"
puts "Stack trace:"
puts reason.backtrace.join("\n")
else
puts "FAILED: #{reason}"
end
smallUsage
end
def quickFail(r1,r2)
$stderr.puts "#{$0}: #{r1}"
puts
fail(r2)
end
def intArg(argName,arg,min,max)
result=arg.to_i
unless result.to_s == arg
quickFail("Expected an integer value for #{argName}, but got #{arg}.",
"Invalid argument for command-line option")
end
if min and result<min
quickFail("Argument for #{argName} cannot be smaller than #{min}.",
"Invalid argument for command-line option")
end
if max and result>max
quickFail("Argument for #{argName} cannot be greater than #{max}.",
"Invalid argument for command-line option")
end
result
end
def computeMean(array)
sum=0.0
array.each {
| value |
sum += value
}
sum/array.length
end
def computeGeometricMean(array)
mult=1.0
array.each {
| value |
mult*=value
}
mult**(1.0/array.length)
end
def computeHarmonicMean(array)
1.0 / computeMean(array.collect{ | value | 1.0 / value })
end
def computeStdDev(array)
case array.length
when 0
0.0/0.0
when 1
0.0
else
begin
mean=computeMean(array)
sum=0.0
array.each {
| value |
sum += (value-mean)**2
}
Math.sqrt(sum/(array.length-1))
rescue
0.0/0.0
end
end
end
class Array
def shuffle!
size.downto(1) { |n| push delete_at(rand(n)) }
self
end
end
def inverseBetaRegularized(n)
IBR_LOOKUP[n-1]
end
def numToStr(num)
"%.4f"%(num.to_f)
end
class NoChange
attr_reader :amountFaster
def initialize(amountFaster)
@amountFaster = amountFaster
end
def shortForm
" "
end
def longForm
" might be #{numToStr(@amountFaster)}x faster"
end
def to_s
if @amountFaster < 1.01
""
else
longForm
end
end
end
class Faster
attr_reader :amountFaster
def initialize(amountFaster)
@amountFaster = amountFaster
end
def shortForm
"^"
end
def longForm
"^ definitely #{numToStr(@amountFaster)}x faster"
end
def to_s
longForm
end
end
class Slower
attr_reader :amountSlower
def initialize(amountSlower)
@amountSlower = amountSlower
end
def shortForm
"!"
end
def longForm
"! definitely #{numToStr(@amountSlower)}x slower"
end
def to_s
longForm
end
end
class MayBeSlower
attr_reader :amountSlower
def initialize(amountSlower)
@amountSlower = amountSlower
end
def shortForm
"?"
end
def longForm
"? might be #{numToStr(@amountSlower)}x slower"
end
def to_s
if @amountSlower < 1.01
"?"
else
longForm
end
end
end
class Stats
def initialize
@array = []
end
def add(value)
if value.is_a? Stats
add(value.array)
elsif value.respond_to? :each
value.each {
| v |
add(v)
}
else
@array << value.to_f
end
end
def array
@array
end
def sum
result=0
@array.each {
| value |
result += value
}
result
end
def min
@array.min
end
def max
@array.max
end
def size
@array.length
end
def mean
computeMean(array)
end
def arithmeticMean
mean
end
def stdDev
computeStdDev(array)
end
def stdErr
stdDev/Math.sqrt(size)
end
# Computes a 95% Student's t distribution confidence interval
def confInt
if size < 2
0.0/0.0
else
raise if size > 1000
Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
end
end
def lower
mean-confInt
end
def upper
mean+confInt
end
def geometricMean
computeGeometricMean(array)
end
def harmonicMean
computeHarmonicMean(array)
end
def compareTo(other)
if upper < other.lower
Faster.new(other.mean/mean)
elsif lower > other.upper
Slower.new(mean/other.mean)
elsif mean > other.mean
MayBeSlower.new(mean/other.mean)
else
NoChange.new(other.mean/mean)
end
end
def to_s
"size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
end
end
def doublePuts(out1,out2,msg)
out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
out2.puts msg
end
class Benchfile < File
@@counter = 0
attr_reader :filename, :basename
def initialize(name)
@basename, @filename = Benchfile.uniqueFilename(name)
super(@filename, "w")
end
def self.uniqueFilename(name)
if name.is_a? Array
basename = name[0] + @@counter.to_s + name[1]
else
basename = name + @@counter.to_s
end
filename = BENCH_DATA_PATH + "/" + basename
@@counter += 1
raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
[basename, filename]
end
def self.create(name)
file = Benchfile.new(name)
yield file
file.close
file.basename
end
end
$dataFiles={}
def ensureFile(key, filename)
unless $dataFiles[key]
$dataFiles[key] = Benchfile.create(key) {
| outp |
doublePuts($stderr,outp,IO::read(filename))
}
end
$dataFiles[key]
end
def emitBenchRunCodeFile(name, plan, benchDataPath, benchPath)
case plan.vm.vmType
when :jsc
Benchfile.create("bencher") {
| file |
case $timeMode
when :preciseTime
doublePuts($stderr,file,"function __bencher_curTimeMS() {")
doublePuts($stderr,file," return preciseTime()*1000")
doublePuts($stderr,file,"}")
when :date
doublePuts($stderr,file,"function __bencher_curTimeMS() {")
doublePuts($stderr,file," return Date.now()")
doublePuts($stderr,file,"}")
else
raise
end
if benchDataPath
doublePuts($stderr,file,"load(#{benchDataPath.inspect});")
doublePuts($stderr,file,"gc();")
doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
doublePuts($stderr,file," before = __bencher_curTimeMS();")
$rerun.times {
doublePuts($stderr,file," load(#{benchPath.inspect});")
}
doublePuts($stderr,file," after = __bencher_curTimeMS();")
doublePuts($stderr,file," if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(after-before));");
doublePuts($stderr,file," gc();") unless plan.vm.shouldMeasureGC
doublePuts($stderr,file,"}")
else
doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
doublePuts($stderr,file," var __bencher_before = __bencher_curTimeMS();")
$rerun.times {
doublePuts($stderr,file," run(__bencher_what);")
}
doublePuts($stderr,file," var __bencher_after = __bencher_curTimeMS();")
doublePuts($stderr,file," return __bencher_after - __bencher_before;")
doublePuts($stderr,file,"}")
$warmup.times {
doublePuts($stderr,file,"__bencher_run(#{benchPath.inspect})")
doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
}
$inner.times {
| innerIndex |
doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchPath.inspect}));")
doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
}
end
}
when :dumpRenderTree
mainCode = Benchfile.create("bencher") {
| file |
doublePuts($stderr,file,"__bencher_count = 0;")
doublePuts($stderr,file,"function __bencher_doNext(result) {")
doublePuts($stderr,file," if (__bencher_count >= #{$warmup})")
doublePuts($stderr,file," debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
doublePuts($stderr,file," __bencher_count++;")
doublePuts($stderr,file," if (__bencher_count < #{$inner+$warmup})")
doublePuts($stderr,file," __bencher_runImpl(__bencher_doNext);")
doublePuts($stderr,file," else")
doublePuts($stderr,file," quit();")
doublePuts($stderr,file,"}")
doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
}
cssCode = Benchfile.create("bencher-css") {
| file |
doublePuts($stderr,file,".pass {\n font-weight: bold;\n color: green;\n}\n.fail {\n font-weight: bold;\n color: red;\n}\n\#console {\n white-space: pre-wrap;\n font-family: monospace;\n}")
}
preCode = Benchfile.create("bencher-pre") {
| file |
doublePuts($stderr,file,"if (window.testRunner) {")
doublePuts($stderr,file," if (window.enablePixelTesting) {")
doublePuts($stderr,file," testRunner.dumpAsTextWithPixelResults();")
doublePuts($stderr,file," } else {")
doublePuts($stderr,file," testRunner.dumpAsText();")
doublePuts($stderr,file," }")
doublePuts($stderr,file,"}")
doublePuts($stderr,file,"")
doublePuts($stderr,file,"function debug(msg)")
doublePuts($stderr,file,"{")
doublePuts($stderr,file," var span = document.createElement(\"span\");")
doublePuts($stderr,file," document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
doublePuts($stderr,file," span.innerHTML = msg + '<br />';")
doublePuts($stderr,file,"}")
doublePuts($stderr,file,"")
doublePuts($stderr,file,"function quit() {")
doublePuts($stderr,file," testRunner.notifyDone();")
doublePuts($stderr,file,"}")
doublePuts($stderr,file,"")
doublePuts($stderr,file,"__bencher_continuation=null;")
doublePuts($stderr,file,"")
doublePuts($stderr,file,"function reportResult(result) {")
doublePuts($stderr,file," __bencher_continuation(result);")
doublePuts($stderr,file,"}")
doublePuts($stderr,file,"")
doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
doublePuts($stderr,file," function doit() {")
doublePuts($stderr,file," document.getElementById(\"frameparent\").innerHTML = \"\";")
doublePuts($stderr,file," document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
doublePuts($stderr,file," var testFrame = document.getElementById(\"testframe\");")
doublePuts($stderr,file," testFrame.contentDocument.open();")
doublePuts($stderr,file," testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
if benchDataPath
doublePuts($stderr,file," testFrame.contentDocument.write(\"<script src=\\\"#{benchDataPath}\\\"></script>\");")
end
doublePuts($stderr,file," testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">__bencher_before = Date.now();</script><script src=\\\"#{benchPath}\\\"></script><script type=\\\"text/javascript\\\">window.parent.reportResult(Date.now() - __bencher_before);</script></body></html>\");")
doublePuts($stderr,file," testFrame.contentDocument.close();")
doublePuts($stderr,file," }")
doublePuts($stderr,file," __bencher_continuation = continuation;")
doublePuts($stderr,file," window.setTimeout(doit, 10);")
doublePuts($stderr,file,"}")
}
Benchfile.create(["bencher-htmldoc",".html"]) {
| file |
doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
}
else
raise
end
end
def emitBenchRunCode(name, plan, benchDataPath, benchPath)
plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchDataPath, benchPath))
end
def planForDescription(plans, benchFullname, vmName, iteration)
raise unless benchFullname =~ /\//
suiteName = $~.pre_match
benchName = $~.post_match
result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
raise unless result.size == 1
result[0]
end
class ParsedResult
attr_reader :plan, :innerIndex, :time
def initialize(plan, innerIndex, time)
@plan = plan
@innerIndex = innerIndex
@time = time
raise unless @plan.is_a? BenchPlan
raise unless @innerIndex.is_a? Integer
raise unless @time.is_a? Numeric
end
def benchmark
plan.benchmark
end
def suite
plan.suite
end
def vm
plan.vm
end
def outerIndex
plan.iteration
end
def self.parse(plans, string)
if string =~ /([a-zA-Z0-9\/-]+): ([a-zA-Z0-9_# ]+): ([0-9]+): ([0-9]+): Time: /
benchFullname = $1
vmName = $2
outerIndex = $3.to_i
innerIndex = $4.to_i
time = $~.post_match.to_f
ParsedResult.new(planForDescription(plans, benchFullname, vmName, outerIndex), innerIndex, time)
else
nil
end
end
end
class VM
def initialize(origPath, name, nameKind, svnRevision)
@origPath = origPath.to_s
@path = origPath.to_s
@name = name
@nameKind = nameKind
if $forceVMKind
@vmType = $forceVMKind
else
if @origPath =~ /DumpRenderTree$/
@vmType = :dumpRenderTree
else
@vmType = :jsc
end
end
@svnRevision = svnRevision
# Try to detect information about the VM.
if path =~ /\/WebKitBuild\/Release\/([a-zA-Z]+)$/
@checkoutPath = $~.pre_match
# FIXME: Use some variant of this:
# <bdash> def retrieve_revision
# <bdash> `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
# <bdash> end
unless @svnRevision
begin
Dir.chdir(@checkoutPath) {
$stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
IO.popen("svn info", "r") {
| inp |
inp.each_line {
| line |
if line =~ /Revision: ([0-9]+)/
@svnRevision = $1
end
}
}
}
unless @svnRevision
$stderr.puts "Warning: running svn info for #{name} silently failed."
end
rescue => e
# Failed to detect svn revision.
$stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
end
end
else
$stderr.puts "Warning: could not identify checkout location for #{name}"
end
if @path =~ /\/Release\/([a-zA-Z]+)$/
@libPath, @relativeBinPath = $~.pre_match+"/Release", "./#{$1}"
elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
@libPath = $~.pre_match
elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
@libPath, @relativeBinPath = $~.pre_match, $&[1..-1]
end
end
def canCopyIntoBenchPath
if @libPath and @relativeBinPath
true
else
false
end
end
def copyIntoBenchPath
raise unless canCopyIntoBenchPath
basename, filename = Benchfile.uniqueFilename("vm")
raise unless Dir.mkdir(filename)
cmd = "cp -a #{@libPath.inspect}/* #{filename.inspect}"
$stderr.puts ">> #{cmd}" if $verbosity>=2
raise unless system(cmd)
@path = "#{basename}/#{@relativeBinPath}"
@libPath = basename
end
def to_s
@name
end
def name
@name
end
def shouldMeasureGC
$measureGC == true or ($measureGC == name)
end
def origPath
@origPath
end
def path
@path
end
def nameKind
@nameKind
end
def vmType
@vmType
end
def checkoutPath
@checkoutPath
end
def svnRevision
@svnRevision
end
def printFunction
case @vmType
when :jsc
"print"
when :dumpRenderTree
"debug"
else
raise @vmType
end
end
def emitRunCode(fileToRun)
myLibPath = @libPath
myLibPath = "" unless myLibPath
$script.puts "export DYLD_LIBRARY_PATH=#{myLibPath.to_s.inspect}"
$script.puts "export DYLD_FRAMEWORK_PATH=#{myLibPath.to_s.inspect}"
$script.puts "#{path} #{fileToRun}"
end
end
class StatsAccumulator
def initialize
@stats = []
($outer*$inner).times {
@stats << Stats.new
}
end
def statsForIteration(outerIteration, innerIteration)
@stats[outerIteration*$inner + innerIteration]
end
def stats
result = Stats.new
@stats.each {
| stat |
result.add(yield stat)
}
result
end
def geometricMeanStats
stats {
| stat |
stat.geometricMean
}
end
def arithmeticMeanStats
stats {
| stat |
stat.arithmeticMean
}
end
end
module Benchmark
attr_accessor :benchmarkSuite
attr_reader :name
def fullname
benchmarkSuite.name + "/" + name
end
def to_s
fullname
end
end
class SunSpiderBenchmark
include Benchmark
def initialize(name)
@name = name
end
def emitRunCode(plan)
emitBenchRunCode(fullname, plan, nil, ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js"))
end
end
class V8Benchmark
include Benchmark
def initialize(name)
@name = name
end
def emitRunCode(plan)
emitBenchRunCode(fullname, plan, nil, ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js"))
end
end
class KrakenBenchmark
include Benchmark
def initialize(name)
@name = name
end
def emitRunCode(plan)
emitBenchRunCode(fullname, plan, ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js"), ensureFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js"))
end
end
class BenchmarkSuite
def initialize(name, path, preferredMean)
@name = name
@path = path
@preferredMean = preferredMean
@benchmarks = []
end
def name
@name
end
def to_s
@name
end
def path
@path
end
def add(benchmark)
if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
benchmark.benchmarkSuite = self
@benchmarks << benchmark
end
end
def benchmarks
@benchmarks
end
def benchmarkForName(name)
result = @benchmarks.select{|v| v.name == name}
raise unless result.length == 1
result[0]
end
def empty?
@benchmarks.empty?
end
def retain_if
@benchmarks.delete_if {
| benchmark |
not yield benchmark
}
end
def preferredMean
@preferredMean
end
def computeMean(stat)
stat.send @preferredMean
end
end
class BenchRunPlan
def initialize(benchmark, vm, iteration)
@benchmark = benchmark
@vm = vm
@iteration = iteration
end
def benchmark
@benchmark
end
def suite
@benchmark.benchmarkSuite
end
def vm
@vm
end
def iteration
@iteration
end
def emitRunCode
@benchmark.emitRunCode(self)
end
end
class BenchmarkOnVM
def initialize(benchmark, suiteOnVM)
@benchmark = benchmark
@suiteOnVM = suiteOnVM
@stats = Stats.new
end
def to_s
"#{@benchmark} on #{@suiteOnVM.vm}"
end
def benchmark
@benchmark
end
def vm
@suiteOnVM.vm
end
def vmStats
@suiteOnVM.vmStats
end
def suite
@benchmark.benchmarkSuite
end
def suiteOnVM
@suiteOnVM
end
def stats
@stats
end
def parseResult(result)
raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
raise unless result.benchmark == @benchmark
@stats.add(result.time)
end
end
class SuiteOnVM < StatsAccumulator
def initialize(vm, vmStats, suite)
super()
@vm = vm
@vmStats = vmStats
@suite = suite
raise unless @vm.is_a? VM
raise unless @vmStats.is_a? StatsAccumulator
raise unless @suite.is_a? BenchmarkSuite
end
def to_s
"#{@suite} on #{@vm}"
end
def suite
@suite
end
def vm
@vm
end
def vmStats
raise unless @vmStats
@vmStats
end
end
class BenchPlan
def initialize(benchmarkOnVM, iteration)
@benchmarkOnVM = benchmarkOnVM
@iteration = iteration
end
def to_s
"#{@benchmarkOnVM} \##{@iteration+1}"
end
def benchmarkOnVM
@benchmarkOnVM
end
def benchmark
@benchmarkOnVM.benchmark
end
def suite
@benchmarkOnVM.suite
end
def vm
@benchmarkOnVM.vm
end
def iteration
@iteration
end
def parseResult(result)
raise unless result.plan == self
@benchmarkOnVM.parseResult(result)
@benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
@benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
end
end
def lpad(str,chars)
if str.length>chars
str
else
"%#{chars}s"%(str)
end
end
def rpad(str,chars)
while str.length<chars
str+=" "
end
str
end
def center(str,chars)
while str.length<chars
str+=" "
if str.length<chars
str=" "+str
end
end
str
end
def statsToStr(stats)
if $inner*$outer == 1
string = numToStr(stats.mean)
raise unless string =~ /\./
left = $~.pre_match
right = $~.post_match
lpad(left,12)+"."+rpad(right,9)
else
lpad(numToStr(stats.mean),11)+"+-"+rpad(numToStr(stats.confInt),9)
end
end
def plural(num)
if num == 1
""
else
"s"
end
end
def wrap(str, columns)
array = str.split
result = ""
curLine = array.shift
array.each {
| curStr |
if (curLine + " " + curStr).size > columns
result += curLine + "\n"
curLine = curStr
else
curLine += " " + curStr
end
}
result + curLine + "\n"
end
def runAndGetResults
results = nil
Dir.chdir(BENCH_DATA_PATH) {
IO.popen("sh ./runscript", "r") {
| inp |
results = inp.read
}
raise "Script did not complete correctly: #{$?}" unless $?.success?
}
raise unless results
results
end
def parseAndDisplayResults(results)
vmStatses = []
$vms.each {
vmStatses << StatsAccumulator.new
}
suitesOnVMs = []
suitesOnVMsForSuite = {}
$suites.each {
| suite |
suitesOnVMsForSuite[suite] = []
}
suitesOnVMsForVM = {}
$vms.each {
| vm |
suitesOnVMsForVM[vm] = []
}
benchmarksOnVMs = []
benchmarksOnVMsForBenchmark = {}
$benchmarks.each {
| benchmark |
benchmarksOnVMsForBenchmark[benchmark] = []
}
$vms.each_with_index {
| vm, vmIndex |
vmStats = vmStatses[vmIndex]
$suites.each {
| suite |
suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
suitesOnVMs << suiteOnVM
suitesOnVMsForSuite[suite] << suiteOnVM
suitesOnVMsForVM[vm] << suiteOnVM
suite.benchmarks.each {
| benchmark |
benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM)
benchmarksOnVMs << benchmarkOnVM
benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
}
}
}
plans = []
benchmarksOnVMs.each {
| benchmarkOnVM |
$outer.times {
| iteration |
plans << BenchPlan.new(benchmarkOnVM, iteration)
}
}
hostname = nil
hwmodel = nil
results.each_line {
| line |
line.chomp!
if line =~ /HOSTNAME:([^.]+)/
hostname = $1
elsif line =~ /HARDWARE:hw\.model: /
hwmodel = $~.post_match.chomp
else
result = ParsedResult.parse(plans, line.chomp)
if result
result.plan.parseResult(result)
end
end
}
# Compute the geomean of the preferred means of results on a SuiteOnVM
overallResults = []
$vms.each {
| vm |
result = Stats.new
$outer.times {
| outerIndex |
$inner.times {
| innerIndex |
curResult = Stats.new
suitesOnVMsForVM[vm].each {
| suiteOnVM |
# For a given iteration, suite, and VM, compute the suite's preferred mean
# over the data collected for all benchmarks in that suite. We'll have one
# sample per benchmark. For example on V8 this will be the geomean of 1
# sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
# splay.
curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
}
# curResult now holds 1 sample for each of the means computed in the above
# loop. Compute the geomean over this, and store it.
result.add(curResult.geometricMean)
}
}
# $overallResults will have a Stats for each VM. That Stats object will hold
# $inner*$outer geomeans, allowing us to compute the arithmetic mean and
# confidence interval of the geomeans of preferred means. Convoluted, but
# useful and probably sound.
overallResults << result
}
if $verbosity >= 2
benchmarksOnVMs.each {
| benchmarkOnVM |
$stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
}
$vms.each_with_index {
| vm, vmIndex |
vmStats = vmStatses[vmIndex]
$stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
$stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
}
end
reportName =
(if ($vms.collect {
| vm |
vm.nameKind
}.index :auto)
""
else
$vms.collect {
| vm |
vm.to_s
}.join("_") + "_"
end) +
($suites.collect {
| suite |
suite.to_s
}.join("")) + "_" +
(if hostname
hostname + "_"
else
""
end)+
(begin
time = Time.now
"%04d%02d%02d_%02d%02d" %
[ time.year, time.month, time.day,
time.hour, time.min ]
end) +
"_benchReport.txt"
unless $brief
puts "Generating benchmark report at #{reportName}"
end
outp = $stdout
begin
outp = File.open(reportName,"w")
rescue => e
$stderr.puts "Error: could not save report to #{reportName}: #{e}"
$stderr.puts
end
def createVMsString
result = ""
result += " " if $suites.size > 1
result += rpad("", $benchpad)
result += " "
$vms.size.times {
| index |
if index != 0
result += " "+NoChange.new(0).shortForm
end
result += lpad(center($vms[index].name, 9+9+2), 11+9+2)
}
result += " "
if $vms.size >= 3
result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
elsif $vms.size >= 2
result += " "*26
end
result
end
columns = [createVMsString.size, 78].max
outp.print "Benchmark report for "
if $suites.size == 1
outp.print $suites[0].to_s
elsif $suites.size == 2
outp.print "#{$suites[0]} and #{$suites[1]}"
else
outp.print "#{$suites[0..-2].join(', ')}, and #{$suites[-1]}"
end
if hostname
outp.print " on #{hostname}"
end
if hwmodel
outp.print " (#{hwmodel})"
end
outp.puts "."
outp.puts
# This looks stupid; revisit later.
if false
$suites.each {
| suite |
outp.puts "#{suite} at #{suite.path}"
}
outp.puts
end
outp.puts "VMs tested:"
$vms.each {
| vm |
outp.print "\"#{vm.name}\" at #{vm.origPath}"
if vm.svnRevision
outp.print " (r#{vm.svnRevision})"
end
outp.puts
}
outp.puts
outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
"with #{$outer} VM invocation#{plural($outer)} per benchmark."+
(if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
"total time of those iterations, for each sample.")
else "" end)+
(if $measureGC == true then (" No manual garbage collection invocations were "+
"emitted.")
elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
"all VMs except #{$measureGC}.")
else (" Emitted a call to gc() between sample measurements.") end)+
(if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
"began with the very first iteration.")
else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
"invocation for warm-up.") end)+
(case $timeMode
when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
"microsecond-level timing.")
when :date then (" Used the portable Date.now() method to get millisecond-"+
"level timing.")
else raise end)+
" Reporting benchmark execution times with 95% confidence "+
"intervals in milliseconds.",
columns)
outp.puts
def printVMs(outp)
outp.puts createVMsString
end
def summaryStats(outp, accumulators, name, &proc)
outp.print " " if $suites.size > 1
outp.print rpad(name, $benchpad)
outp.print " "
accumulators.size.times {
| index |
if index != 0
outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
end
outp.print statsToStr(accumulators[index].stats(&proc))
}
if accumulators.size>=2
outp.print(" "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
end
outp.puts
end
def meanName(currentMean, preferredMean)
result = "<#{currentMean}>"
if "#{currentMean}Mean" == preferredMean.to_s
result += " *"
end
result
end
def allSummaryStats(outp, accumulators, preferredMean)
summaryStats(outp, accumulators, meanName("arithmetic", preferredMean)) {
| stat |
stat.arithmeticMean
}
summaryStats(outp, accumulators, meanName("geometric", preferredMean)) {
| stat |
stat.geometricMean
}
summaryStats(outp, accumulators, meanName("harmonic", preferredMean)) {
| stat |
stat.harmonicMean
}
end
$suites.each {
| suite |
printVMs(outp)
if $suites.size > 1
outp.puts "#{suite.name}:"
else
outp.puts
end
suite.benchmarks.each {
| benchmark |
outp.print " " if $suites.size > 1
outp.print rpad(benchmark.name, $benchpad)
outp.print " "
myConfigs = benchmarksOnVMsForBenchmark[benchmark]
myConfigs.size.times {
| index |
if index != 0
outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
end
outp.print statsToStr(myConfigs[index].stats)
}
if $vms.size>=2
outp.print(" "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
end
outp.puts
}
outp.puts
allSummaryStats(outp, suitesOnVMsForSuite[suite], suite.preferredMean)
outp.puts if $suites.size > 1
}
if $suites.size > 1
printVMs(outp)
outp.puts "All benchmarks:"
allSummaryStats(outp, vmStatses, nil)
outp.puts
printVMs(outp)
outp.puts "Geomean of preferred means:"
outp.print " "
outp.print rpad("<scaled-result>", $benchpad)
outp.print " "
$vms.size.times {
| index |
if index != 0
outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
end
outp.print statsToStr(overallResults[index])
}
if overallResults.size>=2
outp.print(" "+overallResults[-1].compareTo(overallResults[0]).longForm)
end
outp.puts
end
outp.puts
if outp != $stdout
outp.close
end
if outp != $stdout and not $brief
puts
File.open(reportName) {
| inp |
puts inp.read
}
end
if $brief
puts(overallResults.collect{|stats| stats.mean}.join("\t"))
puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
end
end
begin
$sawBenchOptions = false
def resetBenchOptionsIfNecessary
unless $sawBenchOptions
$includeSunSpider = false
$includeV8 = false
$includeKraken = false
$sawBenchOptions = true
end
end
GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
['--inner', GetoptLong::REQUIRED_ARGUMENT],
['--outer', GetoptLong::REQUIRED_ARGUMENT],
['--warmup', GetoptLong::REQUIRED_ARGUMENT],
['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
['--sunspider-only', GetoptLong::NO_ARGUMENT],
['--v8-only', GetoptLong::NO_ARGUMENT],
['--kraken-only', GetoptLong::NO_ARGUMENT],
['--exclude-sunspider', GetoptLong::NO_ARGUMENT],
['--exclude-v8', GetoptLong::NO_ARGUMENT],
['--exclude-kraken', GetoptLong::NO_ARGUMENT],
['--sunspider', GetoptLong::NO_ARGUMENT],
['--v8', GetoptLong::NO_ARGUMENT],
['--kraken', GetoptLong::NO_ARGUMENT],
['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
['--force-vm-copy', GetoptLong::NO_ARGUMENT],
['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
['--verbose', '-v', GetoptLong::NO_ARGUMENT],
['--brief', GetoptLong::NO_ARGUMENT],
['--silent', GetoptLong::NO_ARGUMENT],
['--remote', GetoptLong::REQUIRED_ARGUMENT],
['--local', GetoptLong::NO_ARGUMENT],
['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
['--slave', GetoptLong::NO_ARGUMENT],
['--prepare-only', GetoptLong::NO_ARGUMENT],
['--analyze', GetoptLong::REQUIRED_ARGUMENT],
['--vms', GetoptLong::REQUIRED_ARGUMENT],
['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
| opt, arg |
case opt
when '--rerun'
$rerun = intArg(opt,arg,1,nil)
when '--inner'
$inner = intArg(opt,arg,1,nil)
when '--outer'
$outer = intArg(opt,arg,1,nil)
when '--warmup'
$warmup = intArg(opt,arg,0,nil)
when '--timing-mode'
if arg.upcase == "PRECISETIME"
$timeMode = :preciseTime
elsif arg.upcase == "DATE"
$timeMode = :date
elsif arg.upcase == "AUTO"
$timeMode = :auto
else
quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
"Invalid argument for command-line option")
end
when '--force-vm-kind'
if arg.upcase == "JSC"
$forceVMKind = :jsc
elsif arg.upcase == "DUMPRENDERTREE"
$forceVMKind = :dumpRenderTree
elsif arg.upcase == "AUTO"
$forceVMKind = nil
else
quickFail("Expected either 'jsc' or 'DumpRenderTree' for --force-vm-kind, but got '#{arg}'.",
"Invalid argument for command-line option")
end
when '--force-vm-copy'
$needToCopyVMs = true
when '--dont-copy-vms'
$dontCopyVMs = true
when '--sunspider-only'
$includeV8 = false
$includeKraken = false
when '--v8-only'
$includeSunSpider = false
$includeKraken = false
when '--kraken-only'
$includeSunSpider = false
$includeV8 = false
when '--exclude-sunspider'
$includeSunSpider = false
when '--exclude-v8'
$includeV8 = false
when '--exclude-kraken'
$includeKraken = false
when '--sunspider'
resetBenchOptionsIfNecessary
$includeSunSpider = true
when '--v8'
resetBenchOptionsIfNecessary
$includeV8 = true
when '--kraken'
resetBenchOptionsIfNecessary
$includeKraken = true
when '--benchmarks'
$benchmarkPattern = Regexp.new(arg)
when '--measure-gc'
if arg == ''
$measureGC = true
else
$measureGC = arg
end
when '--verbose'
$verbosity += 1
when '--brief'
$brief = true
when '--silent'
$silent = true
when '--remote'
$remoteHosts += arg.split(',')
$needToCopyVMs = true
when '--ssh-options'
$sshOptions << arg
when '--local'
$alsoLocal = true
when '--prepare-only'
$run = false
when '--analyze'
$prepare = false
$run = false
$analyze << arg
when '--help'
usage
else
raise "bad option: #{opt}"
end
}
# If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
if $dontCopyVMs
$needToCopyVMs = false
end
SUNSPIDER = BenchmarkSuite.new("SunSpider", SUNSPIDER_PATH, :arithmeticMean)
["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
"access-fannkuch", "access-nbody", "access-nsieve",
"bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
"bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
"crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
"math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
"string-base64", "string-fasta", "string-tagcloud",
"string-unpack-code", "string-validate-input"].each {
| name |
SUNSPIDER.add SunSpiderBenchmark.new(name)
}
V8 = BenchmarkSuite.new("V8", V8_PATH, :geometricMean)
["crypto", "deltablue", "earley-boyer", "raytrace",
"regexp", "richards", "splay"].each {
| name |
V8.add V8Benchmark.new(name)
}
KRAKEN = BenchmarkSuite.new("Kraken", KRAKEN_PATH, :arithmeticMean)
["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
"audio-oscillator", "imaging-darkroom", "imaging-desaturate",
"imaging-gaussian-blur", "json-parse-financial",
"json-stringify-tinderbox", "stanford-crypto-aes",
"stanford-crypto-ccm", "stanford-crypto-pbkdf2",
"stanford-crypto-sha256-iterative"].each {
| name |
KRAKEN.add KrakenBenchmark.new(name)
}
ARGV.each {
| vm |
if vm =~ /([a-zA-Z0-9_ ]+):/
name = $1
nameKind = :given
vm = $~.post_match
else
name = "Conf\##{$vms.length+1}"
nameKind = :auto
end
$stderr.puts "#{name}: #{vm}" if $verbosity >= 1
$vms << VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
}
if $vms.empty?
quickFail("Please specify at least on configuraiton on the command line.",
"Insufficient arguments")
end
$vms.each {
| vm |
if vm.vmType != :jsc and $timeMode != :date
$timeMode = :date
$stderr.puts "Warning: using Date.now() instead of preciseTime() because #{vm} doesn't support the latter."
end
}
if FileTest.exist? BENCH_DATA_PATH
cmd = "rm -rf #{BENCH_DATA_PATH}"
$stderr.puts ">> #{cmd}" if $verbosity >= 2
raise unless system cmd
end
Dir.mkdir BENCH_DATA_PATH
if $needToCopyVMs
canCopyIntoBenchPath = true
$vms.each {
| vm |
canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
}
if canCopyIntoBenchPath
$vms.each {
| vm |
$stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
vm.copyIntoBenchPath
}
$stderr.puts "All VMs are in place."
else
$stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
end
end
if $measureGC and $measureGC != true
found = false
$vms.each {
| vm |
if vm.name == $measureGC
found = true
end
}
unless found
$stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
end
end
if $outer*$inner == 1
$stderr.puts "Warning: will only collect one sample per benchmark/VM. Confidence interval calculation will fail."
end
$stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
$suites = []
if $includeSunSpider and not SUNSPIDER.empty?
$suites << SUNSPIDER
end
if $includeV8 and not V8.empty?
$suites << V8
end
if $includeKraken and not KRAKEN.empty?
$suites << KRAKEN
end
$benchmarks = []
$suites.each {
| suite |
$benchmarks += suite.benchmarks
}
$runPlans = []
$vms.each {
| vm |
$benchmarks.each {
| benchmark |
$outer.times {
| iteration |
$runPlans << BenchRunPlan.new(benchmark, vm, iteration)
}
}
}
$runPlans.shuffle!
$suitepad = $suites.collect {
| suite |
suite.to_s.size
}.max + 1
$benchpad = ($benchmarks +
["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
| benchmark |
if benchmark.respond_to? :name
benchmark.name.size
else
benchmark.size
end
}.max + 1
$vmpad = $vms.collect {
| vm |
vm.to_s.size
}.max + 1
if $prepare
File.open("#{BENCH_DATA_PATH}/runscript", "w") {
| file |
file.puts "echo \"HOSTNAME:\\c\""
file.puts "hostname"
file.puts "echo"
file.puts "echo \"HARDWARE:\\c\""
file.puts "/usr/sbin/sysctl hw.model"
file.puts "echo"
file.puts "set -e"
$script = file
$runPlans.each_with_index {
| plan, idx |
if $verbosity == 0 and not $silent
text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
text2 = plan.benchmark.to_s+"/"+plan.vm.to_s
file.puts("echo "+("\r#{text1} #{rpad(text2,$suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2")
file.puts("echo "+("\r#{text1} #{text2}".inspect)[0..-2]+"\\c\" 1>&2")
end
plan.emitRunCode
}
if $verbosity == 0 and not $silent
file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2")
file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size}".inspect)+" 1>&2")
end
}
end
if $run
unless $remoteHosts.empty?
$stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
Dir.chdir(TEMP_PATH) {
cmd = "tar -czf payload.tar.gz benchdata"
$stderr.puts ">> #{cmd}" if $verbosity>=2
raise unless system(cmd)
}
def grokHost(host)
if host =~ /:([0-9]+)$/
"-p " + $1 + " " + $~.pre_match.inspect
else
host.inspect
end
end
def sshRead(host, command)
cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}"
$stderr.puts ">> #{cmd}" if $verbosity>=2
result = ""
IO.popen(cmd, "r") {
| inp |
inp.each_line {
| line |
$stderr.puts "#{host}: #{line}" if $verbosity>=2
result += line
}
}
raise "#{$?}" unless $?.success?
result
end
def sshWrite(host, command, data)
cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}"
$stderr.puts ">> #{cmd}" if $verbosity>=2
IO.popen(cmd, "w") {
| outp |
outp.write(data)
}
raise "#{$?}" unless $?.success?
end
$remoteHosts.each {
| host |
$stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
raise unless remoteTempPath
sshWrite(host, "cd #{remoteTempPath.inspect} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
$stderr.puts "Running on #{host}..." if $verbosity==0
parseAndDisplayResults(sshRead(host, "cd #{(remoteTempPath+'/benchdata').inspect} && sh runscript"))
}
end
if not $remoteHosts.empty? and $alsoLocal
$stderr.puts "Running locally..."
end
if $remoteHosts.empty? or $alsoLocal
parseAndDisplayResults(runAndGetResults)
end
end
$analyze.each_with_index {
| filename, index |
if index >= 1
puts
end
parseAndDisplayResults(IO::read(filename))
}
if $prepare and not $run and $analyze.empty?
puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
"the benchmarks and get the results by doing:", 78)
puts
puts "cd #{BENCH_DATA_PATH}"
puts "sh runscript > results.txt"
puts
puts wrap("Then you can analyze the results by running bencher with the same arguments "+
"as now, but replacing --prepare-only with --analyze results.txt.", 78)
end
rescue => e
fail(e)
end