2015-04-22 02:30:22 +00:00
|
|
|
#!/usr/bin/env ruby
|
|
|
|
|
JetStream should have a more rational story for jitter-oriented latency tests
https://bugs.webkit.org/show_bug.cgi?id=145762
Reviewed by Geoffrey Garen.
JetStream has some latency tests that are meant to measure jitter. Prior to this change, they
did this by computing the RMS. But the RMS is a pretty bad metric. The thing that it rewards
isn't really the thing that you'd want your browser to do. These RMS-based tests involve taking
the geomean of the RMS of some samples and the sample average. The lower the geomean, the better
(in the JetStream harness we then invert the scores so that higher is better, but let's ignore
that for this discussion and assume that lower is better). Here's an example of how this can go
bad. A browser that always computes a task in some horribly long time (say, 1000ms) but never
varies that time will perform better than a browser that usually computes the task super quickly
(say, 10ms) and sometimes just a little bit less quickly (say, 15ms). The former browser will
have an RMS of 0 and an average of 1000. The latter will have a RMS somewhere around 3.5 and an
average of 12.5 (assuming equal probability of 10ms and 15ms). The geomean of (0, 1000) is 0.
The geomean of (3.5, 12.5) is 6.6. Lower is better, so the former browser scores higher - even
though it's obviously never better to have a browser always complete a task in 1000ms when a
different browser can do it in 15ms in the worst case.
JetStream should not have this pathology. The right way of avoiding it is to replace RMS with
some other metric of how bad things get. A good metric is the average of the worst percentile.
The worst 1% or the worst 5% would be good things to average. This will catch cases where the VM
jittered due to JIT or GC, but it never have the pathology that we end up giving the better score
to a VM whose best case is worst than another VM's worst case.
For now, this change uses the highest samples above the 95% percentile. I'm not yet sure if that
is the best thing - it might include too many scores that are around the best-case performance -
but it's certainly better than RMS and it might be good enough to keep. But because of that
uncertainty, I'm setting the version to be "1.1-alpha1" to indicate that we aren't ready to
release this yet.
* JetStream/Octane2/base.js:
(.this.Setup.setup.setup):
(.this.TearDown.tearDown.tearDown):
(BenchmarkSuite.GeometricMeanTime):
(BenchmarkSuite.AverageAbovePercentile):
(BenchmarkSuite.GeometricMeanLatency):
(BenchmarkSuite.prototype.NotifyStep):
(BenchmarkSuite.prototype.RunSingleBenchmark):
* JetStream/Octane2/mandreel.js:
(setupMandreel):
(updateMandreelStats):
(startMandreelTimer):
(latencyMandreel):
(tearDownMandreel):
(RMSMandreel): Deleted.
* JetStream/Octane2/splay.js:
(GenerateKey):
(SplayUpdateStats):
(InsertNewNode):
(SplayTearDown):
(SplayRMS): Deleted.
* JetStream/create.rb:
Canonical link: https://commits.webkit.org/163895@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@185425 268f45cc-cd09-0410-ab3c-d52691b4dbfc
2015-06-10 18:44:50 +00:00
|
|
|
# Copyright (C) 2014, 2015 Apple Inc. All rights reserved.
|
2015-04-22 02:30:22 +00:00
|
|
|
#
|
|
|
|
# Redistribution and use in source and binary forms, with or without
|
|
|
|
# modification, are permitted provided that the following conditions
|
|
|
|
# are met:
|
|
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
|
|
# notice, this list of conditions and the following disclaimer.
|
|
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
|
|
# documentation and/or other materials provided with the distribution.
|
|
|
|
#
|
|
|
|
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
|
|
|
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
|
|
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
|
|
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
|
|
|
|
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
|
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
|
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
|
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
|
|
|
# THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
|
|
require "pathname"
|
|
|
|
require "shellwords"
|
|
|
|
|
2015-07-13 22:09:48 +00:00
|
|
|
VERSION = "1.1"
|
2015-04-22 02:30:22 +00:00
|
|
|
DIRECTORY_NAME = "JetStream-#{VERSION}"
|
|
|
|
|
2015-06-22 17:11:54 +00:00
|
|
|
CDJS_FILES = [
|
|
|
|
"constants.js",
|
|
|
|
"util.js",
|
|
|
|
"red_black_tree.js",
|
|
|
|
"call_sign.js",
|
|
|
|
"vector_2d.js",
|
|
|
|
"vector_3d.js",
|
|
|
|
"motion.js",
|
|
|
|
"reduce_collision_set.js",
|
|
|
|
"simulator.js",
|
|
|
|
"collision.js",
|
|
|
|
"collision_detector.js",
|
|
|
|
"benchmark.js"
|
|
|
|
]
|
|
|
|
|
2015-04-22 02:30:22 +00:00
|
|
|
raise unless system("rm -rf " + DIRECTORY_NAME)
|
|
|
|
raise unless system("mkdir -p " + DIRECTORY_NAME)
|
|
|
|
raise unless system("mkdir -p #{DIRECTORY_NAME}/sunspider")
|
|
|
|
raise unless system("mkdir -p #{DIRECTORY_NAME}/sources")
|
|
|
|
raise unless system("cp sunspider/*.js #{DIRECTORY_NAME}/sunspider")
|
2015-06-22 17:11:54 +00:00
|
|
|
raise unless system("mkdir -p #{DIRECTORY_NAME}/cdjs")
|
|
|
|
CDJS_FILES.each {
|
|
|
|
| filename |
|
|
|
|
raise unless system("cp cdjs/#{filename} #{DIRECTORY_NAME}/cdjs")
|
|
|
|
}
|
|
|
|
raise unless system("cp -r JetStream.css JetStreamDriver.js LLVM-test-suite-LICENSE.txt simple Octane2 Octane2Setup.js SimpleSetup.js SunSpiderSetup.js Octane OctaneSetup.js CDjsSetup.js Reference.js TestingSetup.js JetStream-Logo.png JetStream-Logo@2x.png Swoosh.png Swoosh@2x.png " + DIRECTORY_NAME)
|
2015-04-22 02:30:22 +00:00
|
|
|
|
|
|
|
def detemplatize(basename)
|
|
|
|
File.open(DIRECTORY_NAME + "/#{basename}.html", "w") {
|
|
|
|
| outp |
|
|
|
|
outp.write(IO::read("#{basename}-TEMPLATE.html").gsub(/@VERSION@/, VERSION.to_s))
|
|
|
|
}
|
|
|
|
end
|
|
|
|
detemplatize("index")
|
|
|
|
detemplatize("in-depth")
|
|
|
|
|
|
|
|
def transferSource(benchmarkName, *files)
|
|
|
|
files.each {
|
|
|
|
| filename |
|
|
|
|
pathname = Pathname.new(filename)
|
|
|
|
|
|
|
|
originalBasename = pathname.basename
|
|
|
|
extname = originalBasename.extname
|
|
|
|
basename = originalBasename.basename(extname)
|
|
|
|
|
|
|
|
if basename.to_s.start_with? benchmarkName
|
|
|
|
resultPathname = Pathname.new(DIRECTORY_NAME) + "sources" + "#{basename}"
|
|
|
|
resultName = basename.to_s
|
|
|
|
else
|
|
|
|
resultPathname = Pathname.new(DIRECTORY_NAME) + "sources" + "#{benchmarkName}-#{basename}"
|
|
|
|
resultName = "#{benchmarkName} / #{basename}"
|
|
|
|
end
|
|
|
|
|
|
|
|
raise unless system("cp #{Shellwords.shellescape(pathname.to_s)} #{Shellwords.shellescape(resultPathname.to_s + extname)}")
|
|
|
|
}
|
|
|
|
end
|
|
|
|
|
|
|
|
File.open(DIRECTORY_NAME + "/SimplePayload.js", "w") {
|
|
|
|
| outp |
|
|
|
|
outp.puts "// THIS IS AUTO-GENERATED BY create.rb"
|
|
|
|
outp.puts "var SimplePayload = ["
|
|
|
|
directory = Pathname.new("simple")
|
|
|
|
Dir.foreach(directory) {
|
|
|
|
| filename |
|
|
|
|
next unless filename =~ /\.js$/
|
|
|
|
baseName = $~.pre_match
|
|
|
|
outp.puts " {name: #{baseName.inspect}, content:#{IO::read(directory + filename).inspect}},"
|
|
|
|
if filename.to_s =~ /\.(cp*)\.js/
|
|
|
|
transferSource($~.pre_match, directory + ($~.pre_match + "." + $1), directory + filename)
|
|
|
|
else
|
|
|
|
transferSource(baseName, directory + filename)
|
|
|
|
end
|
|
|
|
}
|
|
|
|
outp.puts "];"
|
|
|
|
}
|
|
|
|
|
|
|
|
File.open(DIRECTORY_NAME + "/SunSpiderPayload.js", "w") {
|
|
|
|
| outp |
|
|
|
|
outp.puts "// THIS IS AUTO-GENERATED BY create.rb"
|
|
|
|
outp.puts "var SunSpiderPayload = ["
|
|
|
|
directory = Pathname.new("sunspider")
|
|
|
|
Dir.foreach(directory) {
|
|
|
|
| filename |
|
|
|
|
next unless filename =~ /\.js$/
|
|
|
|
baseName = $~.pre_match
|
|
|
|
outp.puts " {name: #{baseName.inspect}, content:#{IO::read(directory + filename).inspect}},"
|
|
|
|
transferSource(baseName, directory + filename)
|
|
|
|
}
|
|
|
|
outp.puts "];"
|
|
|
|
}
|
|
|
|
|
|
|
|
transferSource("code-multi-load", "Octane/code-load.js")
|
|
|
|
transferSource("richards", "Octane2/richards.js")
|
|
|
|
transferSource("deltablue", "Octane2/deltablue.js")
|
|
|
|
transferSource("crypto", "Octane2/crypto.js")
|
|
|
|
transferSource("raytrace", "Octane2/raytrace.js")
|
|
|
|
transferSource("earley-boyer", "Octane2/earley-boyer.js")
|
|
|
|
transferSource("regexp", "Octane2/regexp.js")
|
|
|
|
transferSource("splay", "Octane2/splay.js")
|
|
|
|
transferSource("navier-stokes", "Octane2/navier-stokes.js")
|
|
|
|
transferSource("pdfjs", "Octane2/pdfjs.js")
|
|
|
|
transferSource("mandreel", "Octane2/mandreel.js")
|
|
|
|
transferSource("gbemu", "Octane2/gbemu-part1.js", "Octane2/gbemu-part2.js")
|
|
|
|
transferSource("code-first-load", "Octane2/code-load.js")
|
|
|
|
transferSource("box2d", "Octane2/box2d.js")
|
|
|
|
transferSource("zlib", "Octane2/zlib.js", "Octane2/zlib-data.js")
|
|
|
|
transferSource("typescript", "Octane2/typescript.js", "Octane2/typescript-compiler.js", "Octane2/typescript-input.js")
|
2015-06-22 17:11:54 +00:00
|
|
|
transferSource("cdjs", *(CDJS_FILES.collect { | filename | "cdjs/#{filename}" }))
|
2015-04-22 02:30:22 +00:00
|
|
|
|
|
|
|
puts "You can now run JetStream by navigating to file://" + (Pathname.new(DIRECTORY_NAME) + "index.html").realpath.to_s
|
|
|
|
|