292 lines
8.6 KiB
Ruby
292 lines
8.6 KiB
Ruby
#!/usr/bin/env ruby
|
|
|
|
# Copyright (C) 2018 Apple Inc. All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
|
|
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
|
|
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
|
# THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
require 'fileutils'
|
|
require 'pathname'
|
|
require 'open3'
|
|
require "JSON"
|
|
require 'getoptlong'
|
|
|
|
def usage
|
|
puts "run-testmem [options]"
|
|
puts "--build-dir (-b) Pass in a path to your build directory, e.g, WebKitBuild/Release"
|
|
puts "--verbose (-v) Print more information as the benchmark runs"
|
|
puts "--count (-c) Number of outer iterations to run the benchmark for"
|
|
puts "--dry (-d) Print shell output that can be run as a bash script on a different device. When using this option, provide the --script-path and --build-dir options"
|
|
puts "--script-path (-s) The path to the directory where you expect the testmem tests to live. Use this when doing a dry run with --dry"
|
|
puts "--parse (-p) After executing the dry run, capture its stdout and write it to a file. Pass the path to that file for this option and run-testmem will compute the results of the benchmark run"
|
|
puts "--help (-h) Print this message"
|
|
end
|
|
|
|
THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
|
|
SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
|
|
|
|
$buildDir = nil
|
|
$verbose = false
|
|
$outerIterations = 3
|
|
$dryRun = false
|
|
$scriptPath = nil
|
|
$parsePath = nil
|
|
|
|
GetoptLong.new(["--build-dir", "-b", GetoptLong::REQUIRED_ARGUMENT],
|
|
["--verbose", "-v", GetoptLong::NO_ARGUMENT],
|
|
["--count", "-c", GetoptLong::REQUIRED_ARGUMENT],
|
|
["--dry", "-d", GetoptLong::NO_ARGUMENT],
|
|
["--script-path", "-s", GetoptLong::REQUIRED_ARGUMENT],
|
|
["--parse", "-p", GetoptLong::REQUIRED_ARGUMENT],
|
|
["--help", "-h", GetoptLong::NO_ARGUMENT],
|
|
).each {
|
|
| opt, arg |
|
|
case opt
|
|
when "--build-dir"
|
|
$buildDir = arg
|
|
when "--verbose"
|
|
$verbose = true
|
|
when "--count"
|
|
$outerIterations = arg.to_i
|
|
if $outerIterations < 1
|
|
puts "--count must be > 0"
|
|
exit 1
|
|
end
|
|
when "--dry"
|
|
$dryRun = true
|
|
when "--script-path"
|
|
$scriptPath = arg
|
|
when "--parse"
|
|
$parsePath = arg
|
|
when "--help"
|
|
usage
|
|
exit 1
|
|
end
|
|
}
|
|
|
|
if $scriptPath && !$dryRun
|
|
puts "--script-path is only supported when you are doing a --dry run"
|
|
exit 1
|
|
end
|
|
|
|
def getBuildDirectory
|
|
if $buildDir != nil
|
|
return $buildDir
|
|
end
|
|
|
|
command = SCRIPTS_PATH.join("webkit-build-directory").to_s
|
|
command += " --release"
|
|
command += " --executablePath"
|
|
|
|
output = `#{command}`.split("\n")
|
|
if !output.length
|
|
puts "Error: could not find release WebKitBuild"
|
|
exit 1
|
|
end
|
|
output = output[0]
|
|
|
|
$buildDir = Pathname.new(output).to_s
|
|
$buildDir
|
|
end
|
|
|
|
def getTestmemPath
|
|
path = Pathname.new(getBuildDirectory).join("testmem").to_s
|
|
if !File.exists?(path) && !$dryRun
|
|
puts "Error: no testmem binary found in <build>/Release"
|
|
exit 1
|
|
end
|
|
path
|
|
end
|
|
|
|
def iterationCount(path)
|
|
iterationMap = {
|
|
"air" => 4,
|
|
"basic" => 5,
|
|
"splay" => 10,
|
|
"hash-map" => 5,
|
|
"box2d" => 3,
|
|
}
|
|
name = File.basename(path, ".js")
|
|
iterationMap[name] || 20
|
|
end
|
|
|
|
def getTests
|
|
dirPath = Pathname.new(SCRIPTS_PATH).join("../../PerformanceTests/testmem")
|
|
files = []
|
|
Dir.foreach(dirPath) {
|
|
| filename |
|
|
next unless filename =~ /\.js$/
|
|
filePath = dirPath.join(filename).to_s
|
|
filePath = Pathname.new($scriptPath).join(filename).to_s if $scriptPath
|
|
files.push([filePath, iterationCount(filePath)])
|
|
}
|
|
|
|
files.sort_by { | (path) | File.basename(path) }
|
|
end
|
|
|
|
def processRunOutput(stdout, path)
|
|
time, peakFootprint, footprintAtEnd = stdout.split("\n")
|
|
raise unless time.slice!("time:")
|
|
raise unless peakFootprint.slice!("peak footprint:")
|
|
raise unless footprintAtEnd.slice!("footprint at end:")
|
|
time = time.to_f
|
|
peakFootprint = peakFootprint.to_f
|
|
footprintAtEnd = footprintAtEnd.to_f
|
|
|
|
if $verbose
|
|
puts path
|
|
puts "time: #{time}"
|
|
puts "peak footprint: #{peakFootprint/1024/1024} MB"
|
|
puts "end footprint: #{footprintAtEnd/1024/1024} MB\n"
|
|
end
|
|
|
|
{"time"=>time, "peak"=>peakFootprint, "end"=>footprintAtEnd}
|
|
end
|
|
|
|
def runTest(path, iters)
|
|
command = "#{getTestmemPath} #{path} #{iters}"
|
|
environment = {
|
|
"DYLD_FRAMEWORK_PATH" => getBuildDirectory,
|
|
"JSC_useJIT" => "false",
|
|
"JSC_useRegExpJIT" => "false",
|
|
}
|
|
|
|
if $dryRun
|
|
environment.each { | key, value |
|
|
command = "#{key}=#{value} #{command}"
|
|
}
|
|
puts "echo \"#{command}\""
|
|
puts command
|
|
return
|
|
end
|
|
|
|
stdout, stderr, exitCode = Open3.capture3(environment, command)
|
|
|
|
if $verbose
|
|
puts stdout
|
|
puts stderr
|
|
end
|
|
|
|
if exitCode != 0
|
|
puts "testmem failed to run"
|
|
puts stdout
|
|
puts stderr
|
|
exit 1
|
|
end
|
|
|
|
processRunOutput(stdout, path)
|
|
end
|
|
|
|
def geomean(arr)
|
|
score = arr.inject(1.0, :*)
|
|
score ** (1.0 / arr.length)
|
|
end
|
|
|
|
def mean(arr)
|
|
sum = arr.inject(0.0, :+)
|
|
sum / arr.length
|
|
end
|
|
|
|
def processScores(scores)
|
|
peakScore = []
|
|
endScore = []
|
|
timeScore = []
|
|
scores.each { | key, value |
|
|
endAvg = mean(value.map { | element | element["end"] })
|
|
peakAvg = mean(value.map { | element | element["peak"] })
|
|
timeAvg = mean(value.map { | element | element["time"] })
|
|
|
|
peakScore.push(peakAvg)
|
|
endScore.push(endAvg)
|
|
timeScore.push(timeAvg)
|
|
|
|
puts File.basename(key, ".js")
|
|
puts " end: #{(endAvg/1024/1024).round(4)} MB"
|
|
puts " peak: #{(peakAvg/1024/1024).round(4)} MB"
|
|
puts " time: #{(timeAvg*1000).round(2)} ms\n"
|
|
}
|
|
|
|
endScore = geomean(endScore)
|
|
peakScore = geomean(peakScore)
|
|
timeScore = geomean(timeScore)
|
|
|
|
puts
|
|
puts "end score: #{(endScore/1024/1024).round(4)} MB"
|
|
puts "peak score: #{(peakScore/1024/1024).round(4)} MB\n"
|
|
puts "total memory score: #{(geomean([endScore, peakScore])/1024/1024).round(4)} MB"
|
|
puts "time score: #{(timeScore*1000).round(2)} ms\n\n"
|
|
|
|
puts JSON.pretty_generate(scores) if $verbose
|
|
end
|
|
|
|
def run
|
|
tests = getTests
|
|
scores = {}
|
|
tests.each { | (path) | scores[path] = [] }
|
|
count = $outerIterations
|
|
|
|
if $dryRun
|
|
(0..(count-1)).each { | currentIter |
|
|
tests.each { | (path, iters) |
|
|
runTest(path, iters)
|
|
}
|
|
}
|
|
return
|
|
end
|
|
|
|
(0..(count-1)).each { | currentIter |
|
|
tests.each { | (path, iters) |
|
|
statusToPrint = "iteration #{currentIter + 1}: #{File.basename(path, ".js")}"
|
|
print "#{statusToPrint}\r"
|
|
scores[path].push(runTest(path, iters))
|
|
print "#{" ".rjust(statusToPrint.length)}\r"
|
|
}
|
|
}
|
|
|
|
processScores(scores)
|
|
end
|
|
|
|
def parseResultOfDryRun(path)
|
|
contents = IO.read(path).split("\n")
|
|
if !contents.length || contents.length % 4 != 0
|
|
puts "Bad input, expect multiple of 4 number of lines from output of running the result of --dry"
|
|
exit 1
|
|
end
|
|
|
|
scores = {}
|
|
i = 0
|
|
while i < contents.length
|
|
path = contents[i + 0].split(" ")[-2]
|
|
scores[path] = [] if !scores[path]
|
|
stdout = [contents[i + 1], contents[i + 2], contents[i + 3]].join("\n")
|
|
scores[path].push(processRunOutput(stdout, path))
|
|
i += 4
|
|
end
|
|
|
|
processScores(scores)
|
|
end
|
|
|
|
if $parsePath
|
|
parseResultOfDryRun($parsePath)
|
|
else
|
|
run
|
|
end
|