397 lines
10 KiB
Ruby
Executable File
397 lines
10 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
# coding: utf-8
|
|
|
|
require 'getoptlong'
|
|
require 'pathname'
|
|
|
|
$benchmarks_all = [
|
|
# Single-threaded benchmarks.
|
|
"churn",
|
|
"list_allocate",
|
|
"tree_allocate",
|
|
"tree_churn",
|
|
"fragment",
|
|
"fragment_iterate",
|
|
"medium",
|
|
"big",
|
|
|
|
# Benchmarks based on browser recordings.
|
|
"facebook",
|
|
"reddit",
|
|
"flickr",
|
|
"theverge",
|
|
"nimlang",
|
|
|
|
# Multi-threaded benchmark variants.
|
|
"message_one",
|
|
"message_many",
|
|
"churn --parallel",
|
|
"list_allocate --parallel",
|
|
"tree_allocate --parallel",
|
|
"tree_churn --parallel",
|
|
"fragment --parallel",
|
|
"fragment_iterate --parallel",
|
|
|
|
# These tests often crash TCMalloc: <rdar://problem/13657137>.
|
|
"medium --parallel",
|
|
"big --parallel",
|
|
|
|
# Enable these tests to test memory footprint. The way they run is not
|
|
# really compatible with throughput testing.
|
|
# "reddit_memory_warning --runs 0",
|
|
# "flickr_memory_warning --runs 0",
|
|
# "theverge_memory_warning --runs 0",
|
|
|
|
# Enable this test to test shrinking back down from a large heap while a process remains active.
|
|
# The way it runs is not really compatible with throughput testing.
|
|
# "balloon"
|
|
"facebook --parallel",
|
|
"reddit --parallel",
|
|
"flickr --parallel",
|
|
"theverge --parallel",
|
|
# "nimlang --use-thread-id",
|
|
]
|
|
|
|
$benchmarks_memory = [
|
|
"facebook",
|
|
"reddit",
|
|
"flickr",
|
|
"theverge",
|
|
"nimlang"
|
|
]
|
|
|
|
$benchmarks_memory_warning = [
|
|
"reddit_memory_warning --runs 0",
|
|
"flickr_memory_warning --runs 0",
|
|
"theverge_memory_warning --runs 0",
|
|
]
|
|
|
|
$benchmarks = $benchmarks_all
|
|
$heap = 0
|
|
|
|
def usage
|
|
puts "run-malloc-benchmarks [options] /path/to/MallocBench Name:/path/to/libmbmalloc.dylib [ Name:/path/to/libmbmalloc.dylib ]"
|
|
puts
|
|
puts " Runs a suite of memory allocation and access benchmarks."
|
|
puts
|
|
puts " <Name:/path/to/libmbmalloc.dylib> is a symbolic name followed by a path to libmbmalloc.dylib."
|
|
puts
|
|
puts " Specify \"SystemMalloc\" to test the built-in libc malloc."
|
|
puts " Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone."
|
|
puts
|
|
puts " Example usage:"
|
|
puts
|
|
puts " run-malloc-benchmarks /BUILD/MallocBench SystemMalloc:/BUILD/libmbmalloc.dylib NanoMalloc:/BUILD/libmbmalloc.dylib"
|
|
puts " run-malloc-benchmarks /BUILD/MallocBench FastMalloc:/BUILD/FastMalloc/libmbmalloc.dylib"
|
|
puts " run-malloc-benchmarks --benchmark churn SystemMalloc:/BUILD/libmbmalloc.dylib FastMalloc:/BUILD/FastMalloc/libmbmalloc.dylib"
|
|
puts
|
|
puts "Options:"
|
|
puts
|
|
puts " --benchmark <benchmark> Select a single benchmark to run instead of the full suite."
|
|
puts " --heap <heap> Set a baseline heap size."
|
|
puts
|
|
end
|
|
|
|
class Dylib
|
|
attr_reader :name
|
|
attr_reader :path
|
|
|
|
def initialize(name, path)
|
|
@name = name
|
|
@path = path
|
|
end
|
|
end
|
|
|
|
class Results
|
|
attr_reader :executionTime
|
|
attr_reader :peakMemory
|
|
attr_reader :memoryAtEnd
|
|
|
|
def initialize(executionTime, peakMemory, memoryAtEnd)
|
|
@executionTime = executionTime
|
|
@peakMemory = peakMemory
|
|
@memoryAtEnd = memoryAtEnd
|
|
end
|
|
end
|
|
|
|
class Stat
|
|
attr_reader :benchmark
|
|
attr_reader :result
|
|
|
|
def initialize(benchmark, result)
|
|
@benchmark = benchmark
|
|
@result = result[/\d+/].to_i
|
|
end
|
|
end
|
|
|
|
class TimeStat < Stat
|
|
def to_s
|
|
@result + "ms"
|
|
end
|
|
end
|
|
|
|
class MemoryStat < Stat
|
|
def to_s
|
|
@result + "kB"
|
|
end
|
|
end
|
|
|
|
class PeakMemoryStat < Stat
|
|
def to_s
|
|
@result + "kB"
|
|
end
|
|
end
|
|
|
|
def lpad(str, chars)
|
|
if str.length > chars
|
|
str
|
|
else
|
|
"%#{chars}s"%(str)
|
|
end
|
|
end
|
|
|
|
def rpad(str, chars)
|
|
while str.length < chars
|
|
str += " "
|
|
end
|
|
str
|
|
end
|
|
|
|
def computeArithmeticMean(array)
|
|
sum = 0.0
|
|
array.each {
|
|
| value |
|
|
sum += value
|
|
}
|
|
(sum / array.length)
|
|
end
|
|
|
|
def computeGeometricMean(array)
|
|
mult = 1.0
|
|
array.each {
|
|
| value |
|
|
mult *= value ? value : 1.0
|
|
}
|
|
(mult ** (1.0 / array.length))
|
|
end
|
|
|
|
def computeHarmonicMean(array)
|
|
1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value })
|
|
end
|
|
|
|
def lowerIsBetter(a, b, better, worse)
|
|
if b < a
|
|
return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better
|
|
end
|
|
|
|
if b == a
|
|
return ""
|
|
end
|
|
|
|
"! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse
|
|
end
|
|
|
|
|
|
def lowerIsFaster(a, b)
|
|
lowerIsBetter(a, b, "faster", "slower")
|
|
end
|
|
|
|
def lowerIsSmaller(a, b)
|
|
lowerIsBetter(a, b, "smaller", "bigger")
|
|
end
|
|
|
|
def numberWithDelimiter(number)
|
|
number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
|
|
end
|
|
|
|
def prettify(number, suffix)
|
|
numberWithDelimiter(number) + suffix
|
|
end
|
|
|
|
def parseOptions
|
|
GetoptLong.new(
|
|
['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
|
|
['--memory', GetoptLong::NO_ARGUMENT],
|
|
['--memory_warning', GetoptLong::NO_ARGUMENT],
|
|
['--heap', GetoptLong::REQUIRED_ARGUMENT],
|
|
['--help', GetoptLong::NO_ARGUMENT],
|
|
).each {
|
|
| opt, arg |
|
|
case opt
|
|
when '--benchmark'
|
|
$benchmarks = [ arg ]
|
|
when '--memory'
|
|
$benchmarks = $benchmarks_memory
|
|
when '--memory_warning'
|
|
$benchmarks = $benchmarks_memory_warning
|
|
when '--heap'
|
|
$heap = arg
|
|
when '--help'
|
|
usage
|
|
exit 1
|
|
else
|
|
raise "bad option: #{opt}"
|
|
end
|
|
}
|
|
|
|
if ARGV.length < 1
|
|
puts "Error: No MallocBench specified."
|
|
exit 1
|
|
end
|
|
|
|
if ARGV.length < 2
|
|
puts "Error: No dylib specified."
|
|
exit 1
|
|
end
|
|
|
|
$mallocBench = File.absolute_path(ARGV.shift)
|
|
if !File.exists?($mallocBench)
|
|
puts "File not found: #{$mallocBench}."
|
|
exit 1
|
|
end
|
|
|
|
$buildDir = Pathname.new($mallocBench).dirname
|
|
|
|
dylibs = []
|
|
ARGV.each {
|
|
| arg |
|
|
name, path = arg.split(":")
|
|
if !name || name.length < 1 ||
|
|
!path || path.length < 1
|
|
puts "Invalid <Name:/path/to/dylib>: '#{arg}'."
|
|
exit 1
|
|
end
|
|
|
|
dylib = Dylib.new(name, File.expand_path(path))
|
|
|
|
if !File.exists?(dylib.path)
|
|
puts "File not found: #{dylib.path}."
|
|
exit 1
|
|
end
|
|
|
|
dylibs.push(dylib)
|
|
}
|
|
dylibs
|
|
end
|
|
|
|
def runBenchmarks(dylibs)
|
|
executionTime = []
|
|
peakMemory = []
|
|
memoryAtEnd = []
|
|
|
|
$benchmarks.each {
|
|
| benchmark |
|
|
|
|
executionTime.push([])
|
|
peakMemory.push([])
|
|
memoryAtEnd.push([])
|
|
|
|
dylibs.each {
|
|
| dylib |
|
|
|
|
$stderr.print "\rRUNNING #{dylib.name}: #{benchmark}... "
|
|
env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
|
|
env += "LD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
|
|
if dylib.name == "NanoMalloc"
|
|
env += "MallocNanoZone=1 "
|
|
elsif dylib.name == "SystemMalloc"
|
|
env += "MallocNanoZone=0 "
|
|
end
|
|
input = "cd '#{$buildDir}'; #{env} '#{$mallocBench}' --benchmark #{benchmark} --heap #{$heap}}"
|
|
output =`#{input}`
|
|
splitOutput = output.split("\n")
|
|
|
|
executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1]))
|
|
peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[2] : "0"))
|
|
memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[3] : "0"))
|
|
}
|
|
}
|
|
$stderr.print "\r \n"
|
|
|
|
Results.new(executionTime, peakMemory, memoryAtEnd)
|
|
end
|
|
|
|
def printResults(dylibs, results)
|
|
def printHeader(dylibs, fieldSize)
|
|
print
|
|
print lpad("", fieldSize)
|
|
print lpad(dylibs[0].name, fieldSize)
|
|
if dylibs.length > 1
|
|
print lpad(dylibs[1].name, fieldSize)
|
|
print lpad("Δ", fieldSize)
|
|
end
|
|
print "\n"
|
|
end
|
|
|
|
def printMetric(name, results, compareFunction, suffix, fieldSize)
|
|
def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize)
|
|
means = []
|
|
|
|
means.push(meanFunction.call(results.collect { | stats | stats[0].result }))
|
|
print rpad(" " + name, fieldSize)
|
|
print lpad("#{prettify(means[0].round, suffix)}", fieldSize)
|
|
|
|
if results[0][1]
|
|
means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
|
|
print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
|
|
print lpad(compareFunction.call(means[0], means[1]), fieldSize)
|
|
end
|
|
|
|
print "\n"
|
|
end
|
|
|
|
if results[0][0].result == 0
|
|
return
|
|
end
|
|
|
|
print name + ":\n"
|
|
results.each {
|
|
| stats |
|
|
|
|
print rpad(" " + stats[0].benchmark, fieldSize)
|
|
print lpad("#{prettify(stats[0].result, suffix)}", fieldSize)
|
|
|
|
if stats[1]
|
|
print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
|
|
print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
|
|
end
|
|
|
|
print "\n"
|
|
}
|
|
|
|
print "\n"
|
|
|
|
printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize)
|
|
printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize)
|
|
printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize)
|
|
|
|
print "\n"
|
|
end
|
|
|
|
fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect {
|
|
| benchmark |
|
|
benchmark.size
|
|
}.max + 4
|
|
|
|
printHeader(dylibs, fieldSize)
|
|
printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize)
|
|
printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize)
|
|
printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize)
|
|
end
|
|
|
|
def main
|
|
begin
|
|
dylibs = parseOptions()
|
|
results = runBenchmarks(dylibs)
|
|
printResults(dylibs, results)
|
|
rescue => exception
|
|
puts
|
|
puts
|
|
puts exception
|
|
puts exception.backtrace
|
|
puts
|
|
end
|
|
end
|
|
|
|
main()
|