389 lines
15 KiB
JavaScript
389 lines
15 KiB
JavaScript
/*
|
||
* Copyright (C) 2011, 2012 Purdue University
|
||
* Written by Gregor Richards
|
||
* All rights reserved.
|
||
*
|
||
* Redistribution and use in source and binary forms, with or without
|
||
* modification, are permitted provided that the following conditions are met:
|
||
*
|
||
* 1. Redistributions of source code must retain the above copyright notice,
|
||
* this list of conditions and the following disclaimer.
|
||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||
* this list of conditions and the following disclaimer in the documentation
|
||
* and/or other materials provided with the distribution.
|
||
*
|
||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||
* POSSIBILITY OF SUCH DAMAGE.
|
||
*/
|
||
|
||
(function() {
|
||
// put benchmarks here
|
||
var benchmarks = ["amazon-chrome", "amazon-chrome-win", "amazon-firefox",
|
||
"amazon-firefox-win", "amazon-safari", "facebook-chrome",
|
||
"facebook-chrome-win", "facebook-firefox",
|
||
"facebook-firefox-win", "facebook-safari",
|
||
"google-chrome", "google-chrome-win", "google-firefox",
|
||
"google-firefox-win", "google-safari", "twitter-chrome",
|
||
"twitter-chrome-win", "twitter-firefox",
|
||
"twitter-firefox-win", "twitter-safari", "yahoo-chrome",
|
||
"yahoo-chrome-win", "yahoo-firefox", "yahoo-firefox-win",
|
||
"yahoo-safari"];
|
||
var modes = {
|
||
"*": ["urem"],
|
||
"amazon-firefox": ["urm"],
|
||
"amazon-firefox-win": ["urm"],
|
||
"google-firefox": ["uem"],
|
||
"twitter-chrome-win": ["rem"]
|
||
};
|
||
var minRuns = 23;
|
||
var keepRuns = 20;
|
||
var maxRuns = 100;
|
||
var maxCIM = 0.10;
|
||
|
||
// fixups for old engines
|
||
if (!Array.prototype.reduce) {
|
||
Array.prototype.reduce = function(f, val) {
|
||
for (var i = 0; i < this.length; i++) {
|
||
val = f(val, this[i]);
|
||
}
|
||
return val;
|
||
};
|
||
}
|
||
|
||
// all test results
|
||
var results = {};
|
||
|
||
var curRun = 0;
|
||
var curBenchmark = 0;
|
||
var curMode = 0;
|
||
|
||
function getModes(bm) {
|
||
if (bm in modes) return modes[bm];
|
||
return modes["*"];
|
||
}
|
||
|
||
// call this to either rerun or go to another page and come back
|
||
function rerun() {
|
||
try {
|
||
if (window.sessionStorage) {
|
||
// store this for the session and go 'round to another page to force uncaching on Chrome
|
||
sessionStorage.JSBNG_harnessState = JSON.stringify({
|
||
results: results,
|
||
curRun: curRun,
|
||
curBenchmark: curBenchmark,
|
||
curMode: curMode
|
||
});
|
||
window.location.href = window.location.href.replace(/\/[^\/]*$/, "/reload.html");
|
||
return;
|
||
}
|
||
} catch (ex) {}
|
||
runBenchmark();
|
||
}
|
||
|
||
// load our current state and run
|
||
function onload() {
|
||
var gob = document.getElementById("go");
|
||
try {
|
||
if (window.sessionStorage && "JSBNG_harnessState" in sessionStorage) {
|
||
var state = JSON.parse(sessionStorage.JSBNG_harnessState);
|
||
results = state.results;
|
||
curRun = state.curRun;
|
||
curBenchmark = state.curBenchmark;
|
||
curMode = state.curMode;
|
||
setTimeout(runBenchmark, 200);
|
||
gob.style.display = "none";
|
||
return;
|
||
}
|
||
} catch (ex) {}
|
||
gob.onclick = function() {
|
||
gob.style.display = "none";
|
||
setTimeout(runBenchmark, 200);
|
||
};
|
||
if (/#run/.test(window.location.href)) {
|
||
gob.style.display = "none";
|
||
setTimeout(runBenchmark, 200);
|
||
}
|
||
}
|
||
|
||
function runBenchmark() {
|
||
var output = document.getElementById("output");
|
||
var bmframe = document.getElementById("bmframe");
|
||
|
||
// should we stop?
|
||
if (curRun < minRuns) {
|
||
output.innerHTML = (curRun+1) + "/" + minRuns;
|
||
} else {
|
||
// check if our S/M is OK
|
||
var stats = handleResults(false);
|
||
output.innerHTML = (curRun+1) + " " + stats.cim + " (" + maxCIM + ")";
|
||
if (curRun >= maxRuns || stats.cim < maxCIM) {
|
||
bmframe.src = "about:blank";
|
||
bmframe.style.display = "none";
|
||
handleResults(true);
|
||
if (window.sessionStorage) delete sessionStorage.JSBNG_harnessState;
|
||
return;
|
||
}
|
||
}
|
||
|
||
// get out our benchmark and mode
|
||
var benchmark = benchmarks[curBenchmark];
|
||
var modes = getModes(benchmark);
|
||
var mode = modes[curMode];
|
||
if (++curMode >= modes.length) {
|
||
curMode = 0;
|
||
curBenchmark++;
|
||
}
|
||
if (curBenchmark >= benchmarks.length) {
|
||
curBenchmark = 0;
|
||
curRun++;
|
||
}
|
||
|
||
// make sure we have the results space
|
||
if (!(benchmark in results)) results[benchmark] = {};
|
||
if (!(mode in results[benchmark])) results[benchmark][mode] = [];
|
||
|
||
// Expose time measuring function.
|
||
if (window.performance && window.performance.now)
|
||
window.currentTimeInMS = function() { return window.performance.now() };
|
||
else if (typeof preciseTime !== 'undefined')
|
||
window.currentTimeInMS = function() { return preciseTime() * 1000; };
|
||
else
|
||
window.currentTimeInMS = function() { return Date.now(); };
|
||
|
||
// set up the receiver
|
||
if (/v/.test(mode)) {
|
||
// verification mode, we only care if there's an error
|
||
window.JSBNG_handleResult = function(res) {
|
||
if (res.error) {
|
||
if (!("errors" in results)) results.errors = [];
|
||
results.errors.push(benchmark + "." + mode + ": " + res.msg);
|
||
}
|
||
rerun();
|
||
};
|
||
} else {
|
||
window.JSBNG_handleResult = function(res) {
|
||
if (!res.error) {
|
||
results[benchmark][mode].push(res.time);
|
||
}
|
||
rerun();
|
||
};
|
||
}
|
||
|
||
// then load it
|
||
bmframe.src = benchmark + "/" + mode + ".html";
|
||
}
|
||
|
||
// handle all of our results
|
||
function handleResults(pr) {
|
||
var output = document.getElementById("output");
|
||
function print(str) {
|
||
output.appendChild(document.createTextNode(str));
|
||
output.appendChild(document.createElement("br"));
|
||
}
|
||
function printarr(arr) {
|
||
for (var i = 0; i < arr.length; i++) print(arr[i]);
|
||
}
|
||
function percent(num) {
|
||
return (num*100).toFixed(2) + "%";
|
||
}
|
||
|
||
// clear out the intermediate results
|
||
if (pr) output.innerHTML = "";
|
||
|
||
// totals
|
||
var totals = {
|
||
mean: 1,
|
||
stddev: 1,
|
||
sem: 1,
|
||
ci: 1,
|
||
runs: 0
|
||
};
|
||
|
||
// stuff to print later
|
||
var ptotals = [];
|
||
var presults = [];
|
||
var praw = [];
|
||
var spc = "\u00a0\u00a0";
|
||
var spc2 = spc + spc;
|
||
|
||
// calculate all the real results
|
||
for (var b = 0; b < benchmarks.length; b++) {
|
||
var benchmark = benchmarks[b];
|
||
var modes = getModes(benchmark);
|
||
if (pr) {
|
||
presults.push(spc + benchmark + ":");
|
||
praw.push(spc + benchmark + ":");
|
||
}
|
||
|
||
for (var m = 0; m < modes.length; m++) {
|
||
var mode = modes[m];
|
||
var bmresults = results[benchmark][mode].slice(-keepRuns);
|
||
if (bmresults.length == 0) continue;
|
||
|
||
// get the raw results
|
||
var rr = spc2 + mode + ": [";
|
||
for (var i = 0; i < bmresults.length; i++) {
|
||
if (i != 0) rr += ", ";
|
||
rr += bmresults[i];
|
||
}
|
||
rr += "]";
|
||
if (pr) praw.push(rr);
|
||
|
||
// now get the stats for this run
|
||
var bmstats = stats(bmresults);
|
||
|
||
// mul it to the totals
|
||
totals.mean *= bmstats.mean;
|
||
totals.stddev *= bmstats.stddev;
|
||
totals.sem *= bmstats.sem;
|
||
totals.ci *= bmstats.ci;
|
||
totals.runs++;
|
||
|
||
// and output it
|
||
if (pr) presults.push(spc2 + mode + ": " +
|
||
bmstats.mean.toFixed(2) + "ms <20> " + percent(bmstats.cim) +
|
||
" (stddev=" + percent(bmstats.sm) + ", stderr=" +
|
||
percent(bmstats.semm) + ")");
|
||
}
|
||
|
||
if (pr) {
|
||
presults.push("");
|
||
praw.push("");
|
||
}
|
||
}
|
||
|
||
// now calculate the totals
|
||
var power = 1 / totals.runs;
|
||
totals.mean = Math.pow(totals.mean, power);
|
||
totals.stddev = Math.pow(totals.stddev, power);
|
||
totals.sm = totals.stddev / totals.mean;
|
||
totals.sem = Math.pow(totals.sem, power);
|
||
totals.semm = totals.sem / totals.mean;
|
||
totals.ci = Math.pow(totals.ci, power);
|
||
totals.cim = totals.ci / totals.mean;
|
||
ptotals.push("Final results:");
|
||
ptotals.push(spc + totals.mean.toFixed(2) + "ms <20> " + percent(totals.cim) + " (lower is better)");
|
||
ptotals.push(spc + "Standard deviation = " + percent(totals.sm) + " of mean");
|
||
ptotals.push(spc + "Standard error = " + percent(totals.semm) + " of mean");
|
||
if (totals.cim >= maxCIM)
|
||
ptotals.push(spc + "WARNING: These results are not trustworthy! After " + maxRuns + " runs, 95% confidence interval is still greater than " + percent(maxCIM) + " of the mean!");
|
||
else
|
||
ptotals.push(spc + curRun + " runs");
|
||
ptotals.push("");
|
||
|
||
// if there are errors, mark those too
|
||
if ("errors" in results) {
|
||
ptotals.push("ERRORS:");
|
||
for (var i = 0; i < results.errors.length; i++) ptotals.push(spc + results.errors[i]);
|
||
ptotals.push("");
|
||
}
|
||
|
||
if (pr) {
|
||
// and print it all out
|
||
printarr(ptotals);
|
||
print("Result breakdown:");
|
||
printarr(presults);
|
||
print("Raw results:");
|
||
printarr(praw);
|
||
}
|
||
|
||
return totals;
|
||
}
|
||
|
||
// standard t-distribution for normally distributed samples
|
||
var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36,
|
||
2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09,
|
||
2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04,
|
||
2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02,
|
||
2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00,
|
||
2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00,
|
||
2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99,
|
||
1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99,
|
||
1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
|
||
1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
|
||
1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
|
||
1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
|
||
1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
|
||
1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
|
||
1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
|
||
|
||
// t distribution
|
||
function tDist(n) {
|
||
if (n >= tDistribution.length)
|
||
return tDistribution[tDistribution.length-1];
|
||
return tDistribution[n];
|
||
}
|
||
|
||
// get statistics
|
||
function stats(results) {
|
||
var ret = {};
|
||
|
||
function sum(arr) {
|
||
return arr.reduce(function(p, c) { return p + c; }, 0);
|
||
}
|
||
|
||
// mean
|
||
ret.mean = sum(results) / results.length;
|
||
|
||
// sample stddev
|
||
ret.stddev = Math.sqrt(
|
||
sum(
|
||
results.map(function(e) { return Math.pow(e - ret.mean, 2); })
|
||
) / (results.length - 1)
|
||
);
|
||
|
||
// stddev / mean
|
||
ret.sm = ret.stddev / ret.mean;
|
||
|
||
// sample SEM (stderr)
|
||
ret.sem = ret.stddev / Math.sqrt(results.length);
|
||
|
||
// sample SEM/mean
|
||
ret.semm = ret.sem / ret.mean;
|
||
|
||
// sample 95% confidence interval range
|
||
ret.ci = tDist(results.length) * ret.sem;
|
||
|
||
// sample 95% CI / mean
|
||
ret.cim = ret.ci / ret.mean;
|
||
|
||
return ret;
|
||
}
|
||
|
||
window.onload = onload;
|
||
})();
|