haikuwebkit/PerformanceTests/JSBench/harness.js

/*
 * Copyright (C) 2011, 2012 Purdue University
 * Written by Gregor Richards
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

(function() {
    // put benchmarks here
    var benchmarks = ["amazon-chrome", "amazon-chrome-win", "amazon-firefox",
                      "amazon-firefox-win", "amazon-safari", "facebook-chrome",
                      "facebook-chrome-win", "facebook-firefox",
                      "facebook-firefox-win", "facebook-safari",
                      "google-chrome", "google-chrome-win", "google-firefox",
                      "google-firefox-win", "google-safari", "twitter-chrome",
                      "twitter-chrome-win", "twitter-firefox",
                      "twitter-firefox-win", "twitter-safari", "yahoo-chrome",
                      "yahoo-chrome-win", "yahoo-firefox", "yahoo-firefox-win",
                      "yahoo-safari"];
    var modes = {
        "*": ["urem"],
        "amazon-firefox": ["urm"],
        "amazon-firefox-win": ["urm"],
        "google-firefox": ["uem"],
        "twitter-chrome-win": ["rem"]
    };
    var minRuns = 23;
    var keepRuns = 20;
    var maxRuns = 100;
    var maxCIM = 0.10;

    // fixups for old engines
    if (!Array.prototype.reduce) {
        Array.prototype.reduce = function(f, val) {
            for (var i = 0; i < this.length; i++) {
                val = f(val, this[i]);
            }
            return val;
        };
    }

    // all test results
    var results = {};

    var curRun = 0;
    var curBenchmark = 0;
    var curMode = 0;

    function getModes(bm) {
        if (bm in modes) return modes[bm];
        return modes["*"];
    }

    // call this to either rerun or go to another page and come back
    function rerun() {
        try {
            if (window.sessionStorage) {
                // store this for the session and go 'round to another page to force uncaching on Chrome
                sessionStorage.JSBNG_harnessState = JSON.stringify({
                    results: results,
                    curRun: curRun,
                    curBenchmark: curBenchmark,
                    curMode: curMode
                });
                window.location.href = window.location.href.replace(/\/[^\/]*$/, "/reload.html");
                return;
            }
        } catch (ex) {}
        runBenchmark();
    }

    // load our current state and run
    function onload() {
        var gob = document.getElementById("go");
        try {
            if (window.sessionStorage && "JSBNG_harnessState" in sessionStorage) {
                var state = JSON.parse(sessionStorage.JSBNG_harnessState);
                results = state.results;
                curRun = state.curRun;
                curBenchmark = state.curBenchmark;
                curMode = state.curMode;
                setTimeout(runBenchmark, 200);
                gob.style.display = "none";
                return;
            }
        } catch (ex) {}
        gob.onclick = function() {
            gob.style.display = "none";
            setTimeout(runBenchmark, 200);
        };
        if (/#run/.test(window.location.href)) {
            gob.style.display = "none";
            setTimeout(runBenchmark, 200);
        }
    }

    function runBenchmark() {
        var output = document.getElementById("output");
        var bmframe = document.getElementById("bmframe");

        // should we stop?
        if (curRun < minRuns) {
            output.innerHTML = (curRun+1) + "/" + minRuns;
        } else {
            // check if our S/M is OK
            var stats = handleResults(false);
            output.innerHTML = (curRun+1) + " " + stats.cim + " (" + maxCIM + ")";
            if (curRun >= maxRuns || stats.cim < maxCIM) {
                bmframe.src = "about:blank";
                bmframe.style.display = "none";
                handleResults(true);
                if (window.sessionStorage) delete sessionStorage.JSBNG_harnessState;
                return;
            }
        }

        // get out our benchmark and mode
        var benchmark = benchmarks[curBenchmark];
        var modes = getModes(benchmark);
        var mode = modes[curMode];
        if (++curMode >= modes.length) {
            curMode = 0;
            curBenchmark++;
        }
        if (curBenchmark >= benchmarks.length) {
            curBenchmark = 0;
            curRun++;
        }

        // make sure we have the results space
        if (!(benchmark in results)) results[benchmark] = {};
        if (!(mode in results[benchmark])) results[benchmark][mode] = [];

        // Expose time measuring function.
        if (window.performance && window.performance.now)
            window.currentTimeInMS = function() { return window.performance.now() };
        else if (typeof preciseTime !== 'undefined')
            window.currentTimeInMS = function() { return preciseTime() * 1000; };
        else
            window.currentTimeInMS = function() { return Date.now(); };

        // set up the receiver
        if (/v/.test(mode)) {
            // verification mode, we only care if there's an error
            window.JSBNG_handleResult = function(res) {
                if (res.error) {
                    if (!("errors" in results)) results.errors = [];
                    results.errors.push(benchmark + "." + mode + ": " + res.msg);
                }
                rerun();
            };
        } else {
            window.JSBNG_handleResult = function(res) {
                if (!res.error) {
                    results[benchmark][mode].push(res.time);
                }
                rerun();
            };
        }

        // then load it
        bmframe.src = benchmark + "/" + mode + ".html";
    }

    // handle all of our results
    function handleResults(pr) {
        var output = document.getElementById("output");
        function print(str) {
            output.appendChild(document.createTextNode(str));
            output.appendChild(document.createElement("br"));
        }
        function printarr(arr) {
            for (var i = 0; i < arr.length; i++) print(arr[i]);
        }
        function percent(num) {
            return (num*100).toFixed(2) + "%";
        }

        // clear out the intermediate results
        if (pr) output.innerHTML = "";

        // totals
        var totals = {
            mean: 1,
            stddev: 1,
            sem: 1,
            ci: 1,
            runs: 0
        };

        // stuff to print later
        var ptotals = [];
        var presults = [];
        var praw = [];
        var spc = "\u00a0\u00a0";
        var spc2 = spc + spc;

        // calculate all the real results
        for (var b = 0; b < benchmarks.length; b++) {
            var benchmark = benchmarks[b];
            var modes = getModes(benchmark);
            if (pr) {
                presults.push(spc + benchmark + ":");
                praw.push(spc + benchmark + ":");
            }

            for (var m = 0; m < modes.length; m++) {
                var mode = modes[m];
                var bmresults = results[benchmark][mode].slice(-keepRuns);
                if (bmresults.length == 0) continue;

                // get the raw results
                var rr = spc2 + mode + ": [";
                for (var i = 0; i < bmresults.length; i++) {
                    if (i != 0) rr += ", ";
                    rr += bmresults[i];
                }
                rr += "]";
                if (pr) praw.push(rr);

                // now get the stats for this run
                var bmstats = stats(bmresults);

                // mul it to the totals
                totals.mean *= bmstats.mean;
                totals.stddev *= bmstats.stddev;
                totals.sem *= bmstats.sem;
                totals.ci *= bmstats.ci;
                totals.runs++;

                // and output it
                if (pr) presults.push(spc2 + mode + ": " +
                    bmstats.mean.toFixed(2) + "ms <20> " + percent(bmstats.cim) +
                    " (stddev=" + percent(bmstats.sm) + ", stderr=" +
                    percent(bmstats.semm) + ")");
            }

            if (pr) {
                presults.push("");
                praw.push("");
            }
        }

        // now calculate the totals
        var power = 1 / totals.runs;
        totals.mean = Math.pow(totals.mean, power);
        totals.stddev = Math.pow(totals.stddev, power);
        totals.sm = totals.stddev / totals.mean;
        totals.sem = Math.pow(totals.sem, power);
        totals.semm = totals.sem / totals.mean;
        totals.ci = Math.pow(totals.ci, power);
        totals.cim = totals.ci / totals.mean;
        ptotals.push("Final results:");
        ptotals.push(spc + totals.mean.toFixed(2) + "ms <20> " + percent(totals.cim) + " (lower is better)");
        ptotals.push(spc + "Standard deviation = " + percent(totals.sm) + " of mean");
        ptotals.push(spc + "Standard error = " + percent(totals.semm) + " of mean");
        if (totals.cim >= maxCIM)
            ptotals.push(spc + "WARNING: These results are not trustworthy! After " + maxRuns + " runs, 95% confidence interval is still greater than " + percent(maxCIM) + " of the mean!");
        else
            ptotals.push(spc + curRun + " runs");
        ptotals.push("");

        // if there are errors, mark those too
        if ("errors" in results) {
            ptotals.push("ERRORS:");
            for (var i = 0; i < results.errors.length; i++) ptotals.push(spc + results.errors[i]);
            ptotals.push("");
        }

        if (pr) {
            // and print it all out
            printarr(ptotals);
            print("Result breakdown:");
            printarr(presults);
            print("Raw results:");
            printarr(praw);
        }

        return totals;
    }

    // standard t-distribution for normally distributed samples
    var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36,
    2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09,
    2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04,
    2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02,
    2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00,
    2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00,
    2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99,
    1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99,
    1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
    1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
    1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
    1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
    1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
    1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
    1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];

    // t distribution
    function tDist(n) {
        if (n >= tDistribution.length)
            return tDistribution[tDistribution.length-1];
        return tDistribution[n];
    }

    // get statistics
    function stats(results) {
        var ret = {};

        function sum(arr) {
            return arr.reduce(function(p, c) { return p + c; }, 0);
        }

        // mean
        ret.mean = sum(results) / results.length;

        // sample stddev
        ret.stddev = Math.sqrt(
            sum(
                results.map(function(e) { return Math.pow(e - ret.mean, 2); })
            ) / (results.length - 1)
        );

        // stddev / mean
        ret.sm = ret.stddev / ret.mean;

        // sample SEM (stderr)
        ret.sem = ret.stddev / Math.sqrt(results.length);

        // sample SEM/mean
        ret.semm = ret.sem / ret.mean;

        // sample 95% confidence interval range
        ret.ci = tDist(results.length) * ret.sem;

        // sample 95% CI / mean
        ret.cim = ret.ci / ret.mean;

        return ret;
    }

    window.onload = onload;
})();