--- /dev/null
+/*
+ * Copyright (C) 2011, 2012 Purdue University
+ * Written by Gregor Richards
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+(function() {
+ // put benchmarks here
+ var benchmarks = ["amazon/chrome", "amazon/chrome-win", "amazon/firefox",
+ "amazon/firefox-win", "amazon/safari", "facebook/chrome",
+ "facebook/chrome-win", "facebook/firefox",
+ "facebook/firefox-win", "facebook/safari",
+ "google/chrome", "google/chrome-win", "google/firefox",
+ "google/firefox-win", "google/safari", "twitter/chrome",
+ "twitter/chrome-win", "twitter/firefox",
+ "twitter/firefox-win", "twitter/safari", "yahoo/chrome",
+ "yahoo/chrome-win", "yahoo/firefox", "yahoo/firefox-win",
+ "yahoo/safari"];
+ var modes = {
+ "*": ["urem"],
+ "amazon/firefox": ["urm"],
+ "amazon/firefox-win": ["urm"],
+ "google/firefox": ["uem"],
+ "twitter/chrome-win": ["rem"]
+ };
+ var minRuns = 23;
+ var keepRuns = 20;
+ var maxRuns = 100;
+ var maxCIM = 0.10;
+
+ // fixups for old engines
+ if (!Array.prototype.reduce) {
+ Array.prototype.reduce = function(f, val) {
+ for (var i = 0; i < this.length; i++) {
+ val = f(val, this[i]);
+ }
+ return val;
+ };
+ }
+
+ // all test results
+ var results = {};
+
+ var curRun = 0;
+ var curBenchmark = 0;
+ var curMode = 0;
+
+ function getModes(bm) {
+ if (bm in modes) return modes[bm];
+ return modes["*"];
+ }
+
+ // call this to either rerun or go to another page and come back
+ function rerun() {
+ try {
+ if (window.sessionStorage) {
+ // store this for the session and go 'round to another page to force uncaching on Chrome
+ sessionStorage.JSBNG_harnessState = JSON.stringify({
+ results: results,
+ curRun: curRun,
+ curBenchmark: curBenchmark,
+ curMode: curMode
+ });
+ window.location.href = window.location.href.replace(/\/[^\/]*$/, "/reload.html");
+ return;
+ }
+ } catch (ex) {}
+ runBenchmark();
+ }
+
+ // load our current state and run
+ function onload() {
+ var gob = document.getElementById("go");
+ try {
+ if (window.sessionStorage && "JSBNG_harnessState" in sessionStorage) {
+ var state = JSON.parse(sessionStorage.JSBNG_harnessState);
+ results = state.results;
+ curRun = state.curRun;
+ curBenchmark = state.curBenchmark;
+ curMode = state.curMode;
+ setTimeout(runBenchmark, 200);
+ gob.style.display = "none";
+ return;
+ }
+ } catch (ex) {}
+ gob.onclick = function() {
+ gob.style.display = "none";
+ setTimeout(runBenchmark, 200);
+ };
+ if (/#run/.test(window.location.href)) {
+ gob.style.display = "none";
+ setTimeout(runBenchmark, 200);
+ }
+ }
+
+ function runBenchmark() {
+ var output = document.getElementById("output");
+ var bmframe = document.getElementById("bmframe");
+
+ // should we stop?
+ if (curRun < minRuns) {
+ output.innerHTML = (curRun+1) + "/" + minRuns;
+ } else {
+ // check if our S/M is OK
+ var stats = handleResults(false);
+ output.innerHTML = (curRun+1) + " " + stats.cim + " (" + maxCIM + ")";
+ if (curRun >= maxRuns || stats.cim < maxCIM) {
+ bmframe.src = "about:blank";
+ bmframe.style.display = "none";
+ handleResults(true);
+ if (window.sessionStorage) delete sessionStorage.JSBNG_harnessState;
+ return;
+ }
+ }
+
+ // get out our benchmark and mode
+ var benchmark = benchmarks[curBenchmark];
+ var modes = getModes(benchmark);
+ var mode = modes[curMode];
+ if (++curMode >= modes.length) {
+ curMode = 0;
+ curBenchmark++;
+ }
+ if (curBenchmark >= benchmarks.length) {
+ curBenchmark = 0;
+ curRun++;
+ }
+
+ // make sure we have the results space
+ if (!(benchmark in results)) results[benchmark] = {};
+ if (!(mode in results[benchmark])) results[benchmark][mode] = [];
+
+ // set up the receiver
+ if (/v/.test(mode)) {
+ // verification mode, we only care if there's an error
+ window.JSBNG_handleResult = function(res) {
+ if (res.error) {
+ if (!("errors" in results)) results.errors = [];
+ results.errors.push(benchmark + "." + mode + ": " + res.msg);
+ }
+ rerun();
+ };
+ } else {
+ window.JSBNG_handleResult = function(res) {
+ if (!res.error) {
+ results[benchmark][mode].push(res.time);
+ }
+ rerun();
+ };
+ }
+
+ // then load it
+ bmframe.src = benchmark + "/" + mode + ".html";
+ }
+
+ // handle all of our results
+ function handleResults(pr) {
+ var output = document.getElementById("output");
+ function print(str) {
+ output.appendChild(document.createTextNode(str));
+ output.appendChild(document.createElement("br"));
+ }
+ function printarr(arr) {
+ for (var i = 0; i < arr.length; i++) print(arr[i]);
+ }
+ function percent(num) {
+ return (num*100).toFixed(2) + "%";
+ }
+
+ // clear out the intermediate results
+ if (pr) output.innerHTML = "";
+
+ // totals
+ var totals = {
+ mean: 1,
+ stddev: 1,
+ sem: 1,
+ ci: 1,
+ runs: 0
+ };
+
+ // stuff to print later
+ var ptotals = [];
+ var presults = [];
+ var praw = [];
+ var spc = "\u00a0\u00a0";
+ var spc2 = spc + spc;
+
+ // calculate all the real results
+ for (var b = 0; b < benchmarks.length; b++) {
+ var benchmark = benchmarks[b];
+ var modes = getModes(benchmark);
+ if (pr) {
+ presults.push(spc + benchmark + ":");
+ praw.push(spc + benchmark + ":");
+ }
+
+ for (var m = 0; m < modes.length; m++) {
+ var mode = modes[m];
+ var bmresults = results[benchmark][mode].slice(-keepRuns);
+ if (bmresults.length == 0) continue;
+
+ // get the raw results
+ var rr = spc2 + mode + ": [";
+ for (var i = 0; i < bmresults.length; i++) {
+ if (i != 0) rr += ", ";
+ rr += bmresults[i];
+ }
+ rr += "]";
+ if (pr) praw.push(rr);
+
+ // now get the stats for this run
+ var bmstats = stats(bmresults);
+
+ // mul it to the totals
+ totals.mean *= bmstats.mean;
+ totals.stddev *= bmstats.stddev;
+ totals.sem *= bmstats.sem;
+ totals.ci *= bmstats.ci;
+ totals.runs++;
+
+ // and output it
+ if (pr) presults.push(spc2 + mode + ": " +
+ bmstats.mean.toFixed(2) + "ms ± " + percent(bmstats.cim) +
+ " (stddev=" + percent(bmstats.sm) + ", stderr=" +
+ percent(bmstats.semm) + ")");
+ }
+
+ if (pr) {
+ presults.push("");
+ praw.push("");
+ }
+ }
+
+ // now calculate the totals
+ var power = 1 / totals.runs;
+ totals.mean = Math.pow(totals.mean, power);
+ totals.stddev = Math.pow(totals.stddev, power);
+ totals.sm = totals.stddev / totals.mean;
+ totals.sem = Math.pow(totals.sem, power);
+ totals.semm = totals.sem / totals.mean;
+ totals.ci = Math.pow(totals.ci, power);
+ totals.cim = totals.ci / totals.mean;
+ ptotals.push("Final results:");
+ ptotals.push(spc + totals.mean.toFixed(2) + "ms ± " + percent(totals.cim) + " (lower is better)");
+ ptotals.push(spc + "Standard deviation = " + percent(totals.sm) + " of mean");
+ ptotals.push(spc + "Standard error = " + percent(totals.semm) + " of mean");
+ if (totals.cim >= maxCIM)
+ ptotals.push(spc + "WARNING: These results are not trustworthy! After " + maxRuns + " runs, 95% confidence interval is still greater than " + percent(maxCIM) + " of the mean!");
+ else
+ ptotals.push(spc + curRun + " runs");
+ ptotals.push("");
+
+ // if there are errors, mark those too
+ if ("errors" in results) {
+ ptotals.push("ERRORS:");
+ for (var i = 0; i < results.errors.length; i++) ptotals.push(spc + results.errors[i]);
+ ptotals.push("");
+ }
+
+ if (pr) {
+ // and print it all out
+ printarr(ptotals);
+ print("Result breakdown:");
+ printarr(presults);
+ print("Raw results:");
+ printarr(praw);
+ }
+
+ return totals;
+ }
+
+ // standard t-distribution for normally distributed samples
+ var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36,
+ 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09,
+ 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04,
+ 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02,
+ 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00,
+ 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00,
+ 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99,
+ 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99,
+ 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
+ 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
+ 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
+ 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
+ 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
+ 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
+ 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
+
+ // t distribution
+ function tDist(n) {
+ if (n >= tDistribution.length)
+ return tDistribution[tDistribution.length-1];
+ return tDistribution[n];
+ }
+
+ // get statistics
+ function stats(results) {
+ var ret = {};
+
+ function sum(arr) {
+ return arr.reduce(function(p, c) { return p + c; }, 0);
+ }
+
+ // mean
+ ret.mean = sum(results) / results.length;
+
+ // sample stddev
+ ret.stddev = Math.sqrt(
+ sum(
+ results.map(function(e) { return Math.pow(e - ret.mean, 2); })
+ ) / (results.length - 1)
+ );
+
+ // stddev / mean
+ ret.sm = ret.stddev / ret.mean;
+
+ // sample SEM (stderr)
+ ret.sem = ret.stddev / Math.sqrt(results.length);
+
+ // sample SEM/mean
+ ret.semm = ret.sem / ret.mean;
+
+ // sample 95% confidence interval range
+ ret.ci = tDist(results.length) * ret.sem;
+
+ // sample 95% CI / mean
+ ret.cim = ret.ci / ret.mean;
+
+ return ret;
+ }
+
+ window.onload = onload;
+})();
projects / c11concurrency-benchmarks.git / blobdiff