import re
import sys
+benchmarks = [
+ "amazon/chrome", "amazon/chrome-win", "amazon/firefox",
+ "amazon/firefox-win", "amazon/safari",
+ "facebook/chrome", "facebook/chrome-win", "facebook/firefox",
+ "facebook/firefox-win", "facebook/safari",
+ "google/chrome", "google/chrome-win", "google/firefox",
+ "google/firefox-win", "google/safari",
+ "twitter/chrome", "twitter/chrome-win", "twitter/firefox",
+ "twitter/firefox-win", "twitter/safari",
+ "yahoo/chrome", "yahoo/chrome-win", "yahoo/firefox",
+ "yahoo/firefox-win", "yahoo/safari",
+]
+
+modes = {
+ "*": ["urem"],
+ "amazon/firefox": ["urm"],
+ "amazon/firefox-win": ["urm"],
+ "google/firefox": ["uem"],
+ "twitter/chrome-win": ["rem"]
+};
+
+'''
benchmarks = ["amazon/chrome", "amazon/firefox", "amazon/safari",
"facebook/chrome", "facebook/firefox", "facebook/safari",
"google/chrome", "google/firefox", "google/safari",
"*": ["urem"],
"amazon/firefox": ["urm"],
"google/firefox": ["uem"]
-}
-runcount = 25
-keepruns = 20
+}'''
-keepfrom = runcount - keepruns
+#runcount = 25
-if len(sys.argv) != 2:
- print "Use: python harness.py <JS executable>"
+if len(sys.argv) != 3:
+ print "Use: python harness.py <JS executable> <number of runs>"
exit(1)
js = sys.argv[1]
+runcount = int(sys.argv[2])
+
+keepruns = 20
+keepfrom = runcount - keepruns
# standard t-distribution for normally distributed samples
tDistribution = [0, 0, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26,
# Now run it and get the results
print(benchmark + " " + mode + " " + str(runno))
res = os.popen(js + " " + benchmark + "/" + mode + ".js").read()
- time = float(re.match("Time: ([0-9]*)ms", res).group(1))
+ time = float(re.findall("Time: ([0-9]*)ms", res)[0])
+# time = float(re.match("Time: ([0-9]*)ms", res).group(1))
if runno >= keepfrom:
results[benchmark][mode].append(time)
"runs": 0
}
+power = 1.0 / len(benchmarks)
+
for benchmark in benchmarks:
sresults[benchmark] = {}
-
- print("middle");
-
bmodes = modes["*"]
if benchmark in modes:
bmodes = modes[benchmark]
sresult["mode"] = mode
- mean = sresult["mean"] = sum(result) / len(result)
+ mean = sresult["mean"] = float(sum(result)) / len(result)
stddev = sresult["stddev"] = math.sqrt(
sum(
map(lambda e: math.pow(e - mean, 2), result)
- ) / (len(result) - 1)
+ ) / float((len(result) - 1))
)
- sm = sresult["sm"] = stddev / mean
- sem = sresult["sem"] = stddev / math.sqrt(len(result))
- semm = sresult["semm"] = sem / mean
+ sm = sresult["sm"] = stddev / float(mean)
+ sem = sresult["sem"] = stddev / float(math.sqrt(len(result)))
+ semm = sresult["semm"] = sem / float(mean)
ci = sresult["ci"] = tDist(len(result)) * sem
- cim = sresult["cim"] = ci / mean
-
- totals["mean"] *= mean
- totals["stddev"] *= stddev
- totals["sem"] *= sem
- totals["ci"] *= ci
-
-power = 1 / totals["runs"]
-totals["mean"] = math.pow(totals["mean"], power)
-totals["stddev"] = math.pow(totals["stddev"], power)
-totals["sm"] = totals["stddev"] / totals["mean"]
-totals["sem"] = math.pow(totals["sem"], power)
-totals["semm"] = totals["sem"] / totals["mean"]
-totals["ci"] = math.pow(totals["ci"], power)
-totals["cim"] = totals["ci"] / totals["mean"]
+ cim = sresult["cim"] = ci / float(mean)
+
+ totals["mean"] *= math.pow(mean, power)
+
+ if (stddev == 0):
+ totals["stddev"] *= 1
+ print("stddev is 0")
+ else:
+ totals["stddev"] *= math.pow(stddev, power)
+ totals["sem"] *= math.pow(sem, power)
+ totals["ci"] *= math.pow(ci, power)
+
+#power = 1.0 / totals["runs"]
+#totals["mean"] = math.pow(totals["mean"], power)
+#totals["stddev"] = math.pow(totals["stddev"], power)
+#totals["sem"] = math.pow(totals["sem"], power)
+#totals["ci"] = math.pow(totals["ci"], power)
+
+totals["sm"] = totals["stddev"] / float(totals["mean"])
+totals["semm"] = totals["sem"] / float(totals["mean"])
+totals["cim"] = totals["ci"] / float(totals["mean"])
totals["sm"] *= 100
totals["semm"] *= 100
totals["cim"] *= 100
print "Final results:"
-print u" %(mean)fms \u00b1 %(cim)f%% (lower is better)" % totals
+print " %(mean)fms +- %(cim)f%% (lower is better)" % totals
print " Standard deviation = %(sm)f%% of mean" % totals
print " Standard error = %(semm)f%% of mean" % totals
print " %(runs)d runs" % {"runs": runcount}
bmodes = modes[benchmark]
for mode in bmodes:
- print u" %(mode)s: %(mean)fms \u00b1 %(cim)f%% (stddev=%(sm)f%%, stderr=%(semm)f%%)" % sresults[benchmark][mode]
+ print " %(mode)s: %(mean)fms +- %(cim)f%% (stddev=%(sm)f%%, stderr=%(semm)f%%)" % sresults[benchmark][mode]
+
print ""
print "Raw results:"