Add flags to skip benchmarks and predictable mode

Librefox · Feb 3, 2014 · f84a733 · f84a733
1 parent e43071e
commit f84a733
Show file tree

Hide file tree

Showing 18 changed files with 67 additions and 21 deletions.
diff --git a/base.js b/base.js
@@ -46,10 +46,12 @@ performance.now = (function() {
 // arguments are functions that will be invoked before and after
 // running the benchmark, but the running time of these functions will
 // not be accounted for in the benchmark score.
-function Benchmark(name, doWarmup, doDeterministic, run, setup, tearDown, rmsResult, minIterations) {
+function Benchmark(name, doWarmup, doDeterministic, deterministicIterations, 
+                   run, setup, tearDown, rmsResult, minIterations) {
   this.name = name;
   this.doWarmup = doWarmup;
   this.doDeterministic = doDeterministic;
+  this.deterministicIterations = deterministicIterations;
   this.run = run;
   this.Setup = setup ? setup : function() { };
   this.TearDown = tearDown ? tearDown : function() { };
@@ -96,6 +98,16 @@ BenchmarkSuite.suites = [];
 // a new benchmark or change an existing one.
 BenchmarkSuite.version = '9';
 
+
+// Defines global benchsuite running mode that overrides benchmark suite 
+// behavior. Intended to be set by the benchmark driver. Undefined 
+// values here allow a benchmark to define behaviour itself.
+BenchmarkSuite.config = {
+  doWarmup: undefined,
+  doDeterministic: undefined
+};
+
+
 // Override the alert function to throw an exception instead.
 alert = function(s) {
   throw "Alert called with argument: " + s;
@@ -125,7 +137,7 @@ BenchmarkSuite.ResetRNG = function() {
 // each individual benchmark to avoid running for too long in the
 // context of browsers. Once done, the final score is reported to the
 // runner.
-BenchmarkSuite.RunSuites = function(runner) {
+BenchmarkSuite.RunSuites = function(runner, skipBenchmarks) {
   var continuation = null;
   var suites = BenchmarkSuite.suites;
   var length = suites.length;
@@ -138,7 +150,11 @@ BenchmarkSuite.RunSuites = function(runner) {
       } else {
         var suite = suites[index++];
         if (runner.NotifyStart) runner.NotifyStart(suite.name);
-        continuation = suite.RunStep(runner);
+        if (skipBenchmarks.indexOf(suite.name) > -1) {
+          suite.NotifySkipped(runner);
+        } else {
+          continuation = suite.RunStep(runner);
+        }
       }
       if (continuation && typeof window != 'undefined' && window.setTimeout) {
         window.setTimeout(RunStep, 25);
@@ -249,6 +265,14 @@ BenchmarkSuite.prototype.NotifyResult = function() {
 }
 
 
+BenchmarkSuite.prototype.NotifySkipped = function(runner) {
+  BenchmarkSuite.scores.push(1);  // push default reference score.
+  if (runner.NotifyResult) {
+    runner.NotifyResult(this.name, "Skipped");
+  }
+}
+
+
 // Notifies the runner that running a benchmark resulted in an error.
 BenchmarkSuite.prototype.NotifyError = function(error) {
   if (this.runner.NotifyError) {
@@ -263,14 +287,22 @@ BenchmarkSuite.prototype.NotifyError = function(error) {
 // Runs a single benchmark for at least a second and computes the
 // average time it takes to run a single iteration.
 BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) {
+  var config = BenchmarkSuite.config;
+  var doWarmup = config.doWarmup !== undefined 
+                 ? config.doWarmup 
+                 : benchmark.doWarmup;
+  var doDeterministic = config.doDeterministic !== undefined 
+                        ? config.doDeterministic 
+                        : benchmark.doDeterministic;
+
   function Measure(data) {
     var elapsed = 0;
     var start = new Date();
 
   // Run either for 1 second or for the number of iterations specified
   // by minIterations, depending on the config flag doDeterministic.
-    for (var i = 0; (benchmark.doDeterministic ? 
-      i<benchmark.minIterations : elapsed < 1000); i++) {
+    for (var i = 0; (doDeterministic ? 
+      i<benchmark.deterministicIterations : elapsed < 1000); i++) {
       benchmark.run();
       elapsed = new Date() - start;
     }
@@ -281,7 +313,7 @@ BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) {
   }
 
   // Sets up data in order to skip or not the warmup phase.
-  if (!benchmark.doWarmup && data == null) {
+  if (!doWarmup && data == null) {
     data = { runs: 0, elapsed: 0 };
   }
 

diff --git a/box2d.js b/box2d.js
diff --git a/code-load.js b/code-load.js
@@ -69,6 +69,7 @@ var CodeLoad = new BenchmarkSuite('CodeLoad', [450000], [
   new Benchmark('CodeLoadClosure',
                 false,
                 false,
+                1600,
                 runCodeLoadClosure,
                 setupCodeLoad,
                 tearDownCodeLoad,
@@ -77,6 +78,7 @@ var CodeLoad = new BenchmarkSuite('CodeLoad', [450000], [
   new Benchmark('CodeLoadJQuery',
                 false,
                 false,
+                100,
                 runCodeLoadJQuery,
                 setupCodeLoad,
                 tearDownCodeLoad,

diff --git a/crypto.js b/crypto.js
@@ -32,8 +32,8 @@
 
 // The code has been adapted for use as a benchmark by Google.
 var Crypto = new BenchmarkSuite('Crypto', [266181], [
-  new Benchmark("Encrypt", true, false, encrypt),
-  new Benchmark("Decrypt", true, false, decrypt)
+  new Benchmark("Encrypt", true, false, 3900, encrypt),
+  new Benchmark("Decrypt", true, false, 220, decrypt)
 ]);
 
 

diff --git a/deltablue.js b/deltablue.js
@@ -24,7 +24,7 @@
 
 
 var DeltaBlue = new BenchmarkSuite('DeltaBlue', [66118], [
-  new Benchmark('DeltaBlue', true, false, deltaBlue)
+  new Benchmark('DeltaBlue', true, false, 4400, deltaBlue)
 ]);
 
 

diff --git a/earley-boyer.js b/earley-boyer.js
@@ -2,8 +2,8 @@
 // benchmark harness code at the beginning and end of the file.
 
 var EarleyBoyer = new BenchmarkSuite('EarleyBoyer', [666463], [
-  new Benchmark("Earley", true,  false, function () { BgL_earleyzd2benchmarkzd2(); }),
-  new Benchmark("Boyer", true, false, function () { BgL_nboyerzd2benchmarkzd2(); })
+  new Benchmark("Earley", true,  false, 2500, function () { BgL_earleyzd2benchmarkzd2(); }),
+  new Benchmark("Boyer", true, false, 200, function () { BgL_nboyerzd2benchmarkzd2(); })
 ]);
 
 

diff --git a/gbemu-part1.js b/gbemu-part1.js
@@ -16,6 +16,7 @@ var GameboyBenchmark = new BenchmarkSuite('Gameboy', [26288412],
                                           [new Benchmark('Gameboy',
                                                          false,
                                                          false,
+                                                         20,
                                                          runGameboy,
                                                          setupGameboy,
                                                          tearDownGameboy,

diff --git a/index.html b/index.html
@@ -41,6 +41,7 @@
   var benchmarks = BenchmarkSuite.CountBenchmarks();
   var success = true;
   var latencyBenchmarks = ["Splay", "Mandreel"];
+  var skipBenchmarks = [];
 
   function ShowBox(name) {
     var box = document.getElementById("Box-" + name);
@@ -104,7 +105,8 @@
       NotifyError : AddError,
       NotifyResult : AddResult,
       NotifyScore : AddScore
-    });
+    },
+    skipBenchmarks);
   }
 
   function CheckCompatibility() {
@@ -117,6 +119,8 @@
       console.log("Typed Arrays not supported");
       document.getElementById("alertbox").style.display="block";
     }
+    if (window.document.URL.indexOf('skip_zlib=1') >= 0)
+      skipBenchmarks.push("zlib");
     if (window.document.URL.indexOf('auto=1') >= 0)
       Run();
   }

diff --git a/mandreel.js b/mandreel.js
@@ -30,6 +30,7 @@ var MandreelBenchmark = new BenchmarkSuite('Mandreel', [16831872, 49852],
                                             [new Benchmark('Mandreel',
                                                            false,
                                                            false,
+                                                           15,
                                                            runMandreel,
                                                            setupMandreel,
                                                            tearDownMandreel,

diff --git a/navier-stokes.js b/navier-stokes.js
@@ -30,6 +30,7 @@ var NavierStokes = new BenchmarkSuite('NavierStokes', [1484000],
                                       [new Benchmark('NavierStokes',
                                                      true,
                                                      false,
+                                                     180,
                                                      runNavierStokes,
                                                      setupNavierStokes,
                                                      tearDownNavierStokes,

diff --git a/pdfjs.js b/pdfjs.js
@@ -27,7 +27,7 @@ var pdf_file = "test.pdf";
 var canvas_logs = [];
 
 var PdfJS = new BenchmarkSuite("PdfJS", [10124921], [
-  new Benchmark("PdfJS", false, false, 
+  new Benchmark("PdfJS", false, false, 24,
     runPdfJS, setupPdfJS, tearDownPdfJS, null, 4)
 ]);
 

diff --git a/raytrace.js b/raytrace.js
@@ -9,7 +9,7 @@
 // JavaScript framework which is used by the ray tracer.
 
 var RayTrace = new BenchmarkSuite('RayTrace', [739989], [
-  new Benchmark('RayTrace', true, false, renderScene)
+  new Benchmark('RayTrace', true, false, 600, renderScene)
 ]);
 
 

diff --git a/regexp.js b/regexp.js
@@ -38,7 +38,7 @@
 
 
 var RegExpSuite = new BenchmarkSuite('RegExp', [910985], [
-  new Benchmark("RegExp", true, false, 
+  new Benchmark("RegExp", true, false, 50,
     RegExpRun, RegExpSetup, RegExpTearDown, null, 16)
 ]);
 

diff --git a/richards.js b/richards.js
@@ -36,7 +36,7 @@
 
 
 var Richards = new BenchmarkSuite('Richards', [35302], [
-  new Benchmark("Richards", true, false, runRichards)
+  new Benchmark("Richards", true, false, 8200, runRichards)
 ]);
 
 

diff --git a/run.js b/run.js
@@ -1,4 +1,4 @@
-// Copyright 2008 the V8 project authors. All rights reserved.
+// Copyright 2014 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
@@ -69,6 +69,9 @@ function PrintScore(score) {
 }
 
 
+BenchmarkSuite.config.doWarmup = undefined;
+BenchmarkSuite.config.doDeterministic = undefined;
+
 BenchmarkSuite.RunSuites({ NotifyResult: PrintResult,
                            NotifyError: PrintError,
                            NotifyScore: PrintScore });
diff --git a/splay.js b/splay.js
@@ -34,7 +34,7 @@
 // graph.
 
 var Splay = new BenchmarkSuite('Splay', [81491, 2739514], [
-  new Benchmark("Splay", true, false, 
+  new Benchmark("Splay", true, false, 1400,
     SplayRun, SplaySetup, SplayTearDown, SplayRMS)
 ]);
 

diff --git a/typescript.js b/typescript.js
@@ -29,6 +29,7 @@ var typescript = new BenchmarkSuite('Typescript', [255011322], [
   new Benchmark("Typescript",
                 false,
                 true,
+                5,
                 runTypescript,
                 setupTypescript,
                 tearDownTypescript,

diff --git a/zlib.js b/zlib.js
@@ -26,7 +26,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 new BenchmarkSuite('zlib', [152815148], [
-  new Benchmark('zlib', false, true, 
+  new Benchmark('zlib', false, true, 10,
     runZlib, undefined, tearDownZlib, null, 3)]);
 
 // Generate 100kB pseudo-random bytes (compressed 25906 bytes) and