node/deps/v8/test/benchmarks/csuite/csuite.py

#!/usr/bin/python
# Copyright 2018 the V8 project authors. All rights reserved.
'''
C S u i t e                                         because who can remember?
-----------------------------------------------------------------------------
python csuite.py [options] <benchmark> <mode> <d8 path>

Arguments
  benchmark: one of octane, sunspider or kraken.
  mode: baseline or compare.
  d8 path: a valid path to the d8 executable you want to use.

CSuite is a wrapper around benchmark.py and compare-baseline.py, old
friends in the d8 benchmarking world. Unlike those tools, it can be
run in any directory. It's also opinionated about which benchmarks it
will run, currently SunSpider, Octane and Kraken. Furthermore, it
runs the versions we pull into ./test/benchmarks/data.

Examples:

Say you want to see how much optimization buys you:
  ./csuite.py kraken baseline ~/src/v8/out/d8 -x="--noopt"
  ./csuite.py kraken compare ~/src/v8/out/d8

Suppose you are comparing two binaries, quick n' dirty style:
  ./csuite.py -r 3 octane baseline ~/src/v8/out-master/d8
  ./csuite.py -r 3 octane compare ~/src/v8/out-mine/d8

You can run from any place:
  ../../somewhere-strange/csuite.py sunspider baseline ./d8
  ../../somewhere-strange/csuite.py sunspider compare ./d8-better
'''

import os
from optparse import OptionParser
import subprocess
import sys

if __name__ == '__main__':
  parser = OptionParser(usage=__doc__)
  parser.add_option("-r", "--runs", dest="runs",
      help="Override the default number of runs for the benchmark.")
  parser.add_option("-x", "--extra-arguments", dest="extra_args",
      help="Pass these extra arguments to d8.")
  parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
      help="See more output about what magic csuite is doing.")
  (opts, args) = parser.parse_args()

  if len(args) < 3:
    print 'not enough arguments'
    sys.exit(1)

  suite = args[0]
  mode = args[1]

  if suite not in ['octane', 'sunspider', 'kraken']:
    print 'Suite must be octane, sunspider or kraken. Aborting.'
    sys.exit(1)

  if mode != 'baseline' and mode != 'compare':
    print 'mode must be baseline or compare. Aborting.'
    sys.exit(1)

  # Set up paths.
  d8_path = os.path.abspath(args[2])
  if not os.path.exists(d8_path):
    print d8_path + " is not valid."
    sys.exit(1)

  csuite_path = os.path.dirname(os.path.abspath(__file__))
  if not os.path.exists(csuite_path):
    print "The csuite directory is invalid."
    sys.exit(1)

  benchmark_py_path = os.path.join(csuite_path, "benchmark.py")
  if not os.path.exists(benchmark_py_path):
    print "Unable to find benchmark.py in " + output_path_base \
        + ". Aborting."
    sys.exit(1)

  compare_baseline_py_path = os.path.join(csuite_path,
      "compare-baseline.py")

  if not os.path.exists(compare_baseline_py_path):
    print "Unable to find compare-baseline.py in " + output_path_base \
        + ". Aborting."
    sys.exit(1)

  benchmark_path = os.path.abspath(os.path.join(csuite_path, "../data"))
  if not os.path.exists(benchmark_path):
    print "I can't find the benchmark data directory. Aborting."
    sys.exit(1)

  # Gather the remaining arguments into a string of extra args for d8.
  extra_args = ""
  if opts.extra_args:
    extra_args = opts.extra_args

  if suite == "octane":
    runs = 10
    suite_path = os.path.join(benchmark_path, "octane")
    cmd = "run.js"
  elif suite == "kraken":
    runs = 80
    suite_path = os.path.join(benchmark_path, "kraken")
    cmd = os.path.join(csuite_path, "run-kraken.js")
  else:
    runs = 100
    suite_path = os.path.join(benchmark_path, "sunspider")
    cmd = os.path.join(csuite_path, "sunspider-standalone-driver.js")

  if opts.runs:
    if (float(opts.runs) / runs) < 0.6:
      print "Normally, %s requires %d runs to get stable results." \
          % (suite, runs)
    runs = int(opts.runs)

  if opts.verbose:
    print "Running and averaging %s %d times." % (suite, runs)

  # Ensure output directory is setup
  output_path_base = os.path.abspath(os.getcwd())
  output_path = os.path.join(output_path_base, "_results")
  output_file = os.path.join(output_path, "master")
  if not os.path.exists(output_path):
    if opts.verbose:
      print "Creating directory %s." % output_path
    os.mkdir(output_path)

  if opts.verbose:
    print "Working directory for runs is %s." % suite_path

  inner_command = " -c \"%s --expose-gc %s %s \"" \
      % (d8_path, extra_args, cmd)
  if opts.verbose:
    print "calling d8 like so: %s." % inner_command

  cmdline_base = "python %s %s -fv -r %d -d %s" \
      % (benchmark_py_path, inner_command, runs, output_path_base)

  if mode == "baseline":
    cmdline = "%s > %s" % (cmdline_base, output_file)
  else:
    cmdline = "%s | %s %s" \
        % (cmdline_base, compare_baseline_py_path, output_file)

  if opts.verbose:
    print "Spawning subprocess: %s." % cmdline
  return_code = subprocess.call(cmdline, shell=True, cwd=suite_path)
  if return_code < 0:
    print "Error return code: %d." % return_code
  if mode == "baseline":
    print "Wrote %s." % output_file
    print "Run %s again with compare mode to see results." % suite