diff --git a/.gitignore b/.gitignore
index ad3ec64ab5af94aa39ec68235e659fc8db596466..5e38f5fa01b273e81d9fd0526f707a783a66a5bf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -118,3 +118,13 @@ gdb.txt
# ctags file
tags
+
+# perf data
+perf.data
+perf.data.old
+
+# bm_diff
+bm_diff_new/
+bm_diff_old/
+bm_*.json
+
diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh
index dee033a951aaf77bc563836c950ba4e4ab085a41..544e31dcbdbae7ec66c9be8a0c1744e7bc8603b5 100755
--- a/tools/jenkins/run_performance.sh
+++ b/tools/jenkins/run_performance.sh
@@ -32,7 +32,7 @@
set -ex
# List of benchmarks that provide good signal for analyzing performance changes in pull requests
-BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
+BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
# Enter the gRPC repo root
cd $(dirname $0)/../..
diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py
index 35a79593b41903512f4331fa58030341143db29c..ba0c225f6cc08cd9f53ca49581e3b0d1746c152c 100755
--- a/tools/profiling/microbenchmarks/bm_diff.py
+++ b/tools/profiling/microbenchmarks/bm_diff.py
@@ -41,6 +41,22 @@ import pipes
import os
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
import comment_on_pr
+import jobset
+import itertools
+import speedup
+import random
+import shutil
+import errno
+
+_INTERESTING = (
+ 'cpu_time',
+ 'real_time',
+ 'locks_per_iteration',
+ 'allocs_per_iteration',
+ 'writes_per_iteration',
+ 'atm_cas_per_iteration',
+ 'atm_add_per_iteration',
+)
def changed_ratio(n, o):
if float(o) <= .0001: o = 0
@@ -60,26 +76,6 @@ def median(ary):
def min_change(pct):
return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0
-nanos = {
- 'abs_diff': 5,
- 'pct_diff': 10,
-}
-counter = {
- 'abs_diff': 0.5,
- 'pct_diff': 10,
-}
-
-_INTERESTING = {
- 'cpu_time': nanos,
- 'real_time': nanos,
- 'locks_per_iteration': counter,
- 'allocs_per_iteration': counter,
- 'writes_per_iteration': counter,
- 'atm_cas_per_iteration': counter,
- 'atm_add_per_iteration': counter,
-}
-
-
_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong',
'bm_fullstack_streaming_ping_pong',
'bm_fullstack_streaming_pump',
@@ -95,14 +91,15 @@ _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong',
argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks')
argp.add_argument('-t', '--track',
- choices=sorted(_INTERESTING.keys()),
+ choices=sorted(_INTERESTING),
nargs='+',
- default=sorted(_INTERESTING.keys()),
+ default=sorted(_INTERESTING),
help='Which metrics to track')
argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq'])
argp.add_argument('-d', '--diff_base', type=str)
-argp.add_argument('-r', '--repetitions', type=int, default=4)
-argp.add_argument('-p', '--p_threshold', type=float, default=0.01)
+argp.add_argument('-r', '--repetitions', type=int, default=1)
+argp.add_argument('-l', '--loops', type=int, default=12)
+argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count())
args = argp.parse_args()
assert args.diff_base
@@ -117,9 +114,10 @@ def avg(lst):
def make_cmd(cfg):
return ['make'] + args.benchmarks + [
- 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]
+ 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs]
-def build():
+def build(dest):
+ shutil.rmtree('bm_diff_%s' % dest, ignore_errors=True)
subprocess.check_call(['git', 'submodule', 'update'])
try:
subprocess.check_call(make_cmd('opt'))
@@ -128,38 +126,38 @@ def build():
subprocess.check_call(['make', 'clean'])
subprocess.check_call(make_cmd('opt'))
subprocess.check_call(make_cmd('counters'))
+ os.rename('bins', 'bm_diff_%s' % dest)
-def collect1(bm, cfg, ver):
- cmd = ['bins/%s/%s' % (cfg, bm),
- '--benchmark_out=%s.%s.%s.json' % (bm, cfg, ver),
+def collect1(bm, cfg, ver, idx):
+ cmd = ['bm_diff_%s/%s/%s' % (ver, cfg, bm),
+ '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx),
'--benchmark_out_format=json',
'--benchmark_repetitions=%d' % (args.repetitions)
]
- print cmd
- subprocess.check_call(cmd)
+ return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, ver, idx+1, args.loops),
+ verbose_success=True, timeout_seconds=None)
-build()
-for bm in args.benchmarks:
- collect1(bm, 'opt', 'new')
- collect1(bm, 'counters', 'new')
+build('new')
where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
-
try:
- build()
- comparables = []
- for bm in args.benchmarks:
- try:
- collect1(bm, 'opt', 'old')
- collect1(bm, 'counters', 'old')
- comparables.append(bm)
- except subprocess.CalledProcessError, e:
- pass
+ build('old')
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
+jobs = []
+for loop in range(0, args.loops):
+ jobs.extend(x for x in itertools.chain(
+ (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks),
+ (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks),
+ (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks),
+ (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks),
+ ))
+random.shuffle(jobs, random.SystemRandom().random)
+
+jobset.run(jobs, maxjobs=args.jobs)
class Benchmark:
@@ -180,16 +178,11 @@ class Benchmark:
new = self.samples[True][f]
old = self.samples[False][f]
if not new or not old: continue
- p = stats.ttest_ind(new, old)[1]
- new_mdn = median(new)
- old_mdn = median(old)
- delta = new_mdn - old_mdn
- ratio = changed_ratio(new_mdn, old_mdn)
- print '%s: new=%r old=%r new_mdn=%f old_mdn=%f delta=%f(%f:%f) ratio=%f(%f:%f) p=%f' % (
- f, new, old, new_mdn, old_mdn, delta, abs(delta), _INTERESTING[f]['abs_diff'], ratio, abs(ratio), _INTERESTING[f]['pct_diff']/100.0, p
- )
- if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']/100.0:
- self.final[f] = delta
+ mdn_diff = abs(median(new) - median(old))
+ print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff)
+ s = speedup.speedup(new, old)
+ if abs(s) > 3 and mdn_diff > 0.5:
+ self.final[f] = '%+d%%' % s
return self.final.keys()
def skip(self):
@@ -199,28 +192,37 @@ class Benchmark:
return [self.final[f] if f in self.final else '' for f in flds]
+def read_file(filename):
+ while True:
+ try:
+ with open(filename) as f:
+ return f.read()
+ except IOError, e:
+ if e.errno != errno.EINTR:
+ raise
+
+def read_json(filename):
+ return json.loads(read_file(filename))
+
benchmarks = collections.defaultdict(Benchmark)
-for bm in comparables:
- with open('%s.counters.new.json' % bm) as f:
- js_new_ctr = json.loads(f.read())
- with open('%s.opt.new.json' % bm) as f:
- js_new_opt = json.loads(f.read())
- with open('%s.counters.old.json' % bm) as f:
- js_old_ctr = json.loads(f.read())
- with open('%s.opt.old.json' % bm) as f:
- js_old_opt = json.loads(f.read())
-
- for row in bm_json.expand_json(js_new_ctr, js_new_opt):
- print row
- name = row['cpp_name']
- if name.endswith('_mean') or name.endswith('_stddev'): continue
- benchmarks[name].add_sample(row, True)
- for row in bm_json.expand_json(js_old_ctr, js_old_opt):
- print row
- name = row['cpp_name']
- if name.endswith('_mean') or name.endswith('_stddev'): continue
- benchmarks[name].add_sample(row, False)
+for bm in args.benchmarks:
+ for loop in range(0, args.loops):
+ js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop))
+ js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop))
+ js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop))
+ js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop))
+
+ for row in bm_json.expand_json(js_new_ctr, js_new_opt):
+ print row
+ name = row['cpp_name']
+ if name.endswith('_mean') or name.endswith('_stddev'): continue
+ benchmarks[name].add_sample(row, True)
+ for row in bm_json.expand_json(js_old_ctr, js_old_opt):
+ print row
+ name = row['cpp_name']
+ if name.endswith('_mean') or name.endswith('_stddev'): continue
+ benchmarks[name].add_sample(row, False)
really_interesting = set()
for name, bm in benchmarks.items():
diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f9023d2c89cc33c8936a9db7d496327ae8aabd0
--- /dev/null
+++ b/tools/profiling/microbenchmarks/speedup.py
@@ -0,0 +1,67 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from scipy import stats
+import math
+
+_THRESHOLD = 0.0001
+
+def scale(a, mul):
+ return [x*mul for x in a]
+
+def cmp(a, b):
+ return stats.ttest_ind(a, b)
+
+def speedup(new, old):
+ s0, p0 = cmp(new, old)
+ if math.isnan(p0): return 0
+ if s0 == 0: return 0
+ if p0 > _THRESHOLD: return 0
+ if s0 < 0:
+ pct = 1
+ while pct < 101:
+ sp, pp = cmp(new, scale(old, 1 - pct/100.0))
+ if sp > 0: break
+ if pp > _THRESHOLD: break
+ pct += 1
+ return -(pct - 1)
+ else:
+ pct = 1
+ while pct < 101:
+ sp, pp = cmp(new, scale(old, 1 + pct/100.0))
+ if sp < 0: break
+ if pp > _THRESHOLD: break
+ pct += 1
+ return pct - 1
+
+if __name__ == "__main__":
+ new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0]
+ old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0]
+ print speedup(new, old)
+ print speedup(old, new)