diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh
index 177b076650b92611ab8917aa6fb7d81de3c187f8..3ce05cc7f1e8fda214a285d97a551d6fada94447 100755
--- a/tools/jenkins/run_performance.sh
+++ b/tools/jenkins/run_performance.sh
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# This script is invoked by Jenkins and runs performance smoke test.
+# This script is invoked by Jenkins and runs a diff on the microbenchmarks
set -ex
# List of benchmarks that provide good signal for analyzing performance changes in pull requests
diff --git a/tools/jenkins/run_trickle_diff.sh b/tools/jenkins/run_trickle_diff.sh
new file mode 100755
index 0000000000000000000000000000000000000000..da905d024903598f3d2c352575710eff26c6a037
--- /dev/null
+++ b/tools/jenkins/run_trickle_diff.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script is invoked by Jenkins and runs a diff on bm_fullstack_trickle
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../..
+
+tools/run_tests/start_port_server.py
+tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls --no-counters --pr_comment_name trickle
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_build.py b/tools/profiling/microbenchmarks/bm_diff/bm_build.py
index 650ccdc2b21601d5a6a6545e4b145ddd40f0bb83..ce62c09d72f677f7b08df35244d4db731a209960 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_build.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_build.py
@@ -46,6 +46,9 @@ def _args():
type=str,
help='Unique name of this build. To be used as a handle to pass to the other bm* scripts'
)
+ argp.add_argument('--counters', dest='counters', action='store_true')
+ argp.add_argument('--no-counters', dest='counters', action='store_false')
+ argp.set_defaults(counters=True)
args = argp.parse_args()
assert args.name
return args
@@ -55,16 +58,18 @@ def _make_cmd(cfg, benchmarks, jobs):
return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs]
-def build(name, benchmarks, jobs):
+def build(name, benchmarks, jobs, counters):
shutil.rmtree('bm_diff_%s' % name, ignore_errors=True)
subprocess.check_call(['git', 'submodule', 'update'])
try:
subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
- subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
+ if counters:
+ subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
except subprocess.CalledProcessError, e:
subprocess.check_call(['make', 'clean'])
subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
- subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
+ if counters:
+ subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
os.rename(
'bins',
'bm_diff_%s' % name,)
@@ -72,4 +77,4 @@ def build(name, benchmarks, jobs):
if __name__ == '__main__':
args = _args()
- build(args.name, args.benchmarks, args.jobs)
+ build(args.name, args.benchmarks, args.jobs, args.counters)
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py
index 2bbd987b20fcad226cac0a1587d0ea1ca63e2a34..4cd65867c371c643178a0028abb771201058f130 100644
--- a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py
@@ -26,4 +26,5 @@ _AVAILABLE_BENCHMARK_TESTS = [
_INTERESTING = ('cpu_time', 'real_time', 'locks_per_iteration',
'allocs_per_iteration', 'writes_per_iteration',
'atm_cas_per_iteration', 'atm_add_per_iteration',
- 'nows_per_iteration',)
+ 'nows_per_iteration', 'cli_transport_stalls', 'cli_stream_stalls',
+ 'svr_transport_stalls', 'svr_stream_stalls',)
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py
index b8e803749a2f5f625a1dfd86e581c927400c4478..73abf90ff520d7628d881179741a78318f6dd021 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py
@@ -67,6 +67,9 @@ def _args():
default=20,
help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py'
)
+ argp.add_argument('--counters', dest='counters', action='store_true')
+ argp.add_argument('--no-counters', dest='counters', action='store_false')
+ argp.set_defaults(counters=True)
argp.add_argument('-n', '--new', type=str, help='New benchmark name')
argp.add_argument('-o', '--old', type=str, help='Old benchmark name')
argp.add_argument(
@@ -121,7 +124,8 @@ def _read_json(filename, badjson_files, nonexistant_files):
stripped = ".".join(filename.split(".")[:-2])
try:
with open(filename) as f:
- return json.loads(f.read())
+ r = f.read();
+ return json.loads(r)
except IOError, e:
if stripped in nonexistant_files:
nonexistant_files[stripped] += 1
@@ -129,14 +133,17 @@ def _read_json(filename, badjson_files, nonexistant_files):
nonexistant_files[stripped] = 1
return None
except ValueError, e:
+ print r
if stripped in badjson_files:
badjson_files[stripped] += 1
else:
badjson_files[stripped] = 1
return None
+def fmt_dict(d):
+ return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d])
-def diff(bms, loops, track, old, new):
+def diff(bms, loops, track, old, new, counters):
benchmarks = collections.defaultdict(Benchmark)
badjson_files = {}
@@ -148,18 +155,22 @@ def diff(bms, loops, track, old, new):
'--benchmark_list_tests']).splitlines():
stripped_line = line.strip().replace("/", "_").replace(
"<", "_").replace(">", "_").replace(", ", "_")
- js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
- (bm, stripped_line, new, loop),
- badjson_files, nonexistant_files)
js_new_opt = _read_json('%s.%s.opt.%s.%d.json' %
(bm, stripped_line, new, loop),
badjson_files, nonexistant_files)
- js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
- (bm, stripped_line, old, loop),
- badjson_files, nonexistant_files)
js_old_opt = _read_json('%s.%s.opt.%s.%d.json' %
(bm, stripped_line, old, loop),
badjson_files, nonexistant_files)
+ if counters:
+ js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+ (bm, stripped_line, new, loop),
+ badjson_files, nonexistant_files)
+ js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+ (bm, stripped_line, old, loop),
+ badjson_files, nonexistant_files)
+ else:
+ js_new_ctr = None
+ js_old_ctr = None
if js_new_ctr:
for row in bm_json.expand_json(js_new_ctr, js_new_opt):
@@ -187,12 +198,12 @@ def diff(bms, loops, track, old, new):
rows.append([name] + benchmarks[name].row(fields))
note = None
if len(badjson_files):
- note = 'Corrupt JSON data (indicates timeout or crash) = %s' % str(badjson_files)
+ note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(badjson_files)
if len(nonexistant_files):
if note:
- note += '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files)
+ note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
else:
- note = '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files)
+ note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
if rows:
return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
else:
@@ -202,5 +213,5 @@ def diff(bms, loops, track, old, new):
if __name__ == '__main__':
args = _args()
diff, note = diff(args.benchmarks, args.loops, args.track, args.old,
- args.new)
+ args.new, args.counters)
print('%s\n%s' % (note, diff if diff else "No performance differences"))
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py
index 8a54f198ab236c1fcd51e78ba7dca74b03b275ff..8b4e0cb69a4519ba06174d073bc32a81c52b84b8 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py
@@ -80,6 +80,14 @@ def _args():
type=int,
default=multiprocessing.cpu_count(),
help='Number of CPUs to use')
+ argp.add_argument(
+ '--pr_comment_name',
+ type=str,
+ default="microbenchmarks",
+ help='Name that Jenkins will use to commen on the PR')
+ argp.add_argument('--counters', dest='counters', action='store_true')
+ argp.add_argument('--no-counters', dest='counters', action='store_false')
+ argp.set_defaults(counters=True)
args = argp.parse_args()
assert args.diff_base or args.old, "One of diff_base or old must be set!"
if args.loops < 3:
@@ -103,7 +111,7 @@ def eintr_be_gone(fn):
def main(args):
- bm_build.build('new', args.benchmarks, args.jobs)
+ bm_build.build('new', args.benchmarks, args.jobs, args.counters)
old = args.old
if args.diff_base:
@@ -112,20 +120,20 @@ def main(args):
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
- bm_build.build('old', args.benchmarks, args.jobs)
+ bm_build.build(old, args.benchmarks, args.jobs, args.counters)
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
- bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions)
- bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions)
+ bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)
+ bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)
diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old,
- 'new')
+ 'new', args.counters)
if diff:
- text = 'Performance differences noted:\n' + diff
+ text = '[%s] Performance differences noted:\n%s' % (args.pr_comment_name, diff)
else:
- text = 'No significant performance differences'
+ text = '[%s] No significant performance differences' % args.pr_comment_name
if note:
text = note + '\n\n' + text
print('%s' % text)
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py
index 3457af916b45ae9089612326c879416e30f79f91..72b3d3cf10627a5ef7f873ab3e5f7395093346c0 100755
--- a/tools/profiling/microbenchmarks/bm_diff/bm_run.py
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py
@@ -67,6 +67,9 @@ def _args():
default=20,
help='Number of times to loops the benchmarks. More loops cuts down on noise'
)
+ argp.add_argument('--counters', dest='counters', action='store_true')
+ argp.add_argument('--no-counters', dest='counters', action='store_false')
+ argp.set_defaults(counters=True)
args = argp.parse_args()
assert args.name
if args.loops < 3:
@@ -93,21 +96,22 @@ def _collect_bm_data(bm, cfg, name, reps, idx, loops):
shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1,
loops),
verbose_success=True,
- timeout_seconds=60 * 2))
+ timeout_seconds=60 * 60)) # one hour
return jobs_list
-def run(name, benchmarks, jobs, loops, reps):
+def run(name, benchmarks, jobs, loops, reps, counters):
jobs_list = []
for loop in range(0, loops):
for bm in benchmarks:
jobs_list += _collect_bm_data(bm, 'opt', name, reps, loop, loops)
- jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop,
- loops)
+ if counters:
+ jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop,
+ loops)
random.shuffle(jobs_list, random.SystemRandom().random)
jobset.run(jobs_list, maxjobs=jobs)
if __name__ == '__main__':
args = _args()
- run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions)
+ run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)