Merge pull request #6328 from jtattermusch/performance_tests_facelift

Better killing of pending QpsWorkers (in single-machine setup).

tools/run_tests/performance/kill_workers.sh

+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -ex
+
+cd $(dirname $0)/../../..
+
+# Make sure there are no pre-existing QPS workers around before starting
+# the performance test suite
+
+# C++
+killall -9 qps_worker || true
+
+# C#
+ps -C mono -o pid=,cmd= | grep QpsWorker | awk '{print $1}' | xargs kill -9
+
+# Ruby
+ps -C ruby -o pid=,cmd= | grep 'qps/worker.rb' | awk '{print $1}' | xargs kill -9
+
+# Node
+ps -C node -o pid=,cmd= | grep 'performance/worker.js' | awk '{print $1}' | xargs kill -9
+
+# Java
+jps | grep LoadWorker | awk '{print $1}' | xargs kill -9

tools/run_tests/performance/remote_host_prepare.sh

@@ -41,10 +41,9 @@ ssh "${USER_AT_HOST}" "rm -rf ~/performance_workspace && mkdir -p ~/performance_
 # could also kill jenkins.
 ssh "${USER_AT_HOST}" "killall -9 qps_worker mono node ruby || true"
 
-# Kill all java LoadWorker processes. We can't just killall java
-# as one of the processes might be jenkins.
-ssh "${USER_AT_HOST}" 'kill -9 $(jps | grep LoadWorker | cut -f1 -d" ") || true'
-
 # push the current sources to the slave and unpack it.
 scp ../grpc.tar "${USER_AT_HOST}:~/performance_workspace"
 ssh "${USER_AT_HOST}" "tar -xf ~/performance_workspace/grpc.tar -C ~/performance_workspace"
+
+# For consistency with local run, invoke the kill_workers script remotely.
+ssh "${USER_AT_HOST}" "~/performance_workspace/grpc/tools/run_tests/performance/kill_workers.sh"

tools/run_tests/run_performance_tests.py

@@ -157,8 +157,9 @@ def archive_repo(languages):
     sys.exit(1)
 
 
-def prepare_remote_hosts(hosts):
-  """Prepares remote hosts."""
+def prepare_remote_hosts(hosts, prepare_local=False):
+  """Prepares remote hosts (and maybe prepare localhost as well)."""
+  prepare_timeout = 5*60
   prepare_jobs = []
   for host in hosts:
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
@@ -167,13 +168,20 @@ def prepare_remote_hosts(hosts):
             cmdline=['tools/run_tests/performance/remote_host_prepare.sh'],
             shortname='remote_host_prepare.%s' % host,
             environ = {'USER_AT_HOST': user_at_host},
-            timeout_seconds=5*60))
-  jobset.message('START', 'Preparing remote hosts.', do_newline=True)
+            timeout_seconds=prepare_timeout))
+  if prepare_local:
+    # Prepare localhost as well
+    prepare_jobs.append(
+        jobset.JobSpec(
+            cmdline=['tools/run_tests/performance/kill_workers.sh'],
+            shortname='local_prepare',
+            timeout_seconds=prepare_timeout))
+  jobset.message('START', 'Preparing hosts.', do_newline=True)
   num_failures, _ = jobset.run(
       prepare_jobs, newline_on_success=True, maxjobs=10)
   if num_failures == 0:
     jobset.message('SUCCESS',
-                   'Remote hosts ready to start build.',
+                   'Prepare step completed successfully.',
                    do_newline=True)
   else:
     jobset.message('FAILED', 'Failed to prepare remote hosts.',
@@ -322,7 +330,9 @@ if args.remote_driver_host:
 
 if remote_hosts:
   archive_repo(languages=[str(l) for l in languages])
-  prepare_remote_hosts(remote_hosts)
+  prepare_remote_hosts(remote_hosts, prepare_local=True)
+else:
+  prepare_remote_hosts([], prepare_local=True)
 
 build_local = False
 if not args.remote_driver_host: