diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c
index a851d01635523db8a0618604b30b46784fef318a..38054dd1e7480b942a0cf97bc2c4b77f65023e8a 100644
--- a/test/core/util/port_server_client.c
+++ b/test/core/util/port_server_client.c
@@ -162,6 +162,15 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg,
   if (failed) {
     grpc_httpcli_request req;
     memset(&req, 0, sizeof(req));
+    if (pr->retries >= 5) {
+      gpr_mu_lock(pr->mu);
+      pr->port = 0;
+      GRPC_LOG_IF_ERROR(
+          "pollset_kick",
+          grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL));
+      gpr_mu_unlock(pr->mu);
+      return;
+    }
     GPR_ASSERT(pr->retries < 10);
     gpr_sleep_until(gpr_time_add(
         gpr_now(GPR_CLOCK_REALTIME),
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 7c04d228ba3cd14ad88a4fc5750f10b57f2e6679..ee4102c5913bc0bfa6f570ce843b0f8e3103a741 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -101,7 +101,7 @@ def create_qpsworker_job(language, shortname=None, port=10000, remote_host=None,
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
     ssh_cmd = ['ssh']
     cmdline = ['timeout', '%s' % (worker_timeout + 30)] + cmdline
-    ssh_cmd.extend([str(user_at_host), 'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline)])
+    ssh_cmd.extend([str(user_at_host), 'cd ~/performance_workspace/grpc/ && tools/run_tests/start_port_server.py && %s' % ' '.join(cmdline)])
     cmdline = ssh_cmd
 
   jobspec = jobset.JobSpec(