diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c
index a851d01635523db8a0618604b30b46784fef318a..38054dd1e7480b942a0cf97bc2c4b77f65023e8a 100644
--- a/test/core/util/port_server_client.c
+++ b/test/core/util/port_server_client.c
@@ -162,6 +162,15 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg,
   if (failed) {
     grpc_httpcli_request req;
     memset(&req, 0, sizeof(req));
+    if (pr->retries >= 5) {
+      gpr_mu_lock(pr->mu);
+      pr->port = 0;
+      GRPC_LOG_IF_ERROR(
+          "pollset_kick",
+          grpc_pollset_kick(grpc_polling_entity_pollset(&pr->pops), NULL));
+      gpr_mu_unlock(pr->mu);
+      return;
+    }
     GPR_ASSERT(pr->retries < 10);
     gpr_sleep_until(gpr_time_add(
         gpr_now(GPR_CLOCK_REALTIME),