diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c
index 50cbafab5693bb2a346e69d8e16d48bf034a1ecc..dc85d1cf5c7beb7777771a311b1923f7d22bd668 100644
--- a/src/core/ext/transport/chttp2/server/chttp2_server.c
+++ b/src/core/ext/transport/chttp2/server/chttp2_server.c
@@ -338,11 +338,12 @@ error:
   }
   if (tcp_server) {
     grpc_tcp_server_unref(exec_ctx, tcp_server);
+  } else {
+    grpc_channel_args_destroy(state->args);
+    grpc_chttp2_server_handshaker_factory_destroy(exec_ctx,
+                                                  state->handshaker_factory);
+    gpr_free(state);
   }
-  grpc_channel_args_destroy(state->args);
-  grpc_chttp2_server_handshaker_factory_destroy(exec_ctx,
-                                                state->handshaker_factory);
-  gpr_free(state);
   *port_num = 0;
 
 done:
diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
index b9a833803bc2f3d3acec5884a256b32946202093..5f417281321329d1f598c2ddd3f7d1a7be8053a7 100644
--- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
@@ -69,6 +69,7 @@ static void server_security_handshaker_factory_destroy(
       (server_security_handshaker_factory *)hf;
   GRPC_SECURITY_CONNECTOR_UNREF(&handshaker_factory->security_connector->base,
                                 "server");
+  gpr_free(hf);
 }
 
 static const grpc_chttp2_server_handshaker_factory_vtable
@@ -124,9 +125,6 @@ done:
     gpr_log(GPR_ERROR, "%s", msg);
     grpc_error_free_string(msg);
     GRPC_ERROR_UNREF(err);
-    if (sc != NULL) {
-      GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "server");
-    }
   }
   return port_num;
 }