diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index 36f878fdd4ddb7912d6e0c9434f2c3d647198842..5f286a6723ccf780f8f4e229f0d196f7a9d4e84a 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -114,6 +114,8 @@ struct grpc_tcp_server {
 
   /* is this server shutting down? */
   bool shutdown;
+  /* have listeners been shutdown? */
+  bool shutdown_listeners;
   /* use SO_REUSEPORT */
   bool so_reuseport;
   /* expand wildcard addresses to a list of all local addresses */
@@ -161,7 +163,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx,
                                    grpc_tcp_server **server) {
   gpr_once_init(&check_init, init);
 
-  grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server));
+  grpc_tcp_server *s = gpr_zalloc(sizeof(grpc_tcp_server));
   s->so_reuseport = has_so_reuseport;
   s->resource_quota = grpc_resource_quota_create(NULL);
   s->expand_wildcard_addrs = false;
@@ -422,7 +424,14 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
           grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure);
           return;
         default:
-          gpr_log(GPR_ERROR, "Failed accept4: %s", strerror(errno));
+          gpr_mu_lock(&sp->server->mu);
+          if (!sp->server->shutdown_listeners) {
+            gpr_log(GPR_ERROR, "Failed accept4: %s", strerror(errno));
+          } else {
+            /* if we have shutdown listeners, accept4 could fail, and we
+               needn't notify users */
+          }
+          gpr_mu_unlock(&sp->server->mu);
           goto error;
       }
     }
@@ -438,11 +447,6 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
 
     grpc_fd *fdobj = grpc_fd_create(fd, name);
 
-    if (read_notifier_pollset == NULL) {
-      gpr_log(GPR_ERROR, "Read notifier pollset is not set on the fd");
-      goto error;
-    }
-
     grpc_pollset_add_fd(exec_ctx, read_notifier_pollset, fdobj);
 
     // Create acceptor.
@@ -941,6 +945,7 @@ void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
 void grpc_tcp_server_shutdown_listeners(grpc_exec_ctx *exec_ctx,
                                         grpc_tcp_server *s) {
   gpr_mu_lock(&s->mu);
+  s->shutdown_listeners = true;
   /* shutdown all fd's */
   if (s->active_ports) {
     grpc_tcp_listener *sp;