diff --git a/Makefile b/Makefile
index a5b3978109eefbc5d193bc9485e7cc35d2159c45..74e855fe6da4a94faae157c9f3c522244eeb2837 100644
--- a/Makefile
+++ b/Makefile
@@ -1894,6 +1894,7 @@ LIBGRPC_SRC = \
src/core/iomgr/iomgr_windows.c \
src/core/iomgr/pollset_kick.c \
src/core/iomgr/pollset_multipoller_with_poll_posix.c \
+ src/core/iomgr/pollset_multipoller_with_epoll.c \
src/core/iomgr/pollset_posix.c \
src/core/iomgr/pollset_windows.c \
src/core/iomgr/resolve_address.c \
@@ -2029,6 +2030,7 @@ src/core/iomgr/iomgr_posix.c: $(OPENSSL_DEP)
src/core/iomgr/iomgr_windows.c: $(OPENSSL_DEP)
src/core/iomgr/pollset_kick.c: $(OPENSSL_DEP)
src/core/iomgr/pollset_multipoller_with_poll_posix.c: $(OPENSSL_DEP)
+src/core/iomgr/pollset_multipoller_with_epoll.c: $(OPENSSL_DEP)
src/core/iomgr/pollset_posix.c: $(OPENSSL_DEP)
src/core/iomgr/pollset_windows.c: $(OPENSSL_DEP)
src/core/iomgr/resolve_address.c: $(OPENSSL_DEP)
@@ -2186,6 +2188,7 @@ objs/$(CONFIG)/src/core/iomgr/iomgr_posix.o:
objs/$(CONFIG)/src/core/iomgr/iomgr_windows.o:
objs/$(CONFIG)/src/core/iomgr/pollset_kick.o:
objs/$(CONFIG)/src/core/iomgr/pollset_multipoller_with_poll_posix.o:
+objs/$(CONFIG)/src/core/iomgr/pollset_multipoller_with_epoll.o:
objs/$(CONFIG)/src/core/iomgr/pollset_posix.o:
objs/$(CONFIG)/src/core/iomgr/pollset_windows.o:
objs/$(CONFIG)/src/core/iomgr/resolve_address.o:
@@ -2427,6 +2430,7 @@ LIBGRPC_UNSECURE_SRC = \
src/core/iomgr/iomgr_windows.c \
src/core/iomgr/pollset_kick.c \
src/core/iomgr/pollset_multipoller_with_poll_posix.c \
+ src/core/iomgr/pollset_multipoller_with_epoll.c \
src/core/iomgr/pollset_posix.c \
src/core/iomgr/pollset_windows.c \
src/core/iomgr/resolve_address.c \
@@ -2567,6 +2571,7 @@ objs/$(CONFIG)/src/core/iomgr/iomgr_posix.o:
objs/$(CONFIG)/src/core/iomgr/iomgr_windows.o:
objs/$(CONFIG)/src/core/iomgr/pollset_kick.o:
objs/$(CONFIG)/src/core/iomgr/pollset_multipoller_with_poll_posix.o:
+objs/$(CONFIG)/src/core/iomgr/pollset_multipoller_with_epoll.o:
objs/$(CONFIG)/src/core/iomgr/pollset_posix.o:
objs/$(CONFIG)/src/core/iomgr/pollset_windows.o:
objs/$(CONFIG)/src/core/iomgr/resolve_address.o:
diff --git a/build.json b/build.json
index ca41c8dcb3985029097cd1e40c33f0a6efda9f19..2347383b99381980d9757496c7f2060da151a20e 100644
--- a/build.json
+++ b/build.json
@@ -139,6 +139,7 @@
"src/core/iomgr/iomgr_windows.c",
"src/core/iomgr/pollset_kick.c",
"src/core/iomgr/pollset_multipoller_with_poll_posix.c",
+ "src/core/iomgr/pollset_multipoller_with_epoll.c",
"src/core/iomgr/pollset_posix.c",
"src/core/iomgr/pollset_windows.c",
"src/core/iomgr/resolve_address.c",
diff --git a/include/grpc/support/port_platform.h b/include/grpc/support/port_platform.h
index b0b528d28288aaf5ed4afe1c9410b0f4aefc9c8b..9d4bfbee5e83385355f409999b7487349195d940 100644
--- a/include/grpc/support/port_platform.h
+++ b/include/grpc/support/port_platform.h
@@ -73,7 +73,7 @@
#define GPR_CPU_LINUX 1
#define GPR_GCC_ATOMIC 1
#define GPR_LINUX 1
-#define GPR_POSIX_MULTIPOLL_WITH_POLL 1
+#define GPR_LINUX_MULTIPOLL_WITH_EPOLL 1
#define GPR_POSIX_WAKEUP_FD 1
#define GPR_LINUX_EVENTFD 1
#define GPR_POSIX_SOCKET 1
diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c
new file mode 100644
index 0000000000000000000000000000000000000000..9fb28195062925e9de24d5829befd01519a6b08d
--- /dev/null
+++ b/src/core/iomgr/pollset_multipoller_with_epoll.c
@@ -0,0 +1,197 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_LINUX_MULTIPOLL_WITH_EPOLL
+
+#include <errno.h>
+#include <string.h>
+#include <sys/epoll.h>
+#include <unistd.h>
+
+#include "src/core/iomgr/fd_posix.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+typedef struct {
+ int epoll_fd;
+ grpc_wakeup_fd_info wakeup_fd;
+} pollset_hdr;
+
+static void multipoll_with_epoll_pollset_add_fd(grpc_pollset *pollset,
+ grpc_fd *fd) {
+ pollset_hdr *h = pollset->data.ptr;
+ struct epoll_event ev;
+ int err;
+
+ ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
+ ev.data.ptr = fd;
+ err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev);
+ if (err < 0) {
+ /* FDs may be added to a pollset multiple times, so EEXIST is normal. */
+ if (errno != EEXIST) {
+ gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd,
+ strerror(errno));
+ }
+ }
+}
+
+static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset,
+ grpc_fd *fd) {
+ pollset_hdr *h = pollset->data.ptr;
+ int err;
+ /* Note that this can race with concurrent poll, but that should be fine since
+ * at worst it creates a spurious read event on a reused grpc_fd object. */
+ err = epoll_ctl(h->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL);
+ if (err < 0) {
+ gpr_log(GPR_ERROR, "epoll_ctl del for %d failed: %s", fd->fd,
+ strerror(errno));
+ }
+}
+
+/* TODO(klempner): We probably want to turn this down a bit */
+#define GRPC_EPOLL_MAX_EVENTS 1000
+
+static int multipoll_with_epoll_pollset_maybe_work(
+ grpc_pollset *pollset, gpr_timespec deadline, gpr_timespec now,
+ int allow_synchronous_callback) {
+ struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS];
+ int ep_rv;
+ pollset_hdr *h = pollset->data.ptr;
+ int timeout_ms;
+
+ /* If you want to ignore epoll's ability to sanely handle parallel pollers,
+ * for a more apples-to-apples performance comparison with poll, add a
+ * if (pollset->counter == 0) { return 0 }
+ * here.
+ */
+
+ if (gpr_time_cmp(deadline, gpr_inf_future) == 0) {
+ timeout_ms = -1;
+ } else {
+ timeout_ms = gpr_time_to_millis(gpr_time_sub(deadline, now));
+ if (timeout_ms <= 0) {
+ return 1;
+ }
+ }
+ pollset->counter += 1;
+ gpr_mu_unlock(&pollset->mu);
+
+ do {
+ ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms);
+ if (ep_rv < 0) {
+ if (errno != EINTR) {
+ gpr_log(GPR_ERROR, "epoll_wait() failed: %s", strerror(errno));
+ }
+ } else {
+ int i;
+ for (i = 0; i < ep_rv; ++i) {
+ if (ep_ev[i].data.ptr == 0) {
+ grpc_wakeup_fd_consume_wakeup(&h->wakeup_fd);
+ } else {
+ grpc_fd *fd = ep_ev[i].data.ptr;
+ /* TODO(klempner): We might want to consider making err and pri
+ * separate events */
+ int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP);
+ int read = ep_ev[i].events & (EPOLLIN | EPOLLPRI);
+ int write = ep_ev[i].events & EPOLLOUT;
+ if (read || cancel) {
+ grpc_fd_become_readable(fd, allow_synchronous_callback);
+ }
+ if (write || cancel) {
+ grpc_fd_become_writable(fd, allow_synchronous_callback);
+ }
+ }
+ }
+ }
+ timeout_ms = 0;
+ } while (ep_rv == GRPC_EPOLL_MAX_EVENTS);
+
+ gpr_mu_lock(&pollset->mu);
+ pollset->counter -= 1;
+ /* TODO(klempner): This should signal once per event rather than broadcast,
+ * although it probably doesn't matter because threads will generally be
+ * blocked in epoll_wait rather than being blocked on the cv. */
+ gpr_cv_broadcast(&pollset->cv);
+ return 1;
+}
+
+static void multipoll_with_epoll_pollset_destroy(grpc_pollset *pollset) {
+ pollset_hdr *h = pollset->data.ptr;
+ grpc_wakeup_fd_destroy(&h->wakeup_fd);
+ close(h->epoll_fd);
+ gpr_free(h);
+}
+
+static void epoll_kick(grpc_pollset *pollset) {
+ pollset_hdr *h = pollset->data.ptr;
+ grpc_wakeup_fd_wakeup(&h->wakeup_fd);
+}
+
+static const grpc_pollset_vtable multipoll_with_epoll_pollset = {
+ multipoll_with_epoll_pollset_add_fd, multipoll_with_epoll_pollset_del_fd,
+ multipoll_with_epoll_pollset_maybe_work, epoll_kick,
+ multipoll_with_epoll_pollset_destroy};
+
+void grpc_platform_become_multipoller(grpc_pollset *pollset, grpc_fd **fds,
+ size_t nfds) {
+ size_t i;
+ pollset_hdr *h = gpr_malloc(sizeof(pollset_hdr));
+ struct epoll_event ev;
+ int err;
+
+ pollset->vtable = &multipoll_with_epoll_pollset;
+ pollset->data.ptr = h;
+ h->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
+ if (h->epoll_fd < 0) {
+ /* TODO(klempner): Fall back to poll here, especially on ENOSYS */
+ gpr_log(GPR_ERROR, "epoll_create1 failed: %s", strerror(errno));
+ abort();
+ }
+ for (i = 0; i < nfds; i++) {
+ multipoll_with_epoll_pollset_add_fd(pollset, fds[i]);
+ }
+
+ grpc_wakeup_fd_create(&h->wakeup_fd);
+ ev.events = EPOLLIN;
+ ev.data.ptr = 0;
+ err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD,
+ GRPC_WAKEUP_FD_GET_READ_FD(&h->wakeup_fd), &ev);
+ if (err < 0) {
+ gpr_log(GPR_ERROR, "Wakeup fd epoll_ctl failed: %s", strerror(errno));
+ abort();
+ }
+}
+
+#endif /* GPR_LINUX_MULTIPOLL_WITH_EPOLL */
diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c
index 3244ae08db57ada55006ae2ba3a8bc2fd004de09..c136ee0b5286450801df577fe47d33d8ce1d9c8a 100644
--- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c
+++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c
@@ -200,6 +200,10 @@ static int multipoll_with_poll_pollset_maybe_work(
return 1;
}
+static void multipoll_with_poll_pollset_kick(grpc_pollset *p) {
+ grpc_pollset_kick_kick(&p->kick_state);
+}
+
static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) {
size_t i;
pollset_hdr *h = pollset->data.ptr;
@@ -219,7 +223,7 @@ static void multipoll_with_poll_pollset_destroy(grpc_pollset *pollset) {
static const grpc_pollset_vtable multipoll_with_poll_pollset = {
multipoll_with_poll_pollset_add_fd, multipoll_with_poll_pollset_del_fd,
- multipoll_with_poll_pollset_maybe_work,
+ multipoll_with_poll_pollset_maybe_work, multipoll_with_poll_pollset_kick,
multipoll_with_poll_pollset_destroy};
void grpc_platform_become_multipoller(grpc_pollset *pollset, grpc_fd **fds,
diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c
index 2837a0dff3ffd204809c6ffb4e5c4fd69e78e9ed..53c9806fb9bdee1d74cb289c24095963202d93bf 100644
--- a/src/core/iomgr/pollset_posix.c
+++ b/src/core/iomgr/pollset_posix.c
@@ -76,7 +76,7 @@ static void backup_poller(void *p) {
void grpc_pollset_kick(grpc_pollset *p) {
if (p->counter) {
- grpc_pollset_kick_kick(&p->kick_state);
+ p->vtable->kick(p);
}
}
@@ -84,6 +84,10 @@ void grpc_pollset_force_kick(grpc_pollset *p) {
grpc_pollset_kick_kick(&p->kick_state);
}
+static void kick_using_pollset_kick(grpc_pollset *p) {
+ grpc_pollset_kick_kick(&p->kick_state);
+}
+
/* global state management */
grpc_pollset *grpc_backup_pollset(void) { return &g_backup_pollset; }
@@ -186,7 +190,7 @@ static void empty_pollset_destroy(grpc_pollset *pollset) {}
static const grpc_pollset_vtable empty_pollset = {
empty_pollset_add_fd, empty_pollset_del_fd, empty_pollset_maybe_work,
- empty_pollset_destroy};
+ kick_using_pollset_kick, empty_pollset_destroy};
static void become_empty_pollset(grpc_pollset *pollset) {
pollset->vtable = &empty_pollset;
@@ -296,7 +300,8 @@ static void unary_poll_pollset_destroy(grpc_pollset *pollset) {
static const grpc_pollset_vtable unary_poll_pollset = {
unary_poll_pollset_add_fd, unary_poll_pollset_del_fd,
- unary_poll_pollset_maybe_work, unary_poll_pollset_destroy};
+ unary_poll_pollset_maybe_work, kick_using_pollset_kick,
+ unary_poll_pollset_destroy};
static void become_unary_pollset(grpc_pollset *pollset, grpc_fd *fd) {
pollset->vtable = &unary_poll_pollset;
diff --git a/src/core/iomgr/pollset_posix.h b/src/core/iomgr/pollset_posix.h
index cdcb9951675d6d4d7668d7d6ef45cd0664cc6cd0..b1a82fccfe73ba562436110b5fe59989e2b862b9 100644
--- a/src/core/iomgr/pollset_posix.h
+++ b/src/core/iomgr/pollset_posix.h
@@ -66,6 +66,7 @@ struct grpc_pollset_vtable {
void (*del_fd)(grpc_pollset *pollset, struct grpc_fd *fd);
int (*maybe_work)(grpc_pollset *pollset, gpr_timespec deadline,
gpr_timespec now, int allow_synchronous_callback);
+ void (*kick)(grpc_pollset *pollset);
void (*destroy)(grpc_pollset *pollset);
};
diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c
index 00b10f936480e7ec1e456babefe8556f77f05aa4..78709f47fbcb113975b8ff02aae0cab27c6f8874 100644
--- a/test/core/iomgr/tcp_client_posix_test.c
+++ b/test/core/iomgr/tcp_client_posix_test.c
@@ -171,6 +171,7 @@ void test_times_out(void) {
int main(void) {
grpc_iomgr_init();
test_succeeds();
+ gpr_log(GPR_ERROR, "End of first test");
test_fails();
test_times_out();
grpc_iomgr_shutdown();
diff --git a/vsprojects/vs2013/grpc.vcxproj b/vsprojects/vs2013/grpc.vcxproj
index bd87569a3f9309ff90231f19195bce9cc4a3fd5d..fc740fec925c78f9c3ae3f3fa3ddfb4f697b9763 100644
--- a/vsprojects/vs2013/grpc.vcxproj
+++ b/vsprojects/vs2013/grpc.vcxproj
@@ -273,6 +273,8 @@
</ClCompile>
<ClCompile Include="..\..\src\core\iomgr\pollset_multipoller_with_poll_posix.c">
</ClCompile>
+ <ClCompile Include="..\..\src\core\iomgr\pollset_multipoller_with_epoll.c">
+ </ClCompile>
<ClCompile Include="..\..\src\core\iomgr\pollset_posix.c">
</ClCompile>
<ClCompile Include="..\..\src\core\iomgr\pollset_windows.c">
diff --git a/vsprojects/vs2013/grpc.vcxproj.filters b/vsprojects/vs2013/grpc.vcxproj.filters
index fed8fb10bfb69f5e64a9360aa05ebdda301a41f5..75ecc7a822d5c46d99b9d36933613c3d47cfdb8b 100644
--- a/vsprojects/vs2013/grpc.vcxproj.filters
+++ b/vsprojects/vs2013/grpc.vcxproj.filters
@@ -130,6 +130,9 @@
<ClCompile Include="..\..\src\core\iomgr\pollset_multipoller_with_poll_posix.c">
<Filter>src\core\iomgr</Filter>
</ClCompile>
+ <ClCompile Include="..\..\src\core\iomgr\pollset_multipoller_with_epoll.c">
+ <Filter>src\core\iomgr</Filter>
+ </ClCompile>
<ClCompile Include="..\..\src\core\iomgr\pollset_posix.c">
<Filter>src\core\iomgr</Filter>
</ClCompile>
diff --git a/vsprojects/vs2013/grpc_unsecure.vcxproj b/vsprojects/vs2013/grpc_unsecure.vcxproj
index 0e5bdae62f3a2319b28d6a0f8cae5d085374c727..c5130eee5e789a65ee945a91061ecc51133711fa 100644
--- a/vsprojects/vs2013/grpc_unsecure.vcxproj
+++ b/vsprojects/vs2013/grpc_unsecure.vcxproj
@@ -235,6 +235,8 @@
</ClCompile>
<ClCompile Include="..\..\src\core\iomgr\pollset_multipoller_with_poll_posix.c">
</ClCompile>
+ <ClCompile Include="..\..\src\core\iomgr\pollset_multipoller_with_epoll.c">
+ </ClCompile>
<ClCompile Include="..\..\src\core\iomgr\pollset_posix.c">
</ClCompile>
<ClCompile Include="..\..\src\core\iomgr\pollset_windows.c">
diff --git a/vsprojects/vs2013/grpc_unsecure.vcxproj.filters b/vsprojects/vs2013/grpc_unsecure.vcxproj.filters
index b0964b61b33c15cfd7a111cafedba0cce27d36e9..50f319066f145a1086b23eccb1f6dccb07e9264a 100644
--- a/vsprojects/vs2013/grpc_unsecure.vcxproj.filters
+++ b/vsprojects/vs2013/grpc_unsecure.vcxproj.filters
@@ -91,6 +91,9 @@
<ClCompile Include="..\..\src\core\iomgr\pollset_multipoller_with_poll_posix.c">
<Filter>src\core\iomgr</Filter>
</ClCompile>
+ <ClCompile Include="..\..\src\core\iomgr\pollset_multipoller_with_epoll.c">
+ <Filter>src\core\iomgr</Filter>
+ </ClCompile>
<ClCompile Include="..\..\src\core\iomgr\pollset_posix.c">
<Filter>src\core\iomgr</Filter>
</ClCompile>