From cbd1bce45a605df97aa00a42b7568d08419253f9 Mon Sep 17 00:00:00 2001
From: Masood Malekghassemi <atash@google.com>
Date: Tue, 15 Mar 2016 13:24:10 -0700
Subject: [PATCH] Fix two ways tests can hang
Both have to do with the test runner's handling of the tests. With one
it's the read thread somehow outliving the other threads (e.g. with
ctrl-C). The other is due to a filled OS-level pipe's buffer causing a
block while code is still holding the GIL in some gRPC core function. We
can't empty the buffer from Python because the GIL is held, and the OS
can't unblock because it's waiting for the buffer to get cleared:
deadlock.
---
src/python/grpcio/tests/_runner.py | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py
index b0dbd92a49..32a31ce00e 100644
--- a/src/python/grpcio/tests/_runner.py
+++ b/src/python/grpcio/tests/_runner.py
@@ -43,6 +43,13 @@ import uuid
from tests import _loader
from tests import _result
+# This number needs to be large enough to outpace output on stdout and stderr
+# from the gRPC core, otherwise we could end up in a potential deadlock. This
+# stems from the OS waiting on someone to clear a filled pipe buffer while the
+# GIL is held from a write to stderr from gRPC core, but said someone is in
+# Python code thus necessitating GIL acquisition.
+_READ_BYTES = 2**20
+
class CapturePipe(object):
"""A context-manager pipe to redirect output to a byte array.
@@ -76,6 +83,10 @@ class CapturePipe(object):
flags = fcntl.fcntl(self._read_fd, fcntl.F_GETFL)
fcntl.fcntl(self._read_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
self._read_thread = threading.Thread(target=self._read)
+ # If the user wants to exit from the Python program and hits ctrl-C and the
+ # read thread is somehow deadlocked with something else, the Python code may
+ # refuse to exit. This prevents that by making the read thread second-class.
+ self._read_thread.daemon = True
self._read_thread.start()
def stop(self):
@@ -93,7 +104,7 @@ class CapturePipe(object):
self.output = bytearray()
while True:
select.select([self._read_fd], [], [])
- read_bytes = os.read(self._read_fd, 1024)
+ read_bytes = os.read(self._read_fd, _READ_BYTES)
if read_bytes:
self.output.extend(read_bytes)
else:
--
GitLab