Use manylinux

setup.py

@@ -236,7 +236,8 @@ setup_arguments = {
     'ext_modules': CYTHON_EXTENSION_MODULES,
     'packages': list(PACKAGES),
     'package_dir': PACKAGE_DIRECTORIES,
-    'namespace_packages': ['grpc'],
+    # TODO(atash): Figure out why auditwheel doesn't like namespace packages.
+    #'namespace_packages': ['grpc'],
     'package_data': PACKAGE_DATA,
     'install_requires': INSTALL_REQUIRES,
     'setup_requires': SETUP_REQUIRES,

tools/distrib/python/grpcio_tools/grpc/protoc/main.cc

@@ -5,7 +5,7 @@
 
 #include "grpc/protoc/main.h"
 
-int main(int argc, char* argv[]) {
+int protoc_main(int argc, char* argv[]) {
   google::protobuf::compiler::CommandLineInterface cli;
   cli.AllowPlugins("protoc-");
 

tools/distrib/python/grpcio_tools/grpc/protoc/main.h

@@ -28,6 +28,6 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-// We declare `main` here since we want access to it from Cython as an extern
-// but *without* triggering a dllimport declspec when on Windows.
-int main(int argc, char *argv[]);
+// We declare `protoc_main` here since we want access to it from Cython as an
+// extern but *without* triggering a dllimport declspec when on Windows.
+int protoc_main(int argc, char *argv[]);

tools/distrib/python/grpcio_tools/grpc/protoc/protoc_compiler.pyx

@@ -30,10 +30,10 @@
 from libc cimport stdlib
 
 cdef extern from "grpc/protoc/main.h":
-  int main(int argc, char *argv[])
+  int protoc_main(int argc, char *argv[])
 
 def run_main(list args not None):
   cdef char **argv = <char **>stdlib.malloc(len(args)*sizeof(char *))
   for i in range(len(args)):
     argv[i] = args[i]
-  return main(len(args), argv)
+  return protoc_main(len(args), argv)

tools/distrib/python/grpcio_tools/setup.py

@@ -76,7 +76,8 @@ setuptools.setup(
       protoc_ext_module(),
   ]),
   packages=setuptools.find_packages('.'),
-  namespace_packages=['grpc'],
+  # TODO(atash): Figure out why auditwheel doesn't like namespace packages.
+  #namespace_packages=['grpc'],
   install_requires=[
     'protobuf>=3.0.0a3',
   ],

tools/dockerfile/grpc_artifact_python_manylinux_x64/Dockerfile

+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Docker file for building gRPC manylinux Python artifacts.
+
+FROM quay.io/pypa/manylinux1_x86_64
+
+# Update the package manager
+RUN yum update -y
+
+###################################
+# Install Python build requirements
+RUN /opt/python/cp27-cp27m/bin/pip install cython
+RUN /opt/python/cp27-cp27mu/bin/pip install cython
+RUN /opt/python/cp34-cp34m/bin/pip install cython
+RUN /opt/python/cp35-cp35m/bin/pip install cython
+

tools/dockerfile/grpc_artifact_python_manylinux_x86/Dockerfile

+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Docker file for building gRPC manylinux Python artifacts.
+
+FROM quay.io/pypa/manylinux1_i686
+
+# Update the package manager
+RUN yum update -y
+
+###################################
+# Install Python build requirements
+RUN /opt/python/cp27-cp27m/bin/pip install cython
+RUN /opt/python/cp27-cp27mu/bin/pip install cython
+RUN /opt/python/cp34-cp34m/bin/pip install cython
+RUN /opt/python/cp35-cp35m/bin/pip install cython
+

tools/run_tests/artifact_targets.py

@@ -84,12 +84,16 @@ python_version_arch_map = {
 class PythonArtifact:
   """Builds Python artifacts."""
 
-  def __init__(self, platform, arch):
-    self.name = 'python_%s_%s' % (platform, arch)
+  def __init__(self, platform, arch, manylinux_build=None):
+    if manylinux_build:
+      self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build)
+    else:
+      self.name = 'python_%s_%s' % (platform, arch)
     self.platform = platform
     self.arch = arch
     self.labels = ['artifact', 'python', platform, arch]
     self.python_version = python_version_arch_map[arch]
+    self.manylinux_build = manylinux_build
 
   def pre_build_jobspecs(self):
       return []
@@ -99,8 +103,47 @@ class PythonArtifact:
     if self.platform == 'linux':
       if self.arch == 'x86':
         environ['SETARCH_CMD'] = 'linux32'
+      # Inside the manylinux container, the python installations are located in
+      # special places...
+      environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build)
+      environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build)
+      # Our docker image has all the prerequisites pip-installed already.
+      environ['SKIP_PIP_INSTALL'] = '1'
+      # Platform autodetection for the manylinux1 image breaks so we set the
+      # defines ourselves.
+      # TODO(atash) get better platform-detection support in core so we don't
+      # need to do this manually...
+      environ['CFLAGS'] = " ".join([
+        '-DGPR_NO_AUTODETECT_PLATFORM',
+        '-DGPR_PLATFORM_STRING=\\"manylinux\\"',
+        '-DGPR_POSIX_CRASH_HANDLER=1',
+        '-DGPR_CPU_LINUX=1',
+        '-DGPR_GCC_ATOMIC=1',
+        '-DGPR_GCC_TLS=1',
+        '-DGPR_LINUX=1',
+        '-DGPR_LINUX_LOG=1',
+        #'-DGPR_LINUX_MULTIPOLL_WITH_EPOLL=1',
+        '-DGPR_POSIX_SOCKET=1',
+        '-DGPR_POSIX_WAKEUP_FD=1',
+        '-DGPR_POSIX_SOCKETADDR=1',
+        #'-DGPR_LINUX_EVENTFD=1',
+        #'-DGPR_LINUX_SOCKETUTILS=1',
+        '-DGPR_HAVE_UNIX_SOCKET=1',
+        '-DGPR_HAVE_IP_PKTINFO=1',
+        '-DGPR_HAVE_IPV6_RECVPKTINFO=1',
+        '-DGPR_LINUX_ENV=1',
+        '-DGPR_POSIX_FILE=1',
+        '-DGPR_POSIX_TMPFILE=1',
+        '-DGPR_POSIX_STRING=1',
+        '-DGPR_POSIX_SUBPROCESS=1',
+        '-DGPR_POSIX_SYNC=1',
+        '-DGPR_POSIX_TIME=1',
+        '-DGPR_GETPID_IN_UNISTD_H=1',
+        '-DGPR_HAVE_MSG_NOSIGNAL=1',
+        '-DGPR_ARCH_{arch}=1'.format(arch=('32' if self.arch == 'x86' else '64')),
+      ])
       return create_docker_jobspec(self.name,
-          'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
+          'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch,
           'tools/run_tests/build_artifact_python.sh',
           environ=environ)
     elif self.platform == 'windows':
@@ -307,8 +350,10 @@ def targets():
            for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
            for platform in ('linux', 'macos', 'windows')
            for arch in ('x86', 'x64')] +
-          [PythonArtifact('linux', 'x86'),
-           PythonArtifact('linux', 'x64'),
+          [PythonArtifact('linux', 'x86', 'cp27-cp27m'),
+           PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
+           PythonArtifact('linux', 'x64', 'cp27-cp27m'),
+           PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
            PythonArtifact('macos', 'x64'),
            PythonArtifact('windows', 'x86'),
            PythonArtifact('windows', 'x64'),

tools/run_tests/build_artifact_python.sh

@@ -32,43 +32,48 @@ set -ex
 
 cd $(dirname $0)/../..
 
+export GRPC_PYTHON_USE_CUSTOM_BDIST=0
+export GRPC_PYTHON_BUILD_WITH_CYTHON=1
+export PYTHON=${PYTHON:-python}
+export PIP=${PIP:-pip}
+export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
+
+
 if [ "$SKIP_PIP_INSTALL" == "" ]
 then
-  pip install --upgrade six
+  ${PIP} install --upgrade six
   # There's a bug in newer versions of setuptools (see
   # https://bitbucket.org/pypa/setuptools/issues/503/pkg_resources_vendorpackagingrequirementsi)
-  pip install --upgrade 'setuptools==18'
-  pip install -rrequirements.txt
+  ${PIP} pip install --upgrade 'setuptools==18'
+  ${PIP} install -rrequirements.txt
 fi
 
-export GRPC_PYTHON_USE_CUSTOM_BDIST=0
-export GRPC_PYTHON_BUILD_WITH_CYTHON=1
-
 # Build the source distribution first because MANIFEST.in cannot override
 # exclusion of built shared objects among package resources (for some
 # inexplicable reason).
-${SETARCH_CMD} python setup.py  \
+${SETARCH_CMD} ${PYTHON} setup.py  \
     sdist
 
-# The bdist_wheel_grpc_custom command is finicky about command output ordering
-# and thus ought to be run in a shell command separate of others. Further, it
-# trashes the actual bdist_wheel output, so it should be run first so that
-# bdist_wheel may be run unmolested.
-${SETARCH_CMD} python setup.py  \
-    build_tagged_ext
-
 # Wheel has a bug where directories don't get excluded.
 # https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory
-${SETARCH_CMD} python setup.py  \
+${SETARCH_CMD} ${PYTHON} setup.py  \
     bdist_wheel
 
 # Build gRPC tools package
-python tools/distrib/python/make_grpcio_tools.py
-# Build with clang since there's a bug in GCC 4.x where some constant
-# expressions are treated as non-constant in the presence of the fwrapv flag
-# (fixed in at most GCC 5.3).
-CC=clang python tools/distrib/python/grpcio_tools/setup.py bdist_wheel
+${PYTHON} tools/distrib/python/make_grpcio_tools.py
+CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \
+  ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
 
 mkdir -p artifacts
-cp -r dist/* artifacts
-cp -r tools/distrib/python/grpcio_tools/dist/* artifacts
+if command -v ${AUDITWHEEL}
+then
+  for wheel in dist/*.whl; do
+    ${AUDITWHEEL} repair $wheel -w artifacts/
+  done
+  for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do
+    ${AUDITWHEEL} repair $wheel -w artifacts/
+  done
+else
+  cp -r dist/* artifacts
+  cp -r tools/distrib/python/grpcio_tools/dist/* artifacts
+fi