Merge pull request #20092 from jtattermusch/python_fix_docker_build

Make python dockerfiles buildable again

templates/tools/dockerfile/debian_testing_repo.include

-# Add Debian 'testing' repository
-RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local

templates/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile.template

@@ -16,5 +16,5 @@
   
   <%include file="../../python_stretch.include"/>
 
-  RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+  RUN apt-get update && apt-get -t stable install -y python3.7 python3-all-dev
   RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7

templates/tools/dockerfile/python_stretch.include

@@ -3,7 +3,9 @@ FROM debian:stretch
 <%include file="./apt_get_basic.include"/>
 <%include file="./gcp_api_libraries.include"/>
 <%include file="./apt_get_python_27.include"/>
-<%include file="./debian_testing_repo.include"/>
+# Add Debian 'buster' repository, we will need it for installing newer versions of python
+RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 <%include file="./run_tests_addons.include"/>
 # Define the default command.
 CMD ["bash"]

templates/tools/dockerfile/test/python_stretch_3.6_x64/Dockerfile.template

@@ -16,5 +16,17 @@
 
   <%include file="../../python_stretch.include"/>
 
-  RUN apt-get update && apt-get -t testing install -y python3.6 python3-all-dev
-  RUN curl https://bootstrap.pypa.io/get-pip.py | python3.6
+  RUN apt-get install -y jq zlib1g-dev libssl-dev
+
+  RUN apt-get install -y jq build-essential libffi-dev
+
+  RUN cd /tmp && ${'\\'}
+    wget -q https://github.com/python/cpython/archive/v3.6.9.tar.gz && ${'\\'}
+    tar xzvf v3.6.9.tar.gz && ${'\\'}
+    cd cpython-3.6.9 && ${'\\'}
+    ./configure && ${'\\'}
+    make install
+
+  RUN python3.6 -m ensurepip && ${'\\'}
+      python3.6 -m pip install coverage
+

templates/tools/dockerfile/test/python_stretch_3.7_x64/Dockerfile.template

@@ -16,7 +16,7 @@
 
   <%include file="../../python_stretch.include"/>
 
-  RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+  RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev
   RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
 
   # for Python test coverage reporting

templates/tools/dockerfile/test/python_stretch_3.8_x64/Dockerfile.template

@@ -18,11 +18,14 @@
   <%include file="../../python_stretch.include"/>
   RUN apt-get install -y jq zlib1g-dev libssl-dev
 
-  COPY get_cpython.sh /tmp
-  RUN apt-get install -y jq build-essential libffi-dev && ${'\\'}
-    chmod +x /tmp/get_cpython.sh && ${'\\'}
-    /tmp/get_cpython.sh && ${'\\'}
-    rm /tmp/get_cpython.sh
+  RUN apt-get install -y jq build-essential libffi-dev
+
+  RUN cd /tmp && ${'\\'}
+    wget -q https://github.com/python/cpython/archive/v3.8.0b3.tar.gz && ${'\\'}
+    tar xzvf v3.8.0b3.tar.gz && ${'\\'}
+    cd cpython-3.8.0b3 && ${'\\'}
+    ./configure && ${'\\'}
+    make install
 
   RUN python3.8 -m ensurepip && ${'\\'}
       python3.8 -m pip install coverage

templates/tools/dockerfile/test/sanity/Dockerfile.template

@@ -18,7 +18,7 @@
   <%include file="../../cxx_deps.include"/>
   #========================
   # Sanity test dependencies
-  RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+  RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev
   RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
   # Make Python 3.7 the default Python 3 version
   RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1

tools/dockerfile/interoptest/grpc_interop_python/Dockerfile

@@ -57,10 +57,9 @@ RUN pip install --upgrade google-api-python-client oauth2client
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Add Debian 'testing' repository
-RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
-
+# Add Debian 'buster' repository, we will need it for installing newer versions of python
+RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 
@@ -68,5 +67,5 @@ RUN mkdir /var/local/jenkins
 CMD ["bash"]
 
 
-RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+RUN apt-get update && apt-get -t stable install -y python3.7 python3-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7

tools/dockerfile/test/python_stretch_2.7_x64/Dockerfile

@@ -57,10 +57,9 @@ RUN pip install --upgrade google-api-python-client oauth2client
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Add Debian 'testing' repository
-RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
-
+# Add Debian 'buster' repository, we will need it for installing newer versions of python
+RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 

tools/dockerfile/test/python_stretch_3.5_x64/Dockerfile

@@ -57,10 +57,9 @@ RUN pip install --upgrade google-api-python-client oauth2client
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Add Debian 'testing' repository
-RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
-
+# Add Debian 'buster' repository, we will need it for installing newer versions of python
+RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 

tools/dockerfile/test/python_stretch_3.6_x64/Dockerfile

@@ -57,10 +57,9 @@ RUN pip install --upgrade google-api-python-client oauth2client
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Add Debian 'testing' repository
-RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
-
+# Add Debian 'buster' repository, we will need it for installing newer versions of python
+RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 
@@ -68,5 +67,16 @@ RUN mkdir /var/local/jenkins
 CMD ["bash"]
 
 
-RUN apt-get update && apt-get -t testing install -y python3.6 python3-all-dev
-RUN curl https://bootstrap.pypa.io/get-pip.py | python3.6
+RUN apt-get install -y jq zlib1g-dev libssl-dev
+
+RUN apt-get install -y jq build-essential libffi-dev
+
+RUN cd /tmp && \
+  wget -q https://github.com/python/cpython/archive/v3.6.9.tar.gz && \
+  tar xzvf v3.6.9.tar.gz && \
+  cd cpython-3.6.9 && \
+  ./configure && \
+  make install
+
+RUN python3.6 -m ensurepip && \
+    python3.6 -m pip install coverage

tools/dockerfile/test/python_stretch_3.7_x64/Dockerfile

@@ -57,10 +57,9 @@ RUN pip install --upgrade google-api-python-client oauth2client
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Add Debian 'testing' repository
-RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
-
+# Add Debian 'buster' repository, we will need it for installing newer versions of python
+RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 
@@ -68,7 +67,7 @@ RUN mkdir /var/local/jenkins
 CMD ["bash"]
 
 
-RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
 
 # for Python test coverage reporting

tools/dockerfile/test/python_stretch_3.8_x64/Dockerfile

@@ -57,10 +57,9 @@ RUN pip install --upgrade google-api-python-client oauth2client
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Add Debian 'testing' repository
-RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
-
+# Add Debian 'buster' repository, we will need it for installing newer versions of python
+RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 
@@ -69,11 +68,14 @@ CMD ["bash"]
 
 RUN apt-get install -y jq zlib1g-dev libssl-dev
 
-COPY get_cpython.sh /tmp
-RUN apt-get install -y jq build-essential libffi-dev && \
-  chmod +x /tmp/get_cpython.sh && \
-  /tmp/get_cpython.sh && \
-  rm /tmp/get_cpython.sh
+RUN apt-get install -y jq build-essential libffi-dev
+
+RUN cd /tmp && \
+  wget -q https://github.com/python/cpython/archive/v3.8.0b3.tar.gz && \
+  tar xzvf v3.8.0b3.tar.gz && \
+  cd cpython-3.8.0b3 && \
+  ./configure && \
+  make install
 
 RUN python3.8 -m ensurepip && \
     python3.8 -m pip install coverage

tools/dockerfile/test/python_stretch_3.8_x64/get_cpython.sh

-#!/bin/bash
-
-# Copyright 2019 The gRPC Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-VERSION_REGEX="v3.8.*"
-REPO="python/cpython"
-
-LATEST=$(curl -s https://api.github.com/repos/$REPO/tags | \
-          jq -r '.[] | select(.name|test("'$VERSION_REGEX'")) | .name' \
-          | sort | tail -n1)
-
-wget https://github.com/$REPO/archive/$LATEST.tar.gz
-tar xzvf *.tar.gz
-( cd cpython*
-  ./configure
-  make install
-)

tools/dockerfile/test/sanity/Dockerfile

@@ -57,10 +57,9 @@ RUN pip install --upgrade google-api-python-client oauth2client
 RUN apt-get update && apt-get install -y python2.7 python-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7
 
-# Add Debian 'testing' repository
-RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list
-RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local
-
+# Add Debian 'buster' repository, we will need it for installing newer versions of python
+RUN echo 'deb http://ftp.de.debian.org/debian buster main' >> /etc/apt/sources.list
+RUN echo 'APT::Default-Release "stretch";' | tee -a /etc/apt/apt.conf.d/00local
 
 RUN mkdir /var/local/jenkins
 
@@ -73,7 +72,7 @@ RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev c
 
 #========================
 # Sanity test dependencies
-RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev
+RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev
 RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7
 # Make Python 3.7 the default Python 3 version
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1