Merge pull request #3638 from soltanmm/pip-so-cash

Enable `pip` caching

tools/jenkins/build_docker_and_run_tests.sh

@@ -37,8 +37,13 @@ cd `dirname $0`/../..
 git_root=`pwd`
 cd -
 
+# Ensure existence of ccache directory
 mkdir -p /tmp/ccache
 
+# Ensure existence of the home directory for XDG caches (e.g. what pip uses for
+# its cache location now that --download-cache is deprecated).
+mkdir -p /tmp/xdg-cache-home
+
 # Create a local branch so the child Docker script won't complain
 git branch -f jenkins-docker
 
@@ -57,9 +62,11 @@ docker run \
   -e "config=$config" \
   -e "arch=$arch" \
   -e CCACHE_DIR=/tmp/ccache \
+  -e XDG_CACHE_HOME=/tmp/xdg-cache-home \
   -i $TTY_FLAG \
   -v "$git_root:/var/local/jenkins/grpc" \
   -v /tmp/ccache:/tmp/ccache \
+  -v /tmp/xdg-cache-home:/tmp/xdg-cache-home \
   -v /var/run/docker.sock:/var/run/docker.sock \
   -v $(which docker):/bin/docker \
   -w /var/local/git/grpc \

tools/jenkins/docker_run_tests.sh

@@ -36,6 +36,10 @@ set -e
 export CONFIG=$config
 export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.5
 
+# Ensure that programs depending on current-user-ownership of cache directories
+# are satisfied (it's being mounted from outside the image).
+chown `whoami` $XDG_CACHE_HOME
+
 mkdir -p /var/local/git
 git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
 

tools/jenkins/grpc_jenkins_slave/Dockerfile

@@ -126,10 +126,11 @@ RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
 RUN apt-get update && apt-get install -y \
     python-all-dev \
     python3-all-dev \
-    python-pip \
-    python-virtualenv
+    python-pip
 
 # Install Python packages from PyPI
+RUN pip install pip --upgrade
+RUN pip install virtualenv
 RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2
 
 # For sanity test

tools/run_tests/build_python.sh

@@ -39,6 +39,33 @@ GRPCIO=$ROOT/src/python/grpcio
 GRPCIO_TEST=$ROOT/src/python/grpcio_test
 GRPCIO_HEALTH_CHECKING=$ROOT/src/python/grpcio_health_checking
 
+install_grpcio_deps() {
+  cd $GRPCIO
+  pip install -r requirements.txt
+}
+install_grpcio_test_deps() {
+  cd $GRPCIO_TEST
+  pip install -r requirements.txt
+}
+
+install_grpcio() {
+  CFLAGS="-I$ROOT/include -std=c89" LDFLAGS=-L$ROOT/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install $GRPCIO
+}
+install_grpcio_test() {
+  pip install $GRPCIO_TEST
+}
+install_grpcio_health_checking() {
+  pip install $GRPCIO_HEALTH_CHECKING
+}
+
+# Cleans the environment of previous installations
+clean_grpcio_all() {
+  (yes | pip uninstall grpcio) || true
+  (yes | pip uninstall grpcio_test) || true
+  (yes | pip uninstall grpcio_health_checking) || true
+}
+
+# Builds the testing environment.
 make_virtualenv() {
   virtualenv_name="python"$1"_virtual_environment"
   if [ ! -d $virtualenv_name ]
@@ -48,33 +75,29 @@ make_virtualenv() {
     source $virtualenv_name/bin/activate
 
     # Install grpcio
-    cd $GRPCIO
-    pip install -r requirements.txt
-    CFLAGS="-I$ROOT/include -std=c89" LDFLAGS=-L$ROOT/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install $GRPCIO
+    install_grpcio_deps
+    install_grpcio
 
     # Install grpcio_test
-    cd $GRPCIO_TEST
-    pip install -r requirements.txt
-    pip install $GRPCIO_TEST
+    install_grpcio_test_deps
+    install_grpcio_test
 
     # Install grpcio_health_checking
-    pip install $GRPCIO_HEALTH_CHECKING
+    install_grpcio_health_checking
   else
     source $virtualenv_name/bin/activate
     # Uninstall and re-install the packages we care about. Don't use
     # --force-reinstall or --ignore-installed to avoid propagating this
     # unnecessarily to dependencies. Don't use --no-deps to avoid missing
     # dependency upgrades.
-    (yes | pip uninstall grpcio) || true
-    (yes | pip uninstall grpcio_test) || true
-    (yes | pip uninstall grpcio_health_checking) || true
-    (CFLAGS="-I$ROOT/include -std=c89" LDFLAGS=-L$ROOT/libs/$CONFIG GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install $GRPCIO) || (
+    clean_grpcio_all
+    install_grpcio || (
       # Fall back to rebuilding the entire environment
       rm -rf $virtualenv_name
       make_virtualenv $1
     )
-    pip install $GRPCIO_TEST
-    pip install $GRPCIO_HEALTH_CHECKING
+    install_grpcio_test
+    install_grpcio_health_checking
   fi
 }