Merge pull request #5971 from sreecha/stress_test_flexible_config

Stress test flexible config

templates/tools/dockerfile/gcp_api_libraries.include

+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+

templates/tools/dockerfile/grpc_interop_stress_cxx/Dockerfile.template

+%YAML 1.2
+--- |
+  # Copyright 2015-2016, Google Inc.
+  # All rights reserved.
+  #
+  # Redistribution and use in source and binary forms, with or without
+  # modification, are permitted provided that the following conditions are
+  # met:
+  #
+  #     * Redistributions of source code must retain the above copyright
+  # notice, this list of conditions and the following disclaimer.
+  #     * Redistributions in binary form must reproduce the above
+  # copyright notice, this list of conditions and the following disclaimer
+  # in the documentation and/or other materials provided with the
+  # distribution.
+  #     * Neither the name of Google Inc. nor the names of its
+  # contributors may be used to endorse or promote products derived from
+  # this software without specific prior written permission.
+  #
+  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  
+  FROM debian:jessie
+  
+  <%include file="../apt_get_basic.include"/>
+  <%include file="../ccache_setup.include"/>
+  <%include file="../cxx_deps.include"/>
+  <%include file="../gcp_api_libraries.include"/>
+  <%include file="../clang_update.include"/>
+  # Define the default command.
+  CMD ["bash"]

tools/dockerfile/grpc_interop_stress_cxx/Dockerfile

@@ -27,12 +27,9 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# A work-in-progress Dockerfile that allows running gRPC test suites
-# inside a docker container.
-
 FROM debian:jessie
 
-# Install Git.
+# Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
   autoconf \
   autotools-dev \
@@ -43,13 +40,16 @@ RUN apt-get update && apt-get install -y \
   gcc \
   gcc-multilib \
   git \
+  golang \
   gyp \
+  lcov \
   libc6 \
   libc6-dbg \
   libc6-dev \
   libgtest-dev \
   libtool \
   make \
+  perl \
   strace \
   python-dev \
   python-setuptools \
@@ -59,7 +59,9 @@ RUN apt-get update && apt-get install -y \
   wget \
   zip && apt-get clean
 
-RUN easy_install -U pip
+#================
+# Build profiling
+RUN apt-get update && apt-get install -y time && apt-get clean
 
 # Prepare ccache
 RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
@@ -69,12 +71,47 @@ RUN ln -s /usr/bin/ccache /usr/local/bin/c++
 RUN ln -s /usr/bin/ccache /usr/local/bin/clang
 RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
 
-##################
+#=================
 # C++ dependencies
-RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang
+RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean
 
-# Google Cloud platform API libraries (for BigQuery)
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
 RUN pip install --upgrade google-api-python-client
 
+
+#=================
+# Update clang to a version with improved tsan
+
+RUN apt-get update && apt-get -y install python cmake && apt-get clean
+
+RUN git clone -n -b release_38 http://llvm.org/git/llvm.git && \
+  cd llvm && git checkout ad57503 && cd ..
+RUN git clone -n -b release_38 http://llvm.org/git/clang.git && \
+  cd clang && git checkout ad2c56e && cd ..
+RUN git clone -n -b release_38 http://llvm.org/git/compiler-rt.git && \
+  cd compiler-rt && git checkout 3176922 && cd ..
+RUN git clone -n -b release_38 \
+  http://llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && \
+  git checkout c288525 && cd ..
+RUN git clone -n -b release_38 http://llvm.org/git/libcxx.git && \
+  cd libcxx && git checkout fda3549  && cd ..
+RUN git clone -n -b release_38 http://llvm.org/git/libcxxabi.git && \
+  cd libcxxabi && git checkout 8d4e51d && cd ..
+
+RUN mv clang llvm/tools
+RUN mv compiler-rt llvm/projects
+RUN mv clang-tools-extra llvm/tools/clang/tools
+RUN mv libcxx llvm/projects
+RUN mv libcxxabi llvm/projects
+
+RUN mkdir llvm-build
+RUN cd llvm-build && cmake \
+  -DCMAKE_BUILD_TYPE:STRING=Release \
+  -DCMAKE_INSTALL_PREFIX:STRING=/usr \
+  -DLLVM_TARGETS_TO_BUILD:STRING=X86 \
+  ../llvm
+RUN make -C llvm-build && make -C llvm-build install && rm -rf llvm-build
+
 # Define the default command.
 CMD ["bash"]

tools/dockerfile/grpc_interop_stress_cxx/build_interop_stress.sh

@@ -41,5 +41,7 @@ cd /var/local/git/grpc
 
 make install-certs
 
+BUILD_TYPE=${BUILD_TYPE:=opt}
+
 # build C++ interop stress client, interop client and server
-make stress_test metrics_client interop_client interop_server
+make CONFIG=$BUILD_TYPE stress_test metrics_client interop_client interop_server

tools/jenkins/build_interop_stress_image.sh

@@ -34,10 +34,12 @@
 set -x
 
 # Params:
-#  INTEROP_IMAGE - name of tag of the final interop image
+#  INTEROP_IMAGE - Name of tag of the final interop image
 #  INTEROP_IMAGE_TAG - Optional. If set, the created image will be tagged using
 #    the command: 'docker tag $INTEROP_IMAGE $INTEROP_IMAGE_REPOSITORY_TAG'
-#  BASE_NAME - base name used to locate the base Dockerfile and build script
+#  BASE_NAME - Base name used to locate the base Dockerfile and build script
+#  BUILD_TYPE - The 'CONFIG' variable passed to the 'make' command (example:
+#  asan, tsan. Default value: opt).
 #  TTY_FLAG - optional -t flag to make docker allocate tty
 #  BUILD_INTEROP_DOCKER_EXTRA_ARGS - optional args to be passed to the
 #    docker run command
@@ -71,6 +73,7 @@ CONTAINER_NAME="build_${BASE_NAME}_$(uuidgen)"
 (docker run \
   -e CCACHE_DIR=/tmp/ccache \
   -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+  -e BUILD_TYPE=${BUILD_TYPE:=opt} \
   -i $TTY_FLAG \
   $MOUNT_ARGS \
   $BUILD_INTEROP_DOCKER_EXTRA_ARGS \

tools/run_tests/stress_test/README.md

@@ -67,8 +67,10 @@ The script has several parameters and you can find out more details by using the
   - `<grpc_root_dir>$ tools/run_tests/stress_test/run_stress_tests_on_gke.py --help`
 
 > **Example**
-> `$ tools/run_tests/stress_test/run_stress_tests_on_gke.py --project_id=sree-gce --test_duration_secs=180 --num_clients=5`
+> ```bash
+> $ # Change to the grpc root directory
+> $ cd $GRPC_ROOT
+> $ tools/run_tests/stress_test/run_on_gke.py --project_id=sree-gce --config_file=tools/run_tests/stress_test/configs/opt.json
+> ```
 
-> Launches the 5 instances of stress test clients, 1 instance of stress test server and runs the test for 180 seconds. The test would be run on the default container cluster (that you have set in `gcloud`) in the project `sree-gce`.
-
-> Note: we currently do not have the ability to launch multiple instances of the server. This can be added very easily in future
+> The above runs the stress test on GKE under the project `sree-gce` in the default cluster (that you set by `gcloud` command earlier). The test settings (like number of client instances, servers, the parmeters to pass, test cases etc) are all loaded from the config file `$GRPC_ROOT/tools/run_tests/stress_test/opt.json`

tools/run_tests/stress_test/configs/opt-tsan-asan.json

+{
+  "dockerImages": {
+    "grpc_stress_cxx_opt" : {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_cxx",
+      "buildType": "opt"
+    },
+    "grpc_stress_cxx_tsan": {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_cxx",
+      "buildType": "tsan"
+    },
+    "grpc_stress_cxx_asan": {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_cxx",
+      "buildType": "asan"
+    }
+  },
+
+  "clientTemplates": {
+    "baseTemplates": {
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
+        "pollIntervalSecs": 60,
+        "clientArgs": {
+          "num_channels_per_server":5,
+          "num_stubs_per_channel":10,
+          "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
+          "metrics_port": 8081,
+          "metrics_collection_interval_secs":60
+        },
+        "metricsPort": 8081,
+        "metricsArgs": {
+          "metrics_server_address": "localhost:8081",
+          "total_only": "true"
+        }
+      }
+    },
+    "templates": {
+      "cxx_client_opt": {
+        "baseTemplate": "default",
+        "clientImagePath": "/var/local/git/grpc/bins/opt/stress_test",
+        "metricsClientImagePath": "/var/local/git/grpc/bins/opt/metrics_client"
+      },
+      "cxx_client_tsan": {
+        "baseTemplate": "default",
+        "clientImagePath": "/var/local/git/grpc/bins/tsan/stress_test",
+        "metricsClientImagePath": "/var/local/git/grpc/bins/tsan/metrics_client"
+      },
+    "cxx_client_asan": {
+        "baseTemplate": "default",
+        "clientImagePath": "/var/local/git/grpc/bins/asan/stress_test",
+        "metricsClientImagePath": "/var/local/git/grpc/bins/asan/metrics_client"
+      }
+    }
+  },
+
+  "serverTemplates": {
+    "baseTemplates":{
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
+        "serverPort": 8080,
+        "serverArgs": {
+          "port": 8080
+        }
+      }
+    },
+    "templates": {
+      "cxx_server_opt": {
+        "baseTemplate": "default",
+        "serverImagePath": "/var/local/git/grpc/bins/opt/interop_server"
+      },
+      "cxx_server_tsan": {
+        "baseTemplate": "default",
+        "serverImagePath": "/var/local/git/grpc/bins/tsan/interop_server"
+      },
+    "cxx_server_asan": {
+        "baseTemplate": "default",
+        "serverImagePath": "/var/local/git/grpc/bins/asan/interop_server"
+      }
+    }
+  },
+
+  "testMatrix": {
+    "serverPodSpecs": {
+      "stress-server-opt": {
+        "serverTemplate": "cxx_server_opt",
+        "dockerImage": "grpc_stress_cxx_opt",
+        "numInstances": 1
+      },
+      "stress-server-tsan": {
+        "serverTemplate": "cxx_server_tsan",
+        "dockerImage": "grpc_stress_cxx_tsan",
+        "numInstances": 1
+      },
+      "stress-server-asan": {
+        "serverTemplate": "cxx_server_asan",
+        "dockerImage": "grpc_stress_cxx_asan",
+        "numInstances": 1
+      }
+   },
+
+    "clientPodSpecs": {
+      "stress-client-opt": {
+        "clientTemplate": "cxx_client_opt",
+        "dockerImage": "grpc_stress_cxx_opt",
+        "numInstances": 3,
+        "serverPodSpec": "stress-server-opt"
+      },
+      "stress-client-tsan": {
+        "clientTemplate": "cxx_client_tsan",
+        "dockerImage": "grpc_stress_cxx_tsan",
+        "numInstances": 3,
+        "serverPodSpec": "stress-server-tsan"
+      },
+      "stress-client-asan": {
+        "clientTemplate": "cxx_client_asan",
+        "dockerImage": "grpc_stress_cxx_asan",
+        "numInstances": 3,
+        "serverPodSpec": "stress-server-asan"
+      }
+    }
+  },
+
+  "globalSettings": {
+    "buildDockerImages": true,
+    "pollIntervalSecs": 60,
+    "testDurationSecs": 7200,
+    "kubernetesProxyPort": 8001,
+    "datasetIdNamePrefix": "stress_test_opt_tsan",
+    "summaryTableId": "summary",
+    "qpsTableId": "qps",
+    "podWarmupSecs": 60
+  }
+}

tools/run_tests/stress_test/configs/opt.json

+{
+  "dockerImages": {
+    "grpc_stress_cxx_opt" : {
+      "buildScript": "tools/jenkins/build_interop_stress_image.sh",
+      "dockerFileDir": "grpc_interop_stress_cxx",
+      "buildType": "opt"
+    }
+  },
+
+  "clientTemplates": {
+    "baseTemplates": {
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
+        "pollIntervalSecs": 60,
+        "clientArgs": {
+          "num_channels_per_server":5,
+          "num_stubs_per_channel":10,
+          "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
+          "metrics_port": 8081,
+          "metrics_collection_interval_secs":60
+        },
+        "metricsPort": 8081,
+        "metricsArgs": {
+          "metrics_server_address": "localhost:8081",
+          "total_only": "true"
+        }
+      }
+    },
+    "templates": {
+      "cxx_client_opt": {
+        "baseTemplate": "default",
+        "clientImagePath": "/var/local/git/grpc/bins/opt/stress_test",
+        "metricsClientImagePath": "/var/local/git/grpc/bins/opt/metrics_client"
+      }
+    }
+  },
+
+  "serverTemplates": {
+    "baseTemplates":{
+      "default": {
+        "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
+        "serverPort": 8080,
+        "serverArgs": {
+          "port": 8080
+        }
+      }
+    },
+    "templates": {
+      "cxx_server_opt": {
+        "baseTemplate": "default",
+        "serverImagePath": "/var/local/git/grpc/bins/opt/interop_server"
+      }
+    }
+  },
+
+  "testMatrix": {
+    "serverPodSpecs": {
+      "stress-server-opt": {
+        "serverTemplate": "cxx_server_opt",
+        "dockerImage": "grpc_stress_cxx_opt",
+        "numInstances": 1
+      }
+    },
+
+    "clientPodSpecs": {
+      "stress-client-opt": {
+        "clientTemplate": "cxx_client_opt",
+        "dockerImage": "grpc_stress_cxx_opt",
+        "numInstances": 10,
+        "serverPodSpec": "stress-server-opt"
+      }
+    }
+  },
+
+  "globalSettings": {
+    "buildDockerImages": true,
+    "pollIntervalSecs": 10,
+    "testDurationSecs": 120,
+    "kubernetesProxyPort": 8001,
+    "datasetIdNamePrefix": "stress_test_opt",
+    "summaryTableId": "summary",
+    "qpsTableId": "qps",
+    "podWarmupSecs": 60
+  }
+}
+