diff --git a/templates/tools/dockerfile/clang_format.include b/templates/tools/dockerfile/clang_format.include new file mode 100644 index 0000000000000000000000000000000000000000..9a2b60ba8c47bd47906d2279203d472e7ed4234c --- /dev/null +++ b/templates/tools/dockerfile/clang_format.include @@ -0,0 +1,5 @@ +RUN apt-get update && apt-get -y install wget +RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list +RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN apt-get update && apt-get -y install clang-format-3.8 diff --git a/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template new file mode 100644 index 0000000000000000000000000000000000000000..8360fc121c1803828ef96e1e24e3bb942ecd1020 --- /dev/null +++ b/templates/tools/dockerfile/grpc_clang_format/Dockerfile.template @@ -0,0 +1,37 @@ +%YAML 1.2 +--- | + # Copyright 2015, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM ubuntu:15.10 + + <%include file="../clang_format.include"/> + ADD clang_format_all_the_things.sh / + CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"] + diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template index 01683539334cd0826ddba1d8192a65ab5c6b4b61..8617666b21eb2f36395794caf85fa90d4d330f5f 100644 --- a/templates/tools/dockerfile/test/sanity/Dockerfile.template +++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template @@ -52,18 +52,12 @@ # ./compile.sh without a local protoc dependency # TODO(mattkwong): install dependencies to support latest Bazel version if newer # version is needed - RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \ + RUN git clone https://github.com/bazelbuild/bazel.git /bazel && ${"\\"} cd /bazel && git checkout tags/0.4.1 && ./compile.sh RUN ln -s /bazel/output/bazel /bin/ - #=================== - # Docker "inception" - # Note this is quite the ugly hack. - # This makes sure that the docker binary we inject has its dependencies. - RUN curl https://get.docker.com/ | sh - RUN apt-get remove --purge -y docker-engine - - RUN mkdir /var/local/jenkins + <%include file="../../clang_format.include"/> + <%include file="../../run_tests_addons.include"/> # Define the default command. CMD ["bash"] diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index c0707051a60a81ffd08b15eb5919fa1c127e9efe..eb64e23daf760dc5b93603d6208a3e3bc86d162a 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -37,7 +37,7 @@ readonly PROTOBUF_INSTALL_PREFIX="$(mktemp -d)" pushd third_party/protobuf ./autogen.sh ./configure --prefix="$PROTOBUF_INSTALL_PREFIX" -make +make -j 8 make install #ldconfig popd @@ -51,7 +51,7 @@ fi # stack up and change to nanopb's proto generator directory pushd third_party/nanopb/generator/proto export PATH="$PROTOC_BIN_PATH:$PATH" -make +make -j 8 # back to the root directory popd diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh index 858e07489816a7f59f273e058c9d7310db2df319..13e018709fe93a06f2f86fbc83ef40e9a8931aee 100755 --- a/tools/distrib/clang_format_code.sh +++ b/tools/distrib/clang_format_code.sh @@ -32,9 +32,15 @@ set -ex # change to root directory cd $(dirname $0)/../.. +REPO_ROOT=$(pwd) -# build clang-format docker image -docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format +if [ "$CLANG_FORMAT_SKIP_DOCKER" == "" ] +then + # build clang-format docker image + docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format -# run clang-format against the checked out codebase -docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" --rm=true -v ${HOST_GIT_ROOT:-`pwd`}:/local-code -t grpc_clang_format /clang_format_all_the_things.sh + # run clang-format against the checked out codebase + docker run -e TEST=$TEST -e CHANGED_FILES="$CHANGED_FILES" -e CLANG_FORMAT_ROOT="/local-code" --rm=true -v "${REPO_ROOT}":/local-code -t grpc_clang_format /clang_format_all_the_things.sh +else + CLANG_FORMAT_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +fi diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile index ab58017a02af8937b6b1c393e16032c6250618b6..85f5e4db74a639388fb9458f986e45d9aea44ead 100644 --- a/tools/dockerfile/grpc_clang_format/Dockerfile +++ b/tools/dockerfile/grpc_clang_format/Dockerfile @@ -27,13 +27,13 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM ubuntu:wily -RUN apt-get update -RUN apt-get -y install wget +FROM ubuntu:15.10 + +RUN apt-get update && apt-get -y install wget RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - -RUN apt-get update -RUN apt-get -y install clang-format-3.8 +RUN apt-get update && apt-get -y install clang-format-3.8 + ADD clang_format_all_the_things.sh / CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"] diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh index 462c65ab5e9be20dcc582c7916b3391020b83cb7..c6e4aabfe63fcb88f4e961451699275f753c1bb5 100755 --- a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh +++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh @@ -44,7 +44,7 @@ for dir in $DIRS do for glob in $GLOB do - files="$files `find /local-code/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c -and -not -name *.pb.cc`" + files="$files `find ${CLANG_FORMAT_ROOT}/$dir -name $glob -and -not -name *.generated.* -and -not -name *.pb.h -and -not -name *.pb.c -and -not -name *.pb.cc`" done done @@ -54,7 +54,7 @@ if [ -n "$CHANGED_FILES" ]; then files=$(comm -12 <(echo $files | tr ' ' '\n' | sort -u) <(echo $CHANGED_FILES | tr ' ' '\n' | sort -u)) fi -if [ "x$TEST" = "x" ] +if [ "$TEST" == "" ] then echo $files | xargs $CLANG_FORMAT -i else diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 6b19ac845b4813598ab6ef889468c79cb06d6496..811384fda14f996fae5606e83af4cf852e6064d2 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -97,17 +97,27 @@ RUN apt-get install -y openjdk-8-jdk # ./compile.sh without a local protoc dependency # TODO(mattkwong): install dependencies to support latest Bazel version if newer # version is needed -RUN git clone https://github.com/bazelbuild/bazel.git /bazel && cd /bazel && git checkout tags/0.4.1 && ./compile.sh +RUN git clone https://github.com/bazelbuild/bazel.git /bazel && \ + cd /bazel && git checkout tags/0.4.1 && ./compile.sh RUN ln -s /bazel/output/bazel /bin/ -#=================== -# Docker "inception" -# Note this is quite the ugly hack. -# This makes sure that the docker binary we inject has its dependencies. -RUN curl https://get.docker.com/ | sh -RUN apt-get remove --purge -y docker-engine +RUN apt-get update && apt-get -y install wget +RUN echo deb http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list +RUN echo deb-src http://llvm.org/apt/wily/ llvm-toolchain-wily-3.8 main >> /etc/apt/sources.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key| apt-key add - +RUN apt-get update && apt-get -y install clang-format-3.8 + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + RUN mkdir /var/local/jenkins + # Define the default command. CMD ["bash"] diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index c3219c533d5ec834e55020fbda04d8ac675aab48..b68ac89121cb96948ec695f597cae04da860be97 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -77,8 +77,6 @@ docker run \ -v /tmp/ccache:/tmp/ccache \ -v /tmp/npm-cache:/tmp/npm-cache \ -v /tmp/xdg-cache-home:/tmp/xdg-cache-home \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v $(which docker):/bin/docker \ -w /var/local/git/grpc \ --name=$CONTAINER_NAME \ $DOCKER_IMAGE_NAME \ diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 1008c6b6cf578a4eb6eb01fc547ee6a072327e8f..924274191e9e8ec65b92468776e52517c774e431 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -820,8 +820,12 @@ class Sanity(object): def test_specs(self): import yaml with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f: + environ={'TEST': 'true'} + if _is_use_docker_child(): + environ['CLANG_FORMAT_SKIP_DOCKER'] = 'true' return [self.config.job_spec(cmd['script'].split(), - timeout_seconds=30*60, environ={'TEST': 'true'}, + timeout_seconds=30*60, + environ=environ, cpu_cost=cmd.get('cpu_cost', 1)) for cmd in yaml.load(f)]