Merge pull request #9176 from jtattermusch/runtests_refactor

Clean up tools/run_tests directory

.gitignore

@@ -96,7 +96,7 @@ DerivedData
 Pods/
 
 # Artifacts directory
-artifacts/
+/artifacts/
 
 # Git generated files for conflicting
 *.orig

tools/buildgen/generate_projects.py

@@ -36,7 +36,7 @@ import shutil
 import sys
 import tempfile
 import multiprocessing
-sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests'))
+sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests', 'python_utils'))
 
 assert sys.argv[1:], 'run generate_projects.sh instead of this directly'
 

tools/run_tests/prepare_travis.sh → tools/run_tests/artifacts/__init__.py

-#!/bin/bash
-# Copyright 2015, Google Inc.
+# Copyright 2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -27,41 +26,3 @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-cd `dirname $0`/../..
-grpc_dir=`pwd`
-
-distrib=`md5sum /etc/issue | cut -f1 -d\ `
-echo "Configuring for distribution $distrib"
-git submodule | while read sha path extra ; do
-  cd /tmp
-  name=`basename $path`
-  file=$name-$sha-$CONFIG-prebuilt-$distrib.tar.gz
-  echo -n "Looking for $file ..."
-  url=http://storage.googleapis.com/grpc-prebuilt-packages/$file
-  wget -q $url && (
-    echo " Found."
-    tar xfz $file
-  ) || echo " Not found."
-done
-
-mkdir -p bins/$CONFIG/protobuf
-mkdir -p libs/$CONFIG/protobuf
-mkdir -p libs/$CONFIG/openssl
-
-function cpt {
-  cp /tmp/prebuilt/$1 $2/$CONFIG/$3
-  touch $2/$CONFIG/$3/`basename $1`
-}
-
-if [ -e /tmp/prebuilt/bin/protoc ] ; then
-  touch third_party/protobuf/configure
-  cpt bin/protoc bins protobuf
-  cpt lib/libprotoc.a libs protobuf
-  cpt lib/libprotobuf.a libs protobuf
-fi
-
-if [ -e /tmp/prebuilt/lib/libssl.a ] ; then
-  cpt lib/libcrypto.a libs openssl
-  cpt lib/libssl.a libs openssl
-fi

tools/run_tests/artifact_targets.py → tools/run_tests/artifacts/artifact_targets.py

@@ -35,7 +35,8 @@ import random
 import string
 import sys
 
-import jobset
+sys.path.insert(0, os.path.abspath('..'))
+import python_utils.jobset as jobset
 
 
 def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
@@ -113,7 +114,7 @@ class PythonArtifact:
       environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE'
       return create_docker_jobspec(self.name,
           'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch,
-          'tools/run_tests/build_artifact_python.sh',
+          'tools/run_tests/artifacts/build_artifact_python.sh',
           environ=environ,
           timeout_seconds=60*60)
     elif self.platform == 'windows':
@@ -125,7 +126,7 @@ class PythonArtifact:
       # seed.  We create a random temp-dir here
       dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10))
       return create_jobspec(self.name,
-                            ['tools\\run_tests\\build_artifact_python.bat',
+                            ['tools\\run_tests\\artifacts\\build_artifact_python.bat',
                              self.py_version,
                              '32' if self.arch == 'x86' else '64',
                              dir
@@ -136,7 +137,7 @@ class PythonArtifact:
       environ['PYTHON'] = self.py_version
       environ['SKIP_PIP_INSTALL'] = 'TRUE'
       return create_jobspec(self.name,
-                            ['tools/run_tests/build_artifact_python.sh'],
+                            ['tools/run_tests/artifacts/build_artifact_python.sh'],
                             environ=environ)
 
   def __str__(self):
@@ -165,11 +166,11 @@ class RubyArtifact:
           environ['SETARCH_CMD'] = 'linux32'
         return create_docker_jobspec(self.name,
             'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
-            'tools/run_tests/build_artifact_ruby.sh',
+            'tools/run_tests/artifacts/build_artifact_ruby.sh',
             environ=environ)
       else:
         return create_jobspec(self.name,
-                              ['tools/run_tests/build_artifact_ruby.sh'])
+                              ['tools/run_tests/artifacts/build_artifact_ruby.sh'])
 
 
 class CSharpExtArtifact:
@@ -184,7 +185,7 @@ class CSharpExtArtifact:
   def pre_build_jobspecs(self):
     if self.platform == 'windows':
       return [create_jobspec('prebuild_%s' % self.name,
-                             ['tools\\run_tests\\pre_build_c.bat'],
+                             ['tools\\run_tests\\helper_scripts\\pre_build_c.bat'],
                              shell=True,
                              flake_retries=5,
                              timeout_retries=2)]
@@ -195,7 +196,7 @@ class CSharpExtArtifact:
     if self.platform == 'windows':
       msbuild_platform = 'Win32' if self.arch == 'x86' else self.arch
       return create_jobspec(self.name,
-                            ['tools\\run_tests\\build_artifact_csharp.bat',
+                            ['tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
                              'vsprojects\\grpc_csharp_ext.sln',
                              '/p:Configuration=Release',
                              '/p:PlatformToolset=v120',
@@ -210,14 +211,14 @@ class CSharpExtArtifact:
       if self.platform == 'linux':
         return create_docker_jobspec(self.name,
             'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
-            'tools/run_tests/build_artifact_csharp.sh',
+            'tools/run_tests/artifacts/build_artifact_csharp.sh',
             environ=environ)
       else:
         archflag = _ARCH_FLAG_MAP[self.arch]
         environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG)
         environ['LDFLAGS'] += ' %s' % archflag
         return create_jobspec(self.name,
-                              ['tools/run_tests/build_artifact_csharp.sh'],
+                              ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
                               environ=environ)
 
   def __str__(self):
@@ -245,7 +246,7 @@ class NodeExtArtifact:
   def build_jobspec(self):
     if self.platform == 'windows':
       return create_jobspec(self.name,
-                            ['tools\\run_tests\\build_artifact_node.bat',
+                            ['tools\\run_tests\\artifacts\\build_artifact_node.bat',
                              self.gyp_arch],
                             shell=True)
     else:
@@ -253,10 +254,10 @@ class NodeExtArtifact:
         return create_docker_jobspec(
             self.name,
             'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch),
-            'tools/run_tests/build_artifact_node.sh {}'.format(self.gyp_arch))
+            'tools/run_tests/artifacts/build_artifact_node.sh {}'.format(self.gyp_arch))
       else:
         return create_jobspec(self.name,
-                              ['tools/run_tests/build_artifact_node.sh',
+                              ['tools/run_tests/artifacts/build_artifact_node.sh',
                                self.gyp_arch])
 
 class PHPArtifact:
@@ -276,10 +277,10 @@ class PHPArtifact:
       return create_docker_jobspec(
           self.name,
           'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch),
-          'tools/run_tests/build_artifact_php.sh')
+          'tools/run_tests/artifacts/build_artifact_php.sh')
     else:
       return create_jobspec(self.name,
-                            ['tools/run_tests/build_artifact_php.sh'])
+                            ['tools/run_tests/artifacts/build_artifact_php.sh'])
 
 class ProtocArtifact:
   """Builds protoc and protoc-plugin artifacts"""
@@ -306,18 +307,18 @@ class ProtocArtifact:
       if self.platform == 'linux':
         return create_docker_jobspec(self.name,
             'tools/dockerfile/grpc_artifact_protoc',
-            'tools/run_tests/build_artifact_protoc.sh',
+            'tools/run_tests/artifacts/build_artifact_protoc.sh',
             environ=environ)
       else:
         environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
         return create_jobspec(self.name,
-            ['tools/run_tests/build_artifact_protoc.sh'],
+            ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
             environ=environ)
     else:
       generator = 'Visual Studio 12 Win64' if self.arch == 'x64' else 'Visual Studio 12' 
       vcplatform = 'x64' if self.arch == 'x64' else 'Win32'
       return create_jobspec(self.name,
-                            ['tools\\run_tests\\build_artifact_protoc.bat'],
+                            ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
                             environ={'generator': generator,
                                      'Platform': vcplatform})
 

tools/run_tests/build_artifact_csharp.sh → tools/run_tests/artifacts/build_artifact_csharp.sh

@@ -30,7 +30,7 @@
 
 set -ex
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 
 make grpc_csharp_ext
 

tools/run_tests/build_artifact_node.sh → tools/run_tests/artifacts/build_artifact_node.sh

@@ -34,7 +34,7 @@ source ~/.nvm/nvm.sh
 nvm use 4
 set -ex
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 
 rm -rf build || true
 

tools/run_tests/build_artifact_php.sh → tools/run_tests/artifacts/build_artifact_php.sh

@@ -31,7 +31,7 @@
 PHP_TARGET_ARCH=$1
 set -ex
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 
 mkdir -p artifacts
 

tools/run_tests/build_artifact_protoc.bat → tools/run_tests/artifacts/build_artifact_protoc.bat

@@ -34,7 +34,7 @@ cd third_party/protobuf/cmake
 
 mkdir build & cd build
 mkdir solution & cd solution
-cmake -G "%generator%" -Dprotobuf_BUILD_TESTS=OFF ../.. || goto :error
+cmake -G "%generator%" -Dprotobuf_BUILD_TESTS=OFF ../../.. || goto :error
 endlocal
 
 call vsprojects/build_plugins.bat || goto :error

tools/run_tests/build_artifact_protoc.sh → tools/run_tests/artifacts/build_artifact_protoc.sh

@@ -33,7 +33,7 @@ source scl_source enable devtoolset-1.1
 
 set -ex
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 
 make plugins
 

tools/run_tests/build_artifact_python.sh → tools/run_tests/artifacts/build_artifact_python.sh

@@ -30,7 +30,7 @@
 
 set -ex
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 
 export GRPC_PYTHON_USE_CUSTOM_BDIST=0
 export GRPC_PYTHON_BUILD_WITH_CYTHON=1

tools/run_tests/build_artifact_ruby.sh → tools/run_tests/artifacts/build_artifact_ruby.sh

@@ -31,7 +31,7 @@ set -ex
 
 SYSTEM=`uname | cut -f 1 -d_`
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 set +ex
 [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
 [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm"

tools/run_tests/build_package_node.sh → tools/run_tests/artifacts/build_package_node.sh

@@ -33,7 +33,7 @@ source ~/.nvm/nvm.sh
 nvm use 4
 set -ex
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 
 base=$(pwd)
 

tools/run_tests/build_package_php.sh → tools/run_tests/artifacts/build_package_php.sh

@@ -30,7 +30,7 @@
 
 set -ex
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 
 mkdir -p artifacts/
 cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=php,platform={windows,linux,macos}/artifacts/* artifacts/ || true

tools/run_tests/build_package_python.sh → tools/run_tests/artifacts/build_package_python.sh

@@ -30,7 +30,7 @@
 
 set -ex
 
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../..
 
 mkdir -p artifacts/