Adding the tools directory to the git export.

tools/README.md

+buildgen: contains the template renderer for our build system.
+dockerfile: contains all of the docker files to test gRPC.
+gce_setup: contains boilerplate for running the docker files under GCE.
+run_tests: contains python scripts to properly run the tests in parallel.

tools/buildgen/bunch.py

+"""Allows dot-accessible dictionaries."""
+
+
+class Bunch(dict):
+
+  def __init__(self, d):
+    dict.__init__(self, d)
+    self.__dict__.update(d)
+
+
+# Converts any kind of variable to a Bunch
+def to_bunch(var):
+  if isinstance(var, list):
+    return [to_bunch(i) for i in var]
+  if isinstance(var, dict):
+    ret = {}
+    for k, v in var.items():
+      if isinstance(v, (list, dict)):
+        v = to_bunch(v)
+      ret[k] = v
+    return Bunch(ret)
+  else:
+    return var
+
+
+# Merges JSON 'add' into JSON 'dst'
+def merge_json(dst, add):
+  if isinstance(dst, dict) and isinstance(add, dict):
+    for k, v in add.items():
+      if k in dst:
+        merge_json(dst[k], v)
+      else:
+        dst[k] = v
+  elif isinstance(dst, list) and isinstance(add, list):
+    dst.extend(add)
+  else:
+    raise Exception('Tried to merge incompatible objects %r, %r' % (dst, add))

tools/buildgen/generate_projects.sh

+#!/bin/bash
+
+set -ex
+
+if [ "x$TEST" == "x" ] ; then
+  TEST=false
+fi
+
+
+cd `dirname $0`/..
+mako_renderer=tools/buildgen/mako_renderer.py
+gen_build_json=test/core/end2end/gen_build_json.py
+
+end2end_test_build=`mktemp`
+$gen_build_json > $end2end_test_build
+
+global_plugins=`find ./tools/buildgen/plugins -name '*.py' |
+  sort | grep -v __init__ |
+  while read p ; do echo -n "-p $p " ; done`
+
+for dir in . ; do
+  local_plugins=`find $dir/templates -name '*.py' |
+    sort | grep -v __init__ |
+    while read p ; do echo -n "-p $p " ; done`
+
+  plugins="$global_plugins $local_plugins"
+
+  find -L $dir/templates -type f -and -name *.template | while read file ; do
+    out=${dir}/${file#$dir/templates/}  # strip templates dir prefix
+    out=${out%.*}  # strip template extension
+    json_files="build.json $end2end_test_build"
+    data=`for i in $json_files; do echo -n "-d $i "; done`
+    if [ $TEST == true ] ; then
+      actual_out=$out
+      out=`mktemp`
+    else
+      g4 open $out || true
+    fi
+    $mako_renderer $plugins $data -o $out $file
+    if [ $TEST == true ] ; then
+      diff -q $out $actual_out
+      rm $out
+    fi
+  done
+done
+
+rm $end2end_test_build
+

tools/buildgen/mako_renderer.py

+#!/usr/bin/python
+
+"""Simple Mako renderer.
+
+Just a wrapper around the mako rendering library.
+
+"""
+
+import getopt
+import imp
+import os
+import sys
+
+
+from mako.lookup import TemplateLookup
+from mako.runtime import Context
+from mako.template import Template
+import simplejson
+import bunch
+
+
+# Imports a plugin
+def import_plugin(name):
+  _, base_ex = os.path.split(name)
+  base, _ = os.path.splitext(base_ex)
+
+  with open(name, 'r') as plugin_file:
+    plugin_code = plugin_file.read()
+  plugin_module = imp.new_module(base)
+  exec plugin_code in plugin_module.__dict__
+  return plugin_module
+
+
+def out(msg):
+  print >> sys.stderr, msg
+
+
+def showhelp():
+  out('mako-renderer.py [-o out] [-m cache] [-d dict] [-d dict...] template')
+
+
+def main(argv):
+  got_input = False
+  module_directory = None
+  dictionary = {}
+  json_dict = {}
+  got_output = False
+  output_file = sys.stdout
+  plugins = []
+
+  try:
+    opts, args = getopt.getopt(argv, 'hm:d:o:p:')
+  except getopt.GetoptError:
+    out('Unknown option')
+    showhelp()
+    sys.exit(2)
+
+  for opt, arg in opts:
+    if opt == '-h':
+      out('Displaying showhelp')
+      showhelp()
+      sys.exit()
+    elif opt == '-o':
+      if got_output:
+        out('Got more than one output')
+        showhelp()
+        sys.exit(3)
+      got_output = True
+      output_file = open(arg, 'w')
+    elif opt == '-m':
+      if module_directory is not None:
+        out('Got more than one cache directory')
+        showhelp()
+        sys.exit(4)
+      module_directory = arg
+    elif opt == '-d':
+      dict_file = open(arg, 'r')
+      bunch.merge_json(json_dict, simplejson.loads(dict_file.read()))
+      dict_file.close()
+    elif opt == '-p':
+      plugins.append(import_plugin(arg))
+
+  for plugin in plugins:
+    plugin.mako_plugin(json_dict)
+
+  for k, v in json_dict.items():
+    dictionary[k] = bunch.to_bunch(v)
+
+  ctx = Context(output_file, **dictionary)
+
+  for arg in args:
+    got_input = True
+    template = Template(filename=arg,
+                        module_directory=module_directory,
+                        lookup=TemplateLookup(directories=['.']))
+    template.render_context(ctx)
+
+  if not got_input:
+    out('Got nothing to do')
+    showhelp()
+
+  output_file.close()
+
+if __name__ == '__main__':
+  main(sys.argv[1:])

tools/buildgen/plugins/expand_filegroups.py

+"""Buildgen expand filegroups plugin.
+
+This takes the list of libs from our json dictionary,
+and expands any and all filegroup.
+
+"""
+
+
+def excluded(filename, exclude_res):
+  for r in exclude_res:
+    if r.search(filename):
+      return True
+  return False
+
+
+def mako_plugin(dictionary):
+  """The exported plugin code for expand_filegroups.
+
+  The list of libs in the build.json file can contain "filegroups" tags.
+  These refer to the filegroups in the root object. We will expand and
+  merge filegroups on the src, headers and public_headers properties.
+
+  """
+  libs = dictionary.get('libs')
+  filegroups_list = dictionary.get('filegroups')
+  filegroups = {}
+
+  for fg in filegroups_list:
+    filegroups[fg['name']] = fg
+
+  for lib in libs:
+    for fg_name in lib.get('filegroups', []):
+      fg = filegroups[fg_name]
+
+      src = lib.get('src', [])
+      src.extend(fg.get('src', []))
+      lib['src'] = src
+
+      headers = lib.get('headers', [])
+      headers.extend(fg.get('headers', []))
+      lib['headers'] = headers
+
+      public_headers = lib.get('public_headers', [])
+      public_headers.extend(fg.get('public_headers', []))
+      lib['public_headers'] = public_headers

tools/buildgen/plugins/list_protos.py

+"""Buildgen .proto files list plugin.
+
+This parses the list of targets from the json build file, and creates
+a list called "protos" that contains all of the proto file names.
+
+"""
+
+
+import re
+
+
+def mako_plugin(dictionary):
+  """The exported plugin code for list_protos.
+
+  Some projects generators may want to get the full list of unique .proto files
+  that are being included in a project. This code extracts all files referenced
+  in any library or target that ends in .proto, and builds and exports that as
+  a list called "protos".
+
+  """
+
+  libs = dictionary.get('libs', [])
+  targets = dictionary.get('targets', [])
+
+  proto_re = re.compile('(.*)\\.proto')
+
+  protos = set()
+  for lib in libs:
+    for src in lib.get('src', []):
+      m = proto_re.match(src)
+      if m:
+        protos.add(m.group(1))
+  for tgt in targets:
+    for src in tgt.get('src', []):
+      m = proto_re.match(src)
+      if m:
+        protos.add(m.group(1))
+
+  protos = sorted(protos)
+
+  dictionary['protos'] = protos

tools/dockerfile/grpc_base/Dockerfile

+# Base Dockerfile for gRPC dev images
+FROM debian:latest
+
+# Install Git.
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  curl \
+  gcc \
+  git \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libevent-dev \
+  libtool \
+  make \
+  strace \
+  python-dev \
+  python-setuptools \
+  telnet \
+  unzip \
+  wget \
+  zip && apt-get clean
+
+# Install useful useful python modules
+RUN easy_install -U pip
+RUN pip install -U crcmod  # makes downloads from cloud storage faster
+
+# Install GCloud
+RUN wget https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip \
+  && unzip google-cloud-sdk.zip && rm google-cloud-sdk.zip
+ENV CLOUD_SDK /google-cloud-sdk
+RUN $CLOUD_SDK/install.sh --usage-reporting=true --path-update=true --bash-completion=true --rc-path=/.bashrc --disable-installation-options
+ENV PATH $CLOUD_SDK/bin:$PATH
+
+# Install gcompute-tools to allow access to private git-on-borg repos
+RUN git clone https://gerrit.googlesource.com/gcompute-tools /var/local/git/gcompute-tools
+
+# Start the daemon that allows access to private git-on-borg repos
+RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+
+# Install the grpc-tools scripts dir from git
+RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-tools /var/local/git/grpc-tools
+
+# Install the grpc-protobuf dir that has the protoc patch
+RUN git clone https://team.googlesource.com/one-platform-grpc-team/protobuf /var/local/git/protobuf
+
+# Install the patched version of protoc
+RUN cd /var/local/git/protobuf && \
+  ./autogen.sh && \
+  ./configure --prefix=/usr && \
+  make && make check && make install && make clean
+
+# Define the default command.
+CMD ["bash"]

tools/dockerfile/grpc_base/README.md

+Base GRPC Dockerfile
+====================
+
+Dockerfile for creating the base gRPC development Docker instance.
+For now, this assumes that the development will be done on GCE instances, with source code on Git-on-Borg.
+
+As of 2014/09/29, it includes
+- git
+- some useful tools like curl, emacs, strace, telnet etc
+- downloads the gerrit-compute-tools and installs the script that allows access to gerrit when on git-on-borg
+- a patched version of protoc, to allow protos with stream tags to work

tools/dockerfile/grpc_cxx/Dockerfile

+# Dockerfile for gRPC C++
+FROM grpc/base
+
+# Start the daemon that allows access to the protected git-on-borg repos
+RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+
+RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc
+RUN cd /var/local/git/grpc \
+  && git pull --recurse-submodules \
+  && git submodule update --init --recursive
+
+RUN make install -C /var/local/git/grpc
+
+# Define the default command.
+CMD ["bash"]
\ No newline at end of file

tools/dockerfile/grpc_java/Dockerfile

+# Dockerfile for the gRPC Java dev image
+FROM grpc/java_base
+
+# Start the daemon that allows access to private git-on-borg repos
+RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+
+RUN  cd /var/local/git/grpc-java/lib/okhttp && \
+  mvn -pl okhttp -am install
+RUN  cd /var/local/git/grpc-java/lib/netty && \
+  mvn -pl codec-http2 -am -DskipTests install
+RUN cd /var/local/git/grpc-java && \
+  protoc --version>ver.txt && \
+  mvn install
+
+# Specify the default command such that the interop server runs on its known testing port
+CMD ["/var/local/git/grpc-java/run-test-server.sh", "--transport=NETTY_TLS", "--grpc_version=2", "--port=8030"]

tools/dockerfile/grpc_java/README.md

+GRPC Java Dockerfile
+====================
+
+Dockerfile for creating the Java development image
+
+As of 2014/12 this
+ - is based on the gRPC Java base
+ - pulls from gRPC Java on git-on-borg
+ - installs it and runs the tests
\ No newline at end of file

tools/dockerfile/grpc_java_base/Dockerfile

+# Base Dockerfile for the gRPC Java dev image
+FROM grpc/base
+
+RUN apt-get update && apt-get -y install java7-jdk
+
+# Install maven
+RUN wget http://mirror.olnevhost.net/pub/apache/maven/binaries/apache-maven-3.2.1-bin.tar.gz && \
+  tar xvf apache-maven-3.2.1-bin.tar.gz -C /var/local
+
+ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64
+ENV M2_HOME /var/local/apache-maven-3.2.1
+ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin
+ENV LD_LIBRARY_PATH /usr/local/lib
+
+# Start the daemon that allows access to the protected git-on-borg repos
+RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+
+RUN git clone --recursive https://team.googlesource.com/one-platform-grpc-team/grpc-java /var/local/git/grpc-java
+
+RUN cd /var/local/git/grpc-java/lib/okhttp && \
+  mvn -pl okhttp -am validate
+RUN cd /var/local/git/grpc-java/lib/netty && \
+  mvn -pl codec-http2 -am validate
+RUN cd /var/local/git/grpc-java && \
+  mvn validate
\ No newline at end of file

tools/dockerfile/grpc_java_base/README.md

+GRPC Java Base Dockerfile
+=========================
+
+Dockerfile for creating the Java gRPC development Docker instance.
+
+As of 2014/12 this
+ - installs tools and dependencies needed to build gRPC Java
+ - does not install gRPC Java itself; a separate Dockerfile that depends on
+   this one will do that.
\ No newline at end of file

tools/dockerfile/grpc_php/Dockerfile

+# Dockerfile for gRPC PHP
+FROM grpc/php_base
+
+# Start the daemon that allows access to the protected git-on-borg repos
+RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+
+RUN cd /var/local/git/grpc \
+  && git pull --recurse-submodules \
+  && git submodule update --init --recursive
+
+RUN make install_c -j12 -C /var/local/git/grpc
+
+RUN cd /var/local/git/grpc/src/php/ext/grpc && git pull && phpize
+
+# Build the grpc PHP extension
+RUN cd /var/local/git/grpc/src/php/ext/grpc \
+  && ./configure \
+  && make
\ No newline at end of file

tools/dockerfile/grpc_php/README.md

+GRPC PHP Dockerfile
+===================
+
+Dockerfile for creating the PHP development instances
+
+As of 2014/10 this
+- is based on the GRPC PHP base
+- adds a pull of the HEAD GRPC PHP source from git-on-borg
+- it builds it
+- runs the tests, i.e, the image won't be created if the tests don't pass

tools/dockerfile/grpc_php_base/Dockerfile

+# Base Dockerfile for gRPC PHP.
+#
+# Includes PHP installation dependencies, things that are unlikely to vary.
+FROM grpc/base
+
+# Install RVM dependencies and other packages
+RUN apt-get update && apt-get install -y \
+    autoconf \
+    automake \
+    bison \
+    curl \
+    g++ \
+    gawk \
+    gcc \
+    groff \
+    libc6-dev \
+    libffi-dev \
+    libgdbm-dev \
+    libncurses5-dev \
+    libreadline6-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libtool \
+    libyaml-dev \
+    make \
+    patch \
+    procps \
+# TODO(mlumish): Uncomment these lines when building against them works
+#    php5-common \
+#    php5-cli \
+#    php5-dev \
+#    php-pear \
+    pkg-config \
+    procps \
+    sqlite3 \
+    zlib1g-dev
+
+# Install the version of PHP gRPC is tested against
+ENV DEBIAN_FRONTEND noniteractive
+RUN apt-get update && apt-get install -y libxml2 libxml2-dev  # used by PHP
+RUN cd /var/local \
+  && curl -o php-5.5.17.tar.gz http://php.net/distributions/php-5.5.17.tar.gz \
+  && tar -xf php-5.5.17.tar.gz \
+  && cd php-5.5.17 \
+  && ./configure --with-zlib=/usr --with-libxml-dir=ext/libxml \
+  && make && make install
+
+# Start the daemon that allows access to the protected git-on-borg repos
+RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+
+# Download the patched PHP protobuf so that PHP gRPC clients can be generated
+# from proto3 schemas.
+RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-php-protobuf-php /var/local/git/protobuf-php
+
+# Install ruby (via RVM) as ruby tools are dependencies for building Protobuf
+# PHP extensions.
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3  # Needed for RVM
+RUN \curl -sSL https://get.rvm.io | bash -s stable --ruby
+ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+# ronn: a ruby tool used to convert markdown to man pages, used during the
+# install of Protobuf extensions
+#
+# rake: a ruby version of make used to build the PHP Protobuf extension
+RUN rvm all do gem install ronn rake
+
+# Install the patched PHP protobuf so that PHP gRPC clients can be generated
+# from proto3 schemas.
+RUN cd /var/local/git/protobuf-php \
+  && rvm all do rake pear:package version=1.0 \
+  && pear install Protobuf-1.0.tgz
+
+# Install PHPUnit, used to run the PHP unit tests
+RUN wget https://phar.phpunit.de/phpunit.phar \
+  && chmod +x phpunit.phar \
+  && mv phpunit.phar /usr/local/bin/phpunit
+
+RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc
+RUN cd /var/local/git/grpc \
+  && git submodule update --init --recursive
+
+RUN make static_c shared_c -j12 -C /var/local/git/grpc
+
+# Define the default command.
+CMD ["bash"]

tools/dockerfile/grpc_php_base/README.md

+GRPC PHP Base Dockerfile
+========================
+
+Dockerfile for creating the PHP gRPC development Docker instance.
+
+As of 2014/10 this
+- it installs tools and dependencies needed to build gRPC PHP
+- it does not install gRPC PHP itself; a separate Dockerfile that depends on
+  this one will do that

tools/dockerfile/grpc_ruby/Dockerfile

+# Dockerfile for gRPC Ruby
+FROM grpc/ruby_base
+
+RUN cd /var/local/git/grpc \
+  && git pull --recurse-submodules \
+  && git submodule update --init --recursive
+
+RUN make install_c -C /var/local/git/grpc
+
+# Install the grpc gem locally with its dependencies and build the extension.
+RUN /bin/bash -l -c 'cd /var/local/git/beefcake && bundle && gem build beefcake.gemspec && gem install beefcake'
+RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake compile:grpc && gem build grpc.gemspec && gem install grpc'
+
+# TODO add a command to run the unittest tests when the bug below is fixed
+# - the tests fail due to an error in the C threading library:
+#   they fail with 'ruby: __pthread_mutex_cond_lock_adjust for unknown reasons' at the end of a testcase
+# - however, the interop server and client run OK, so this bug can be investigated
+# RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake'
+
+# Specify the default command such that the interop server runs on its known testing port
+CMD ["/bin/bash", "-l", "-c", "ruby /var/local/git/grpc/src/ruby/bin/interop/interop_server.rb --port 8060"]

tools/dockerfile/grpc_ruby/README.md

+GRPC Ruby Dockerfile
+====================
+
+Dockerfile for creating the Ruby development instances
+
+As of 2014/10 this
+- is based on the GRPC Ruby base
+- adds a pull of the HEAD gRPC Ruby source from git-on-borg
+- it builds it
+- runs the tests, i.e, the image won't be created if the tests don't pass

tools/dockerfile/grpc_ruby_base/Dockerfile

+# Base Dockerfile for gRPC Ruby.
+#
+# Includes Ruby installation dependencies, things that are unlikely to vary.
+FROM grpc/base
+
+# Allows 'source' to work
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+
+# Install RVM dependencies
+RUN apt-get update && apt-get install -y \
+    autoconf \
+    automake \
+    bison \
+    curl \
+    g++ \
+    gawk \
+    gcc \
+    libc6-dev \
+    libffi-dev \
+    libgdbm-dev \
+    libncurses5-dev \
+    libreadline6-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libtool \
+    libyaml-dev \
+    make \
+    patch \
+    pkg-config \
+    procps \
+    sqlite3 \
+    zlib1g-dev
+
+
+# Start the daemon that allows access to the protected git-on-borg repos
+RUN /var/local/git/gcompute-tools/git-cookie-authdaemon
+
+# Download the patched Ruby protobuf (beefcake) so that Ruby gRPC clients can
+# be generated from proto3 schemas.
+RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-ruby-beefcake \
+              /var/local/git/beefcake
+
+# Install RVM, use this to install ruby
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3  # Needed for RVM
+RUN /bin/bash -l -c "curl -L get.rvm.io | bash -s stable"
+
+# Install Ruby 2.1
+RUN /bin/bash -l -c "rvm install ruby-2.1"
+RUN /bin/bash -l -c "rvm use --default ruby-2.1"
+RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
+RUN /bin/bash -l -c "echo 'source /home/grpc_ruby/.rvm/scripts/rvm' >> ~/.bashrc"
+RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc"
+RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
+
+RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc
+RUN cd /var/local/git/grpc \
+  && git submodule update --init --recursive
+
+RUN make static_c shared_c -C /var/local/git/grpc
\ No newline at end of file