Add HTTP/2 Interop tests to be run

tools/http2_interop/http2interop.go

@@ -2,15 +2,38 @@ package http2interop
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"io"
-	"log"
+	"net"
+	"testing"
+	"time"
 )
 
 const (
 	Preface = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
 )
 
+var (
+	defaultTimeout = 1 * time.Second
+)
+
+type HTTP2InteropCtx struct {
+	// Inputs
+	ServerHost             string
+	ServerPort             int
+	UseTLS                 bool
+	UseTestCa              bool
+	ServerHostnameOverride string
+
+	T *testing.T
+
+	// Derived
+	serverSpec string
+	authority  string
+	rootCAs    *x509.CertPool
+}
+
 func parseFrame(r io.Reader) (Frame, error) {
 	fh := FrameHeader{}
 	if err := fh.Parse(r); err != nil {
@@ -49,22 +72,8 @@ func streamFrame(w io.Writer, f Frame) error {
 	return nil
 }
 
-func getHttp2Conn(addr string) (*tls.Conn, error) {
-	config := &tls.Config{
-		InsecureSkipVerify: true,
-		NextProtos:         []string{"h2"},
-	}
-
-	conn, err := tls.Dial("tcp", addr, config)
-	if err != nil {
-		return nil, err
-	}
-
-	return conn, nil
-}
-
-func testClientShortSettings(addr string, length int) error {
-	c, err := getHttp2Conn(addr)
+func testClientShortSettings(ctx *HTTP2InteropCtx, length int) error {
+	c, err := connect(ctx)
 	if err != nil {
 		return err
 	}
@@ -82,22 +91,22 @@ func testClientShortSettings(addr string, length int) error {
 		Data: make([]byte, length),
 	}
 	if err := streamFrame(c, sf); err != nil {
+		ctx.T.Log("Unable to stream frame", sf)
 		return err
 	}
 
 	for {
-		frame, err := parseFrame(c)
-		if err != nil {
+		if _, err := parseFrame(c); err != nil {
+			ctx.T.Log("Unable to parse frame")
 			return err
 		}
-		log.Println(frame)
 	}
 
 	return nil
 }
 
-func testClientPrefaceWithStreamId(addr string) error {
-	c, err := getHttp2Conn(addr)
+func testClientPrefaceWithStreamId(ctx *HTTP2InteropCtx) error {
+	c, err := connect(ctx)
 	if err != nil {
 		return err
 	}
@@ -119,18 +128,16 @@ func testClientPrefaceWithStreamId(addr string) error {
 	}
 
 	for {
-		frame, err := parseFrame(c)
-		if err != nil {
+		if _, err := parseFrame(c); err != nil {
 			return err
 		}
-		log.Println(frame)
 	}
 
 	return nil
 }
 
-func testUnknownFrameType(addr string) error {
-	c, err := getHttp2Conn(addr)
+func testUnknownFrameType(ctx *HTTP2InteropCtx) error {
+	c, err := connect(ctx)
 	if err != nil {
 		return err
 	}
@@ -143,6 +150,7 @@ func testUnknownFrameType(addr string) error {
 	// Send some settings, which are part of the client preface
 	sf := &SettingsFrame{}
 	if err := streamFrame(c, sf); err != nil {
+		ctx.T.Log("Unable to stream frame", sf)
 		return err
 	}
 
@@ -154,6 +162,7 @@ func testUnknownFrameType(addr string) error {
 			},
 		}
 		if err := streamFrame(c, fh); err != nil {
+			ctx.T.Log("Unable to stream frame", fh)
 			return err
 		}
 	}
@@ -162,12 +171,14 @@ func testUnknownFrameType(addr string) error {
 		Data: []byte("01234567"),
 	}
 	if err := streamFrame(c, pf); err != nil {
+		ctx.T.Log("Unable to stream frame", sf)
 		return err
 	}
 
 	for {
 		frame, err := parseFrame(c)
 		if err != nil {
+			ctx.T.Log("Unable to parse frame")
 			return err
 		}
 		if npf, ok := frame.(*PingFrame); !ok {
@@ -183,8 +194,8 @@ func testUnknownFrameType(addr string) error {
 	return nil
 }
 
-func testShortPreface(addr string, prefacePrefix string) error {
-	c, err := getHttp2Conn(addr)
+func testShortPreface(ctx *HTTP2InteropCtx, prefacePrefix string) error {
+	c, err := connect(ctx)
 	if err != nil {
 		return err
 	}
@@ -201,17 +212,15 @@ func testShortPreface(addr string, prefacePrefix string) error {
 	return err
 }
 
-func testTLSMaxVersion(addr string, version uint16) error {
-	config := &tls.Config{
-		InsecureSkipVerify: true,
-		NextProtos:         []string{"h2"},
-		MaxVersion:         version,
-	}
-	conn, err := tls.Dial("tcp", addr, config)
+func testTLSMaxVersion(ctx *HTTP2InteropCtx, version uint16) error {
+	config := buildTlsConfig(ctx)
+	config.MaxVersion = version
+	conn, err := connectWithTls(ctx, config)
 	if err != nil {
 		return err
 	}
 	defer conn.Close()
+	conn.SetDeadline(time.Now().Add(defaultTimeout))
 
 	buf := make([]byte, 256)
 	if n, err := conn.Read(buf); err != nil {
@@ -223,16 +232,15 @@ func testTLSMaxVersion(addr string, version uint16) error {
 	return nil
 }
 
-func testTLSApplicationProtocol(addr string) error {
-	config := &tls.Config{
-		InsecureSkipVerify: true,
-		NextProtos:         []string{"h2c"},
-	}
-	conn, err := tls.Dial("tcp", addr, config)
+func testTLSApplicationProtocol(ctx *HTTP2InteropCtx) error {
+	config := buildTlsConfig(ctx)
+	config.NextProtos = []string{"h2c"}
+	conn, err := connectWithTls(ctx, config)
 	if err != nil {
 		return err
 	}
 	defer conn.Close()
+	conn.SetDeadline(time.Now().Add(defaultTimeout))
 
 	buf := make([]byte, 256)
 	if n, err := conn.Read(buf); err != nil {
@@ -243,3 +251,48 @@ func testTLSApplicationProtocol(addr string) error {
 	}
 	return nil
 }
+
+func connect(ctx *HTTP2InteropCtx) (net.Conn, error) {
+	var conn net.Conn
+	var err error
+	if !ctx.UseTLS {
+		conn, err = connectWithoutTls(ctx)
+	} else {
+		config := buildTlsConfig(ctx)
+		conn, err = connectWithTls(ctx, config)
+	}
+	if err != nil {
+		return nil, err
+	}
+	conn.SetDeadline(time.Now().Add(defaultTimeout))
+
+	return conn, nil
+}
+
+func buildTlsConfig(ctx *HTTP2InteropCtx) *tls.Config {
+	return &tls.Config{
+		RootCAs:    ctx.rootCAs,
+		NextProtos: []string{"h2"},
+		ServerName: ctx.authority,
+		MinVersion: tls.VersionTLS12,
+		// TODO(carl-mastrangelo): remove this once all test certificates have been updated.
+		InsecureSkipVerify: true,
+	}
+}
+
+func connectWithoutTls(ctx *HTTP2InteropCtx) (net.Conn, error) {
+	conn, err := net.DialTimeout("tcp", ctx.serverSpec, defaultTimeout)
+	if err != nil {
+		return nil, err
+	}
+	return conn, nil
+}
+
+func connectWithTls(ctx *HTTP2InteropCtx, config *tls.Config) (*tls.Conn, error) {
+	conn, err := connectWithoutTls(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return tls.Client(conn, config), nil
+}

tools/http2_interop/http2interop_test.go

@@ -2,46 +2,117 @@ package http2interop
 
 import (
 	"crypto/tls"
+	"crypto/x509"
+	"strings"
 	"flag"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"strconv"
 	"testing"
 )
 
 var (
-	serverSpec = flag.String("spec", ":50051", "The server spec to test")
+	serverHost = flag.String("server_host", "", "The host to test")
+	serverPort = flag.Int("server_port", 443, "The port to test")
+	useTls     = flag.Bool("use_tls", true, "Should TLS tests be run")
+	// TODO: implement
+	testCase              = flag.String("test_case", "", "What test cases to run")
+
+	// The rest of these are unused, but present to fulfill the client interface
+	serverHostOverride    = flag.String("server_host_override", "", "Unused")
+	useTestCa             = flag.Bool("use_test_ca", false, "Unused")
+	defaultServiceAccount = flag.String("default_service_account", "", "Unused")
+	oauthScope            = flag.String("oauth_scope", "", "Unused")
+	serviceAccountKeyFile = flag.String("service_account_key_file", "", "Unused")
 )
 
+func InteropCtx(t *testing.T) *HTTP2InteropCtx {
+	ctx := &HTTP2InteropCtx{
+		ServerHost:             *serverHost,
+		ServerPort:             *serverPort,
+		ServerHostnameOverride: *serverHostOverride,
+		UseTLS:                 *useTls,
+		UseTestCa:              *useTestCa,
+		T:                      t,
+	}
+
+	ctx.serverSpec = ctx.ServerHost
+	if ctx.ServerPort != -1 {
+		ctx.serverSpec += ":" + strconv.Itoa(ctx.ServerPort)
+	}
+	if ctx.ServerHostnameOverride == "" {
+		ctx.authority = ctx.ServerHost
+	} else {
+		ctx.authority = ctx.ServerHostnameOverride
+	}
+
+	if ctx.UseTestCa {
+		// It would be odd if useTestCa was true, but not useTls.  meh
+		certData, err := ioutil.ReadFile("src/core/tsi/test_creds/ca.pem")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		ctx.rootCAs = x509.NewCertPool()
+		if !ctx.rootCAs.AppendCertsFromPEM(certData) {
+			t.Fatal(fmt.Errorf("Unable to parse pem data"))
+		}
+	}
+
+	return ctx
+}
+
+func (ctx *HTTP2InteropCtx) Close() error {
+	// currently a noop
+	return nil
+}
+
 func TestShortPreface(t *testing.T) {
+	ctx := InteropCtx(t)
 	for i := 0; i < len(Preface)-1; i++ {
-		if err := testShortPreface(*serverSpec, Preface[:i]+"X"); err != io.EOF {
+		if err := testShortPreface(ctx, Preface[:i]+"X"); err != io.EOF {
 			t.Error("Expected an EOF but was", err)
 		}
 	}
 }
 
 func TestUnknownFrameType(t *testing.T) {
-	if err := testUnknownFrameType(*serverSpec); err != nil {
+	ctx := InteropCtx(t)
+	if err := testUnknownFrameType(ctx); err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestTLSApplicationProtocol(t *testing.T) {
-	if err := testTLSApplicationProtocol(*serverSpec); err != io.EOF {
-		t.Fatal("Expected an EOF but was", err)
-	}
+	ctx := InteropCtx(t)
+	err := testTLSApplicationProtocol(ctx); 
+	matchError(t, err, "EOF")
 }
 
 func TestTLSMaxVersion(t *testing.T) {
-	if err := testTLSMaxVersion(*serverSpec, tls.VersionTLS11); err != io.EOF {
-		t.Fatal("Expected an EOF but was", err)
-	}
+	ctx := InteropCtx(t)
+	err := testTLSMaxVersion(ctx, tls.VersionTLS11);
+	matchError(t, err, "EOF", "server selected unsupported protocol")
 }
 
 func TestClientPrefaceWithStreamId(t *testing.T) {
-	if err := testClientPrefaceWithStreamId(*serverSpec); err != io.EOF {
-		t.Fatal("Expected an EOF but was", err)
-	}
+	ctx := InteropCtx(t)
+	err := testClientPrefaceWithStreamId(ctx)
+	matchError(t, err, "EOF")
+}
+
+func matchError(t *testing.T, err error, matches  ... string) {
+  if err == nil {
+    t.Fatal("Expected an error")
+  }
+  for _, s := range matches {
+    if strings.Contains(err.Error(), s) {
+      return
+    }
+  }
+  t.Fatalf("Error %v not in %+v", err, matches)
 }
 
 func TestMain(m *testing.M) {

tools/jenkins/grpc_interop_http2/Dockerfile

+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM golang:1.4
+
+# Using login shell removes Go from path, so we add it.
+RUN ln -s /usr/src/go/bin/go /usr/local/bin
+
+# Define the default command.
+CMD ["bash"]

tools/jenkins/grpc_interop_http2/build_interop.sh

+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Builds http2 interop client in a base image.
+set -e
+
+mkdir -p /var/local/git
+git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+
+# copy service account keys if available
+cp -r /var/local/jenkins/service_account $HOME || true
+
+# compile the tests
+(cd /var/local/git/grpc/tools/http2_interop && go test -c)
+

tools/run_tests/report_utils.py

@@ -108,10 +108,12 @@ def fill_one_test_result(shortname, resultset, html_str):
 
 
 def render_html_report(client_langs, server_langs, test_cases, auth_test_cases,
-                       resultset, num_failures, cloud_to_prod):
+                       http2_cases, resultset, num_failures, cloud_to_prod, 
+                       http2_interop):
   """Generate html report."""
   sorted_test_cases = sorted(test_cases)
   sorted_auth_test_cases = sorted(auth_test_cases)
+  sorted_http2_cases = sorted(http2_cases)
   sorted_client_langs = sorted(client_langs)
   sorted_server_langs = sorted(server_langs)
   html_str = ('<!DOCTYPE html>\n'
@@ -170,6 +172,30 @@ def render_html_report(client_langs, server_langs, test_cases, auth_test_cases,
           html_str = fill_one_test_result(shortname, resultset, html_str)
         html_str = '%s</tr>\n' % html_str
       html_str = '%s</table>\n' % html_str
+  if http2_interop:
+    # Each column header is the server language.
+    html_str = ('%s<h2>HTTP/2 Interop</h2>\n' 
+                '<table style=\"width:100%%\" border=\"1\">\n'
+                '<tr bgcolor=\"#00BFFF\">\n'
+                '<th>Servers &#9658;<br/>'
+                'Test Cases &#9660;</th>\n') % html_str
+    for server_lang in sorted_server_langs:
+      html_str = '%s<th>%s\n' % (html_str, server_lang)
+    if cloud_to_prod:
+      html_str = '%s<th>%s\n' % (html_str, "prod")
+    html_str = '%s</tr>\n' % html_str
+    for test_case in sorted_http2_cases:
+      html_str = '%s<tr><td><b>%s</b></td>\n' % (html_str, test_case)
+      # Fill up the cells with test result.
+      for server_lang in sorted_server_langs:
+        shortname = 'cloud_to_cloud:%s:%s_server:%s' % (
+            "http2", server_lang, test_case)
+        html_str = fill_one_test_result(shortname, resultset, html_str)
+      if cloud_to_prod:
+        shortname = 'cloud_to_prod:%s:%s' % ("http2", test_case)
+        html_str = fill_one_test_result(shortname, resultset, html_str)
+      html_str = '%s</tr>\n' % html_str
+    html_str = '%s</table>\n' % html_str
 
   html_str = ('%s\n'
               '<script>\n'

tools/run_tests/run_interop_tests.py

@@ -159,6 +159,31 @@ class GoLanguage:
     return 'go'
 
 
+class Http2Client:
+  """Represents the HTTP/2 Interop Test
+
+  This pretends to be a language in order to be built and run, but really it
+  isn't.
+  """
+  def __init__(self):
+    self.client_cwd = None
+    self.safename = str(self)
+
+  def client_args(self):
+    return ['tools/http2_interop/http2_interop.test']
+
+  def cloud_to_prod_env(self):
+    return {}
+
+  def global_env(self):
+    return {}
+
+  def unimplemented_test_cases(self):
+    return _TEST_CASES
+
+  def __str__(self):
+    return 'http2'
+
 class NodeLanguage:
 
   def __init__(self):
@@ -281,6 +306,7 @@ _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong',
 _AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds',
                     'oauth2_auth_token', 'per_rpc_creds']
 
+_HTTP2_TEST_CASES = ["tls"]
 
 def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None):
   """Wraps given cmdline array to create 'docker run' cmdline from it."""
@@ -439,6 +465,7 @@ def server_jobspec(language, docker_image):
                                       environ=environ,
                                       docker_args=['-p', str(_DEFAULT_SERVER_PORT),
                                                    '--name', container_name])
+
   server_job = jobset.JobSpec(
           cmdline=docker_cmdline,
           environ=environ,
@@ -516,6 +543,12 @@ argp.add_argument('--allow_flakes',
                   action='store_const',
                   const=True,
                   help='Allow flaky tests to show as passing (re-runs failed tests up to five times)')
+argp.add_argument('--http2_interop',
+                  default=False,
+                  action='store_const',
+                  const=True,
+                  help='Enable HTTP/2 interop tests')
+                  
 args = argp.parse_args()
 
 servers = set(s for s in itertools.chain.from_iterable(_SERVERS
@@ -539,12 +572,16 @@ languages = set(_LANGUAGES[l]
                 for l in itertools.chain.from_iterable(
                       _LANGUAGES.iterkeys() if x == 'all' else [x]
                       for x in args.language))
+                      
+http2Interop = Http2Client() if args.http2_interop else None
 
 docker_images={}
 if args.use_docker:
   # languages for which to build docker images
   languages_to_build = set(_LANGUAGES[k] for k in set([str(l) for l in languages] +
                                                     [s for s in servers]))
+  if args.http2_interop:
+    languages_to_build.add(http2Interop)
 
   build_jobs = []
   for l in languages_to_build:
@@ -586,6 +623,13 @@ try:
           test_job = cloud_to_prod_jobspec(language, test_case,
                                            docker_image=docker_images.get(str(language)))
           jobs.append(test_job)
+          
+    if args.http2_interop:
+      for test_case in _HTTP2_TEST_CASES:
+        test_job = cloud_to_prod_jobspec(http2Interop, test_case,
+                                         docker_image=docker_images.get(str(http2Interop)))
+        jobs.append(test_job)
+     
 
   if args.cloud_to_prod_auth:
     for language in languages:
@@ -613,6 +657,16 @@ try:
                                             server_port,
                                             docker_image=docker_images.get(str(language)))
           jobs.append(test_job)
+          
+    if args.http2_interop:
+      for test_case in _HTTP2_TEST_CASES:
+        test_job = cloud_to_cloud_jobspec(http2Interop,
+                                          test_case,
+                                          server_name,
+                                          server_host,
+                                          server_port,
+                                          docker_image=docker_images.get(str(http2Interop)))
+        jobs.append(test_job)
 
   if not jobs:
     print 'No jobs to run.'
@@ -631,7 +685,8 @@ try:
   
   report_utils.render_html_report(
       set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, 
-      resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod)
+      _HTTP2_TEST_CASES, resultset, num_failures,
+      args.cloud_to_prod_auth or args.cloud_to_prod, args.http2_interop)
 
 finally:
   # Check if servers are still running.