Add optional passing more

tools/http2_interop/http2interop_test.go

@@ -69,7 +69,7 @@ func (ctx *HTTP2InteropCtx) Close() error {
 }
 
 func TestClientShortSettings(t *testing.T) {
-	defer Report(t)()
+	defer Report(t)
 	if *testCase != "framing" {
 		t.SkipNow()
 	}
@@ -81,7 +81,7 @@ func TestClientShortSettings(t *testing.T) {
 }
 
 func TestShortPreface(t *testing.T) {
-	defer Report(t)()
+	defer Report(t)
 	if *testCase != "framing" {
 		t.SkipNow()
 	}
@@ -93,7 +93,7 @@ func TestShortPreface(t *testing.T) {
 }
 
 func TestUnknownFrameType(t *testing.T) {
-	defer Report(t)()
+	defer Report(t)
 	if *testCase != "framing" {
 		t.SkipNow()
 	}
@@ -104,7 +104,7 @@ func TestUnknownFrameType(t *testing.T) {
 }
 
 func TestClientPrefaceWithStreamId(t *testing.T) {
-	defer Report(t)()
+	defer Report(t)
 	if *testCase != "framing" {
 		t.SkipNow()
 	}
@@ -114,7 +114,7 @@ func TestClientPrefaceWithStreamId(t *testing.T) {
 }
 
 func TestTLSApplicationProtocol(t *testing.T) {
-	defer Report(t)()
+	defer Report(t)
 	if *testCase != "tls" {
 		t.SkipNow()
 	}
@@ -124,7 +124,7 @@ func TestTLSApplicationProtocol(t *testing.T) {
 }
 
 func TestTLSMaxVersion(t *testing.T) {
-	defer Report(t)()
+	defer Report(t)
 	if *testCase != "tls" {
 		t.SkipNow()
 	}
@@ -136,7 +136,7 @@ func TestTLSMaxVersion(t *testing.T) {
 }
 
 func TestTLSBadCipherSuites(t *testing.T) {
-	defer Report(t)()
+	defer Report(t)
 	if *testCase != "tls" {
 		t.SkipNow()
 	}
@@ -160,9 +160,24 @@ func matchError(t *testing.T, err error, matches ...string) {
 func TestMain(m *testing.M) {
 	flag.Parse()
 	m.Run()
+	var fatal bool
+	var any bool
+	for _, ci := range allCaseInfos.Cases {
+		if ci.Skipped {
+			continue
+		}
+		any = true
+		if !ci.Passed && ci.Fatal {
+			fatal = true
+		}
+	}
+
 	if err := json.NewEncoder(os.Stderr).Encode(&allCaseInfos); err != nil {
 		fmt.Println("Failed to encode", err)
 	}
-	// Always pass
-	os.Exit(0)
+	var code int
+	if !any || fatal {
+		code = 1
+	}
+	os.Exit(code)
 }

tools/http2_interop/s6.5.go

@@ -11,7 +11,6 @@ func testSmallMaxFrameSize(ctx *HTTP2InteropCtx) error {
 		return err
 	}
 	defer conn.Close()
-	conn.Log = ctx.T.Log
 	conn.SetDeadline(time.Now().Add(defaultTimeout))
 
 	sf := &SettingsFrame{

tools/http2_interop/s6.5_test.go

@@ -4,8 +4,9 @@ import (
 	"testing"
 )
 
-func TestSmallMaxFrameSize(t *testing.T) {
-	if *testCase != "experimental" {
+func TestSoonSmallMaxFrameSize(t *testing.T) {
+	defer Report(t)
+	if *testCase != "framing" {
 		t.SkipNow()
 	}
 	ctx := InteropCtx(t)

tools/http2_interop/testsuite.go

 package http2interop
 
 import (
+	"path"
 	"runtime"
 	"strings"
 	"sync"
@@ -9,29 +10,40 @@ import (
 
 // When a test is skipped or fails, runtime.Goexit() is called which destroys the callstack.
 // This means the name of the test case is lost, so we need to grab a copy of pc before.
-func Report(t testing.TB) func() {
-	pc, _, _, ok := runtime.Caller(1)
-	if !ok {
-		t.Fatal("Can't get caller info")
-	}
-	return func() {
+func Report(t testing.TB) {
+	// If the goroutine panics, Fatal()s, or Skip()s, the function name is at the 3rd callstack
+	// layer.  On success, its at 1st.  Since it's hard to check which happened, just try both.
+	pcs := make([]uintptr, 10)
+	total := runtime.Callers(1, pcs)
+	var name string
+	for _, pc := range pcs[:total] {
 		fn := runtime.FuncForPC(pc)
 		fullName := fn.Name()
-		name := strings.Split(fullName, ".")[1]
-		allCaseInfos.lock.Lock()
-		defer allCaseInfos.lock.Unlock()
-		allCaseInfos.Cases = append(allCaseInfos.Cases, &caseInfo{
-			Name:    name,
-			Passed:  !t.Failed(),
-			Skipped: t.Skipped(),
-		})
+		if strings.HasPrefix(path.Ext(fullName), ".Test") {
+			// Skip the leading .
+			name = string([]byte(path.Ext(fullName))[1:])
+			break
+		}
+	}
+	if name == "" {
+		return
 	}
+
+	allCaseInfos.lock.Lock()
+	defer allCaseInfos.lock.Unlock()
+	allCaseInfos.Cases = append(allCaseInfos.Cases, &caseInfo{
+		Name:    name,
+		Passed:  !t.Failed() && !t.Skipped(),
+		Skipped: t.Skipped(),
+		Fatal:   t.Failed() && !strings.HasPrefix(name, "TestSoon"),
+	})
 }
 
 type caseInfo struct {
 	Name    string `json:"name"`
 	Passed  bool   `json:"passed"`
-	Skipped bool   `json:"skipped"`
+	Skipped bool   `json:"skipped,omitempty"`
+	Fatal   bool   `json:"fatal,omitempty"`
 }
 
 type caseInfos struct {