Run sub-tests in a group so teardownsuite is called in the right order

Define new interface `Copy` to create copies of suite object for parallel subtests

suite/doc.go

@@ -6,10 +6,7 @@
 // implement).
 //
 // A testing suite is usually built by first extending the built-in
-// suite functionality from suite.Suite in testify.  Alternatively,
-// you could reproduce that logic on your own if you wanted (you
-// just need to implement the TestingSuite interface from
-// suite/interfaces.go).
+// suite functionality from suite.Suite in testify.
 //
 // After that, you can implement any of the interfaces in
 // suite/interfaces.go to add setup/teardown functionality to your
@@ -23,6 +20,10 @@
 // identity that "go test" is already looking for (i.e.
 // func(*testing.T)).
 //
+// To be able to run parallel sub-tests, your testing suite should
+// implement "CopySuite". This may or may not be a deepcopy depending
+// on the fields in the struct.
+//
 // Regular expression to select test suites specified command-line
 // argument "-run". Regular expression to select the methods
 // of test suites specified command-line argument "-m".

suite/interfaces.go

@@ -6,7 +6,16 @@ import "testing"
 // generated by 'go test'.
 type TestingSuite interface {
 	T() *testing.T
-	SetT(*testing.T)
+	setT(*testing.T)
+	clearT()
+}
+
+// CopySuite indicates a copyable struct, deepcopy vs shallow is
+// implementation detail of the application.
+type CopySuite interface {
+	// Copy creates a copy of the calling suite object. The returned
+	// object must be the same concrete type as caller
+	Copy() TestingSuite
 }
 
 // SetupAllSuite has a SetupSuite method, which will run before the

suite/suite.go

@@ -30,13 +30,22 @@ func (suite *Suite) T() *testing.T {
 	return suite.t
 }
 
-// SetT sets the current *testing.T context.
-func (suite *Suite) SetT(t *testing.T) {
+// setT sets the current *testing.T context.
+func (suite *Suite) setT(t *testing.T) {
+	if suite.t != nil {
+		panic("suite.t already set, can't overwrite")
+	}
 	suite.t = t
 	suite.Assertions = assert.New(t)
 	suite.require = require.New(t)
 }
 
+func (suite *Suite) clearT() {
+	suite.t = nil
+	suite.Assertions = nil
+	suite.require = nil
+}
+
 // Require returns a require context for suite.
 func (suite *Suite) Require() *require.Assertions {
 	if suite.require == nil {
@@ -69,11 +78,16 @@ func failOnPanic(t *testing.T) {
 // called in place of t.Run(name, func(t *testing.T)) in test suite code.
 // The passed-in func will be executed as a subtest with a fresh instance of t.
 // Provides compatibility with go test pkg -run TestSuite/TestName/SubTestName.
+// Deprecated: This method doesn't handle parallel sub-tests and will be removed in v2.
 func (suite *Suite) Run(name string, subtest func()) bool {
 	oldT := suite.T()
-	defer suite.SetT(oldT)
+	defer func() {
+		suite.clearT()
+		suite.setT(oldT)
+	}()
 	return oldT.Run(name, func(t *testing.T) {
-		suite.SetT(t)
+		suite.clearT()
+		suite.setT(t)
 		subtest()
 	})
 }
@@ -83,8 +97,6 @@ func (suite *Suite) Run(name string, subtest func()) bool {
 func Run(t *testing.T, suite TestingSuite) {
 	defer failOnPanic(t)
 
-	suite.SetT(t)
-
 	var suiteSetupDone bool
 
 	var stats *SuiteInformation
@@ -96,84 +108,95 @@ func Run(t *testing.T, suite TestingSuite) {
 	methodFinder := reflect.TypeOf(suite)
 	suiteName := methodFinder.Elem().Name()
 
-	for i := 0; i < methodFinder.NumMethod(); i++ {
-		method := methodFinder.Method(i)
+	t.Run("All", func(t *testing.T) {
+		defer failOnPanic(t)
 
-		ok, err := methodFilter(method.Name)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "testify: invalid regexp for -m: %s\n", err)
-			os.Exit(1)
-		}
+		suite.setT(t)
 
-		if !ok {
-			continue
-		}
+		for i := 0; i < methodFinder.NumMethod(); i++ {
+			method := methodFinder.Method(i)
 
-		if !suiteSetupDone {
-			if stats != nil {
-				stats.Start = time.Now()
+			ok, err := methodFilter(method.Name)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "testify: invalid regexp for -m: %s\n", err)
+				os.Exit(1)
 			}
 
-			if setupAllSuite, ok := suite.(SetupAllSuite); ok {
-				setupAllSuite.SetupSuite()
+			if !ok {
+				continue
 			}
 
-			suiteSetupDone = true
-		}
+			if !suiteSetupDone {
+				if stats != nil {
+					stats.Start = time.Now()
+				}
 
-		test := testing.InternalTest{
-			Name: method.Name,
-			F: func(t *testing.T) {
-				parentT := suite.T()
-				suite.SetT(t)
-				defer failOnPanic(t)
-				defer func() {
-					if stats != nil {
-						passed := !t.Failed()
-						stats.end(method.Name, passed)
-					}
+				if setupAllSuite, ok := suite.(SetupAllSuite); ok {
+					setupAllSuite.SetupSuite()
+				}
 
-					if afterTestSuite, ok := suite.(AfterTest); ok {
-						afterTestSuite.AfterTest(suiteName, method.Name)
-					}
+				suiteSetupDone = true
+			}
 
-					if tearDownTestSuite, ok := suite.(TearDownTestSuite); ok {
-						tearDownTestSuite.TearDownTest()
+			test := testing.InternalTest{
+				Name: method.Name,
+				F: func(t *testing.T) {
+					defer failOnPanic(t)
+					childSuite := suite
+					if c, ok := suite.(CopySuite); ok {
+						childSuite = c.Copy()
+						childSuite.clearT()
+					}
+					childSuite.setT(t)
+					defer func() {
+						if childSuite == suite {
+							defer suite.clearT()
+						}
+						if stats != nil {
+							passed := !t.Failed()
+							stats.end(method.Name, passed)
+						}
+
+						if tearDownTestSuite, ok := childSuite.(TearDownTestSuite); ok {
+							tearDownTestSuite.TearDownTest()
+						}
+
+						if afterTestSuite, ok := childSuite.(AfterTest); ok {
+							afterTestSuite.AfterTest(suiteName, method.Name)
+						}
+					}()
+
+					if beforeTestSuite, ok := childSuite.(BeforeTest); ok {
+						beforeTestSuite.BeforeTest(methodFinder.Elem().Name(), method.Name)
+					}
+					if setupTestSuite, ok := childSuite.(SetupTestSuite); ok {
+						setupTestSuite.SetupTest()
 					}
 
-					suite.SetT(parentT)
-				}()
-
-				if setupTestSuite, ok := suite.(SetupTestSuite); ok {
-					setupTestSuite.SetupTest()
-				}
-				if beforeTestSuite, ok := suite.(BeforeTest); ok {
-					beforeTestSuite.BeforeTest(methodFinder.Elem().Name(), method.Name)
-				}
-
-				if stats != nil {
-					stats.start(method.Name)
-				}
+					if stats != nil {
+						stats.start(method.Name)
+					}
 
-				method.Func.Call([]reflect.Value{reflect.ValueOf(suite)})
-			},
+					method.Func.Call([]reflect.Value{reflect.ValueOf(childSuite)})
+				},
+			}
+			tests = append(tests, test)
 		}
-		tests = append(tests, test)
-	}
+
+		suite.clearT()
+		runTests(t, tests)
+	})
 	if suiteSetupDone {
-		defer func() {
-			if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok {
-				tearDownAllSuite.TearDownSuite()
-			}
+		suite.setT(t)
+		if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok {
+			tearDownAllSuite.TearDownSuite()
+		}
 
-			if suiteWithStats, measureStats := suite.(WithStats); measureStats {
-				stats.End = time.Now()
-				suiteWithStats.HandleStats(suiteName, stats)
-			}
-		}()
+		if suiteWithStats, measureStats := suite.(WithStats); measureStats {
+			stats.End = time.Now()
+			suiteWithStats.HandleStats(suiteName, stats)
+		}
 	}
-
-	runTests(t, tests)
 }
 
 // Filtering method according to set regular expression
@@ -185,25 +208,13 @@ func methodFilter(name string) (bool, error) {
 	return regexp.MatchString(*matchMethod, name)
 }
 
-func runTests(t testing.TB, tests []testing.InternalTest) {
+func runTests(t *testing.T, tests []testing.InternalTest) {
 	if len(tests) == 0 {
 		t.Log("warning: no tests to run")
 		return
 	}
 
-	r, ok := t.(runner)
-	if !ok { // backwards compatibility with Go 1.6 and below
-		if !testing.RunTests(allTestsFilter, tests) {
-			t.Fail()
-		}
-		return
-	}
-
 	for _, test := range tests {
-		r.Run(test.Name, test.F)
+		t.Run(test.Name, test.F)
 	}
 }
-
-type runner interface {
-	Run(name string, f func(t *testing.T)) bool
-}