From 527a11a2bbe59e172649d5c22b87878d86859806 Mon Sep 17 00:00:00 2001 From: Cian Hatton Date: Wed, 6 Jul 2022 11:04:28 +0100 Subject: [PATCH] Script to dynamically generate list of e2e tests (E2E #1) (#1644) --- .github/scripts/build_test_matrix.go | 130 +++++++++++++++++ .github/scripts/build_test_matrix_test.go | 162 ++++++++++++++++++++++ .github/workflows/test.yml | 2 +- Makefile | 2 +- 4 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/build_test_matrix.go create mode 100644 .github/scripts/build_test_matrix_test.go diff --git a/.github/scripts/build_test_matrix.go b/.github/scripts/build_test_matrix.go new file mode 100644 index 00000000000..c86324b31f8 --- /dev/null +++ b/.github/scripts/build_test_matrix.go @@ -0,0 +1,130 @@ +package main + +import ( + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "strings" +) + +const ( + testNamePrefix = "Test" + testFileNameSuffix = "_test.go" + e2eTestDirectory = "e2e" +) + +// GithubActionTestMatrix represents +type GithubActionTestMatrix struct { + Include []TestSuitePair `json:"include"` +} + +type TestSuitePair struct { + Test string `json:"test"` + Suite string `json:"suite"` +} + +func main() { + githubActionMatrix, err := getGithubActionMatrixForTests(e2eTestDirectory) + if err != nil { + fmt.Printf("error generating github action json: %s", err) + os.Exit(1) + } + + ghBytes, err := json.Marshal(githubActionMatrix) + if err != nil { + fmt.Printf("error marshalling github action json: %s", err) + os.Exit(1) + } + fmt.Println(string(ghBytes)) +} + +// getGithubActionMatrixForTests returns a json string representing the contents that should go in the matrix +// field in a github action workflow. This string can be used with `fromJSON(str)` to dynamically build +// the workflow matrix to include all E2E tests under the e2eRootDirectory directory. +func getGithubActionMatrixForTests(e2eRootDirectory string) (GithubActionTestMatrix, error) { + testSuiteMapping := map[string][]string{} + fset := token.NewFileSet() + err := filepath.Walk(e2eRootDirectory, func(path string, info fs.FileInfo, err error) error { + // only look at test files + if !strings.HasSuffix(path, testFileNameSuffix) { + return nil + } + + f, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + return fmt.Errorf("failed parsing file: %s", err) + } + + suiteNameForFile, testCases, err := extractSuiteAndTestNames(f) + if err != nil { + return fmt.Errorf("failed extracting test suite name and test cases: %s", err) + } + + testSuiteMapping[suiteNameForFile] = testCases + + return nil + }) + + if err != nil { + return GithubActionTestMatrix{}, err + } + + gh := GithubActionTestMatrix{ + Include: []TestSuitePair{}, + } + + for testSuiteName, testCases := range testSuiteMapping { + for _, testCaseName := range testCases { + gh.Include = append(gh.Include, TestSuitePair{ + Test: testCaseName, + Suite: testSuiteName, + }) + } + } + + return gh, nil +} + +// extractSuiteAndTestNames extracts the name of the test suite function as well +// as all tests associated with it in the same file. +func extractSuiteAndTestNames(file *ast.File) (string, []string, error) { + var suiteNameForFile string + var testCases []string + + for _, d := range file.Decls { + if f, ok := d.(*ast.FuncDecl); ok { + functionName := f.Name.Name + if isTestSuiteMethod(f) { + if suiteNameForFile != "" { + return "", nil, fmt.Errorf("found a second test function: %s when %s was already found", f.Name.Name, suiteNameForFile) + } + suiteNameForFile = functionName + continue + } + if isTestFunction(f) { + testCases = append(testCases, functionName) + } + } + } + if suiteNameForFile == "" { + return "", nil, fmt.Errorf("file %s had no test suite test case", file.Name.Name) + } + return suiteNameForFile, testCases, nil +} + +// isTestSuiteMethod returns true if the function is a test suite function. +// e.g. func TestFeeMiddlewareTestSuite(t *testing.T) { ... } +func isTestSuiteMethod(f *ast.FuncDecl) bool { + return strings.HasPrefix(f.Name.Name, testNamePrefix) && len(f.Type.Params.List) == 1 +} + +// isTestFunction returns true if the function name starts with "Test" and has no parameters. +// as test suite functions do not accept a *testing.T. +func isTestFunction(f *ast.FuncDecl) bool { + return strings.HasPrefix(f.Name.Name, testNamePrefix) && len(f.Type.Params.List) == 0 +} diff --git a/.github/scripts/build_test_matrix_test.go b/.github/scripts/build_test_matrix_test.go new file mode 100644 index 00000000000..47de4d981c2 --- /dev/null +++ b/.github/scripts/build_test_matrix_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "os" + "path" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + nonTestFile = "not_test_file.go" + goTestFileNameOne = "first_go_file_test.go" + goTestFileNameTwo = "second_go_file_test.go" +) + +func TestGetGithubActionMatrixForTests(t *testing.T) { + t.Run("empty dir does not fail", func(t *testing.T) { + testingDir := t.TempDir() + _, err := getGithubActionMatrixForTests(testingDir) + assert.NoError(t, err) + }) + + t.Run("only test functions are picked up", func(t *testing.T) { + testingDir := t.TempDir() + createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, goTestFileNameOne) + + gh, err := getGithubActionMatrixForTests(testingDir) + assert.NoError(t, err) + + expected := GithubActionTestMatrix{ + Include: []TestSuitePair{ + { + Suite: "TestFeeMiddlewareTestSuite", + Test: "TestA", + }, + { + Suite: "TestFeeMiddlewareTestSuite", + Test: "TestB", + }, + }, + } + assertGithubActionTestMatricesEqual(t, expected, gh) + }) + + t.Run("all files are picked up", func(t *testing.T) { + testingDir := t.TempDir() + createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, goTestFileNameOne) + createFileWithTestSuiteAndTests(t, "TransferTestSuite", "TestC", "TestD", testingDir, goTestFileNameTwo) + + gh, err := getGithubActionMatrixForTests(testingDir) + assert.NoError(t, err) + + expected := GithubActionTestMatrix{ + Include: []TestSuitePair{ + { + Suite: "TestTransferTestSuite", + Test: "TestC", + }, + { + Suite: "TestFeeMiddlewareTestSuite", + Test: "TestA", + }, + { + Suite: "TestFeeMiddlewareTestSuite", + Test: "TestB", + }, + { + Suite: "TestTransferTestSuite", + Test: "TestD", + }, + }, + } + + assertGithubActionTestMatricesEqual(t, expected, gh) + }) + + t.Run("non test files are not picked up", func(t *testing.T) { + testingDir := t.TempDir() + createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, nonTestFile) + + gh, err := getGithubActionMatrixForTests(testingDir) + assert.NoError(t, err) + assert.Empty(t, gh.Include) + }) + + t.Run("fails when there are multiple suite runs", func(t *testing.T) { + testingDir := t.TempDir() + createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, nonTestFile) + + fileWithTwoSuites := `package foo +func SuiteOne(t *testing.T) { + suite.Run(t, new(FeeMiddlewareTestSuite)) +} + +func SuiteTwo(t *testing.T) { + suite.Run(t, new(FeeMiddlewareTestSuite)) +} + +type FeeMiddlewareTestSuite struct {} +` + + err := os.WriteFile(path.Join(testingDir, goTestFileNameOne), []byte(fileWithTwoSuites), os.FileMode(777)) + assert.NoError(t, err) + + _, err = getGithubActionMatrixForTests(testingDir) + assert.Error(t, err) + }) +} + +func assertGithubActionTestMatricesEqual(t *testing.T, expected, actual GithubActionTestMatrix) { + // sort by both suite and test as the order of the end result does not matter as + // all tests will be run. + sort.SliceStable(expected.Include, func(i, j int) bool { + memberI := expected.Include[i] + memberJ := expected.Include[j] + if memberI.Suite == memberJ.Suite { + return memberI.Test < memberJ.Test + } + return memberI.Suite < memberJ.Suite + }) + + sort.SliceStable(actual.Include, func(i, j int) bool { + memberI := actual.Include[i] + memberJ := actual.Include[j] + if memberI.Suite == memberJ.Suite { + return memberI.Test < memberJ.Test + } + return memberI.Suite < memberJ.Suite + }) + assert.Equal(t, expected.Include, actual.Include) +} + +func goTestFileContents(suiteName, fnName1, fnName2 string) string { + + replacedSuiteName := strings.ReplaceAll(`package foo + +func TestSuiteName(t *testing.T) { + suite.Run(t, new(SuiteName)) +} + +type SuiteName struct {} + +func (s *SuiteName) fnName1() {} +func (s *SuiteName) fnName2() {} + +func (s *SuiteName) suiteHelper() {} + +func helper() {} +`, "SuiteName", suiteName) + + replacedFn1Name := strings.ReplaceAll(replacedSuiteName, "fnName1", fnName1) + return strings.ReplaceAll(replacedFn1Name, "fnName2", fnName2) +} + +func createFileWithTestSuiteAndTests(t *testing.T, suiteName, fn1Name, fn2Name, dir, filename string) { + goFileContents := goTestFileContents(suiteName, fn1Name, fn2Name) + err := os.WriteFile(path.Join(dir, filename), []byte(goFileContents), os.FileMode(777)) + assert.NoError(t, err) +} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c915db779cd..ff8ed4d65d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Create a file with all the pkgs - run: go list ./... > pkgs.txt + run: go list ./... ./.github/scripts > pkgs.txt - name: Split pkgs into 4 files run: split -d -n l/4 pkgs.txt pkgs.txt.part. # cache multiple diff --git a/Makefile b/Makefile index 0ea7f0c10c5..9beb4ab61bc 100644 --- a/Makefile +++ b/Makefile @@ -216,7 +216,7 @@ view-docs: test: test-unit test-all: test-unit test-ledger-mock test-race test-cover -TEST_PACKAGES=./... +TEST_PACKAGES=./... ./.github/scripts TEST_TARGETS := test-unit test-unit-amino test-unit-proto test-ledger-mock test-race test-ledger test-race # Test runs-specific rules. To add a new test target, just add