chore: standardise how unit/integration tests is run on CI, adding a build command to compress the system-tests files

Jenkinsfile

@@ -664,7 +664,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
         error("Error '${err.toString()}'")
       } finally {
         if (archive) {
-          archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
+          archiveTestOutput(directory: directory, testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
         }
         tearDown()
       }
@@ -752,6 +752,8 @@ def getCommonModuleInTheChangeSet(String directory) {
 * to bypass some issues when working with big repositories.
 */
 def archiveTestOutput(Map args = [:]) {
+  def directory = args.get('directory', '')
+
   catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
     if (isUnix()) {
       fixPermissions("${WORKSPACE}")
@@ -775,13 +777,19 @@ def archiveTestOutput(Map args = [:]) {
     }
     if (args.upload) {
       catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
-        def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
-        log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
-        if (folder.trim()) {
-          // TODO: nodeOS() should support ARM
-          def os_suffix = isArm() ? 'linux' : nodeOS()
-          def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix
-          tarAndUploadArtifacts(file: "${name}.tgz", location: folder)
+        withMageEnv(version: "${env.GO_VERSION}"){
+          dir(directory){
+            cmd(label: "Archive system tests files", script: 'mage packageSystemTests')
+          }
+        }
+        def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/pkg.go#PackageSystemTests method
+        dir("${BASE_DIR}"){
+          googleStorageUploadExt(
+            bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
+            credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+            pattern: "${fileName}",
+            sharedPublicly: true
+          )
         }
       }
     }

dev-tools/mage/common.go

@@ -353,6 +353,77 @@ func unzip(sourceFile, destinationDir string) error {
 	return nil
 }
 
+// Tar compress a directory using tar + gzip algorithms
+func Tar(src string, targetFile string) error {
+	var buf bytes.Buffer
+
+	// tar > gzip > buf
+	zr := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zr)
+
+	// walk through every file in the folder
+	filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error {
+		// if a symlink, skip file
+		if fi.Mode().Type() == os.ModeSymlink {
+			fmt.Printf(">> skipping symlink: %s\n", file)
+			return nil
+		}
+
+		// generate tar header
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			fmt.Printf(">> error getting file info header: %s\n", err)
+			return err
+		}
+
+		// must provide real name
+		// (see https://golang.org/src/archive/tar/common.go?#L626)
+		header.Name = filepath.ToSlash(file)
+
+		// write header
+		if err := tw.WriteHeader(header); err != nil {
+			fmt.Printf(">> error writing header: %s\n", err)
+			return err
+		}
+
+		// if not a dir, write file content
+		if !fi.IsDir() {
+			data, err := os.Open(file)
+			if err != nil {
+				fmt.Printf(">> error opening file: %s\n", err)
+				return err
+			}
+			if _, err := io.Copy(tw, data); err != nil {
+				fmt.Printf(">> error compressing file: %s\n", err)
+				return err
+			}
+		}
+		return nil
+	})
+
+	// produce tar
+	if err := tw.Close(); err != nil {
+		fmt.Printf(">> error closing tar file: %s\n", err)
+		return err
+	}
+	// produce gzip
+	if err := zr.Close(); err != nil {
+		fmt.Printf(">> error closing gzip file: %s\n", err)
+		return err
+	}
+
+	// write the .tar.gzip
+	fileToWrite, err := os.OpenFile(targetFile, os.O_CREATE|os.O_RDWR, os.FileMode(0600))
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(fileToWrite, &buf); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func untar(sourceFile, destinationDir string) error {
 	file, err := os.Open(sourceFile)
 	if err != nil {

dev-tools/mage/pkg.go

@@ -21,8 +21,10 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
@@ -142,6 +144,33 @@ func (b packageBuilder) Build() error {
 		b.Spec.Name, b.Type, b.Platform.Name)
 }
 
+// PackageSystemTests packages the python system tests results
+func PackageSystemTests() error {
+	excludeds := []string{".ci", ".git", ".github", "vendor", "dev-tools"}
+
+	// include run as it's the directory we want to compress
+	systemTestsDir := fmt.Sprintf("build%[1]csystem-tests%[1]crun", os.PathSeparator)
+	files, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool {
+		base := filepath.Base(path)
+		for _, excluded := range excludeds {
+			if strings.HasPrefix(base, excluded) {
+				return false
+			}
+		}
+
+		return strings.HasPrefix(path, systemTestsDir)
+	})
+	if err != nil {
+		return err
+	}
+
+	if len(files) == 0 {
+		return fmt.Errorf("there are no system test files under %s", systemTestsDir)
+	}
+
+	return Tar(systemTestsDir, MustExpand("{{ elastic_beats_dir }}/build/system-tests-"+MustExpand("{{ repo.SubDir }}")+".tar.gz"))
+}
+
 type testPackagesParams struct {
 	HasModules           bool
 	HasMonitorsD         bool

filebeat/Jenkinsfile.yml

@@ -35,9 +35,16 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
-        withModule: true       ## run the ITs only if the changeset affects a specific module.
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"       ## run the ITs only if the changeset affects a specific module.
+        withModule: true
         stage: mandatory
     macos:
         mage: "mage build unitTest"

filebeat/magefile.go

@@ -210,6 +210,11 @@ func GoIntegTest(ctx context.Context) error {
 	})
 }
 
+// PackageSystemTests packages the system tests results into a TAR file
+func PackageSystemTests() error {
+	return devtools.PackageSystemTests()
+}
+
 // PythonIntegTest executes the python system tests in the integration environment (Docker).
 // Use GENERATE=true to generate expected log files.
 // Use TESTING_FILEBEAT_MODULES=module[,module] to limit what modules to test.

metricbeat/Jenkinsfile.yml

@@ -68,15 +68,15 @@ stages:
         platforms:             ## override default labels in this specific stage.
             - "windows-10"
         stage: extended
-    windows-8:
+    windows-2008:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
-            - "windows-8"
+            - "windows-2008-r2"
         stage: extended
-    windows-2008:
+    windows-8:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
-            - "windows-2008-r2"
+            - "windows-8"
         stage: extended
     windows-7:
         mage: "mage build unitTest"

metricbeat/magefile.go

@@ -188,6 +188,11 @@ func GoIntegTest(ctx context.Context) error {
 	return devtools.GoTestIntegrationForModule(ctx)
 }
 
+// PackageSystemTests packages the system tests results into a TAR file
+func PackageSystemTests() error {
+	return devtools.PackageSystemTests()
+}
+
 // PythonIntegTest executes the python system tests in the integration
 // environment (Docker).
 // Use MODULE=module to run only tests for `module`.

x-pack/filebeat/Jenkinsfile.yml

@@ -35,9 +35,16 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
-        withModule: true       ## run the ITs only if the changeset affects a specific module.
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"       ## run the ITs only if the changeset affects a specific module.
+        withModule: true
         stage: mandatory
     macos:
         mage: "mage build unitTest"

x-pack/filebeat/magefile.go

@@ -166,6 +166,11 @@ func GoIntegTest(ctx context.Context) error {
 	})
 }
 
+// PackageSystemTests packages the system tests results into a TAR file
+func PackageSystemTests() error {
+	return devtools.PackageSystemTests()
+}
+
 // PythonIntegTest executes the python system tests in the integration environment (Docker).
 // Use GENERATE=true to generate expected log files.
 // Use TESTING_FILEBEAT_MODULES=module[,module] to limit what modules to test.

x-pack/metricbeat/Jenkinsfile.yml

@@ -24,6 +24,14 @@ stages:
     unitTest:
         mage: "mage build unitTest"
         stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"
+        withModule: true
+        stage: mandatory
     cloud:
         cloud: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

x-pack/metricbeat/magefile.go

@@ -218,6 +218,11 @@ func GoIntegTest(ctx context.Context) error {
 	return devtools.GoTestIntegrationForModule(ctx)
 }
 
+// PackageSystemTests packages the system tests results into a TAR file
+func PackageSystemTests() error {
+	return devtools.PackageSystemTests()
+}
+
 // PythonIntegTest executes the python system tests in the integration
 // environment (Docker).
 // Use MODULE=module to run only tests for `module`.