From 746e2597334a660c09c7ee17175922b1f70d76d0 Mon Sep 17 00:00:00 2001
From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com>
Date: Thu, 16 Jun 2022 09:29:17 +0100
Subject: [PATCH] [Security Solution] fixes flaky-test-runner Cypress Security
 Solution tests (#134205)

## Summary

Fixes issue with security solution cypress flaky test runner

After introducing [dynamic split for cypress tests in Security Solution](https://github.com/elastic/kibana/pull/125986), there was discovered an issue with flaky test runner, which is using parallelism in slightly different manner: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/697#_: tests were evenly split between N jobs, instead of repeating tests N times(as N jobs)

So, for flaky runner, I introducing a new ENV variable that would disable split for tests between parallel jobs and instead would run all test per each job.

Here is link to flaky runner build from this PR, which shows that all test runs per job
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/736
---
 .buildkite/pipelines/flaky_tests/pipeline.js             | 6 ++++++
 .buildkite/scripts/steps/functional/security_solution.sh | 4 ++--
 x-pack/test/security_solution_cypress/runner.ts          | 9 +++++++++
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/flaky_tests/pipeline.js b/.buildkite/pipelines/flaky_tests/pipeline.js
index d62156a08c55ad..56a6f668068387 100644
--- a/.buildkite/pipelines/flaky_tests/pipeline.js
+++ b/.buildkite/pipelines/flaky_tests/pipeline.js
@@ -174,6 +174,12 @@ for (const testSuite of testSuites) {
         concurrency: concurrency,
         concurrency_group: process.env.UUID,
         concurrency_method: 'eager',
+        env: {
+          // disable split of test cases between parallel jobs when running them in flaky test runner
+          // by setting chunks vars to value 1, which means all test will run in one job
+          CLI_NUMBER: 1,
+          CLI_COUNT: 1,
+        },
       });
       break;
     default:
diff --git a/.buildkite/scripts/steps/functional/security_solution.sh b/.buildkite/scripts/steps/functional/security_solution.sh
index 5e3b1513826f96..fa1908683d1578 100755
--- a/.buildkite/scripts/steps/functional/security_solution.sh
+++ b/.buildkite/scripts/steps/functional/security_solution.sh
@@ -5,8 +5,8 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
-export CLI_NUMBER=$((BUILDKITE_PARALLEL_JOB+1))
-export CLI_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
+export CLI_NUMBER=${CLI_NUMBER:-$((BUILDKITE_PARALLEL_JOB+1))}
+export CLI_COUNT=${CLI_COUNT:-$BUILDKITE_PARALLEL_JOB_COUNT}
 
 echo "--- Security Solution tests (Chrome)"
 
diff --git a/x-pack/test/security_solution_cypress/runner.ts b/x-pack/test/security_solution_cypress/runner.ts
index 2f4f76de53ced0..0074adb4a19d15 100644
--- a/x-pack/test/security_solution_cypress/runner.ts
+++ b/x-pack/test/security_solution_cypress/runner.ts
@@ -57,6 +57,15 @@ export async function SecuritySolutionConfigurableCypressTestRunner(
   });
 }
 
+/**
+ * Takes total CI jobs number(totalCiJobs) between which tests will be split and sequential number of the job(ciJobNumber).
+ * This helper will split file list cypress integrations into chunks, and run integrations in chunk which match ciJobNumber
+ * If both totalCiJobs === 1 && ciJobNumber === 1, this function will run all existing tests, without splitting them
+ * @param context FtrProviderContext
+ * @param {number} totalCiJobs - total number of jobs between which tests will be split
+ * @param {number} ciJobNumber - number of job
+ * @returns
+ */
 export async function SecuritySolutionCypressCliTestRunnerCI(
   context: FtrProviderContext,
   totalCiJobs: number,