Code coverage setup on CI

.ci/Jenkinsfile_coverage

@@ -0,0 +1,112 @@
+#!/bin/groovy
+
+library 'kibana-pipeline-library'
+kibanaLibrary.load() // load from the Jenkins instance
+
+stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
+  timeout(time: 180, unit: 'MINUTES') {
+    timestamps {
+      ansiColor('xterm') {
+        catchError {
+          withEnv([
+            'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc.
+          ]) {
+            parallel([
+              'kibana-intake-agent': {
+                withEnv([
+                  'NODE_ENV=test' // Needed for jest tests only
+                ]) {
+                  kibanaPipeline.legacyJobRunner('kibana-intake')()
+                }
+              },
+              'x-pack-intake-agent': {
+                withEnv([
+                  'NODE_ENV=test' // Needed for jest tests only
+                ]) {
+                  kibanaPipeline.legacyJobRunner('x-pack-intake')()
+                }
+              },
+              'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+                'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
+                'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
+                'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
+                'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
+                'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
+                'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
+                'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
+                'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
+                'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
+                'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
+                'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
+                'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
+              ]),
+              'kibana-xpack-agent-1': kibanaPipeline.withWorkers('kibana-xpack-tests-1', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
+                'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
+              ]),
+              'kibana-xpack-agent-2': kibanaPipeline.withWorkers('kibana-xpack-tests-2', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
+                'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
+              ]),
+
+              'kibana-xpack-agent-3': kibanaPipeline.withWorkers('kibana-xpack-tests-3', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
+                'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
+                'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
+                'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
+                'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
+                'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
+              ]),
+            ])
+            kibanaPipeline.jobRunner('tests-l', false) {
+              kibanaPipeline.downloadCoverageArtifacts()
+              kibanaPipeline.bash(
+                """
+                  # bootstrap from x-pack folder
+                  source src/dev/ci_setup/setup_env.sh
+                  cd x-pack
+                  yarn kbn bootstrap --prefer-offline
+                  cd ..
+                  # extract archives
+                  mkdir -p /tmp/extracted_coverage
+                  echo extracting intakes
+                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  tar -xzf /tmp/downloaded_coverage/coverage/x-pack-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  echo extracting kibana-oss-tests
+                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  echo extracting kibana-xpack-tests
+                  for i in {1..3}; do
+                    tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests-\${i}/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  done
+                  # replace path in json files to have valid html report
+                  pwd=\$(pwd)
+                  du -sh /tmp/extracted_coverage/target/kibana-coverage/
+                  echo replacing path in json files
+                  for i in {1..9}; do
+                    sed -i "s|/dev/shm/workspace/kibana|\$pwd|g" /tmp/extracted_coverage/target/kibana-coverage/functional/\${i}*.json &
+                  done
+                  wait
+                  # merge oss & x-pack reports
+                  echo merging coverage reports
+                  yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/jest --report-dir target/kibana-coverage/jest-combined --reporter=html --reporter=json-summary
+                  yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/functional --report-dir target/kibana-coverage/functional-combined --reporter=html --reporter=json-summary
+                  echo copy mocha reports
+                  mkdir -p target/kibana-coverage/mocha-combined
+                  cp -r /tmp/extracted_coverage/target/kibana-coverage/mocha target/kibana-coverage/mocha-combined
+                """,
+                "run `yarn kbn bootstrap && merge coverage`"
+              )
+              sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*'
+              kibanaPipeline.uploadGcsArtifact("coverage/jest-combined", 'kibana-jest-coverage.tar.gz')
+              sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*'
+              kibanaPipeline.uploadGcsArtifact("coverage/functional-combined", 'kibana-functional-coverage.tar.gz')
+              sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*'
+              kibanaPipeline.uploadGcsArtifact("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz')
+            }
+          }
+        }
+        kibanaPipeline.sendMail()
+      }
+    }
+  }
+}

scripts/functional_tests.js

@@ -18,11 +18,19 @@
  */
 
 require('../src/setup_node_env');
-require('@kbn/test').runTestsCli([
-  require.resolve('../test/functional/config.js'),
-  require.resolve('../test/api_integration/config.js'),
-  require.resolve('../test/plugin_functional/config.js'),
-  require.resolve('../test/interpreter_functional/config.ts'),
-  require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'),
-  require.resolve('../test/examples/config.js'),
-]);
+require('@kbn/test').runTestsCli(
+  !!process.env.CODE_COVERAGE
+    ? [
+        require.resolve('../test/functional/config.js'),
+        require.resolve('../test/plugin_functional/config.js'),
+        require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'),
+      ]
+    : [
+        require.resolve('../test/functional/config.js'),
+        require.resolve('../test/api_integration/config.js'),
+        require.resolve('../test/plugin_functional/config.js'),
+        require.resolve('../test/interpreter_functional/config.ts'),
+        require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'),
+        require.resolve('../test/examples/config.js'),
+      ]
+);

src/dev/jest/config.js

@@ -69,7 +69,7 @@ export default {
   ],
   setupFilesAfterEnv: ['<rootDir>/src/dev/jest/setup/mocks.js'],
   coverageDirectory: '<rootDir>/target/kibana-coverage/jest',
-  coverageReporters: ['html', 'text'],
+  coverageReporters: !!process.env.CODE_COVERAGE ? ['json'] : ['html', 'text'],
   moduleFileExtensions: ['js', 'json', 'ts', 'tsx'],
   modulePathIgnorePatterns: ['__fixtures__/', 'target/'],
   testMatch: ['**/*.test.{js,ts,tsx}'],

src/legacy/server/config/schema.js

@@ -193,7 +193,7 @@ export default () =>
         .default('localhost'),
       watchPrebuild: Joi.boolean().default(false),
       watchProxyTimeout: Joi.number().default(10 * 60000),
-      useBundleCache: Joi.boolean().default(Joi.ref('$prod')),
+      useBundleCache: Joi.boolean().default(!!process.env.CODE_COVERAGE ? true : Joi.ref('$prod')),
       sourceMaps: Joi.when('$prod', {
         is: true,
         then: Joi.boolean().valid(false),

tasks/config/run.js

@@ -152,6 +152,7 @@ module.exports = function(grunt) {
       args: [
         'nyc',
         '--reporter=html',
+        '--reporter=json-summary',
         '--report-dir=./target/kibana-coverage/mocha',
         NODE,
         'scripts/mocha',

test/functional/services/remote/remote.ts

@@ -100,7 +100,9 @@ export async function RemoteProvider({ getService }: FtrProviderContext) {
     .subscribe({
       next({ message, level }) {
         const msg = message.replace(/\\n/g, '\n');
-        log[level === 'SEVERE' ? 'error' : 'debug'](`browser[${level}] ${msg}`);
+        log[level === 'SEVERE' || level === 'error' ? 'error' : 'debug'](
+          `browser[${level}] ${msg}`
+        );
       },
     });
 

test/scripts/jenkins_build_kibana.sh

@@ -8,5 +8,8 @@ node scripts/es snapshot --license=oss --download-only;
 echo " -> Ensuring all functional tests are in a ciGroup"
 yarn run grunt functionalTests:ensureAllTestsInCiGroup;
 
-echo " -> building and extracting OSS Kibana distributable for use in functional tests"
-node scripts/build --debug --oss
+# Do not build kibana for code coverage run
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  echo " -> building and extracting OSS Kibana distributable for use in functional tests"
+  node scripts/build --debug --oss
+fi

test/scripts/jenkins_ci_group.sh

@@ -2,22 +2,33 @@
 
 source test/scripts/jenkins_test_setup.sh
 
-if [[ -z "$IS_PIPELINE_JOB" ]] ; then
-  yarn run grunt functionalTests:ensureAllTestsInCiGroup;
-  node scripts/build --debug --oss;
-else
-  installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
-  destDir=${installDir}-${CI_WORKER_NUMBER}
-  cp -R "$installDir" "$destDir"
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+    yarn run grunt functionalTests:ensureAllTestsInCiGroup;
+    node scripts/build --debug --oss;
+  else
+    installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
+    destDir=${installDir}-${CI_WORKER_NUMBER}
+    cp -R "$installDir" "$destDir"
 
-  export KIBANA_INSTALL_DIR="$destDir"
-fi
+    export KIBANA_INSTALL_DIR="$destDir"
+  fi
+
+  checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
+
+  if [ "$CI_GROUP" == "1" ]; then
+    source test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh
+    yarn run grunt run:pluginFunctionalTestsRelease --from=source;
+    yarn run grunt run:exampleFunctionalTestsRelease --from=source;
+    yarn run grunt run:interpreterFunctionalTestsRelease;
+  fi
+else
+  echo " -> Running Functional tests with code coverage"
 
-checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
+  export NODE_OPTIONS=--max_old_space_size=8192
 
-if [ "$CI_GROUP" == "1" ]; then
-  source test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh
-  yarn run grunt run:pluginFunctionalTestsRelease --from=source;
-  yarn run grunt run:exampleFunctionalTestsRelease --from=source;
-  yarn run grunt run:interpreterFunctionalTestsRelease;
+  checks-reporter-with-killswitch " Functional tests with code coverage / Group ${CI_GROUP}" \
+  node scripts/functional_tests \
+    --debug \
+    --include-tag "ciGroup$CI_GROUP"
 fi

test/scripts/jenkins_unit.sh

@@ -4,4 +4,16 @@ set -e
 
 export TEST_BROWSER_HEADLESS=1
 
-"$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:unit --dev;
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:unit --dev;
+else
+  echo "NODE_ENV=$NODE_ENV"
+  echo " -> Running jest tests with coverage"
+  checks-reporter-with-killswitch "Jest Coverage" node scripts/jest --ci --verbose --coverage
+  echo ""
+  echo ""
+  echo " -> Running mocha tests with coverage"
+  checks-reporter-with-killswitch "Mocha Coverage" yarn run grunt "test:mochaCoverage";
+  echo ""
+  echo ""
+fi

test/scripts/jenkins_xpack.sh

@@ -4,33 +4,46 @@ set -e
 
 export TEST_BROWSER_HEADLESS=1
 
-echo " -> Running mocha tests"
-cd "$XPACK_DIR"
-checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:browser
-echo ""
-echo ""
-
-echo " -> Running jest tests"
-cd "$XPACK_DIR"
-checks-reporter-with-killswitch "X-Pack Jest" node scripts/jest --ci --verbose
-echo ""
-echo ""
-
-echo " -> Running SIEM cyclic dependency test"
-cd "$XPACK_DIR"
-checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node legacy/plugins/siem/scripts/check_circular_deps
-echo ""
-echo ""
-
-# FAILING: https://github.com/elastic/kibana/issues/44250
-# echo " -> Running jest contracts tests"
-# cd "$XPACK_DIR"
-# SLAPSHOT_ONLINE=true CONTRACT_ONLINE=true node scripts/jest_contract.js --ci --verbose
-# echo ""
-# echo ""
-
-# echo " -> Running jest integration tests"
-# cd "$XPACK_DIR"
-# node scripts/jest_integration --ci --verbose
-# echo ""
-# echo ""
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  echo " -> Running mocha tests"
+  cd "$XPACK_DIR"
+  checks-reporter-with-killswitch "X-Pack Karma Tests" yarn test:browser
+  echo ""
+  echo ""
+
+  echo " -> Running jest tests"
+  cd "$XPACK_DIR"
+  checks-reporter-with-killswitch "X-Pack Jest" node scripts/jest --ci --verbose
+  echo ""
+  echo ""
+
+  echo " -> Running SIEM cyclic dependency test"
+  cd "$XPACK_DIR"
+  checks-reporter-with-killswitch "X-Pack SIEM cyclic dependency test" node legacy/plugins/siem/scripts/check_circular_deps
+  echo ""
+  echo ""
+
+  # FAILING: https://github.com/elastic/kibana/issues/44250
+  # echo " -> Running jest contracts tests"
+  # cd "$XPACK_DIR"
+  # SLAPSHOT_ONLINE=true CONTRACT_ONLINE=true node scripts/jest_contract.js --ci --verbose
+  # echo ""
+  # echo ""
+
+  # echo " -> Running jest integration tests"
+  # cd "$XPACK_DIR"
+  # node scripts/jest_integration --ci --verbose
+  # echo ""
+  # echo ""
+else
+  echo " -> Running jest tests with coverage"
+  cd "$XPACK_DIR"
+  # build runtime for canvas
+  echo "NODE_ENV=$NODE_ENV"
+  node ./legacy/plugins/canvas/scripts/shareable_runtime
+  checks-reporter-with-killswitch "X-Pack Jest Coverage" node scripts/jest --ci --verbose --coverage
+  # rename file in order to be unique one
+  mv ../target/kibana-coverage/jest/coverage-final.json ../target/kibana-coverage/jest/xpack-coverage-final.json
+  echo ""
+  echo ""
+fi

test/scripts/jenkins_xpack_build_kibana.sh

@@ -20,10 +20,13 @@ node scripts/functional_tests --assert-none-excluded \
   --include-tag ciGroup9 \
   --include-tag ciGroup10
 
-echo " -> building and extracting default Kibana distributable for use in functional tests"
-cd "$KIBANA_DIR"
-node scripts/build --debug --no-oss
-linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
-installDir="$PARENT_DIR/install/kibana"
-mkdir -p "$installDir"
-tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+# Do not build kibana for code coverage run
+if [[ -z "$CODE_COVERAGE" ]] ; then
+  echo " -> building and extracting default Kibana distributable for use in functional tests"
+  cd "$KIBANA_DIR"
+  node scripts/build --debug --no-oss
+  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+  installDir="$PARENT_DIR/install/kibana"
+  mkdir -p "$installDir"
+  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+fi