Merge remote-tracking branch 'upstream/main' into issue-change-index-…

…pattern-to-data-view

.buildkite/ftr_configs.yml

@@ -34,7 +34,10 @@ disabled:
   - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
+
+  # Elastic Synthetics configs
   - x-pack/plugins/synthetics/e2e/config.ts
+  - x-pack/plugins/synthetics/e2e/playwright_run.ts
 
   # Configs that exist but weren't running in CI when this file was introduced
   - test/visual_regression/config.ts
@@ -159,10 +162,12 @@ enabled:
   - x-pack/test/functional/apps/maps/group2/config.ts
   - x-pack/test/functional/apps/maps/group3/config.ts
   - x-pack/test/functional/apps/maps/group4/config.ts
+  - x-pack/test/functional/apps/ml/anomaly_detection/config.ts
+  - x-pack/test/functional/apps/ml/data_frame_analytics/config.ts
   - x-pack/test/functional/apps/ml/data_visualizer/config.ts
-  - x-pack/test/functional/apps/ml/group1/config.ts
-  - x-pack/test/functional/apps/ml/group2/config.ts
-  - x-pack/test/functional/apps/ml/group3/config.ts
+  - x-pack/test/functional/apps/ml/permissions/config.ts
+  - x-pack/test/functional/apps/ml/short_tests/config.ts
+  - x-pack/test/functional/apps/ml/stack_management_jobs/config.ts
   - x-pack/test/functional/apps/monitoring/config.ts
   - x-pack/test/functional/apps/remote_clusters/config.ts
   - x-pack/test/functional/apps/reporting_management/config.ts

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#f5381bea52e0a71f50a6919cb6357ff3262cf2d6"
+    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#b9f6b423059cac7554a7402277f2ad3ecfe132a4"
   }
 }

.buildkite/pipelines/es_snapshots/verify.yml

@@ -32,6 +32,9 @@ steps:
     agents:
       queue: kibana-default
     env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
       LIMIT_CONFIG_TYPE: integration,functional
     retry:
       automatic:
@@ -44,10 +47,8 @@ steps:
     agents:
       queue: kibana-default
     depends_on:
-      - default-cigroup
-      - oss-cigroup
+      - ftr-configs
       - jest-integration
-      - api-integration
 
   - wait: ~
     continue_on_failure: true

.buildkite/pipelines/on_merge.yml

@@ -53,6 +53,10 @@ steps:
     label: 'Pick Test Group Run Order'
     agents:
       queue: kibana-default
+    env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
     retry:
       automatic:
         - exit_status: '*'
@@ -78,10 +82,19 @@ steps:
 
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
+    agents:
+      queue: n2-2-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/check_types.sh
+    label: 'Check Types'
     agents:
       queue: c2-8
-    key: checks
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
 
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'

.buildkite/pipelines/performance/daily.yml

@@ -17,6 +17,13 @@ steps:
     agents:
       queue: kb-static-ubuntu
     depends_on: build
+    key: tests
+
+  - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
+    command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
+    agents:
+      queue: n2-2
+    depends_on: tests
 
   - wait: ~
     continue_on_failure: true

.buildkite/pipelines/pull_request/base.yml

@@ -19,6 +19,10 @@ steps:
     label: 'Pick Test Group Run Order'
     agents:
       queue: kibana-default
+    env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
     retry:
       automatic:
         - exit_status: '*'
@@ -40,10 +44,19 @@ steps:
 
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
+    agents:
+      queue: n2-2-spot
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/check_types.sh
+    label: 'Check Types'
     agents:
       queue: c2-8
-    key: checks
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
 
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'

.buildkite/scripts/bootstrap.sh

@@ -6,6 +6,15 @@ source .buildkite/scripts/common/util.sh
 source .buildkite/scripts/common/setup_bazel.sh
 
 echo "--- yarn install and bootstrap"
+
+# Use the node_modules that is baked into the agent image, if it exists, as a cache
+# But only for agents not mounting the workspace on a local ssd or in memory
+# It actually ends up being slower to move all of the tiny files between the disks vs extracting archives from the yarn cache
+if [[ -d ~/.kibana/node_modules && "$(pwd)" != *"/local-ssd/"* && "$(pwd)" != "/dev/shm"* ]]; then
+  echo "Using ~/.kibana/node_modules as a starting point"
+  mv ~/.kibana/node_modules ./
+fi
+
 if ! yarn kbn bootstrap; then
   echo "bootstrap failed, trying again in 15 seconds"
   sleep 15

.buildkite/scripts/common/env.sh

@@ -101,3 +101,8 @@ export DISABLE_BOOTSTRAP_VALIDATION=true
 
 # Prevent Browserlist from logging on CI about outdated database versions
 export BROWSERSLIST_IGNORE_OLD_DATA=true
+
+# keys used to associate test group data in ci-stats with Jest execution order
+export TEST_GROUP_TYPE_UNIT="Jest Unit Tests"
+export TEST_GROUP_TYPE_INTEGRATION="Jest Integration Tests"
+export TEST_GROUP_TYPE_FUNCTIONAL="Functional Tests"

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -9,19 +9,7 @@
 const execSync = require('child_process').execSync;
 const fs = require('fs');
 const { areChangesSkippable, doAnyChangesMatch } = require('kibana-buildkite-library');
-
-const SKIPPABLE_PATHS = [
-  /^docs\//,
-  /^rfcs\//,
-  /^.ci\/.+\.yml$/,
-  /^.ci\/es-snapshots\//,
-  /^.ci\/pipeline-library\//,
-  /^.ci\/Jenkinsfile_[^\/]+$/,
-  /^\.github\//,
-  /\.md$/,
-  /^\.backportrc\.json$/,
-  /^nav-kibana-dev\.docnav\.json$/,
-];
+const { SKIPPABLE_PR_MATCHERS } = require('./skippable_pr_matchers');
 
 const REQUIRED_PATHS = [
   // this file is auto-generated and changes to it need to be validated with CI
@@ -47,7 +35,7 @@ const uploadPipeline = (pipelineContent) => {
 
 (async () => {
   try {
-    const skippable = await areChangesSkippable(SKIPPABLE_PATHS, REQUIRED_PATHS);
+    const skippable = await areChangesSkippable(SKIPPABLE_PR_MATCHERS, REQUIRED_PATHS);
 
     if (skippable) {
       console.log('All changes in PR are skippable. Skipping CI.');

.buildkite/scripts/pipelines/pull_request/skippable_pr_matchers.js

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  SKIPPABLE_PR_MATCHERS: [
+    /^docs\//,
+    /^rfcs\//,
+    /^.ci\/.+\.yml$/,
+    /^.ci\/es-snapshots\//,
+    /^.ci\/pipeline-library\//,
+    /^.ci\/Jenkinsfile_[^\/]+$/,
+    /^\.github\//,
+    /\.md$/,
+    /^\.backportrc\.json$/,
+    /^nav-kibana-dev\.docnav\.json$/,
+    /^src\/dev\/prs\/kibana_qa_pr_list\.json$/,
+    /^\.buildkite\/scripts\/pipelines\/pull_request\/skippable_pr_matchers\.js$/,
+  ],
+};

.buildkite/scripts/steps/checks/check_types.sh → .buildkite/scripts/steps/check_types.sh

@@ -4,6 +4,8 @@ set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
 
+.buildkite/scripts/bootstrap.sh
+
 echo --- Check Types
 checks-reporter-with-killswitch "Check Types" \
   node scripts/type_check

.buildkite/scripts/steps/checks.sh

@@ -13,7 +13,6 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/doc_api_changes.sh
 .buildkite/scripts/steps/checks/kbn_pm_dist.sh
 .buildkite/scripts/steps/checks/plugin_list_docs.sh
-.buildkite/scripts/steps/checks/check_types.sh
 .buildkite/scripts/steps/checks/bundle_limits.sh
 .buildkite/scripts/steps/checks/i18n.sh
 .buildkite/scripts/steps/checks/file_casing.sh

.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/apm_parser_performance)"
+PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/apm_parser_performance)"
+ES_SERVER_URL="https://kibana-ops-e2e-perf.es.us-central1.gcp.cloud.es.io:9243"
+BUILD_ID=${BUILDKITE_BUILD_ID}
+GCS_BUCKET="gs://kibana-performance/scalability-tests"
+
+.buildkite/scripts/bootstrap.sh
+
+echo "--- Extract APM metrics"
+journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover")
+
+for i in "${journeys[@]}"; do
+    JOURNEY_NAME="${i}"
+    echo "Looking for JOURNEY=${JOURNEY_NAME} and BUILD_ID=${BUILD_ID} in APM traces"
+
+    node scripts/extract_performance_testing_dataset \
+        --journeyName "${JOURNEY_NAME}" \
+        --buildId "${BUILD_ID}" \
+        --es-url "${ES_SERVER_URL}" \
+        --es-username "${USER_FROM_VAULT}" \
+        --es-password "${PASS_FROM_VAULT}"
+done
+
+echo "--- Upload Kibana build, plugins and scalability traces to the public bucket"
+mkdir "${BUILD_ID}"
+# Archive json files with traces and upload as build artifacts
+tar -czf "${BUILD_ID}/scalability_traces.tar.gz" -C target scalability_traces
+buildkite-agent artifact upload "${BUILD_ID}/scalability_traces.tar.gz"
+# Upload Kibana build, plugins, commit sha and traces to the bucket
+buildkite-agent artifact download kibana-default.tar.gz ./"${BUILD_ID}"
+buildkite-agent artifact download kibana-default-plugins.tar.gz ./"${BUILD_ID}"
+echo "${BUILDKITE_COMMIT}" > "${BUILD_ID}/KIBANA_COMMIT_HASH"
+gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
+echo "--- Update reference to the latest CI build"
+echo "${BUILD_ID}" > LATEST
+gsutil cp LATEST "${GCS_BUCKET}"

.buildkite/scripts/steps/on_merge_build_and_metrics.sh

@@ -7,6 +7,4 @@ set -euo pipefail
 .buildkite/scripts/build_kibana_plugins.sh
 .buildkite/scripts/post_build_kibana_plugins.sh
 .buildkite/scripts/post_build_kibana.sh
-
-source ".buildkite/scripts/steps/test/test_group_env.sh"
 .buildkite/scripts/saved_object_field_metrics.sh

.buildkite/scripts/steps/storybooks/build_and_upload.js

@@ -38,6 +38,7 @@ const STORYBOOKS = [
   'security_solution',
   'shared_ux',
   'ui_actions_enhanced',
+  'unified_search',
 ];
 
 const GITHUB_CONTEXT = 'Build and Publish Storybooks';

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/test/test_group_env.sh
 
 export JOB_NUM=$BUILDKITE_PARALLEL_JOB
 export JOB=ftr-configs-${JOB_NUM}

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/steps/test/test_group_env.sh
-
 export JOB=$BUILDKITE_PARALLEL_JOB
 
 # a jest failure will result in the script returning an exit code of 10

.buildkite/scripts/steps/test/pick_test_group_run_order.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/test/test_group_env.sh
 
 echo '--- Pick Test Group Run Order'
 node "$(dirname "${0}")/pick_test_group_run_order.js"

.github/CODEOWNERS

@@ -128,6 +128,7 @@
 /src/apm.js @elastic/kibana-core @vigneshshanmugam
 /packages/kbn-apm-config-loader/ @elastic/kibana-core @vigneshshanmugam
 /src/core/types/elasticsearch @elastic/apm-ui
+/packages/elastic-apm-synthtrace/  @elastic/apm-ui
 #CC# /src/plugins/apm_oss/ @elastic/apm-ui
 #CC# /x-pack/plugins/observability/ @elastic/apm-ui
 

docs/api/actions-and-connectors/legacy/index.asciidoc

@@ -1,4 +1,4 @@
 [[actions-and-connectors-legacy-apis]]
 === Deprecated 7.x APIs
 
-These APIs are deprecated and will be removed as of 8.0.
+These APIs are deprecated and will be removed in a future release.

docs/api/alerting/legacy/index.asciidoc

@@ -1,7 +1,7 @@
 [[alerts-api]]
 === Deprecated 7.x APIs
 
-These APIs are deprecated and will be removed as of 8.0.
+These APIs are deprecated and will be removed in a future release.
 
 include::create.asciidoc[leveloffset=+1]
 include::delete.asciidoc[leveloffset=+1]