Merge branch 'main' into fix/synthetics-functional-tests

.bazelrc.common

@@ -132,6 +132,9 @@ coverage --instrument_test_targets
 # Metadata settings
 build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
 
+# Load remote cache settings, if they exist
+try-import %workspace%/.bazelrc.cache
+
 # Load any settings specific to the current user.
 # .bazelrc.user should appear in .gitignore so that settings are not shared with team members
 # This needs to be last statement in this

.buildkite/pipelines/performance/nightly.yml → .buildkite/pipelines/performance/daily.yml

@@ -6,7 +6,7 @@ steps:
         key: "performance-test-iteration-count"
         hint: "How many times you want to run tests? "
         required: true
-    if: build.env('ITERATION_COUNT_ENV') == null
+    if: build.env('PERF_TEST_COUNT') == null
 
   - label: ":male-mechanic::skin-tone-2: Pre-Build"
     command: .buildkite/scripts/lifecycle/pre_build.sh
@@ -19,10 +19,10 @@ steps:
       queue: c2-16
     key: build
 
-  - label: ":muscle: Performance Tests"
-    command: .buildkite/scripts/steps/functional/performance.sh
+  - label: ":muscle: Performance Tests with Playwright config"
+    command: .buildkite/scripts/steps/functional/performance_playwright.sh
     agents:
-      queue: ci-group-6
+      queue: c2-16
     depends_on: build
 
   - wait: ~

.buildkite/scripts/common/env.sh

@@ -22,6 +22,9 @@ KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_BRANCH
 export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
 
+KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
+export KIBANA_PKG_VERSION
+
 export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
 export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
 export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"

.buildkite/scripts/lifecycle/pre_command.sh

@@ -92,6 +92,15 @@ export KIBANA_DOCKER_USERNAME
 KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
 export KIBANA_DOCKER_PASSWORD
 
+SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_USERNAME
+
+SYNTHETICS_SERVICE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_PASSWORD
+
+SYNTHETICS_SERVICE_MANIFEST="$(retry 5 5 vault read -field=manifest secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_MANIFEST
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)

.buildkite/scripts/steps/functional/performance.sh → .buildkite/scripts/steps/functional/performance_playwright.sh

@@ -2,20 +2,22 @@
 
 set -uo pipefail
 
-if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
-  ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
+if [ -z "${PERF_TEST_COUNT+x}" ]; then
+  TEST_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
 else
-  ITERATION_COUNT=$ITERATION_COUNT_ENV
+  TEST_COUNT=$PERF_TEST_COUNT
 fi
 
-tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
+tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${TEST_COUNT} times"
 
 cat << EOF | buildkite-agent pipeline upload
 steps:
-  - command: .buildkite/scripts/steps/functional/performance_sub.sh
-    parallelism: "$ITERATION_COUNT"
+  - command: .buildkite/scripts/steps/functional/performance_sub_playwright.sh
+    parallelism: "$TEST_COUNT"
     concurrency: 20
     concurrency_group: 'performance-test-group'
+    agents:
+      queue: c2-16
 EOF
 
 

.buildkite/scripts/steps/functional/performance_sub_playwright.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+echo --- Run Performance Tests with Playwright config
+
+node scripts/es snapshot&
+
+esPid=$!
+
+export TEST_PERFORMANCE_PHASE=WARMUP
+export TEST_ES_URL=http://elastic:changeme@localhost:9200
+export TEST_ES_DISABLE_STARTUP=true
+export ELASTIC_APM_ACTIVE=false
+
+sleep 120
+
+cd "$XPACK_DIR"
+
+# warmup round 1
+checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Phase: WARMUP)" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config "test/performance/config.playwright.ts";
+
+export TEST_PERFORMANCE_PHASE=TEST
+export ELASTIC_APM_ACTIVE=true
+
+checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Phase: TEST)" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config "test/performance/config.playwright.ts";
+
+kill "$esPid"

.buildkite/scripts/steps/package_testing/build.sh

@@ -4,9 +4,10 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-echo "--- Build Kibana Distribution"
 node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-ubi --skip-docker-contexts
 
+DOCKER_FILE="kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz"
+
 cd target
-buildkite-agent artifact upload "./*-docker-image.tar.gz;./*.deb;./*.rpm"
+buildkite-agent artifact upload "./$DOCKER_FILE;./*.deb;./*.rpm"
 cd ..

.buildkite/scripts/steps/package_testing/test.sh

@@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
   buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.50.6"
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
-  buildkite-agent artifact download 'kibana-*-docker-image.tar.gz' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.50.7"
 fi
 cd ..

.gitignore

@@ -87,8 +87,9 @@ report.asciidoc
 /bazel
 /bazel-*
 .bazelrc.user
+.bazelrc.cache
 
 elastic-agent-*
 fleet-server-*
 elastic-agent.yml
-fleet-server.yml
+fleet-server.yml

api_docs/cases.mdx

@@ -12,13 +12,13 @@ import casesObj from './cases.json';
 
 The Case management system in Kibana
 
-Contact [Security Solution Threat Hunting](https://github.com/orgs/elastic/teams/security-threat-hunting) for questions regarding this plugin.
+Contact [ResponseOps](https://github.com/orgs/elastic/teams/response-ops) for questions regarding this plugin.
 
 **Code health stats**
 
-| Public API count  | Any count | Items lacking comments | Missing exports |
-|-------------------|-----------|------------------------|-----------------|
-| 83 | 0 | 57 | 23 |
+| Public API count | Any count | Items lacking comments | Missing exports |
+| ---------------- | --------- | ---------------------- | --------------- |
+| 83               | 0         | 57                     | 23              |
 
 ## Client
 

api_docs/plugin_directory.mdx

@@ -31,7 +31,7 @@ warning: This document is auto-generated and is meant to be viewed inside our ex
 | <DocLink id="kibBannersPluginApi" text="banners"/> | [Kibana Core](https://github.com/orgs/elastic/teams/kibana-core) | - | 9 | 0 | 9 | 0 |
 | <DocLink id="kibBfetchPluginApi" text="bfetch"/> | [App Services](https://github.com/orgs/elastic/teams/kibana-app-services) | Considering using bfetch capabilities when fetching large amounts of data. This services supports batching HTTP requests and streaming responses back. | 76 | 1 | 67 | 2 |
 | <DocLink id="kibCanvasPluginApi" text="canvas"/> | [Kibana Presentation](https://github.com/orgs/elastic/teams/kibana-presentation) | Adds Canvas application to Kibana | 9 | 0 | 8 | 3 |
-| <DocLink id="kibCasesPluginApi" text="cases"/> | [Security Solution Threat Hunting](https://github.com/orgs/elastic/teams/security-threat-hunting) | The Case management system in Kibana | 83 | 0 | 57 | 23 |
+| <DocLink id="kibCasesPluginApi" text="cases"/> | [ResponseOps](https://github.com/orgs/elastic/teams/response-ops) | The Case management system in Kibana | 83 | 0 | 57 | 23 |
 | <DocLink id="kibChartsPluginApi" text="charts"/> | [Vis Editors](https://github.com/orgs/elastic/teams/kibana-vis-editors) | - | 314 | 2 | 281 | 4 |
 | <DocLink id="kibCloudPluginApi" text="cloud"/> | [Kibana Core](https://github.com/orgs/elastic/teams/kibana-core) | - | 22 | 0 | 22 | 0 |
 | <DocLink id="kibConsolePluginApi" text="console"/> | [Stack Management](https://github.com/orgs/elastic/teams/kibana-stack-management) | - | 13 | 0 | 13 | 1 |

docs/apm/troubleshooting.asciidoc

@@ -47,37 +47,14 @@ This section can help with any of the following:
 There are a number of factors that could be at play here.
 One important thing to double-check first is your index template.
 
-*Index template*
-An APM index template must exist for the APM app to work correctly.
-By default, this index template is created by APM Server on startup.
-However, this only happens if `setup.template.enabled` is `true` in `apm-server.yml`.
-You can create the index template manually by running `apm-server setup`.
+*Index templates*
+For the APM app to work correctly, an index template must exist for each APM data stream.
+By default, {fleet} sets up APM index templates when the APM integration is installed.
 Take note that index templates *cannot* be applied retroactively -- they are only applied at index creation time.
 More information is available in {apm-guide-ref}/apm-server-configuration.html[Set up and configure].
 
-You can check for the existence of an APM index template using the
-{ref}/indices-get-template.html[Get index template API].
-If you're using the default index naming pattern, that request would be:
-
-[source,js]
---------------------------------------------------
-GET /_template/apm-{version}
---------------------------------------------------
-// CONSOLE
-
-*Using Logstash, Kafka, etc.*
-If you're not outputting data directly from APM Server to Elasticsearch (perhaps you're using Logstash or Kafka),
-then the index template will not be set up automatically. Instead, you'll need to
-{apm-guide-ref}/apm-server-template.html[load the template manually].
-
-*Using a custom index names*
-This problem can also occur if you've customized the index name that you write APM data to.
-If you change the default, you must also configure the `setup.template.name` and `setup.template.pattern` options.
-See {apm-guide-ref}/configuration-template.html[Load the Elasticsearch index template].
-If the Elasticsearch index template has already been successfully loaded to the index,
-you can customize the indices that the APM app uses to display data.
-Navigate to *APM* > *Settings* > *Indices*, and change all `xpack.apm.indices.*` values to
-include the new index pattern. For example: `customIndexName-*`.
+You can check for the existence of APM index templates in Kibana.
+Go to **Stack Management** > **Index Management** > **Index Templates** and search for `apm`.
 
 [float]
 [[troubleshooting-too-many-transactions]]
@@ -164,39 +141,34 @@ You can also use the Agent's public API to manually set a name for the transacti
 === Fields are not searchable
 
 In Elasticsearch, index templates are used to define settings and mappings that determine how fields should be analyzed.
-The recommended index template file for APM Server is installed by the APM Server packages.
-This template, by default, enables and disables indexing on certain fields.
+The recommended index templates for APM are installed by {fleet} when the Elastic APM integration is installed.
+These templates, by default, enable and disable indexing on certain fields.
 
 As an example, some agents store cookie values in `http.request.cookies`.
 Since `http.request` has disabled dynamic indexing, and `http.request.cookies` is not declared in a custom mapping,
 the values in `http.request.cookies` are not indexed and thus not searchable.
 
-*Ensure an index pattern exists*
-As a first step, you should ensure the correct index pattern exists.
-Open the main menu, then click *Stack Management > Index Patterns*.
-In the pattern list, you should see an apm index pattern; The default is `apm-*`.
-If you don't, the index pattern doesn't exist. See <<no-apm-data-found>> for information on how to fix this problem.
+*Ensure an APM data view exists*
+As a first step, you should ensure the correct data view exists.
+In {kib}, go to *Stack Management* > *Data views*.
+You should see the APM data view--the default is
+`traces-apm*,apm-*,logs-apm*,apm-*,metrics-apm*,apm-*`.
+If you don't, the data view doesn't exist.
+To fix this, navigate to the APM app in {kib} and select *Add data*.
+In the APM tutorial, click *Load Kibana objects* to create the APM data view.
 
-Selecting the `apm-*` index pattern shows a listing of every field defined in the pattern.
+If creating an APM data view doesn't solve the problem,
+see <<no-apm-data-found>> for further troubleshooting.
 
 *Ensure a field is searchable*
 There are two things you can do to if you'd like to ensure a field is searchable:
 
 1. Index your additional data as {apm-guide-ref}/metadata.html[labels] instead.
 These are dynamic by default, which means they will be indexed and become searchable and aggregatable.
 
-2. Use the `append_fields` feature. As an example,
-adding the following to `apm-server.yml` will enable dynamic indexing for `http.request.cookies`:
-
-[source,yml]
-----
-setup.template.enabled: true
-setup.template.overwrite: true
-setup.template.append_fields:
-  - name: http.request.cookies
-    type: object
-    dynamic: true
-----
+2. Create a custom mapping for the field.
+// link will be added in a later PR.
+// docs will be added in https://github.com/elastic/apm-server/pull/6940
 
 [float]
 [[service-map-rum-connections]]

docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md

@@ -147,6 +147,7 @@ readonly links: {
             readonly significant_terms: string;
             readonly terms: string;
             readonly terms_doc_count_error: string;
+            readonly rare_terms: string;
             readonly avg: string;
             readonly avg_bucket: string;
             readonly max_bucket: string;