Merge branch '7.x' into ua/fix_terms_used_in_systemindices_migration_api

elastic · Oct 19, 2021 · 38a7276 · 38a7276
2 parents e612778 + 3e17aff
commit 38a7276
Show file tree

Hide file tree

Showing 791 changed files with 40,128 additions and 42,248 deletions.
diff --git a/.buildkite/pipelines/performance/nightly.yml b/.buildkite/pipelines/performance/nightly.yml
@@ -0,0 +1,35 @@
+steps:
+  - block: ":gear: Performance Tests Configuration"
+    prompt: "Fill out the details for performance test"
+    fields:
+      - text: ":arrows_counterclockwise: Iterations"
+        key: "performance-test-iteration-count"
+        hint: "How many times you want to run tests? "
+        required: true
+    if: build.env('ITERATION_COUNT_ENV') == null
+
+  - label: ":male-mechanic::skin-tone-2: Pre-Build"
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+
+  - wait
+
+  - label: ":factory_worker: Build Kibana Distribution and Plugins"
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      queue: c2-16
+    key: build
+
+  - label: ":muscle: Performance Tests"
+    command: .buildkite/scripts/steps/functional/performance.sh
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    concurrency: 50
+    concurrency_group: 'performance-test-group'
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":male_superhero::skin-tone-2: Post-Build"
+    command: .buildkite/scripts/lifecycle/post_build.sh
+
diff --git a/.buildkite/scripts/steps/functional/performance.sh b/.buildkite/scripts/steps/functional/performance.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -uo pipefail
+
+if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
+  ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
+else
+  ITERATION_COUNT=$ITERATION_COUNT_ENV
+fi
+
+tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
+
+cat << EOF | buildkite-agent pipeline upload
+steps:
+  - command: .buildkite/scripts/steps/functional/performance_sub.sh
+    parallelism: "$ITERATION_COUNT"
+EOF
+
+
+
diff --git a/.buildkite/scripts/steps/functional/performance_sub.sh b/.buildkite/scripts/steps/functional/performance_sub.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+cd "$XPACK_DIR"
+
+echo --- Run Performance Tests
+checks-reporter-with-killswitch "Run Performance Tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config test/performance/config.ts;
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -156,6 +156,78 @@ const DEV_PATTERNS = [
   'x-pack/plugins/*/server/scripts/**/*',
 ];
 
+/** Restricted imports with suggested alternatives */
+const RESTRICTED_IMPORTS = [
+  {
+    name: 'lodash',
+    importNames: ['set', 'setWith'],
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash.set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash.setwith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/setWith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp',
+    importNames: ['set', 'setWith', 'assoc', 'assocPath'],
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/setWith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/assoc',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/assocPath',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash',
+    importNames: ['template'],
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash.template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/fp',
+    importNames: ['template'],
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/fp/template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'react-use',
+    message: 'Please use react-use/lib/{method} instead.',
+  },
+];
+
 module.exports = {
   root: true,
 
@@ -671,81 +743,7 @@ module.exports = {
         'no-restricted-imports': [
           2,
           {
-            paths: [
-              {
-                name: 'lodash',
-                importNames: ['set', 'setWith'],
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash.set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash.setwith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/setWith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp',
-                importNames: ['set', 'setWith', 'assoc', 'assocPath'],
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/setWith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/assoc',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/assocPath',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash',
-                importNames: ['template'],
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash.template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/fp',
-                importNames: ['template'],
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/fp/template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'react-use',
-                message: 'Please use react-use/lib/{method} instead.',
-              },
-            ],
+            paths: RESTRICTED_IMPORTS,
           },
         ],
         'no-restricted-modules': [
@@ -838,6 +836,23 @@ module.exports = {
         ],
       },
     },
+    {
+      files: ['**/common/**/*.{js,mjs,ts,tsx}', '**/public/**/*.{js,mjs,ts,tsx}'],
+      rules: {
+        'no-restricted-imports': [
+          2,
+          {
+            paths: [
+              ...RESTRICTED_IMPORTS,
+              {
+                name: 'semver',
+                message: 'Please use "semver/*/{function}" instead',
+              },
+            ],
+          },
+        ],
+      },
+    },
 
     /**
      * APM and Observability overrides
@@ -1585,8 +1600,8 @@ module.exports = {
       plugins: ['react', '@typescript-eslint'],
       files: ['x-pack/plugins/osquery/**/*.{js,mjs,ts,tsx}'],
       rules: {
-        // 'arrow-body-style': ['error', 'as-needed'],
-        // 'prefer-arrow-callback': 'error',
+        'arrow-body-style': ['error', 'as-needed'],
+        'prefer-arrow-callback': 'error',
         'no-unused-vars': 'off',
         'react/prop-types': 'off',
         '@typescript-eslint/explicit-module-boundary-types': 'off',

diff --git a/docs/developer/plugin/external-plugin-localization.asciidoc b/docs/developer/plugin/external-plugin-localization.asciidoc
@@ -135,27 +135,6 @@ export const Component = () => {
 
 Full details are {kib-repo}tree/master/packages/kbn-i18n#react[here].
 
-
-
-[discrete]
-==== i18n for Angular
-
-You are encouraged to use `i18n.translate()` by statically importing `i18n` from `@kbn/i18n` wherever possible in your Angular code. Angular wrappers use the translation `service` with the i18n engine under the hood.
-
-The translation directive has the following syntax:
-["source","js"]
------------
-<ANY
-  i18n-id="{string}"
-  i18n-default-message="{string}"
-  [i18n-values="{object}"]
-  [i18n-description="{string}"]
-></ANY>
------------
-
-Full details are {kib-repo}tree/master/packages/kbn-i18n#angularjs[here].
-
-
 [discrete]
 === Resources
 

diff --git a/docs/getting-started/quick-start-guide.asciidoc b/docs/getting-started/quick-start-guide.asciidoc
@@ -138,7 +138,7 @@ image::images/dashboard_sampleDataAddFilter_7.15.0.png[The [eCommerce] Revenue D
 [[quick-start-whats-next]]
 == What's next?
 
-*Add your own data.* Ready to add your own data? Go to {fleet-guide}/fleet-quick-start.html[Quick start: Get logs and metrics into the Elastic Stack] to learn how to ingest your data, or go to <<connect-to-elasticsearch,Add data to {kib}>> and learn about all the other ways you can add data.
+*Add your own data.* Ready to add your own data? Go to {observability-guide}/ingest-logs-metrics-uptime.html[Ingest logs, metrics, and uptime data with {agent}], or go to <<connect-to-elasticsearch,Add data to {kib}>> and learn about all the other ways you can add data.
 
 *Explore your own data in Discover.* Ready to learn more about exploring your data in *Discover*? Go to <<discover, Discover>>.
 

diff --git a/docs/management/connectors/action-types/pagerduty.asciidoc b/docs/management/connectors/action-types/pagerduty.asciidoc
@@ -68,7 +68,7 @@ PagerDuty actions have the following properties.
 
 Severity::      The perceived severity of on the affected system. This can be one of `Critical`, `Error`, `Warning` or `Info`(default).
 Event action::  One of `Trigger` (default), `Resolve`, or `Acknowledge`. See https://v2.developer.pagerduty.com/docs/events-api-v2#event-action[event action] for more details.
-Dedup Key::     All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `<alert ID>:<alert instance ID>`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. 
+Dedup Key::     All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `<rule ID>:<alert ID>`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. 
 Timestamp::     An *optional* https://v2.developer.pagerduty.com/v2/docs/types#datetime[ISO-8601 format date-time], indicating the time the event was detected or generated.
 Component::     An *optional* value indicating the component of the source machine that is responsible for the event, for example `mysql` or `eth0`.
 Group::         An *optional* value indicating the logical grouping of components of a service, for example `app-stack`.

diff --git a/docs/management/connectors/action-types/servicenow-sir.asciidoc b/docs/management/connectors/action-types/servicenow-sir.asciidoc
@@ -72,13 +72,11 @@ image::management/connectors/images/servicenow-sir-params-test.png[ServiceNow Se
 ServiceNow SecOps actions have the following configuration properties.
 
 Short description::    A short description for the incident, used for searching the contents of the knowledge base.
-Source Ips::           A list of source IPs related to the incident. The IPs will be added as observables to the security incident.
-Destination Ips::      A list of destination IPs related to the incident. The IPs will be added as observables to the security incident.
-Malware URLs::         A list of malware URLs related to the incident. The URLs will be added as observables to the security incident.
-Malware Hashes::       A list of malware hashes related to the incident. The hashes  will be added as observables to the security incident.
 Priority::             The priority of the incident.
 Category::             The category of the incident.
 Subcategory::          The subcategory of the incident.
+Correlation ID::       All actions sharing this ID will be associated with the same ServiceNow security incident. If an incident exists in ServiceNow with the same correlation ID the security incident will be updated. Default value: `<rule ID>:<alert instance ID>`.
+Correlation Display::  A descriptive label of the alert for correlation purposes in ServiceNow.
 Description::          The details about the incident.
 Additional comments::  Additional information for the client, such as how to troubleshoot the issue.
 

diff --git a/docs/management/connectors/action-types/servicenow.asciidoc b/docs/management/connectors/action-types/servicenow.asciidoc
@@ -76,6 +76,8 @@ Severity::             The severity of the incident.
 Impact::               The effect an incident has on business. Can be measured by the number of affected users or by how critical it is to the business in question.
 Category::             The category of the incident.
 Subcategory::          The category of the incident.
+Correlation ID::       All actions sharing this ID will be associated with the same ServiceNow incident. If an incident exists in ServiceNow with the same correlation ID the incident will be updated. Default value: `<rule ID>:<alert instance ID>`.
+Correlation Display::  A descriptive label of the alert for correlation purposes in ServiceNow.
 Short description::    A short description for the incident, used for searching the contents of the knowledge base.
 Description::          The details about the incident.
 Additional comments::  Additional information for the client, such as how to troubleshoot the issue.

diff --git a/docs/management/connectors/images/servicenow-sir-params-test.png b/docs/management/connectors/images/servicenow-sir-params-test.png
diff --git a/docs/osquery/osquery.asciidoc b/docs/osquery/osquery.asciidoc
@@ -365,7 +365,7 @@ The following is an example of an **error response** for an undefined action que
 == System requirements
 
 * {fleet-guide}/fleet-overview.html[Fleet] is enabled on your cluster, and
-one or more {fleet-guide}/elastic-agent-installation-configuration.html[Elastic Agents] is enrolled.
+one or more {fleet-guide}/elastic-agent-installation.html[Elastic Agents] is enrolled.
 * The https://docs.elastic.co/en/integrations/osquery_manager[*Osquery Manager*] integration
 has been added and configured
 for an agent policy through Fleet.

diff --git a/docs/setup/connect-to-elasticsearch.asciidoc b/docs/setup/connect-to-elasticsearch.asciidoc
@@ -47,7 +47,7 @@ so you can quickly get insights into your data, and {fleet} mode offers several
 image::images/addData_fleet_7.15.0.png[Add data using Fleet]
 
 To get started, refer to
-{fleet-guide}/fleet-quick-start.html[Quick start: Get logs and metrics into the Elastic Stack].
+{observability-guide}/ingest-logs-metrics-uptime.html[Ingest logs, metrics, and uptime data with {agent}].
 
 [discrete]
 [[upload-data-kibana]]

diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc
@@ -291,7 +291,7 @@ that the {kib} server uses to perform maintenance on the {kib} index at startup.
 is an alternative to `elasticsearch.username` and `elasticsearch.password`.
 
 | `enterpriseSearch.host`
-  | The URL of your Enterprise Search instance
+  | The http(s) URL of your Enterprise Search instance. For example, in a local self-managed setup, set this to `http://localhost:3002`. Authentication between Kibana and the Enterprise Search host URL, such as via OAuth, is not supported. You can also {enterprise-search-ref}/configure-ssl-tls.html#configure-ssl-tls-in-kibana[configure Kibana to trust your Enterprise Search TLS certificate authority].
 
 | `interpreter.enableInVisualize`
   | Enables use of interpreter in Visualize. *Default: `true`*

diff --git a/package.json b/package.json
@@ -341,7 +341,7 @@
     "react-moment-proptypes": "^1.7.0",
     "react-monaco-editor": "^0.41.2",
     "react-popper-tooltip": "^2.10.1",
-    "react-query": "^3.21.1",
+    "react-query": "^3.27.0",
     "react-redux": "^7.2.0",
     "react-resizable": "^1.7.5",
     "react-resize-detector": "^4.2.0",
@@ -496,7 +496,6 @@
     "@testing-library/react-hooks": "^5.1.1",
     "@testing-library/user-event": "^13.1.1",
     "@types/angular": "^1.6.56",
-    "@types/angular-mocks": "^1.7.0",
     "@types/apidoc": "^0.22.3",
     "@types/archiver": "^5.1.0",
     "@types/babel__core": "^7.1.16",
@@ -651,7 +650,6 @@
     "@yarnpkg/lockfile": "^1.1.0",
     "abab": "^2.0.4",
     "aggregate-error": "^3.1.0",
-    "angular-mocks": "^1.7.9",
     "antlr4ts-cli": "^0.5.0-alpha.3",
     "apidoc": "^0.29.0",
     "apidoc-markdown": "^6.0.0",