Merge branch 'main' into fixing-text-classification-model-testing

elastic · Apr 27, 2022 · a2755bb · a2755bb
2 parents dde7160 + 29d6e3e
commit a2755bb
Show file tree

Hide file tree

Showing 834 changed files with 14,751 additions and 13,261 deletions.
diff --git a/.buildkite/pipelines/es_snapshots/verify.yml b/.buildkite/pipelines/es_snapshots/verify.yml
@@ -42,7 +42,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: ci-group-4d
     depends_on: build

diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -66,7 +66,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: n2-4-spot
     depends_on: build

diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml
@@ -32,7 +32,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
-    parallelism: 11
+    parallelism: 12
     agents:
       queue: n2-4-spot
     depends_on: build

diff --git a/.buildkite/scripts/steps/functional/synthetics.sh b/.buildkite/scripts/steps/functional/synthetics.sh
@@ -14,4 +14,4 @@ echo "--- Uptime @elastic/synthetics Tests"
 cd "$XPACK_DIR"
 
 checks-reporter-with-killswitch "Uptime @elastic/synthetics Tests" \
-  node plugins/uptime/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --grep "MonitorManagement-monitor*"
+  node plugins/synthetics/scripts/e2e.js --kibana-install-dir "$KIBANA_BUILD_LOCATION" --grep "MonitorManagement-monitor*"
diff --git a/.ci/ci_groups.yml b/.ci/ci_groups.yml
@@ -10,6 +10,7 @@ root:
   - ciGroup9
   - ciGroup10
   - ciGroup11
+  - ciGroup12
 
 xpack:
   - ciGroup1

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -415,6 +415,7 @@
 /x-pack/plugins/security_solution/cypress/integration/urls @elastic/security-threat-hunting-investigations
 
 /x-pack/plugins/security_solution/public/common/components/alerts_viewer @elastic/security-threat-hunting-investigations
+/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_action @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/common/components/event_details @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/common/components/events_viewer @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/common/components/markdown_editor @elastic/security-threat-hunting-investigations

diff --git a/docs/api/actions-and-connectors/create.asciidoc b/docs/api/actions-and-connectors/create.asciidoc
@@ -74,6 +74,7 @@ The API returns the following:
     "executionTimeField": null
   },
   "is_preconfigured": false,
+  "is_deprecated": false,
   "is_missing_secrets": false
 }
 --------------------------------------------------
diff --git a/docs/api/actions-and-connectors/get.asciidoc b/docs/api/actions-and-connectors/get.asciidoc
@@ -51,6 +51,7 @@ The API returns the following:
     "executionTimeField": null
   },
   "is_preconfigured": false,
+  "is_deprecated": false,
   "is_missing_secrets": false
 }
 --------------------------------------------------
diff --git a/docs/api/actions-and-connectors/get_all.asciidoc b/docs/api/actions-and-connectors/get_all.asciidoc
@@ -44,6 +44,7 @@ The API returns the following:
     "connector_type_id": ".email",
     "name": "email: preconfigured-mail-connector",
     "is_preconfigured": true,
+    "is_deprecated": false,
     "referenced_by_count": 0 <1>
   },
   {
@@ -56,6 +57,7 @@ The API returns the following:
       "executionTimeField": null
     },
     "is_preconfigured": false,
+    "is_deprecated": false,
     "is_missing_secrets": false,
     "referenced_by_count": 3
   }

diff --git a/docs/api/actions-and-connectors/legacy/create.asciidoc b/docs/api/actions-and-connectors/legacy/create.asciidoc
@@ -76,6 +76,7 @@ The API returns the following:
     "executionTimeField": null
   },
   "isPreconfigured": false,
+  "isDeprecated": false,
   "isMissingSecrets": false
 }
 --------------------------------------------------
diff --git a/docs/api/actions-and-connectors/legacy/get.asciidoc b/docs/api/actions-and-connectors/legacy/get.asciidoc
@@ -53,6 +53,7 @@ The API returns the following:
     "executionTimeField": null
   },
   "isPreconfigured": false,
+  "isDeprecated": false,
   "isMissingSecrets": false
 }
 --------------------------------------------------
diff --git a/docs/api/actions-and-connectors/legacy/get_all.asciidoc b/docs/api/actions-and-connectors/legacy/get_all.asciidoc
@@ -45,7 +45,8 @@ The API returns the following:
     "id": "preconfigured-mail-action",
     "actionTypeId": ".email",
     "name": "email: preconfigured-mail-action",
-    "isPreconfigured": true
+    "isPreconfigured": true,
+    "isDeprecated": false
   },
   {
     "id": "c55b6eb0-6bad-11eb-9f3b-611eebc6c3ad",
@@ -57,6 +58,7 @@ The API returns the following:
       "executionTimeField": null
     },
     "isPreconfigured": false,
+    "isDeprecated": false,
     "isMissingSecrets": false
   }
 ]

diff --git a/docs/api/actions-and-connectors/legacy/update.asciidoc b/docs/api/actions-and-connectors/legacy/update.asciidoc
@@ -71,6 +71,7 @@ The API returns the following:
     "executionTimeField": null
   },
   "isPreconfigured": false,
+  "isDeprecated": false,
   "isMissingSecrets": false
 }
 --------------------------------------------------
diff --git a/docs/api/actions-and-connectors/update.asciidoc b/docs/api/actions-and-connectors/update.asciidoc
@@ -69,6 +69,7 @@ The API returns the following:
     "executionTimeField": null
   },
   "is_preconfigured": false,
+  "is_deprecated": false,
   "is_missing_secrets": false
 }
 --------------------------------------------------
diff --git a/docs/api/cases/cases-api-find-connectors.asciidoc b/docs/api/cases/cases-api-find-connectors.asciidoc
@@ -55,6 +55,7 @@ The API returns a JSON object describing the connectors and their settings:
     "projectKey":"ES"
   },
   "isPreconfigured":false,
+  "isDeprecated": false,
   "referencedByCount":0
 }]
 --------------------------------------------------
diff --git a/docs/management/connectors/action-types/email.asciidoc b/docs/management/connectors/action-types/email.asciidoc
@@ -9,6 +9,8 @@ The email connector uses the SMTP protocol to send mail messages, using an integ
 
 NOTE: For emails to have a footer with a link back to {kib}, set the <<server-publicBaseUrl, `server.publicBaseUrl`>> configuration setting.
 
+NOTE: When the <<action-config-email-domain-allowlist, `xpack.actions.email.domain_allowlist`>> configuration setting is used, the email addresses used for all of the Sender (from), To, CC, and BCC properties must have email domains specified in the configuration setting.
+
 [float]
 [[email-connector-configuration]]
 ==== Connector configuration

diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc
@@ -125,6 +125,11 @@ The contents of a PEM-encoded certificate file, or multiple files appended
 into a single string.  This configuration can be used for environments where
 the files cannot be made available.
 
+[[action-config-email-domain-allowlist]] `xpack.actions.email.domain_allowlist`  {ess-icon}::
+A list of allowed email domains which can be used with the email connector.  When this setting is not used, all email domains are allowed.  When this setting is used, if any email is attempted to be sent that includes an addressee with an email domain that is not in the allowlist, or the from address domain is not in the allowlist, the run of the connector will fail with a message indicating the emails not allowed.
+
+WARNING: This feature is available in {kib} 7.17.4 and 8.3.0 onwards but is not supported in {kib} 8.0, 8.1 or 8.2. As such this settings should be removed before upgrading from 7.17 to 8.0, 8.1 or 8.2. It is possible to configure the settings in 7.17.4 and then upgrade to 8.3.0 directly.
+
 `xpack.actions.enabledActionTypes` {ess-icon}::
 A list of action types that are enabled. It defaults to `[*]`, enabling all types. The names for built-in {kib} action types are prefixed with a `.` and include: `.email`, `.index`, `.jira`, `.pagerduty`, `.resilient`, `.server-log`, `.servicenow`, .`servicenow-itom`, `.servicenow-sir`, `.slack`, `.swimlane`, `.teams`, `.xmatters`, and `.webhook`. An empty list `[]` will disable all action types.
 +
@@ -211,13 +216,26 @@ For example, `20m`, `24h`, `7d`, `1w`. Default: `5m`.
 `xpack.alerting.rules.run.ruleTypeOverrides`::
 Overrides the configs under `xpack.alerting.rules.run` for the rule type with the given ID. List the rule identifier and its settings in an array of objects.
 +
---
 For example:
-```
+[source,yaml]
+--
 xpack.alerting.rules.run:
     timeout: '5m'
     ruleTypeOverrides:
         - id: '.index-threshold'
           timeout: '15m'
-```
---
+--
+
+`xpack.alerting.rules.run.actions.connectorTypeOverrides`::
+Overrides the configs under `xpack.alerting.rules.run.actions` for the connector type with the given ID. List the connector type identifier and its settings in an array of objects.
++
+For example:
+[source,yaml]
+--
+xpack.alerting.rules.run:
+    actions:
+        max: 10
+        connectorTypeOverrides:
+            - id: '.server-log'
+              max: 5
+--
diff --git a/docs/user/alerting/troubleshooting/testing-connectors.asciidoc b/docs/user/alerting/troubleshooting/testing-connectors.asciidoc
@@ -39,6 +39,7 @@ $ kbn-action ls
             "service": null
         },
         "isPreconfigured": false,
+        "isDeprecated": false,
         "referencedByCount": 0
     }
 ]

diff --git a/package.json b/package.json
@@ -250,6 +250,7 @@
     "deepmerge": "^4.2.2",
     "del": "^5.1.0",
     "elastic-apm-node": "^3.31.0",
+    "email-addresses": "^5.0.0",
     "execa": "^4.0.2",
     "exit-hook": "^2.2.0",
     "expiry-js": "0.1.7",

diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml
@@ -1,5 +1,6 @@
 pageLoadAssetSize:
   advancedSettings: 27596
+  actions: 20000
   alerting: 106936
   apm: 64385
   canvas: 1066647

diff --git a/...-optimizer/src/babel_runtime_helpers/find_node_libs_browser_polyfills_in_entry_bundles.ts b/...-optimizer/src/babel_runtime_helpers/find_node_libs_browser_polyfills_in_entry_bundles.ts
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+
+import { run } from '@kbn/dev-utils';
+import { REPO_ROOT } from '@kbn/utils';
+
+import { OptimizerConfig } from '../optimizer';
+import { parseStats, inAnyEntryChunk } from './parse_stats';
+
+export async function runFindNodeLibsBrowserPolyfillsInEntryBundlesCli() {
+  run(async ({ log }) => {
+    const config = OptimizerConfig.create({
+      includeCoreBundle: true,
+      repoRoot: REPO_ROOT,
+    });
+
+    const paths = config.bundles.map((b) => Path.resolve(b.outputDir, 'stats.json'));
+
+    log.info('analyzing', paths.length, 'stats files');
+    log.verbose(paths);
+
+    const imports = new Set();
+    for (const path of paths) {
+      const stats = parseStats(path);
+
+      for (const module of stats.modules) {
+        if (!inAnyEntryChunk(stats, module)) {
+          continue;
+        }
+
+        // Relying on module name instead of actual imports because these are usual polyfills that assume the global
+        // Node.js environment when development (i.e.: Buffer doesn't require an import to be used).
+        if (module.name.includes('node-libs-browser/node_modules/')) {
+          imports.add(module.name);
+        }
+      }
+    }
+
+    log.success('found', imports.size, 'node-libs-browser/* imports in entry bundles');
+    log.write(
+      Array.from(imports, (i) => `'${i}',`)
+        .sort()
+        .join('\n')
+    );
+  });
+}
diff --git a/packages/kbn-optimizer/src/babel_runtime_helpers/index.ts b/packages/kbn-optimizer/src/babel_runtime_helpers/index.ts
@@ -7,3 +7,4 @@
  */
 
 export * from './find_babel_runtime_helpers_in_entry_bundles';
+export * from './find_node_libs_browser_polyfills_in_entry_bundles';
diff --git a/packages/kbn-optimizer/src/babel_runtime_helpers/parse_stats.ts b/packages/kbn-optimizer/src/babel_runtime_helpers/parse_stats.ts
@@ -20,6 +20,7 @@ const partialObject = <P extends Props>(props: P) => {
 export type Module = TypeOf<typeof moduleSchema>;
 const moduleSchema = partialObject({
   identifier: schema.string(),
+  name: schema.string(),
   chunks: schema.arrayOf(schema.oneOf([schema.string(), schema.number()])),
   reasons: schema.arrayOf(
     partialObject({

diff --git a/packages/kbn-tooling-log/BUILD.bazel b/packages/kbn-tooling-log/BUILD.bazel
@@ -54,6 +54,7 @@ TYPES_DEPS = [
   "@npm//tslib",
   "@npm//@types/node",
   "@npm//@types/jest",
+  "//packages/kbn-jest-serializers:npm_module_types" # needed for windows development, only used in tests
 ]
 
 jsts_transpiler(

diff --git a/packages/kbn-ui-shared-deps-npm/webpack.config.js b/packages/kbn-ui-shared-deps-npm/webpack.config.js
@@ -37,6 +37,10 @@ module.exports = (_, argv) => {
         'regenerator-runtime/runtime',
         'whatwg-fetch',
         'symbol-observable',
+        // Parts of node-libs-browser that are used in many places across Kibana
+        'buffer',
+        'punycode',
+        'util',
 
         /**
          * babel runtime helpers referenced from entry chunks

diff --git a/scripts/find_node_libs_browser_polyfills_in_use.js b/scripts/find_node_libs_browser_polyfills_in_use.js
@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../src/setup_node_env/no_transpilation');
+require('@kbn/optimizer').runFindNodeLibsBrowserPolyfillsInEntryBundlesCli();
diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker
@@ -188,6 +188,7 @@ kibana_vars=(
     vis_type_vega.enableExternalUrls
     xpack.actions.allowedHosts
     xpack.actions.customHostSettings
+    xpack.actions.email.domain_allowlist
     xpack.actions.enabledActionTypes
     xpack.actions.maxResponseContentLength
     xpack.actions.preconfigured
@@ -211,6 +212,7 @@ kibana_vars=(
     xpack.alerting.rules.minimumScheduleInterval.value
     xpack.alerting.rules.minimumScheduleInterval.enforce
     xpack.alerting.rules.run.actions.max
+    xpack.alerting.rules.run.actions.connectorTypeOverrides
     xpack.alerts.healthCheck.interval
     xpack.alerts.invalidateApiKeysTask.interval
     xpack.alerts.invalidateApiKeysTask.removalDelay

diff --git a/src/dev/prs/README.md b/src/dev/prs/README.md
@@ -0,0 +1,13 @@
+# Pulling a list of PRs
+
+This folder contains files used to pull lists of Kibana PRs for release testing.  
+
+`scripts/download_pr_list.js` is the cli wrapper.
+
+You must have a `GITHUB_TOKEN` either set in your environment or on command line like
+
+`GITHUB_TOKEN=<your token here> node scripts/download_pr_list.js`
+
+Run it with `--help` or without arguments for help.
+
+`kibana_qa_pr_list.json` is the file currently used by the Kibana QA team and also serves as an example.