Merge branch 'master' into expression_error

.github/CODEOWNERS

@@ -315,7 +315,6 @@
 /src/plugins/es_ui_shared/  @elastic/kibana-stack-management
 /x-pack/plugins/cross_cluster_replication/  @elastic/kibana-stack-management
 /x-pack/plugins/index_lifecycle_management/  @elastic/kibana-stack-management
-/x-pack/plugins/console_extensions/  @elastic/kibana-stack-management
 /x-pack/plugins/grokdebugger/  @elastic/kibana-stack-management
 /x-pack/plugins/index_management/  @elastic/kibana-stack-management
 /x-pack/plugins/license_api_guard/  @elastic/kibana-stack-management
@@ -330,7 +329,6 @@
 /x-pack/plugins/ingest_pipelines/  @elastic/kibana-stack-management
 /packages/kbn-ace/ @elastic/kibana-stack-management
 /packages/kbn-monaco/ @elastic/kibana-stack-management
-#CC# /x-pack/plugins/console_extensions/ @elastic/kibana-stack-management
 #CC# /x-pack/plugins/cross_cluster_replication/ @elastic/kibana-stack-management
 
 # Security Solution

docs/developer/plugin-list.asciidoc

@@ -358,10 +358,6 @@ The plugin exposes the static DefaultEditorController class to consume.
 The client-side plugin configures following values:
 
 
-|{kib-repo}blob/{branch}/x-pack/plugins/console_extensions/README.md[consoleExtensions]
-|This plugin provides autocomplete definitions of licensed APIs to the OSS Console plugin.
-
-
 |{kib-repo}blob/{branch}/x-pack/plugins/cross_cluster_replication/README.md[crossClusterReplication]
 |You can run a local cluster and simulate a remote cluster within a single Kibana directory.
 

docs/maps/vector-layer.asciidoc

@@ -20,10 +20,10 @@ The index must contain at least one field mapped as {ref}/geo-point.html[geo_poi
 Results are limited to the `index.max_result_window` index setting, which defaults to 10000.
 Select the appropriate *Scaling* option for your use case.
 +
-* *Limit results to 10000.* The layer displays features from the first `index.max_result_window` documents.
+* *Limit results to 10,000* The layer displays features from the first `index.max_result_window` documents.
 Results exceeding `index.max_result_window` are not displayed.
 
-* *Show clusters when results exceed 10000.* When results exceed `index.max_result_window`, the layer uses {ref}/search-aggregations-bucket-geotilegrid-aggregation.html[GeoTile grid aggregation] to group your documents into clusters and displays metrics for each cluster. When results are less then `index.max_result_window`, the layer displays features from individual documents.
+* *Show clusters when results exceed 10,000* When results exceed `index.max_result_window`, the layer uses {ref}/search-aggregations-bucket-geotilegrid-aggregation.html[GeoTile grid aggregation] to group your documents into clusters and displays metrics for each cluster. When results are less then `index.max_result_window`, the layer displays features from individual documents.
 
 * *Use vector tiles.* Vector tiles partition your map into 6 to 8 tiles.
 Each tile request is limited to the `index.max_result_window` index setting.

docs/user/management.asciidoc

@@ -82,9 +82,10 @@ connectors>> for triggering actions.
 | Monitor the generation of reports—PDF, PNG, and CSV—and download reports that you previously generated.
 A report can contain a dashboard, visualization, saved search, or Canvas workpad.
 
-| {ml-docs}/ml-jobs.html[Machine Learning Jobs]
-| View your {anomaly-jobs} and {dfanalytics-jobs}. Open the Single Metric
-Viewer or Anomaly Explorer to see your {ml} results.
+| Machine Learning Jobs
+| View your <<xpack-ml-anomalies,{anomaly-detect}>> and
+<<xpack-ml-dfanalytics,{dfanalytics}>> jobs. Open the Single Metric
+Viewer or Anomaly Explorer to see your {anomaly-detect} results.
 
 | <<watcher-ui, Watcher>>
 | Detect changes in your data by creating, managing, and monitoring alerts.

docs/user/ml/index.asciidoc

@@ -48,8 +48,9 @@ pane:
 image::user/ml/images/ml-job-management.png[Job Management]
 
 You can use the *Settings* pane to create and edit
-{ml-docs}/ml-calendars.html[calendars] and the filters that are used in
-{ml-docs}/ml-rules.html[custom rules]:
+{ml-docs}/ml-ad-finding-anomalies.html#ml-ad-calendars[calendars] and the
+filters that are used in
+{ml-docs}/ml-ad-finding-anomalies.html#ml-ad-rules[custom rules]:
 
 [role="screenshot"]
 image::user/ml/images/ml-settings.png[Calendar Management]

package.json

@@ -99,7 +99,7 @@
   "dependencies": {
     "@elastic/apm-rum": "^5.8.0",
     "@elastic/apm-rum-react": "^1.2.11",
-    "@elastic/charts": "31.1.0",
+    "@elastic/charts": "32.0.0",
     "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath",
     "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.13",
     "@elastic/ems-client": "7.14.0",

packages/kbn-securitysolution-list-utils/src/autocomplete_operators/index.ts

@@ -11,8 +11,7 @@ import {
   ListOperatorEnum as OperatorEnum,
   ListOperatorTypeEnum as OperatorTypeEnum,
 } from '@kbn/securitysolution-io-ts-list-types';
-
-import { OperatorOption } from './types';
+import { OperatorOption } from '../types';
 
 export const isOperator: OperatorOption = {
   message: i18n.translate('lists.exceptions.isOperatorLabel', {

packages/kbn-securitysolution-list-utils/src/helpers/index.ts

@@ -43,7 +43,6 @@ import {
   isOneOfOperator,
   isOperator,
 } from '../autocomplete_operators';
-import { OperatorOption } from '../autocomplete_operators/types';
 
 import {
   BuilderEntry,
@@ -52,6 +51,7 @@ import {
   EmptyNestedEntry,
   ExceptionsBuilderExceptionItem,
   FormattedBuilderEntry,
+  OperatorOption,
 } from '../types';
 
 export const isEntryNested = (item: BuilderEntry): item is EntryNested => {

packages/kbn-securitysolution-list-utils/src/types/index.ts

@@ -23,7 +23,12 @@ import {
   EXCEPTION_LIST_NAMESPACE_AGNOSTIC,
 } from '@kbn/securitysolution-list-constants';
 
-import type { OperatorOption } from '../autocomplete_operators/types';
+export interface OperatorOption {
+  message: string;
+  value: string;
+  operator: OperatorEnum;
+  type: OperatorTypeEnum;
+}
 
 /**
  * @deprecated Use the one from core once it is in its own package which will be from:

packages/kbn-spec-to-console/README.md

@@ -18,15 +18,10 @@ git pull --depth=1 origin master
 
 ### Usage
 
-You need to run the command twice: once for the **OSS** specs and once for the **X-Pack** specs  
 At the root of the Kibana repository, run the following commands:
 
 ```sh
-# OSS
 yarn spec_to_console -g "<ELASTICSEARCH-REPO-FOLDER>/rest-api-spec/src/main/resources/rest-api-spec/api/*" -d "src/plugins/console/server/lib/spec_definitions/json/generated"
-
-# X-pack
-yarn spec_to_console -g "<ELASTICSEARCH-REPO-FOLDER>/x-pack/plugin/src/test/resources/rest-api-spec/api/*" -d "x-pack/plugins/console_extensions/server/lib/spec_definitions/json/generated"
 ```
 
 ### Information used in Console that is not available in the REST spec

packages/kbn-spec-to-console/lib/convert/params.js

@@ -37,6 +37,7 @@ module.exports = (params) => {
       case 'string':
       case 'number':
       case 'number|string':
+      case 'boolean|long':
         result[param] = defaultValue || '';
         break;
       case 'list':

src/core/public/doc_links/doc_links_service.ts

@@ -232,20 +232,20 @@ export class DocLinksService {
           guide: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/index.html`,
           aggregations: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-configuring-aggregation.html`,
           anomalyDetection: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-overview.html`,
-          anomalyDetectionJobs: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-jobs.html`,
+          anomalyDetectionJobs: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html`,
           anomalyDetectionConfiguringCategories: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-configuring-categories.html`,
-          anomalyDetectionBucketSpan: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/create-jobs.html#bucket-span`,
-          anomalyDetectionCardinality: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/create-jobs.html#cardinality`,
-          anomalyDetectionCreateJobs: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/create-jobs.html`,
-          anomalyDetectionDetectors: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/create-jobs.html#detectors`,
-          anomalyDetectionInfluencers: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-influencers.html`,
+          anomalyDetectionBucketSpan: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-bucket-span`,
+          anomalyDetectionCardinality: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-cardinality`,
+          anomalyDetectionCreateJobs: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-create-job`,
+          anomalyDetectionDetectors: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-detectors`,
+          anomalyDetectionInfluencers: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-influencers`,
           anomalyDetectionJobResource: `${ELASTICSEARCH_DOCS}ml-put-job.html#ml-put-job-path-parms`,
           anomalyDetectionJobResourceAnalysisConfig: `${ELASTICSEARCH_DOCS}ml-put-job.html#put-analysisconfig`,
-          anomalyDetectionJobTips: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/create-jobs.html#job-tips`,
-          anomalyDetectionModelMemoryLimits: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/create-jobs.html#model-memory-limits`,
-          calendars: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-calendars.html`,
+          anomalyDetectionJobTips: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-job-tips`,
+          anomalyDetectionModelMemoryLimits: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-model-memory-limits`,
+          calendars: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-calendars`,
           classificationEvaluation: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-dfa-classification.html#ml-dfanalytics-classification-evaluation`,
-          customRules: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-rules.html`,
+          customRules: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-rules`,
           customUrls: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-configuring-url.html`,
           dataFrameAnalytics: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-dfanalytics.html`,
           featureImportance: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-feature-importance.html`,

src/core/server/core_app/bundle_routes/file_hash.test.ts

@@ -19,7 +19,8 @@ const mockedCache = (): jest.Mocked<IFileHashCache> => ({
   set: jest.fn(),
 });
 
-describe('getFileHash', () => {
+// FLAKY: https://github.com/elastic/kibana/issues/105174
+describe.skip('getFileHash', () => {
   const sampleFilePath = resolve(__dirname, 'foo.js');
   const fd = 42;
   const stats: Stats = { ino: 42, size: 9000 } as any;

src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts

@@ -137,14 +137,15 @@ describe('migrateRawDocsSafely', () => {
     const transform = jest.fn<any, any>((doc: any) => [
       set(_.cloneDeep(doc), 'attributes.name', 'HOI!'),
     ]);
-    const task = migrateRawDocsSafely(
-      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
-      transform,
-      [
+    const task = migrateRawDocsSafely({
+      serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      knownTypes: new Set(['a', 'c']),
+      migrateDoc: transform,
+      rawDocs: [
         { _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } },
         { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
-      ]
-    );
+      ],
+    });
     const result = (await task()) as Either.Right<DocumentsTransformSuccess>;
     expect(result._tag).toEqual('Right');
     expect(result.right.processedDocs).toEqual([
@@ -181,14 +182,15 @@ describe('migrateRawDocsSafely', () => {
     const transform = jest.fn<any, any>((doc: any) => [
       set(_.cloneDeep(doc), 'attributes.name', 'TADA'),
     ]);
-    const task = migrateRawDocsSafely(
-      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
-      transform,
-      [
+    const task = migrateRawDocsSafely({
+      serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      knownTypes: new Set(['a', 'c']),
+      migrateDoc: transform,
+      rawDocs: [
         { _id: 'foo:b', _source: { type: 'a', a: { name: 'AAA' } } },
         { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
-      ]
-    );
+      ],
+    });
     const result = (await task()) as Either.Left<DocumentsTransformFailed>;
     expect(transform).toHaveBeenCalledTimes(1);
     expect(result._tag).toEqual('Left');
@@ -202,11 +204,12 @@ describe('migrateRawDocsSafely', () => {
       set(_.cloneDeep(doc), 'attributes.name', 'HOI!'),
       { id: 'bar', type: 'foo', attributes: { name: 'baz' } },
     ]);
-    const task = migrateRawDocsSafely(
-      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
-      transform,
-      [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }]
-    );
+    const task = migrateRawDocsSafely({
+      serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      knownTypes: new Set(['a', 'c']),
+      migrateDoc: transform,
+      rawDocs: [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }],
+    });
     const result = (await task()) as Either.Right<DocumentsTransformSuccess>;
     expect(result._tag).toEqual('Right');
     expect(result.right.processedDocs).toEqual([
@@ -235,11 +238,12 @@ describe('migrateRawDocsSafely', () => {
     const transform = jest.fn<any, any>((doc: any) => {
       throw new TransformSavedObjectDocumentError(new Error('error during transform'), '8.0.0');
     });
-    const task = migrateRawDocsSafely(
-      new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
-      transform,
-      [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }] // this is the raw doc
-    );
+    const task = migrateRawDocsSafely({
+      serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      knownTypes: new Set(['a', 'c']),
+      migrateDoc: transform,
+      rawDocs: [{ _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } }], // this is the raw doc
+    });
     const result = (await task()) as Either.Left<DocumentsTransformFailed>;
     expect(transform).toHaveBeenCalledTimes(1);
     expect(result._tag).toEqual('Left');
@@ -252,4 +256,43 @@ describe('migrateRawDocsSafely', () => {
       }
     `);
   });
+
+  test('skips documents of unknown types', async () => {
+    const transform = jest.fn<any, any>((doc: any) => [
+      set(_.cloneDeep(doc), 'attributes.name', 'HOI!'),
+    ]);
+    const task = migrateRawDocsSafely({
+      serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()),
+      knownTypes: new Set(['a']),
+      migrateDoc: transform,
+      rawDocs: [
+        { _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } },
+        { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
+      ],
+    });
+
+    const result = (await task()) as Either.Right<DocumentsTransformSuccess>;
+    expect(result._tag).toEqual('Right');
+    expect(result.right.processedDocs).toEqual([
+      {
+        _id: 'a:b',
+        _source: { type: 'a', a: { name: 'HOI!' }, migrationVersion: {}, references: [] },
+      },
+      {
+        _id: 'c:d',
+        // name field is not migrated on unknown type
+        _source: { type: 'c', c: { name: 'DDD' } },
+      },
+    ]);
+
+    const obj1 = {
+      id: 'b',
+      type: 'a',
+      attributes: { name: 'AAA' },
+      migrationVersion: {},
+      references: [],
+    };
+    expect(transform).toHaveBeenCalledTimes(1);
+    expect(transform).toHaveBeenNthCalledWith(1, obj1);
+  });
 });

src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts

@@ -81,26 +81,40 @@ export async function migrateRawDocs(
   return processedDocs;
 }
 
+interface MigrateRawDocsSafelyDeps {
+  serializer: SavedObjectsSerializer;
+  knownTypes: ReadonlySet<string>;
+  migrateDoc: MigrateAndConvertFn;
+  rawDocs: SavedObjectsRawDoc[];
+}
+
 /**
  * Applies the specified migration function to every saved object document provided
  * and converts the saved object to a raw document.
  * Captures the ids and errors from any documents that are not valid saved objects or
  * for which the transformation function failed.
  * @returns {TaskEither.TaskEither<DocumentsTransformFailed, DocumentsTransformSuccess>}
  */
-export function migrateRawDocsSafely(
-  serializer: SavedObjectsSerializer,
-  migrateDoc: MigrateAndConvertFn,
-  rawDocs: SavedObjectsRawDoc[]
-): TaskEither.TaskEither<DocumentsTransformFailed, DocumentsTransformSuccess> {
+export function migrateRawDocsSafely({
+  serializer,
+  knownTypes,
+  migrateDoc,
+  rawDocs,
+}: MigrateRawDocsSafelyDeps): TaskEither.TaskEither<
+  DocumentsTransformFailed,
+  DocumentsTransformSuccess
+> {
   return async () => {
     const migrateDocNonBlocking = transformNonBlocking(migrateDoc);
     const processedDocs: SavedObjectsRawDoc[] = [];
     const transformErrors: TransformErrorObjects[] = [];
     const corruptSavedObjectIds: string[] = [];
     const options = { namespaceTreatment: 'lax' as const };
     for (const raw of rawDocs) {
-      if (serializer.isRawSavedObject(raw, options)) {
+      // Do not transform documents of unknown types
+      if (raw?._source?.type && !knownTypes.has(raw._source.type)) {
+        processedDocs.push(raw);
+      } else if (serializer.isRawSavedObject(raw, options)) {
         try {
           const savedObject = convertToRawAddMigrationVersion(raw, options, serializer);
           processedDocs.push(

src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts

@@ -184,11 +184,12 @@ export class KibanaMigrator {
               logger: this.log,
               preMigrationScript: indexMap[index].script,
               transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) =>
-                migrateRawDocsSafely(
-                  this.serializer,
-                  this.documentMigrator.migrateAndConvert,
-                  rawDocs
-                ),
+                migrateRawDocsSafely({
+                  serializer: this.serializer,
+                  knownTypes: new Set(this.typeRegistry.getAllTypes().map((t) => t.name)),
+                  migrateDoc: this.documentMigrator.migrateAndConvert,
+                  rawDocs,
+                }),
               migrationVersionPerType: this.documentMigrator.migrationVersion,
               indexPrefix: index,
               migrationsConfig: this.soMigrationsConfig,