Add migrations.discardCorruptObjects flag

docs/setup/upgrade/resolving-migration-failures.asciidoc

@@ -85,6 +85,16 @@ DELETE .kibana_7.12.0_001/_doc/marketing_space:dashboard:e3c5fc71-ac71-4805-bcab
 +
 The dashboard with the `e3c5fc71-ac71-4805-bcab-2bcc9cc93275` ID that belongs to the `marketing_space` space **is no longer available**.
 
+You can configure {kib} to automatically discard all corrupt objects and transform errors that occur during a migration. When setting the configuration option `migrations.discardCorruptObjects`, {kib} will delete the conflicting objects and proceed with the migration.
+ For instance, for an upgrade to 8.4.0, you can define the following setting in kibana.yml:
+
+[source,yaml]
+--------------------------------------------
+migrations.discardCorruptObjects: "8.4.0"
+--------------------------------------------
+
+**WARNING:** Enabling the flag above will cause the system to discard all corrupt objects, as well as those causing transform errors. Thus, it is HIGHLY recommended that you carefully review the list of conflicting objects in the logs.
+
 [float]
 [[unknown-saved-object-types]]
 ==== Documents for unknown saved objects
@@ -113,12 +123,12 @@ Unable to complete saved object migrations for the [.kibana] index: Migration fa
 To proceed with the migration you can configure Kibana to discard unknown saved objects for this migration.
 --------------------------------------------
 
-To proceed with the migration, re-enable any plugins that previously created these saved objects. Alternatively, carefully review the list of unknown saved objects in the Kibana log entry. If the corresponding disabled plugins and their associated saved objects will no longer be used, they can be deleted by setting the configuration option `migrations.discardUnknownObjects` to the version you are upgrading to.
-For instance, for an upgrade to 8.3.0, you can define the following setting in kibana.yml:
+To proceed with the migration, re-enable any plugins that previously created these saved objects. Alternatively, carefully review the list of unknown saved objects in the {kib} log entry. If the corresponding disabled plugins and their associated saved objects will no longer be used, they can be deleted by setting the configuration option `migrations.discardUnknownObjects` to the version you are upgrading to.
+For instance, for an upgrade to 8.4.0, you can define the following setting in kibana.yml:
 
 [source,yaml]
 --------------------------------------------
-migrations.discardUnknownObjects: "8.3.0"
+migrations.discardUnknownObjects: "8.4.0"
 --------------------------------------------
 
 [float]

src/core/server/saved_objects/migrations/core/migrate_raw_docs.test.ts

@@ -192,9 +192,15 @@ describe('migrateRawDocsSafely', () => {
     const result = (await task()) as Either.Left<DocumentsTransformFailed>;
     expect(transform).toHaveBeenCalledTimes(1);
     expect(result._tag).toEqual('Left');
-    expect(Object.keys(result.left)).toEqual(['type', 'corruptDocumentIds', 'transformErrors']);
+    expect(Object.keys(result.left)).toEqual([
+      'type',
+      'corruptDocumentIds',
+      'transformErrors',
+      'processedDocs',
+    ]);
     expect(result.left.corruptDocumentIds.length).toEqual(1);
     expect(result.left.transformErrors.length).toEqual(0);
+    expect(result.left.processedDocs.length).toEqual(1);
   });
 
   test('handles when one document is transformed into multiple documents', async () => {

src/core/server/saved_objects/migrations/core/migrate_raw_docs.ts

@@ -24,6 +24,7 @@ export interface DocumentsTransformFailed {
   readonly type: string;
   readonly corruptDocumentIds: string[];
   readonly transformErrors: TransformErrorObjects[];
+  readonly processedDocs: SavedObjectsRawDoc[];
 }
 
 export interface DocumentsTransformSuccess {
@@ -139,6 +140,7 @@ export function migrateRawDocsSafely({
         type: 'documents_transform_failed',
         corruptDocumentIds: [...corruptSavedObjectIds],
         transformErrors,
+        processedDocs,
       });
     }
     return Either.right({ processedDocs });

src/core/server/saved_objects/migrations/initial_state.test.ts

@@ -26,11 +26,14 @@ describe('createInitialState', () => {
     docLinks = docLinksServiceMock.createSetupContract();
   });
 
+  afterEach(() => jest.clearAllMocks());
+
   const migrationsConfig = {
     retryAttempts: 15,
     batchSize: 1000,
     maxBatchSizeBytes: ByteSizeValue.parse('100mb'),
   } as unknown as SavedObjectsMigrationConfigType;
+
   it('creates the initial state for the model based on the passed in parameters', () => {
     expect(
       createInitialState({
@@ -51,6 +54,7 @@ describe('createInitialState', () => {
         "batchSize": 1000,
         "controlState": "INIT",
         "currentAlias": ".kibana_task_manager",
+        "discardCorruptObjects": false,
         "discardUnknownObjects": false,
         "excludeFromUpgradeFilterHooks": Object {},
         "excludeOnUpgradeQuery": Object {
@@ -400,7 +404,7 @@ describe('createInitialState', () => {
     expect(initialState.discardUnknownObjects).toEqual(false);
     expect(logger.warn).toBeCalledTimes(1);
     expect(logger.warn).toBeCalledWith(
-      'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be ignored.'
+      'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be discarded.'
     );
   });
 
@@ -424,4 +428,51 @@ describe('createInitialState', () => {
 
     expect(initialState.discardUnknownObjects).toEqual(true);
   });
+
+  it('initializes the `discardCorruptObjects` flag to false if the value provided in the config does not match the current kibana version', () => {
+    const logger = mockLogger.get();
+    const initialState = createInitialState({
+      kibanaVersion: '8.1.0',
+      targetMappings: {
+        dynamic: 'strict',
+        properties: { my_type: { properties: { title: { type: 'text' } } } },
+      },
+      migrationVersionPerType: {},
+      indexPrefix: '.kibana_task_manager',
+      migrationsConfig: {
+        ...migrationsConfig,
+        discardCorruptObjects: '8.0.0',
+      },
+      typeRegistry,
+      docLinks,
+      logger,
+    });
+
+    expect(initialState.discardCorruptObjects).toEqual(false);
+    expect(logger.warn).toBeCalledTimes(1);
+    expect(logger.warn).toBeCalledWith(
+      'The flag `migrations.discardCorruptObjects` is defined but does not match the current kibana version; corrupt objects will NOT be discarded.'
+    );
+  });
+
+  it('initializes the `discardCorruptObjects` flag to true if the value provided in the config matches the current kibana version', () => {
+    const initialState = createInitialState({
+      kibanaVersion: '8.1.0',
+      targetMappings: {
+        dynamic: 'strict',
+        properties: { my_type: { properties: { title: { type: 'text' } } } },
+      },
+      migrationVersionPerType: {},
+      indexPrefix: '.kibana_task_manager',
+      migrationsConfig: {
+        ...migrationsConfig,
+        discardCorruptObjects: '8.1.0',
+      },
+      typeRegistry,
+      docLinks,
+      logger: mockLogger.get(),
+    });
+
+    expect(initialState.discardCorruptObjects).toEqual(true);
+  });
 });

src/core/server/saved_objects/migrations/initial_state.ts

@@ -78,7 +78,16 @@ export const createInitialState = ({
     migrationsConfig.discardUnknownObjects !== kibanaVersion
   ) {
     logger.warn(
-      'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be ignored.'
+      'The flag `migrations.discardUnknownObjects` is defined but does not match the current kibana version; unknown objects will NOT be discarded.'
+    );
+  }
+
+  if (
+    migrationsConfig.discardCorruptObjects &&
+    migrationsConfig.discardCorruptObjects !== kibanaVersion
+  ) {
+    logger.warn(
+      'The flag `migrations.discardCorruptObjects` is defined but does not match the current kibana version; corrupt objects will NOT be discarded.'
     );
   }
 
@@ -101,6 +110,7 @@ export const createInitialState = ({
     batchSize: migrationsConfig.batchSize,
     maxBatchSizeBytes: migrationsConfig.maxBatchSizeBytes.getValueInBytes(),
     discardUnknownObjects: migrationsConfig.discardUnknownObjects === kibanaVersion,
+    discardCorruptObjects: migrationsConfig.discardCorruptObjects === kibanaVersion,
     logs: [],
     excludeOnUpgradeQuery: excludeUnusedTypesQuery,
     knownTypes,