[Ingest Manager] Support registration of server side callbacks for Create Datasource API

x-pack/plugins/ingest_manager/server/index.ts

@@ -11,6 +11,7 @@ export {
   IngestManagerSetupContract,
   IngestManagerSetupDeps,
   IngestManagerStartContract,
+  ExternalCallback,
 } from './plugin';
 
 export const config = {
@@ -42,6 +43,8 @@ export const config = {
 
 export type IngestManagerConfigType = TypeOf<typeof config.schema>;
 
+export { DatasourceServiceInterface } from './services/datasource';
+
 export const plugin = (initializerContext: PluginInitializerContext) => {
   return new IngestManagerPlugin(initializerContext);
 };

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -45,13 +45,14 @@ import {
   registerSettingsRoutes,
   registerAppRoutes,
 } from './routes';
-import { IngestManagerConfigType } from '../common';
+import { IngestManagerConfigType, NewDatasource } from '../common';
 import {
   appContextService,
   licenseService,
   ESIndexPatternSavedObjectService,
   ESIndexPatternService,
   AgentService,
+  datasourceService,
 } from './services';
 import { getAgentStatusById } from './services/agents';
 import { CloudSetup } from '../../cloud/server';
@@ -92,12 +93,31 @@ const allSavedObjectTypes = [
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
 ];
 
+/**
+ * Callbacks supported by the Ingest plugin
+ */
+export type ExternalCallback = [
+  'datasourceCreate',
+  (newDatasource: NewDatasource) => Promise<NewDatasource>
+];
+
+export type ExternalCallbacksStorage = Map<ExternalCallback[0], Set<ExternalCallback[1]>>;
+
 /**
  * Describes public IngestManager plugin contract returned at the `startup` stage.
  */
 export interface IngestManagerStartContract {
   esIndexPatternService: ESIndexPatternService;
   agentService: AgentService;
+  /**
+   * Services for Ingest's Datasources
+   */
+  datasourceService: typeof datasourceService;
+  /**
+   * Register callbacks for inclusion in ingest API processing
+   * @param args
+   */
+  registerExternalCallback: (...args: ExternalCallback) => void;
 }
 
 export class IngestManagerPlugin
@@ -237,6 +257,10 @@ export class IngestManagerPlugin
       agentService: {
         getAgentStatusById,
       },
+      datasourceService,
+      registerExternalCallback: (...args: ExternalCallback) => {
+        return appContextService.addExternalCallback(...args);
+      },
     };
   }
 

x-pack/plugins/ingest_manager/server/routes/datasource/handlers.ts

@@ -14,6 +14,7 @@ import {
   CreateDatasourceRequestSchema,
   UpdateDatasourceRequestSchema,
   DeleteDatasourcesRequestSchema,
+  NewDatasource,
 } from '../../types';
 import { CreateDatasourceResponse, DeleteDatasourcesResponse } from '../../../common';
 
@@ -76,23 +77,43 @@ export const createDatasourceHandler: RequestHandler<
   const soClient = context.core.savedObjects.client;
   const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser;
   const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined;
-  const newData = { ...request.body };
+  const logger = appContextService.getLogger();
+  let newData = { ...request.body };
   try {
+    // If we have external callbacks, then process those now before creating the actual datasource
+    const externalCallbacks = appContextService.getExternalCallbacks('datasourceCreate');
+    if (externalCallbacks && externalCallbacks.size > 0) {
+      let updatedNewData: NewDatasource = newData;
+
+      for (const callback of externalCallbacks) {
+        try {
+          updatedNewData = await callback(updatedNewData);
+        } catch (error) {
+          // Log the error, but keep going and process the other callbacks
+          logger.error('An external registered [datasourceCreate] callback failed when executed');
+          logger.error(error);
+        }
+      }
+
+      // @ts-ignore
+      newData = updatedNewData as typeof CreateDatasourceRequestSchema.body;
+    }
+
     // Make sure the datasource package is installed
-    if (request.body.package?.name) {
+    if (newData.package?.name) {
       await ensureInstalledPackage({
         savedObjectsClient: soClient,
-        pkgName: request.body.package.name,
+        pkgName: newData.package.name,
         callCluster,
       });
       const pkgInfo = await getPackageInfo({
         savedObjectsClient: soClient,
-        pkgName: request.body.package.name,
-        pkgVersion: request.body.package.version,
+        pkgName: newData.package.name,
+        pkgVersion: newData.package.version,
       });
       newData.inputs = (await datasourceService.assignPackageStream(
         pkgInfo,
-        request.body.inputs
+        newData.inputs
       )) as TypeOf<typeof CreateDatasourceRequestSchema.body>['inputs'];
     }
 

x-pack/plugins/ingest_manager/server/services/app_context.ts

@@ -12,7 +12,7 @@ import {
 } from '../../../encrypted_saved_objects/server';
 import { SecurityPluginSetup } from '../../../security/server';
 import { IngestManagerConfigType } from '../../common';
-import { IngestManagerAppContext } from '../plugin';
+import { ExternalCallback, ExternalCallbacksStorage, IngestManagerAppContext } from '../plugin';
 import { CloudSetup } from '../../../cloud/server';
 
 class AppContextService {
@@ -27,6 +27,7 @@ class AppContextService {
   private cloud?: CloudSetup;
   private logger: Logger | undefined;
   private httpSetup?: HttpServiceSetup;
+  private externalCallbacks: ExternalCallbacksStorage = new Map();
 
   public async start(appContext: IngestManagerAppContext) {
     this.encryptedSavedObjects = appContext.encryptedSavedObjectsStart?.getClient();
@@ -47,7 +48,9 @@ class AppContextService {
     }
   }
 
-  public stop() {}
+  public stop() {
+    this.externalCallbacks.clear();
+  }
 
   public getEncryptedSavedObjects() {
     if (!this.encryptedSavedObjects) {
@@ -121,6 +124,19 @@ class AppContextService {
     }
     return this.kibanaVersion;
   }
+
+  public addExternalCallback(type: ExternalCallback[0], callback: ExternalCallback[1]) {
+    if (!this.externalCallbacks.has(type)) {
+      this.externalCallbacks.set(type, new Set());
+    }
+    this.externalCallbacks.get(type)!.add(callback);
+  }
+
+  public getExternalCallbacks(type: ExternalCallback[0]) {
+    if (this.externalCallbacks) {
+      return this.externalCallbacks.get(type);
+    }
+  }
 }
 
 export const appContextService = new AppContextService();

x-pack/plugins/ingest_manager/server/services/datasource.ts

@@ -307,4 +307,5 @@ async function _assignPackageStreamToStream(
   return { ...stream };
 }
 
+export type DatasourceServiceInterface = DatasourceService;
 export const datasourceService = new DatasourceService();

x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts

@@ -17,7 +17,6 @@ import {
   sendPutDatasource,
 } from '../policy_list/services/ingest';
 import { NewPolicyData, PolicyData } from '../../../../../../common/endpoint/types';
-import { factory as policyConfigFactory } from '../../../../../../common/endpoint/models/policy_config';
 import { ImmutableMiddlewareFactory } from '../../../../../common/store';
 
 export const policyDetailsMiddlewareFactory: ImmutableMiddlewareFactory<PolicyDetailsState> = (
@@ -43,23 +42,6 @@ export const policyDetailsMiddlewareFactory: ImmutableMiddlewareFactory<PolicyDe
         return;
       }
 
-      // Until we get the Default configuration into the Endpoint package so that the datasource has
-      // the expected data structure, we will add it here manually.
-      if (!policyItem.inputs.length) {
-        policyItem.inputs = [
-          {
-            type: 'endpoint',
-            enabled: true,
-            streams: [],
-            config: {
-              policy: {
-                value: policyConfigFactory(),
-              },
-            },
-          },
-        ];
-      }
-
       dispatch({
         type: 'serverReturnedPolicyDetailsData',
         payload: {

x-pack/plugins/security_solution/server/endpoint/alerts/handlers/alerts.test.ts

@@ -11,7 +11,7 @@ import {
 } from '../../../../../../../src/core/server/mocks';
 import { registerAlertRoutes } from '../routes';
 import { alertingIndexGetQuerySchema } from '../../../../common/endpoint_alerts/schema/alert_index';
-import { createMockAgentService } from '../../mocks';
+import { createMockEndpointAppContextServiceStartContract } from '../../mocks';
 import { EndpointAppContextService } from '../../endpoint_app_context_services';
 import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
 
@@ -28,9 +28,7 @@ describe('test alerts route', () => {
     routerMock = httpServiceMock.createRouter();
 
     endpointAppContextService = new EndpointAppContextService();
-    endpointAppContextService.start({
-      agentService: createMockAgentService(),
-    });
+    endpointAppContextService.start(createMockEndpointAppContextServiceStartContract());
 
     registerAlertRoutes(routerMock, {
       logFactory: loggingServiceMock.create(),

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts

@@ -3,7 +3,15 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { AgentService } from '../../../ingest_manager/server';
+import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server';
+import { handleDatasourceCreate } from './ingest_integration';
+
+export type EndpointAppContextServiceStartContract = Pick<
+  IngestManagerStartContract,
+  'agentService'
+> & {
+  registerIngestCallback: IngestManagerStartContract['registerExternalCallback'];
+};
 
 /**
  * A singleton that holds shared services that are initialized during the start up phase
@@ -12,8 +20,9 @@ import { AgentService } from '../../../ingest_manager/server';
 export class EndpointAppContextService {
   private agentService: AgentService | undefined;
 
-  public start(dependencies: { agentService: AgentService }) {
+  public start(dependencies: EndpointAppContextServiceStartContract) {
     this.agentService = dependencies.agentService;
+    dependencies.registerIngestCallback('datasourceCreate', handleDatasourceCreate);
   }
 
   public stop() {}

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
+import { NewPolicyData } from '../../common/endpoint/types';
+import { NewDatasource } from '../../../ingest_manager/common/types/models';
+
+/**
+ * Callback to handle creation of Datasources in Ingest Manager
+ * @param newDatasource
+ */
+export const handleDatasourceCreate = async (
+  newDatasource: NewDatasource
+): Promise<NewDatasource> => {
+  // We only care about Endpoint datasources
+  if (newDatasource.package?.name !== 'endpoint') {
+    return newDatasource;
+  }
+
+  // We cast the type here so that any changes to the Endpoint specific data
+  // follow the types/schema expected
+  let updatedDatasource = newDatasource as NewPolicyData;
+
+  // Until we get the Default Policy Configuration in the Endpoint package,
+  // we will add it here manually at creation time.
+  // @ts-ignore
+  if (newDatasource.inputs.length === 0) {
+    updatedDatasource = {
+      ...newDatasource,
+      inputs: [
+        {
+          type: 'endpoint',
+          enabled: true,
+          streams: [],
+          config: {
+            policy: {
+              value: policyConfigFactory(),
+            },
+          },
+        },
+      ],
+    };
+  }
+
+  return updatedDatasource;
+};

x-pack/plugins/security_solution/server/endpoint/mocks.ts

@@ -6,7 +6,28 @@
 
 import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server';
 import { xpackMocks } from '../../../../mocks';
-import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server';
+import {
+  AgentService,
+  IngestManagerStartContract,
+  ExternalCallback,
+  DatasourceServiceInterface,
+} from '../../../ingest_manager/server';
+import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services';
+
+/**
+ * Crates a mocked input contract for the `EndpointAppContextService#start()` method
+ */
+export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked<
+  EndpointAppContextServiceStartContract
+> => {
+  return {
+    agentService: createMockAgentService(),
+    registerIngestCallback: jest.fn<
+      ReturnType<IngestManagerStartContract['registerExternalCallback']>,
+      Parameters<IngestManagerStartContract['registerExternalCallback']>
+    >(),
+  };
+};
 
 /**
  * Creates a mock AgentService
@@ -17,6 +38,20 @@ export const createMockAgentService = (): jest.Mocked<AgentService> => {
   };
 };
 
+const createMockDatasourceService = (): jest.Mocked<DatasourceServiceInterface> => {
+  return {
+    assignPackageStream: jest.fn(),
+    buildDatasourceFromPackage: jest.fn(),
+    bulkCreate: jest.fn(),
+    create: jest.fn(),
+    delete: jest.fn(),
+    get: jest.fn(),
+    getByIDs: jest.fn(),
+    list: jest.fn(),
+    update: jest.fn(),
+  };
+};
+
 /**
  * Creates a mock IndexPatternService for use in tests that need to interact with the Ingest Manager's
  * ESIndexPatternService.
@@ -32,6 +67,8 @@ export const createMockIngestManagerStartContract = (
       getESIndexPattern: jest.fn().mockResolvedValue(indexPattern),
     },
     agentService: createMockAgentService(),
+    registerExternalCallback: jest.fn((...args: ExternalCallback) => {}),
+    datasourceService: createMockDatasourceService(),
   };
 };