[Security Solution][DE] Migrate investigation_fields (elastic#169061)

## Summary

**TLDR:** SO will support both `string[]` and `{ field_names: string[]
}`, but detection engine APIs will only support the object format in
8.11+.

x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.test.ts

@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { InvestigationFields } from '../../../../common/api/detection_engine';
+import type { Rule } from './types';
+import { transformRuleFromAlertHit } from './use_rule_with_fallback';
+
+export const getMockAlertSearchResponse = (rule: Rule) => ({
+  took: 1,
+  timeout: false,
+  _shards: {
+    total: 1,
+    successful: 1,
+    skipped: 0,
+    failed: 0,
+  },
+  hits: {
+    total: {
+      value: 75,
+      relation: 'eq',
+    },
+    max_score: null,
+    hits: [
+      {
+        _id: '1234',
+        _index: '.kibana',
+        _source: {
+          '@timestamp': '12334232132',
+          kibana: {
+            alert: {
+              rule,
+            },
+          },
+        },
+      },
+    ],
+  },
+});
+
+describe('use_rule_with_fallback', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+  describe('transformRuleFromAlertHit', () => {
+    // Testing edge case, where if hook does not find the rule and turns to the alert document,
+    // the alert document could still have an unmigrated, legacy version of investigation_fields.
+    // We are not looking to do any migrations to these legacy fields in the alert document, so need
+    // to transform it on read in this case.
+    describe('investigation_fields', () => {
+      it('sets investigation_fields to undefined when set as legacy array', () => {
+        const mockRule = getMockRule({
+          investigation_fields: ['foo'] as unknown as InvestigationFields,
+        });
+        const mockHit = getMockAlertSearchResponse(mockRule);
+        const result = transformRuleFromAlertHit(mockHit);
+        expect(result?.investigation_fields).toBeUndefined();
+      });
+
+      it('sets investigation_fields to undefined when set as legacy empty array', () => {
+        // Ideally, we would have the client side types pull from the same types
+        // as server side so we could denote here that the SO can have investigation_fields
+        // as array or object, but our APIs now only support object. We don't have that here
+        // and would need to adjust the client side type to support both, which we do not want
+        // to do in this instance as we try to migrate folks away from the array version.
+        const mockRule = getMockRule({
+          investigation_fields: [] as unknown as InvestigationFields,
+        });
+        const mockHit = getMockAlertSearchResponse(mockRule);
+        const result = transformRuleFromAlertHit(mockHit);
+        expect(result?.investigation_fields).toBeUndefined();
+      });
+
+      it('does no transformation when "investigation_fields" is intended type', () => {
+        const mockRule = getMockRule({ investigation_fields: { field_names: ['bar'] } });
+        const mockHit = getMockAlertSearchResponse(mockRule);
+        const result = transformRuleFromAlertHit(mockHit);
+        expect(result?.investigation_fields).toEqual({ field_names: ['bar'] });
+      });
+    });
+  });
+});
+
+const getMockRule = (overwrites: Partial<Rule>): Rule => ({
+  id: 'myfakeruleid',
+  author: [],
+  severity_mapping: [],
+  risk_score_mapping: [],
+  rule_id: 'rule-1',
+  risk_score: 50,
+  description: 'some description',
+  from: 'now-5m',
+  to: 'now',
+  name: 'some-name',
+  severity: 'low',
+  type: 'query',
+  query: 'some query',
+  index: ['index-1'],
+  interval: '5m',
+  references: [],
+  actions: [],
+  enabled: false,
+  false_positives: [],
+  max_signals: 100,
+  tags: [],
+  threat: [],
+  throttle: null,
+  version: 1,
+  exceptions_list: [],
+  created_at: '2020-04-09T09:43:51.778Z',
+  created_by: 'elastic',
+  immutable: false,
+  updated_at: '2020-04-09T09:43:51.778Z',
+  updated_by: 'elastic',
+  related_integrations: [],
+  required_fields: [],
+  setup: '',
+  ...overwrites,
+});

x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.ts

@@ -8,6 +8,8 @@
 import { ALERT_RULE_UUID } from '@kbn/rule-data-utils';
 import { isNotFoundError } from '@kbn/securitysolution-t-grid';
 import { useEffect, useMemo } from 'react';
+import type { InvestigationFieldsCombined } from '../../../../server/lib/detection_engine/rule_schema';
+import type { InvestigationFields } from '../../../../common/api/detection_engine';
 import { expandDottedObject } from '../../../../common/utils/expand_dotted';
 import { useAppToasts } from '../../../common/hooks/use_app_toasts';
 import { ALERTS_QUERY_NAMES } from '../../../detections/containers/detection_engine/alerts/constants';
@@ -114,11 +116,50 @@ export const useRuleWithFallback = (ruleId: string): UseRuleWithFallback => {
   };
 };
 
+/**
+ * In 8.10.x investigation_fields is mapped as alert, moving forward, it will be mapped
+ * as an object. This util is being used for the use case where a rule is deleted and the
+ * hook falls back to using the alert document to retrieve rule information. In this scenario
+ * we are going to return undefined if field is in legacy format to avoid any possible complexity
+ * in the UI for such flows. See PR 169061
+ * @param investigationFields InvestigationFieldsCombined | undefined
+ * @returns InvestigationFields | undefined
+ */
+export const migrateLegacyInvestigationFields = (
+  investigationFields: InvestigationFieldsCombined | undefined
+): InvestigationFields | undefined => {
+  if (investigationFields && Array.isArray(investigationFields)) {
+    return undefined;
+  }
+
+  return investigationFields;
+};
+
+/**
+ * In 8.10.x investigation_fields is mapped as alert, moving forward, it will be mapped
+ * as an object. This util is being used for the use case where a rule is deleted and the
+ * hook falls back to using the alert document to retrieve rule information. In this scenario
+ * we are going to return undefined if field is in legacy format to avoid any possible complexity
+ * in the UI for such flows. See PR 169061
+ * @param rule Rule
+ * @returns Rule
+ */
+export const migrateRuleWithLegacyInvestigationFieldsFromAlertHit = (rule: Rule): Rule => {
+  if (!rule) return rule;
+
+  return {
+    ...rule,
+    investigation_fields: migrateLegacyInvestigationFields(rule.investigation_fields),
+  };
+};
+
 /**
  * Transforms an alertHit into a Rule
  * @param data raw response containing single alert
  */
-const transformRuleFromAlertHit = (data: AlertSearchResponse<AlertHit>): Rule | undefined => {
+export const transformRuleFromAlertHit = (
+  data: AlertSearchResponse<AlertHit>
+): Rule | undefined => {
   // if results empty, return rule as undefined
   if (data.hits.hits.length === 0) {
     return undefined;
@@ -136,8 +177,8 @@ const transformRuleFromAlertHit = (data: AlertSearchResponse<AlertHit>): Rule |
       ...expandedRuleWithParams?.kibana?.alert?.rule?.parameters,
     };
     delete expandedRule.parameters;
-    return expandedRule as Rule;
+    return migrateRuleWithLegacyInvestigationFieldsFromAlertHit(expandedRule as Rule);
   }
 
-  return rule;
+  return migrateRuleWithLegacyInvestigationFieldsFromAlertHit(rule);
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_exceptions/api/find_exception_references/route.ts

@@ -22,9 +22,7 @@ import {
   findExceptionReferencesOnRuleSchema,
   rulesReferencedByExceptionListsSchema,
 } from '../../../../../../common/api/detection_engine/rule_exceptions';
-
-import { enrichFilterWithRuleTypeMapping } from '../../../rule_management/logic/search/enrich_filter_with_rule_type_mappings';
-import type { RuleParams } from '../../../rule_schema';
+import { findRules } from '../../../rule_management/logic/search/find_rules';
 
 export const findRuleExceptionReferencesRoute = (router: SecuritySolutionPluginRouter) => {
   router.versioned
@@ -92,15 +90,18 @@ export const findRuleExceptionReferencesRoute = (router: SecuritySolutionPluginR
           }
           const references: RuleReferencesSchema[] = await Promise.all(
             foundExceptionLists.data.map(async (list, index) => {
-              const foundRules = await rulesClient.find<RuleParams>({
-                options: {
-                  perPage: 10000,
-                  filter: enrichFilterWithRuleTypeMapping(null),
-                  hasReference: {
-                    id: list.id,
-                    type: getSavedObjectType({ namespaceType: list.namespace_type }),
-                  },
+              const foundRules = await findRules({
+                rulesClient,
+                perPage: 10000,
+                hasReference: {
+                  id: list.id,
+                  type: getSavedObjectType({ namespaceType: list.namespace_type }),
                 },
+                filter: undefined,
+                fields: undefined,
+                sortField: undefined,
+                sortOrder: undefined,
+                page: undefined,
               });
 
               const ruleData = foundRules.data.map(({ name, id, params }) => ({

x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts

@@ -16,6 +16,7 @@ import {
   getSampleDetailsAsNdjson,
 } from '../../../../../../common/api/detection_engine/rule_management/mocks';
 import type { RuleExceptionsPromiseFromStreams } from './import_rules_utils';
+import type { InvestigationFields } from '../../../../../../common/api/detection_engine';
 
 export const getOutputSample = (): Partial<RuleToImport> => ({
   rule_id: 'rule-1',
@@ -319,5 +320,62 @@ describe('create_rules_stream_from_ndjson', () => {
       const resultOrError = result as BadRequestError[];
       expect(resultOrError[1] instanceof BadRequestError).toEqual(true);
     });
+
+    test('migrates investigation_fields', async () => {
+      const sample1 = {
+        ...getOutputSample(),
+        investigation_fields: ['foo', 'bar'] as unknown as InvestigationFields,
+      };
+      const sample2 = {
+        ...getOutputSample(),
+        rule_id: 'rule-2',
+        investigation_fields: [] as unknown as InvestigationFields,
+      };
+      sample2.rule_id = 'rule-2';
+      const ndJsonStream = new Readable({
+        read() {
+          this.push(getSampleAsNdjson(sample1));
+          this.push(getSampleAsNdjson(sample2));
+          this.push(null);
+        },
+      });
+      const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000);
+      const [{ rules: result }] = await createPromiseFromStreams<
+        RuleExceptionsPromiseFromStreams[]
+      >([ndJsonStream, ...rulesObjectsStream]);
+      expect(result).toEqual([
+        {
+          rule_id: 'rule-1',
+          output_index: '.siem-signals',
+          risk_score: 50,
+          description: 'some description',
+          from: 'now-5m',
+          to: 'now',
+          index: ['index-1'],
+          name: 'some-name',
+          severity: 'low',
+          interval: '5m',
+          type: 'query',
+          immutable: false,
+          investigation_fields: {
+            field_names: ['foo', 'bar'],
+          },
+        },
+        {
+          rule_id: 'rule-2',
+          output_index: '.siem-signals',
+          risk_score: 50,
+          description: 'some description',
+          from: 'now-5m',
+          to: 'now',
+          index: ['index-1'],
+          name: 'some-name',
+          severity: 'low',
+          interval: '5m',
+          type: 'query',
+          immutable: false,
+        },
+      ]);
+    });
   });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts

@@ -29,6 +29,7 @@ import {
   RuleToImport,
   validateRuleToImport,
 } from '../../../../../../common/api/detection_engine/rule_management';
+import type { RulesObjectsExportResultDetails } from '../../../../../utils/read_stream/create_stream_from_ndjson';
 import {
   parseNdjsonStrings,
   createRulesLimitStream,
@@ -103,6 +104,25 @@ export const sortImports = (): Transform => {
   );
 };
 
+export const migrateLegacyInvestigationFields = (): Transform => {
+  return createMapStream<RuleToImport | RulesObjectsExportResultDetails>((obj) => {
+    if (obj != null && 'investigation_fields' in obj && Array.isArray(obj.investigation_fields)) {
+      if (obj.investigation_fields.length) {
+        return {
+          ...obj,
+          investigation_fields: {
+            field_names: obj.investigation_fields,
+          },
+        };
+      } else {
+        const { investigation_fields: _, ...rest } = obj;
+        return rest;
+      }
+    }
+    return obj;
+  });
+};
+
 // TODO: Capture both the line number and the rule_id if you have that information for the error message
 // eventually and then pass it down so we can give error messages on the line number
 
@@ -111,6 +131,7 @@ export const createRulesAndExceptionsStreamFromNdJson = (ruleLimit: number) => {
     createSplitStream('\n'),
     parseNdjsonStrings(),
     filterExportedCounts(),
+    migrateLegacyInvestigationFields(),
     sortImports(),
     validateRulesStream(),
     createRulesLimitStream(ruleLimit),

x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/find_rules.ts

@@ -5,7 +5,10 @@
  * 2.0.
  */
 
+import * as t from 'io-ts';
+
 import type { FindResult, RulesClient } from '@kbn/alerting-plugin/server';
+import { NonEmptyString, UUID } from '@kbn/securitysolution-io-ts-types';
 import type { FindRulesSortFieldOrUndefined } from '../../../../../../common/api/detection_engine/rule_management';
 
 import type {
@@ -20,6 +23,15 @@ import type { RuleParams } from '../../../rule_schema';
 import { enrichFilterWithRuleTypeMapping } from './enrich_filter_with_rule_type_mappings';
 import { transformSortField } from './transform_sort_field';
 
+type HasReferences = t.TypeOf<typeof HasReferences>;
+const HasReferences = t.type({
+  type: NonEmptyString,
+  id: UUID,
+});
+
+type HasReferencesOrUndefined = t.TypeOf<typeof HasReferencesOrUndefined>;
+const HasReferencesOrUndefined = t.union([HasReferences, t.undefined]);
+
 export interface FindRuleOptions {
   rulesClient: RulesClient;
   filter: QueryFilterOrUndefined;
@@ -28,6 +40,7 @@ export interface FindRuleOptions {
   sortOrder: SortOrderOrUndefined;
   page: PageOrUndefined;
   perPage: PerPageOrUndefined;
+  hasReference?: HasReferencesOrUndefined;
 }
 
 export const findRules = ({
@@ -38,6 +51,7 @@ export const findRules = ({
   filter,
   sortField,
   sortOrder,
+  hasReference,
 }: FindRuleOptions): Promise<FindResult<RuleParams>> => {
   return rulesClient.find({
     options: {
@@ -47,6 +61,7 @@ export const findRules = ({
       filter: enrichFilterWithRuleTypeMapping(filter),
       sortOrder,
       sortField: transformSortField(sortField),
+      hasReference,
     },
   });
 };