From 931fb596c83b8af4b0554051b5df973811faaa1e Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Thu, 26 Oct 2023 06:58:35 -0700 Subject: [PATCH] [Security Solution][DE] Migrate investigation_fields (#169061) ## Summary **TLDR:** SO will support both `string[]` and `{ field_names: string[] }`, but detection engine APIs will only support the object format in 8.11+. --- .../logic/use_rule_with_fallback.test.ts | 122 +++++ .../logic/use_rule_with_fallback.ts | 47 +- .../api/find_exception_references/route.ts | 23 +- .../create_rules_stream_from_ndjson.test.ts | 58 +++ .../import/create_rules_stream_from_ndjson.ts | 21 + .../logic/search/find_rules.ts | 15 + .../normalization/rule_converters.ts | 8 +- .../rule_management/utils/utils.test.ts | 23 + .../rule_management/utils/utils.ts | 29 +- .../rule_schema/model/rule_schemas.ts | 23 +- .../group1/create_rules_bulk.ts | 15 + .../group1/delete_rules.ts | 73 ++- .../group1/delete_rules_bulk.ts | 75 +++- .../group1/export_rules.ts | 124 +++++ .../security_and_spaces/group1/find_rules.ts | 86 +++- .../group10/import_rules.ts | 134 +++++- .../group10/patch_rules.ts | 421 ++++++++++------- .../group10/patch_rules_bulk.ts | 145 ++++++ .../group10/perform_bulk_action.ts | 422 ++++++++++++++++++ .../security_and_spaces/group10/read_rules.ts | 113 ++++- .../group10/update_rules.ts | 104 ++++- .../group10/update_rules_bulk.ts | 126 +++++- .../usage_collector/detection_rules.ts | 27 +- .../rule_execution_logic/query.ts | 58 ++- .../utils/create_rule_saved_object.ts | 35 ++ .../utils/get_rule_so_by_id.ts | 3 +- ...t_rule_with_legacy_investigation_fields.ts | 124 +++++ .../utils/index.ts | 2 + .../legacy_investigation_fields/data.json | 271 ----------- .../workflows/create_rule_exceptions.ts | 51 ++- .../rule_creation/create_rules.ts | 38 ++ .../utils/create_rule_saved_object.ts | 35 ++ .../utils/get_rule_so_by_id.ts | 29 ++ ...t_rule_with_legacy_investigation_fields.ts | 124 +++++ .../detections_response/utils/index.ts | 3 + .../tsconfig.json | 4 +- 36 files changed, 2548 insertions(+), 463 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.test.ts create mode 100644 x-pack/test/detection_engine_api_integration/utils/create_rule_saved_object.ts create mode 100644 x-pack/test/detection_engine_api_integration/utils/get_rule_with_legacy_investigation_fields.ts delete mode 100644 x-pack/test/functional/es_archives/security_solution/legacy_investigation_fields/data.json create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/create_rule_saved_object.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/get_rule_so_by_id.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/get_rule_with_legacy_investigation_fields.ts diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.test.ts new file mode 100644 index 000000000000000..99ee0056b5d21be --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InvestigationFields } from '../../../../common/api/detection_engine'; +import type { Rule } from './types'; +import { transformRuleFromAlertHit } from './use_rule_with_fallback'; + +export const getMockAlertSearchResponse = (rule: Rule) => ({ + took: 1, + timeout: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 75, + relation: 'eq', + }, + max_score: null, + hits: [ + { + _id: '1234', + _index: '.kibana', + _source: { + '@timestamp': '12334232132', + kibana: { + alert: { + rule, + }, + }, + }, + }, + ], + }, +}); + +describe('use_rule_with_fallback', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + describe('transformRuleFromAlertHit', () => { + // Testing edge case, where if hook does not find the rule and turns to the alert document, + // the alert document could still have an unmigrated, legacy version of investigation_fields. + // We are not looking to do any migrations to these legacy fields in the alert document, so need + // to transform it on read in this case. + describe('investigation_fields', () => { + it('sets investigation_fields to undefined when set as legacy array', () => { + const mockRule = getMockRule({ + investigation_fields: ['foo'] as unknown as InvestigationFields, + }); + const mockHit = getMockAlertSearchResponse(mockRule); + const result = transformRuleFromAlertHit(mockHit); + expect(result?.investigation_fields).toBeUndefined(); + }); + + it('sets investigation_fields to undefined when set as legacy empty array', () => { + // Ideally, we would have the client side types pull from the same types + // as server side so we could denote here that the SO can have investigation_fields + // as array or object, but our APIs now only support object. We don't have that here + // and would need to adjust the client side type to support both, which we do not want + // to do in this instance as we try to migrate folks away from the array version. + const mockRule = getMockRule({ + investigation_fields: [] as unknown as InvestigationFields, + }); + const mockHit = getMockAlertSearchResponse(mockRule); + const result = transformRuleFromAlertHit(mockHit); + expect(result?.investigation_fields).toBeUndefined(); + }); + + it('does no transformation when "investigation_fields" is intended type', () => { + const mockRule = getMockRule({ investigation_fields: { field_names: ['bar'] } }); + const mockHit = getMockAlertSearchResponse(mockRule); + const result = transformRuleFromAlertHit(mockHit); + expect(result?.investigation_fields).toEqual({ field_names: ['bar'] }); + }); + }); + }); +}); + +const getMockRule = (overwrites: Partial): Rule => ({ + id: 'myfakeruleid', + author: [], + severity_mapping: [], + risk_score_mapping: [], + rule_id: 'rule-1', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + name: 'some-name', + severity: 'low', + type: 'query', + query: 'some query', + index: ['index-1'], + interval: '5m', + references: [], + actions: [], + enabled: false, + false_positives: [], + max_signals: 100, + tags: [], + threat: [], + throttle: null, + version: 1, + exceptions_list: [], + created_at: '2020-04-09T09:43:51.778Z', + created_by: 'elastic', + immutable: false, + updated_at: '2020-04-09T09:43:51.778Z', + updated_by: 'elastic', + related_integrations: [], + required_fields: [], + setup: '', + ...overwrites, +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.ts index 4ab65ae0fd8c7b0..59196fa5264d697 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_rule_with_fallback.ts @@ -8,6 +8,8 @@ import { ALERT_RULE_UUID } from '@kbn/rule-data-utils'; import { isNotFoundError } from '@kbn/securitysolution-t-grid'; import { useEffect, useMemo } from 'react'; +import type { InvestigationFieldsCombined } from '../../../../server/lib/detection_engine/rule_schema'; +import type { InvestigationFields } from '../../../../common/api/detection_engine'; import { expandDottedObject } from '../../../../common/utils/expand_dotted'; import { useAppToasts } from '../../../common/hooks/use_app_toasts'; import { ALERTS_QUERY_NAMES } from '../../../detections/containers/detection_engine/alerts/constants'; @@ -114,11 +116,50 @@ export const useRuleWithFallback = (ruleId: string): UseRuleWithFallback => { }; }; +/** + * In 8.10.x investigation_fields is mapped as alert, moving forward, it will be mapped + * as an object. This util is being used for the use case where a rule is deleted and the + * hook falls back to using the alert document to retrieve rule information. In this scenario + * we are going to return undefined if field is in legacy format to avoid any possible complexity + * in the UI for such flows. See PR 169061 + * @param investigationFields InvestigationFieldsCombined | undefined + * @returns InvestigationFields | undefined + */ +export const migrateLegacyInvestigationFields = ( + investigationFields: InvestigationFieldsCombined | undefined +): InvestigationFields | undefined => { + if (investigationFields && Array.isArray(investigationFields)) { + return undefined; + } + + return investigationFields; +}; + +/** + * In 8.10.x investigation_fields is mapped as alert, moving forward, it will be mapped + * as an object. This util is being used for the use case where a rule is deleted and the + * hook falls back to using the alert document to retrieve rule information. In this scenario + * we are going to return undefined if field is in legacy format to avoid any possible complexity + * in the UI for such flows. See PR 169061 + * @param rule Rule + * @returns Rule + */ +export const migrateRuleWithLegacyInvestigationFieldsFromAlertHit = (rule: Rule): Rule => { + if (!rule) return rule; + + return { + ...rule, + investigation_fields: migrateLegacyInvestigationFields(rule.investigation_fields), + }; +}; + /** * Transforms an alertHit into a Rule * @param data raw response containing single alert */ -const transformRuleFromAlertHit = (data: AlertSearchResponse): Rule | undefined => { +export const transformRuleFromAlertHit = ( + data: AlertSearchResponse +): Rule | undefined => { // if results empty, return rule as undefined if (data.hits.hits.length === 0) { return undefined; @@ -136,8 +177,8 @@ const transformRuleFromAlertHit = (data: AlertSearchResponse): Rule | ...expandedRuleWithParams?.kibana?.alert?.rule?.parameters, }; delete expandedRule.parameters; - return expandedRule as Rule; + return migrateRuleWithLegacyInvestigationFieldsFromAlertHit(expandedRule as Rule); } - return rule; + return migrateRuleWithLegacyInvestigationFieldsFromAlertHit(rule); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_exceptions/api/find_exception_references/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_exceptions/api/find_exception_references/route.ts index be7fbc83aa1ffe2..f8714c4e260ee46 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_exceptions/api/find_exception_references/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_exceptions/api/find_exception_references/route.ts @@ -22,9 +22,7 @@ import { findExceptionReferencesOnRuleSchema, rulesReferencedByExceptionListsSchema, } from '../../../../../../common/api/detection_engine/rule_exceptions'; - -import { enrichFilterWithRuleTypeMapping } from '../../../rule_management/logic/search/enrich_filter_with_rule_type_mappings'; -import type { RuleParams } from '../../../rule_schema'; +import { findRules } from '../../../rule_management/logic/search/find_rules'; export const findRuleExceptionReferencesRoute = (router: SecuritySolutionPluginRouter) => { router.versioned @@ -92,15 +90,18 @@ export const findRuleExceptionReferencesRoute = (router: SecuritySolutionPluginR } const references: RuleReferencesSchema[] = await Promise.all( foundExceptionLists.data.map(async (list, index) => { - const foundRules = await rulesClient.find({ - options: { - perPage: 10000, - filter: enrichFilterWithRuleTypeMapping(null), - hasReference: { - id: list.id, - type: getSavedObjectType({ namespaceType: list.namespace_type }), - }, + const foundRules = await findRules({ + rulesClient, + perPage: 10000, + hasReference: { + id: list.id, + type: getSavedObjectType({ namespaceType: list.namespace_type }), }, + filter: undefined, + fields: undefined, + sortField: undefined, + sortOrder: undefined, + page: undefined, }); const ruleData = foundRules.data.map(({ name, id, params }) => ({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts index 594047d237bc770..22c7130d8a15fd7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.test.ts @@ -16,6 +16,7 @@ import { getSampleDetailsAsNdjson, } from '../../../../../../common/api/detection_engine/rule_management/mocks'; import type { RuleExceptionsPromiseFromStreams } from './import_rules_utils'; +import type { InvestigationFields } from '../../../../../../common/api/detection_engine'; export const getOutputSample = (): Partial => ({ rule_id: 'rule-1', @@ -319,5 +320,62 @@ describe('create_rules_stream_from_ndjson', () => { const resultOrError = result as BadRequestError[]; expect(resultOrError[1] instanceof BadRequestError).toEqual(true); }); + + test('migrates investigation_fields', async () => { + const sample1 = { + ...getOutputSample(), + investigation_fields: ['foo', 'bar'] as unknown as InvestigationFields, + }; + const sample2 = { + ...getOutputSample(), + rule_id: 'rule-2', + investigation_fields: [] as unknown as InvestigationFields, + }; + sample2.rule_id = 'rule-2'; + const ndJsonStream = new Readable({ + read() { + this.push(getSampleAsNdjson(sample1)); + this.push(getSampleAsNdjson(sample2)); + this.push(null); + }, + }); + const rulesObjectsStream = createRulesAndExceptionsStreamFromNdJson(1000); + const [{ rules: result }] = await createPromiseFromStreams< + RuleExceptionsPromiseFromStreams[] + >([ndJsonStream, ...rulesObjectsStream]); + expect(result).toEqual([ + { + rule_id: 'rule-1', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + immutable: false, + investigation_fields: { + field_names: ['foo', 'bar'], + }, + }, + { + rule_id: 'rule-2', + output_index: '.siem-signals', + risk_score: 50, + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'low', + interval: '5m', + type: 'query', + immutable: false, + }, + ]); + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts index dcb22706c3c59bb..af90eb560579fdf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/create_rules_stream_from_ndjson.ts @@ -29,6 +29,7 @@ import { RuleToImport, validateRuleToImport, } from '../../../../../../common/api/detection_engine/rule_management'; +import type { RulesObjectsExportResultDetails } from '../../../../../utils/read_stream/create_stream_from_ndjson'; import { parseNdjsonStrings, createRulesLimitStream, @@ -103,6 +104,25 @@ export const sortImports = (): Transform => { ); }; +export const migrateLegacyInvestigationFields = (): Transform => { + return createMapStream((obj) => { + if (obj != null && 'investigation_fields' in obj && Array.isArray(obj.investigation_fields)) { + if (obj.investigation_fields.length) { + return { + ...obj, + investigation_fields: { + field_names: obj.investigation_fields, + }, + }; + } else { + const { investigation_fields: _, ...rest } = obj; + return rest; + } + } + return obj; + }); +}; + // TODO: Capture both the line number and the rule_id if you have that information for the error message // eventually and then pass it down so we can give error messages on the line number @@ -111,6 +131,7 @@ export const createRulesAndExceptionsStreamFromNdJson = (ruleLimit: number) => { createSplitStream('\n'), parseNdjsonStrings(), filterExportedCounts(), + migrateLegacyInvestigationFields(), sortImports(), validateRulesStream(), createRulesLimitStream(ruleLimit), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/find_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/find_rules.ts index f47a2e497a5d1c7..8fb5f348ae2248d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/find_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/find_rules.ts @@ -5,7 +5,10 @@ * 2.0. */ +import * as t from 'io-ts'; + import type { FindResult, RulesClient } from '@kbn/alerting-plugin/server'; +import { NonEmptyString, UUID } from '@kbn/securitysolution-io-ts-types'; import type { FindRulesSortFieldOrUndefined } from '../../../../../../common/api/detection_engine/rule_management'; import type { @@ -20,6 +23,15 @@ import type { RuleParams } from '../../../rule_schema'; import { enrichFilterWithRuleTypeMapping } from './enrich_filter_with_rule_type_mappings'; import { transformSortField } from './transform_sort_field'; +type HasReferences = t.TypeOf; +const HasReferences = t.type({ + type: NonEmptyString, + id: UUID, +}); + +type HasReferencesOrUndefined = t.TypeOf; +const HasReferencesOrUndefined = t.union([HasReferences, t.undefined]); + export interface FindRuleOptions { rulesClient: RulesClient; filter: QueryFilterOrUndefined; @@ -28,6 +40,7 @@ export interface FindRuleOptions { sortOrder: SortOrderOrUndefined; page: PageOrUndefined; perPage: PerPageOrUndefined; + hasReference?: HasReferencesOrUndefined; } export const findRules = ({ @@ -38,6 +51,7 @@ export const findRules = ({ filter, sortField, sortOrder, + hasReference, }: FindRuleOptions): Promise> => { return rulesClient.find({ options: { @@ -47,6 +61,7 @@ export const findRules = ({ filter: enrichFilterWithRuleTypeMapping(filter), sortOrder, sortField: transformSortField(sortField), + hasReference, }, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index d3dc0e08371f38c..2de991d700e7012 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -77,7 +77,11 @@ import type { NewTermsSpecificRuleParams, } from '../../rule_schema'; import { transformFromAlertThrottle, transformToActionFrequency } from './rule_actions'; -import { convertAlertSuppressionToCamel, convertAlertSuppressionToSnake } from '../utils/utils'; +import { + convertAlertSuppressionToCamel, + convertAlertSuppressionToSnake, + migrateLegacyInvestigationFields, +} from '../utils/utils'; import { createRuleExecutionSummary } from '../../rule_monitoring'; import type { PrebuiltRuleAsset } from '../../prebuilt_rules'; @@ -661,7 +665,7 @@ export const commonParamsCamelToSnake = (params: BaseRuleParams) => { rule_name_override: params.ruleNameOverride, timestamp_override: params.timestampOverride, timestamp_override_fallback_disabled: params.timestampOverrideFallbackDisabled, - investigation_fields: params.investigationFields, + investigation_fields: migrateLegacyInvestigationFields(params.investigationFields), author: params.author, false_positives: params.falsePositives, from: params.from, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts index 328dd8d0ea24ff5..61436a04c267538 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.test.ts @@ -26,6 +26,7 @@ import { getInvalidConnectors, swapActionIds, migrateLegacyActionsIds, + migrateLegacyInvestigationFields, } from './utils'; import { getRuleMock } from '../../routes/__mocks__/request_responses'; import type { PartialFilter } from '../../types'; @@ -1259,4 +1260,26 @@ describe('utils', () => { ]); }); }); + + describe('migrateLegacyInvestigationFields', () => { + test('should return undefined if value not set', () => { + const result = migrateLegacyInvestigationFields(undefined); + expect(result).toEqual(undefined); + }); + + test('should migrate array to object', () => { + const result = migrateLegacyInvestigationFields(['foo']); + expect(result).toEqual({ field_names: ['foo'] }); + }); + + test('should migrate empty array to undefined', () => { + const result = migrateLegacyInvestigationFields([]); + expect(result).toEqual(undefined); + }); + + test('should not migrate if already intended type', () => { + const result = migrateLegacyInvestigationFields({ field_names: ['foo'] }); + expect(result).toEqual({ field_names: ['foo'] }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts index c443a66880454ce..569e6f605a6c197 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/utils/utils.ts @@ -20,11 +20,12 @@ import type { } from '../../../../../common/api/detection_engine/rule_management'; import type { AlertSuppression, - RuleResponse, AlertSuppressionCamel, + InvestigationFields, + RuleResponse, } from '../../../../../common/api/detection_engine/model/rule_schema'; -import type { RuleAlertType, RuleParams } from '../../rule_schema'; +import type { InvestigationFieldsCombined, RuleAlertType, RuleParams } from '../../rule_schema'; import { isAlertType } from '../../rule_schema'; import type { BulkError, OutputError } from '../../routes/utils'; import { createBulkErrorObject } from '../../routes/utils'; @@ -380,3 +381,27 @@ export const convertAlertSuppressionToSnake = ( missing_fields_strategy: input.missingFieldsStrategy, } : undefined; + +/** + * In ESS 8.10.x "investigation_fields" are mapped as string[]. + * For 8.11+ logic is added on read in our endpoints to migrate + * the data over to it's intended type of { field_names: string[] }. + * The SO rule type will continue to support both types until we deprecate, + * but APIs will only support intended object format. + * See PR 169061 + */ +export const migrateLegacyInvestigationFields = ( + investigationFields: InvestigationFieldsCombined | undefined +): InvestigationFields | undefined => { + if (investigationFields && Array.isArray(investigationFields)) { + if (investigationFields.length) { + return { + field_names: investigationFields, + }; + } + + return undefined; + } + + return investigationFields; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index cd511d938647212..fbebf8374fabfbe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -38,6 +38,7 @@ import { } from '@kbn/securitysolution-rules'; import type { SanitizedRuleConfig } from '@kbn/alerting-plugin/common'; +import { NonEmptyString } from '@kbn/securitysolution-io-ts-types'; import { AlertsIndex, AlertsIndexNamespace, @@ -58,7 +59,6 @@ import { RuleAuthorArray, RuleDescription, RuleFalsePositiveArray, - InvestigationFields, RuleFilterArray, RuleLicense, RuleMetadata, @@ -78,6 +78,7 @@ import { TimestampField, TimestampOverride, TimestampOverrideFallbackDisabled, + InvestigationFields, } from '../../../../../common/api/detection_engine/model/rule_schema'; import { savedIdOrUndefined, @@ -87,6 +88,24 @@ import { import { SERVER_APP_ID } from '../../../../../common/constants'; import { ResponseActionRuleParamsOrUndefined } from '../../../../../common/api/detection_engine/model/rule_response_actions'; +// 8.10.x is mapped as an array of strings +export type LegacyInvestigationFields = t.TypeOf; +export const LegacyInvestigationFields = t.array(NonEmptyString); + +/* + * In ESS 8.10.x "investigation_fields" are mapped as string[]. + * For 8.11+ logic is added on read in our endpoints to migrate + * the data over to it's intended type of { field_names: string[] }. + * The SO rule type will continue to support both types until we deprecate, + * but APIs will only support intended object format. + * See PR 169061 + */ +export type InvestigationFieldsCombined = t.TypeOf; +export const InvestigationFieldsCombined = t.union([ + InvestigationFields, + LegacyInvestigationFields, +]); + const nonEqlLanguages = t.keyof({ kuery: null, lucene: null }); export const baseRuleParams = t.exact( @@ -99,7 +118,7 @@ export const baseRuleParams = t.exact( falsePositives: RuleFalsePositiveArray, from: RuleIntervalFrom, ruleId: RuleSignatureId, - investigationFields: t.union([InvestigationFields, t.undefined]), + investigationFields: t.union([InvestigationFieldsCombined, t.undefined]), immutable: IsRuleImmutable, license: t.union([RuleLicense, t.undefined]), outputIndex: AlertsIndex, diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_rules_bulk.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_rules_bulk.ts index 9b7e558a1a57a1e..648281e10ebd01f 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_rules_bulk.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_rules_bulk.ts @@ -436,6 +436,21 @@ export default ({ getService }: FtrProviderContext): void => { }); }); }); + + describe('legacy investigation fields', () => { + it('should error trying to create a rule with legacy investigation fields format', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_BULK_CREATE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([{ ...getSimpleRule(), investigation_fields: ['foo'] }]) + .expect(400); + + expect(body.message).to.eql( + '[request body]: Invalid value "["foo"]" supplied to "investigation_fields"' + ); + }); + }); }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/delete_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/delete_rules.ts index c69433f0fe150c9..655299c0808a6b4 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/delete_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/delete_rules.ts @@ -6,7 +6,8 @@ */ import expect from '@kbn/expect'; - +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { BASE_ALERTING_API_PATH } from '@kbn/alerting-plugin/common'; import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -25,6 +26,9 @@ import { removeServerGeneratedProperties, removeServerGeneratedPropertiesIncludingRuleId, getLegacyActionSO, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFields, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -239,5 +243,72 @@ export default ({ getService }: FtrProviderContext): void => { expect(sidecarActionsPostResults.hits.hits.length).to.eql(0); }); }); + + describe('legacy investigation fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('deletes rule with investigation fields as array', async () => { + const { body } = await supertest + .delete( + `${DETECTION_ENGINE_RULES_URL}?rule_id=${ruleWithLegacyInvestigationField.params.ruleId}` + ) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + }); + + it('deletes rule with investigation fields as empty array', async () => { + const { body } = await supertest + .delete( + `${DETECTION_ENGINE_RULES_URL}?rule_id=${ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId}` + ) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql(undefined); + }); + + it('deletes rule with investigation fields as intended object type', async () => { + const { body } = await supertest + .delete(`${DETECTION_ENGINE_RULES_URL}?rule_id=rule-with-investigation-field`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql({ field_names: ['host.name'] }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/delete_rules_bulk.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/delete_rules_bulk.ts index 5051f6024cb1432..dd476803a9f2484 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/delete_rules_bulk.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/delete_rules_bulk.ts @@ -6,9 +6,11 @@ */ import expect from '@kbn/expect'; - +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { BASE_ALERTING_API_PATH } from '@kbn/alerting-plugin/common'; import { DETECTION_ENGINE_RULES_BULK_DELETE } from '@kbn/security-solution-plugin/common/constants'; +import { RuleResponse } from '@kbn/security-solution-plugin/common/api/detection_engine'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createLegacyRuleAction, @@ -25,6 +27,9 @@ import { removeServerGeneratedProperties, removeServerGeneratedPropertiesIncludingRuleId, getLegacyActionSO, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFields, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -444,5 +449,73 @@ export default ({ getService }: FtrProviderContext): void => { expect(sidecarActionsPostResults.hits.hits.length).to.eql(0); }); }); + + describe('legacy investigation fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('DELETE - should delete a single rule with investigation field', async () => { + // delete the rule in bulk + const { body } = await supertest + .delete(DETECTION_ENGINE_RULES_BULK_DELETE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([ + { rule_id: 'rule-with-investigation-field' }, + { rule_id: ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId }, + { rule_id: ruleWithLegacyInvestigationField.params.ruleId }, + ]) + .expect(200); + const investigationFields = body.map((rule: RuleResponse) => rule.investigation_fields); + expect(investigationFields).to.eql([ + { field_names: ['host.name'] }, + undefined, + { field_names: ['client.address', 'agent.name'] }, + ]); + }); + + it('POST - should delete a single rule with investigation field', async () => { + // delete the rule in bulk + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_BULK_DELETE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([ + { rule_id: 'rule-with-investigation-field' }, + { rule_id: ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId }, + { rule_id: ruleWithLegacyInvestigationField.params.ruleId }, + ]) + .expect(200); + const investigationFields = body.map((rule: RuleResponse) => rule.investigation_fields); + expect(investigationFields).to.eql([ + { field_names: ['host.name'] }, + undefined, + { field_names: ['client.address', 'agent.name'] }, + ]); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/export_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/export_rules.ts index 80719b25ef29532..d03cb681dd62c69 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/export_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/export_rules.ts @@ -7,6 +7,9 @@ import expect from 'expect'; +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; + import { DETECTION_ENGINE_RULES_URL, UPDATE_OR_CREATE_LEGACY_ACTIONS, @@ -24,6 +27,10 @@ import { getWebHookAction, removeServerGeneratedProperties, waitForRulePartialFailure, + getRuleSavedObjectWithLegacyInvestigationFields, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, + getRuleSOById, + createRuleThroughAlertingEndpoint, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -720,6 +727,123 @@ export default ({ getService }: FtrProviderContext): void => { }); }); }); + + describe('legacy investigation fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + await deleteAllRules(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('exports a rule that has legacy investigation_field and transforms field in response', async () => { + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_export`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ + objects: [{ rule_id: ruleWithLegacyInvestigationField.params.ruleId }], + }) + .expect(200) + .parse(binaryToString); + + const exportedRule = JSON.parse(body.toString().split(/\n/)[0]); + + expect(exportedRule.investigation_fields).toEqual({ + field_names: ['client.address', 'agent.name'], + }); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * NOT include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationField.id); + expect(ruleSO?.alert?.params?.investigationFields).toEqual([ + 'client.address', + 'agent.name', + ]); + }); + + it('exports a rule that has a legacy investigation field set to empty array and unsets field in response', async () => { + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_export`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ + objects: [{ rule_id: ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId }], + }) + .expect(200) + .parse(binaryToString); + + const exportedRule = JSON.parse(body.toString().split(/\n/)[0]); + + expect(exportedRule.investigation_fields).toEqual(undefined); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * NOT include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationFieldEmptyArray.id); + expect(ruleSO?.alert?.params?.investigationFields).toEqual([]); + }); + + it('exports rule with investigation fields as intended object type', async () => { + const { body } = await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_export`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ + objects: [{ rule_id: 'rule-with-investigation-field' }], + }) + .expect(200) + .parse(binaryToString); + + const exportedRule = JSON.parse(body.toString().split(/\n/)[0]); + + expect(exportedRule.investigation_fields).toEqual({ + field_names: ['host.name'], + }); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * NOT include a migration on SO. + */ const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, exportedRule.id); + expect(ruleSO?.alert?.params?.investigationFields).toEqual({ field_names: ['host.name'] }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/find_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/find_rules.ts index 81a7c8c74917216..78b4c7f70fc0b6f 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/find_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/find_rules.ts @@ -6,7 +6,9 @@ */ import expect from '@kbn/expect'; - +import { RuleResponse } from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { DETECTION_ENGINE_RULES_URL, UPDATE_OR_CREATE_LEGACY_ACTIONS, @@ -14,19 +16,24 @@ import { import { FtrProviderContext } from '../../common/ftr_provider_context'; import { createRule, + createRuleThroughAlertingEndpoint, deleteAllRules, getComplexRule, getComplexRuleOutput, + getRuleSOById, getSimpleRule, getSimpleRuleOutput, getWebHookAction, removeServerGeneratedProperties, + getRuleSavedObjectWithLegacyInvestigationFields, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, } from '../../utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); + const es = getService('es'); describe('find_rules', () => { beforeEach(async () => { @@ -267,5 +274,82 @@ export default ({ getService }: FtrProviderContext): void => { }); }); }); + + describe('legacy investigation fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + await deleteAllRules(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); + }); + + it('should return a rule with the migrated investigation fields', async () => { + const { body } = await supertest + .get(`${DETECTION_ENGINE_RULES_URL}/_find`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send() + .expect(200); + + const [ruleWithFieldAsArray] = body.data.filter( + (rule: RuleResponse) => rule.rule_id === ruleWithLegacyInvestigationField.params.ruleId + ); + + const [ruleWithFieldAsEmptyArray] = body.data.filter( + (rule: RuleResponse) => + rule.rule_id === ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId + ); + + const [ruleWithExpectedTyping] = body.data.filter( + (rule: RuleResponse) => rule.rule_id === 'rule-with-investigation-field' + ); + + expect(ruleWithFieldAsArray.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + expect(ruleWithFieldAsEmptyArray.investigation_fields).to.eql(undefined); + expect(ruleWithExpectedTyping.investigation_fields).to.eql({ + field_names: ['host.name'], + }); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * NOT include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationField.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + const { + hits: { + hits: [{ _source: ruleSO2 }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationFieldEmptyArray.id); + expect(ruleSO2?.alert?.params?.investigationFields).to.eql([]); + const { + hits: { + hits: [{ _source: ruleSO3 }], + }, + } = await getRuleSOById(es, ruleWithExpectedTyping.id); + expect(ruleSO3?.alert?.params?.investigationFields).to.eql({ field_names: ['host.name'] }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts index b07a54f86d2f64e..dae7835c1602068 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/import_rules.ts @@ -7,7 +7,11 @@ import expect from '@kbn/expect'; -import { RuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine'; +import { + InvestigationFields, + QueryRuleCreateProps, + RuleCreateProps, +} from '@kbn/security-solution-plugin/common/api/detection_engine'; import { EXCEPTION_LIST_ITEM_URL, EXCEPTION_LIST_URL } from '@kbn/securitysolution-list-constants'; import { getCreateExceptionListMinimalSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_exception_list_schema.mock'; import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; @@ -34,6 +38,8 @@ import { createLegacyRuleAction, getLegacyActionSO, createRule, + getRule, + getRuleSOById, } from '../../utils'; import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution'; @@ -177,6 +183,18 @@ const getImportRuleWithConnectorsBuffer = (connectorId: string) => { return buffer; }; +export const getSimpleRuleAsNdjsonWithLegacyInvestigationField = ( + ruleIds: string[], + enabled = false, + overwrites: Partial +): Buffer => { + const stringOfRules = ruleIds.map((ruleId) => { + const simpleRule = { ...getSimpleRule(ruleId, enabled), ...overwrites }; + return JSON.stringify(simpleRule); + }); + return Buffer.from(stringOfRules.join('\n')); +}; + // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); @@ -1885,5 +1903,119 @@ export default ({ getService }: FtrProviderContext): void => { .expect(200); }); }); + + describe('legacy investigation fields', () => { + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + }); + + it('imports rule with investigation fields as array', async () => { + await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach( + 'file', + getSimpleRuleAsNdjsonWithLegacyInvestigationField(['rule-1'], false, { + // mimicking what an 8.10 rule would look like + // we don't want to support this type in our APIs any longer, but do + // want to allow users to import rules from 8.10 + investigation_fields: ['foo', 'bar'] as unknown as InvestigationFields, + }), + 'rules.ndjson' + ) + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + + const rule = await getRule(supertest, log, 'rule-1'); + expect(rule.investigation_fields).to.eql({ field_names: ['foo', 'bar'] }); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, rule.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql({ field_names: ['foo', 'bar'] }); + }); + + it('imports rule with investigation fields as empty array', async () => { + await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach( + 'file', + getSimpleRuleAsNdjsonWithLegacyInvestigationField(['rule-1'], false, { + // mimicking what an 8.10 rule would look like + // we don't want to support this type in our APIs any longer, but do + // want to allow users to import rules from 8.10 + investigation_fields: [] as unknown as InvestigationFields, + }), + 'rules.ndjson' + ) + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + + const rule = await getRule(supertest, log, 'rule-1'); + expect(rule.investigation_fields).to.eql(undefined); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, rule.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(undefined); + }); + + it('imports rule with investigation fields as intended object type', async () => { + await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/_import`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .attach( + 'file', + getSimpleRuleAsNdjsonWithLegacyInvestigationField(['rule-1'], false, { + investigation_fields: { + field_names: ['foo'], + }, + }), + 'rules.ndjson' + ) + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + + const rule = await getRule(supertest, log, 'rule-1'); + expect(rule.investigation_fields).to.eql({ field_names: ['foo'] }); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, rule.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql({ field_names: ['foo'] }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/patch_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/patch_rules.ts index 09e2c7d84c0a84c..0babf1dd3535a96 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/patch_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/patch_rules.ts @@ -6,6 +6,8 @@ */ import expect from '@kbn/expect'; +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { DETECTION_ENGINE_RULES_URL, @@ -31,6 +33,10 @@ import { createLegacyRuleAction, getLegacyActionSO, getSimpleRuleWithoutRuleId, + getRuleSOById, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFields, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, } from '../../utils'; import { getActionsWithFrequencies, @@ -444,7 +450,182 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('investigation_fields', () => { + describe('patch per-action frequencies', () => { + const patchSingleRule = async ( + ruleId: string, + throttle: RuleActionThrottle | undefined, + actions: RuleActionArray + ) => { + const { body: patchedRule } = await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ rule_id: ruleId, throttle, actions }) + .expect(200); + + patchedRule.actions = removeUUIDFromActions(patchedRule.actions); + return removeServerGeneratedPropertiesIncludingRuleId(patchedRule); + }; + + describe('actions without frequencies', () => { + [undefined, NOTIFICATION_THROTTLE_NO_ACTIONS, NOTIFICATION_THROTTLE_RULE].forEach( + (throttle) => { + it(`it sets each action's frequency attribute to default value when 'throttle' is ${throttle}`, async () => { + const actionsWithoutFrequencies = await getActionsWithoutFrequencies(supertest); + + // create simple rule + const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); + + // patch a simple rule's `throttle` and `actions` + const patchedRule = await patchSingleRule( + createdRule.rule_id, + throttle, + actionsWithoutFrequencies + ); + + const expectedRule = getSimpleRuleOutputWithoutRuleId(); + expectedRule.revision = 1; + expectedRule.actions = actionsWithoutFrequencies.map((action) => ({ + ...action, + frequency: NOTIFICATION_DEFAULT_FREQUENCY, + })); + + expect(patchedRule).to.eql(expectedRule); + }); + } + ); + + // Action throttle cannot be shorter than the schedule interval which is by default is 5m + ['300s', '5m', '3h', '4d'].forEach((throttle) => { + it(`it correctly transforms 'throttle = ${throttle}' and sets it as a frequency of each action`, async () => { + const actionsWithoutFrequencies = await getActionsWithoutFrequencies(supertest); + + // create simple rule + const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); + + // patch a simple rule's `throttle` and `actions` + const patchedRule = await patchSingleRule( + createdRule.rule_id, + throttle, + actionsWithoutFrequencies + ); + + const expectedRule = getSimpleRuleOutputWithoutRuleId(); + expectedRule.revision = 1; + expectedRule.actions = actionsWithoutFrequencies.map((action) => ({ + ...action, + frequency: { summary: true, throttle, notifyWhen: 'onThrottleInterval' }, + })); + + expect(patchedRule).to.eql(expectedRule); + }); + }); + }); + + describe('actions with frequencies', () => { + [ + undefined, + NOTIFICATION_THROTTLE_NO_ACTIONS, + NOTIFICATION_THROTTLE_RULE, + '321s', + '6m', + '10h', + '2d', + ].forEach((throttle) => { + it(`it does not change actions frequency attributes when 'throttle' is '${throttle}'`, async () => { + const actionsWithFrequencies = await getActionsWithFrequencies(supertest); + + // create simple rule + const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); + + // patch a simple rule's `throttle` and `actions` + const patchedRule = await patchSingleRule( + createdRule.rule_id, + throttle, + actionsWithFrequencies + ); + + const expectedRule = getSimpleRuleOutputWithoutRuleId(); + expectedRule.revision = 1; + expectedRule.actions = actionsWithFrequencies; + + expect(patchedRule).to.eql(expectedRule); + }); + }); + }); + + describe('some actions with frequencies', () => { + [undefined, NOTIFICATION_THROTTLE_NO_ACTIONS, NOTIFICATION_THROTTLE_RULE].forEach( + (throttle) => { + it(`it overrides each action's frequency attribute to default value when 'throttle' is ${throttle}`, async () => { + const someActionsWithFrequencies = await getSomeActionsWithFrequencies(supertest); + + // create simple rule + const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); + + // patch a simple rule's `throttle` and `actions` + const patchedRule = await patchSingleRule( + createdRule.rule_id, + throttle, + someActionsWithFrequencies + ); + + const expectedRule = getSimpleRuleOutputWithoutRuleId(); + expectedRule.revision = 1; + expectedRule.actions = someActionsWithFrequencies.map((action) => ({ + ...action, + frequency: action.frequency ?? NOTIFICATION_DEFAULT_FREQUENCY, + })); + + expect(patchedRule).to.eql(expectedRule); + }); + } + ); + + // Action throttle cannot be shorter than the schedule interval which is by default is 5m + ['430s', '7m', '1h', '8d'].forEach((throttle) => { + it(`it correctly transforms 'throttle = ${throttle}' and overrides frequency attribute of each action`, async () => { + const someActionsWithFrequencies = await getSomeActionsWithFrequencies(supertest); + + // create simple rule + const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); + + // patch a simple rule's `throttle` and `actions` + const patchedRule = await patchSingleRule( + createdRule.rule_id, + throttle, + someActionsWithFrequencies + ); + + const expectedRule = getSimpleRuleOutputWithoutRuleId(); + expectedRule.revision = 1; + expectedRule.actions = someActionsWithFrequencies.map((action) => ({ + ...action, + frequency: action.frequency ?? { + summary: true, + throttle, + notifyWhen: 'onThrottleInterval', + }, + })); + + expect(patchedRule).to.eql(expectedRule); + }); + }); + }); + }); + }); + + describe('investigation fields', () => { + describe('investigation_field', () => { + beforeEach(async () => { + await createSignalsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + }); + it('should overwrite investigation_fields value on patch - non additive', async () => { await createRule(supertest, log, { ...getSimpleRule('rule-1'), @@ -506,168 +687,100 @@ export default ({ getService }: FtrProviderContext) => { expect(body.investigation_fields.field_names).to.eql(['blob', 'boop']); }); }); - }); - - describe('patch per-action frequencies', () => { - const patchSingleRule = async ( - ruleId: string, - throttle: RuleActionThrottle | undefined, - actions: RuleActionArray - ) => { - const { body: patchedRule } = await supertest - .patch(DETECTION_ENGINE_RULES_URL) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .send({ rule_id: ruleId, throttle, actions }) - .expect(200); - - patchedRule.actions = removeUUIDFromActions(patchedRule.actions); - return removeServerGeneratedPropertiesIncludingRuleId(patchedRule); - }; - - describe('actions without frequencies', () => { - [undefined, NOTIFICATION_THROTTLE_NO_ACTIONS, NOTIFICATION_THROTTLE_RULE].forEach( - (throttle) => { - it(`it sets each action's frequency attribute to default value when 'throttle' is ${throttle}`, async () => { - const actionsWithoutFrequencies = await getActionsWithoutFrequencies(supertest); - // create simple rule - const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); - - // patch a simple rule's `throttle` and `actions` - const patchedRule = await patchSingleRule( - createdRule.rule_id, - throttle, - actionsWithoutFrequencies - ); - - const expectedRule = getSimpleRuleOutputWithoutRuleId(); - expectedRule.revision = 1; - expectedRule.actions = actionsWithoutFrequencies.map((action) => ({ - ...action, - frequency: NOTIFICATION_DEFAULT_FREQUENCY, - })); - - expect(patchedRule).to.eql(expectedRule); - }); - } - ); - - // Action throttle cannot be shorter than the schedule interval which is by default is 5m - ['300s', '5m', '3h', '4d'].forEach((throttle) => { - it(`it correctly transforms 'throttle = ${throttle}' and sets it as a frequency of each action`, async () => { - const actionsWithoutFrequencies = await getActionsWithoutFrequencies(supertest); - - // create simple rule - const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); - - // patch a simple rule's `throttle` and `actions` - const patchedRule = await patchSingleRule( - createdRule.rule_id, - throttle, - actionsWithoutFrequencies - ); - - const expectedRule = getSimpleRuleOutputWithoutRuleId(); - expectedRule.revision = 1; - expectedRule.actions = actionsWithoutFrequencies.map((action) => ({ - ...action, - frequency: { summary: true, throttle, notifyWhen: 'onThrottleInterval' }, - })); - - expect(patchedRule).to.eql(expectedRule); - }); + describe('investigation_fields legacy', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); }); - }); - describe('actions with frequencies', () => { - [ - undefined, - NOTIFICATION_THROTTLE_NO_ACTIONS, - NOTIFICATION_THROTTLE_RULE, - '321s', - '6m', - '10h', - '2d', - ].forEach((throttle) => { - it(`it does not change actions frequency attributes when 'throttle' is '${throttle}'`, async () => { - const actionsWithFrequencies = await getActionsWithFrequencies(supertest); - - // create simple rule - const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); - - // patch a simple rule's `throttle` and `actions` - const patchedRule = await patchSingleRule( - createdRule.rule_id, - throttle, - actionsWithFrequencies - ); - - const expectedRule = getSimpleRuleOutputWithoutRuleId(); - expectedRule.revision = 1; - expectedRule.actions = actionsWithFrequencies; - - expect(patchedRule).to.eql(expectedRule); - }); + afterEach(async () => { + await deleteAllRules(supertest, log); }); - }); - describe('some actions with frequencies', () => { - [undefined, NOTIFICATION_THROTTLE_NO_ACTIONS, NOTIFICATION_THROTTLE_RULE].forEach( - (throttle) => { - it(`it overrides each action's frequency attribute to default value when 'throttle' is ${throttle}`, async () => { - const someActionsWithFrequencies = await getSomeActionsWithFrequencies(supertest); - - // create simple rule - const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); + it('errors if trying to patch investigation fields using legacy format', async () => { + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ + rule_id: ruleWithLegacyInvestigationField.params.ruleId, + name: 'some other name', + investigation_fields: ['client.foo'], + }) + .expect(400); + + expect(body.message).to.eql( + '[request body]: Invalid value "["client.foo"]" supplied to "investigation_fields"' + ); + }); - // patch a simple rule's `throttle` and `actions` - const patchedRule = await patchSingleRule( - createdRule.rule_id, - throttle, - someActionsWithFrequencies - ); + it('should patch a rule with a legacy investigation field and transform response', async () => { + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ + rule_id: ruleWithLegacyInvestigationField.params.ruleId, + name: 'some other name', + }) + .expect(200); - const expectedRule = getSimpleRuleOutputWithoutRuleId(); - expectedRule.revision = 1; - expectedRule.actions = someActionsWithFrequencies.map((action) => ({ - ...action, - frequency: action.frequency ?? NOTIFICATION_DEFAULT_FREQUENCY, - })); + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * NOT include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql([ + 'client.address', + 'agent.name', + ]); + }); - expect(patchedRule).to.eql(expectedRule); - }); - } - ); - - // Action throttle cannot be shorter than the schedule interval which is by default is 5m - ['430s', '7m', '1h', '8d'].forEach((throttle) => { - it(`it correctly transforms 'throttle = ${throttle}' and overrides frequency attribute of each action`, async () => { - const someActionsWithFrequencies = await getSomeActionsWithFrequencies(supertest); - - // create simple rule - const createdRule = await createRule(supertest, log, getSimpleRuleWithoutRuleId()); - - // patch a simple rule's `throttle` and `actions` - const patchedRule = await patchSingleRule( - createdRule.rule_id, - throttle, - someActionsWithFrequencies - ); - - const expectedRule = getSimpleRuleOutputWithoutRuleId(); - expectedRule.revision = 1; - expectedRule.actions = someActionsWithFrequencies.map((action) => ({ - ...action, - frequency: action.frequency ?? { - summary: true, - throttle, - notifyWhen: 'onThrottleInterval', - }, - })); + it('should patch a rule with a legacy investigation field - empty array - and transform response', async () => { + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ + rule_id: ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId, + name: 'some other name', + }) + .expect(200); - expect(patchedRule).to.eql(expectedRule); - }); + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql(undefined); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * NOT include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql([]); }); }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/patch_rules_bulk.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/patch_rules_bulk.ts index 225afd21716574c..28c5a92080c933f 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/patch_rules_bulk.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/patch_rules_bulk.ts @@ -9,6 +9,8 @@ import expect from '@kbn/expect'; import { DETECTION_ENGINE_RULES_BULK_UPDATE } from '@kbn/security-solution-plugin/common/constants'; import { ExceptionListTypeEnum } from '@kbn/securitysolution-io-ts-list-types'; +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { @@ -23,6 +25,10 @@ import { createRule, createLegacyRuleAction, getLegacyActionSO, + getRuleSOById, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, + getRuleSavedObjectWithLegacyInvestigationFields, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -499,5 +505,144 @@ export default ({ getService }: FtrProviderContext) => { ]); }); }); + + describe('legacy investigation fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('errors if trying to patch investigation fields using legacy format', async () => { + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_BULK_UPDATE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([ + { + rule_id: ruleWithLegacyInvestigationField.params.ruleId, + name: 'some other name', + investigation_fields: ['foobar'], + }, + ]) + .expect(400); + + expect(body.message).to.eql( + '[request body]: Invalid value "["foobar"]" supplied to "investigation_fields"' + ); + }); + + it('should patch a rule with a legacy investigation field and transform field in response', async () => { + // patch a simple rule's name + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_BULK_UPDATE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([ + { rule_id: ruleWithLegacyInvestigationField.params.ruleId, name: 'some other name' }, + ]) + .expect(200); + + const bodyToCompareLegacyField = removeServerGeneratedProperties(body[0]); + expect(bodyToCompareLegacyField.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + expect(bodyToCompareLegacyField.name).to.eql('some other name'); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * NOT include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body[0].id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + }); + + it('should patch a rule with a legacy investigation field - empty array - and transform field in response', async () => { + // patch a simple rule's name + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_BULK_UPDATE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([ + { + rule_id: ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId, + name: 'some other name 2', + }, + ]) + .expect(200); + + const bodyToCompareLegacyFieldEmptyArray = removeServerGeneratedProperties(body[0]); + expect(bodyToCompareLegacyFieldEmptyArray.investigation_fields).to.eql(undefined); + expect(bodyToCompareLegacyFieldEmptyArray.name).to.eql('some other name 2'); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * NOT include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body[0].id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql([]); + }); + + it('should patch a rule with an investigation field', async () => { + await createRule(supertest, log, { + ...getSimpleRule('rule-1'), + investigation_fields: { + field_names: ['host.name'], + }, + }); + + // patch a simple rule's name + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_BULK_UPDATE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([ + { + rule_id: 'rule-1', + name: 'some other name 3', + }, + ]) + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body[0]); + expect(bodyToCompare.investigation_fields).to.eql({ + field_names: ['host.name'], + }); + expect(bodyToCompare.name).to.eql('some other name 3'); + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body[0].id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql({ + field_names: ['host.name'], + }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts index a5fe52e6d4736e0..e9234d47a1816d0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/perform_bulk_action.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import expect from '@kbn/expect'; import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks'; import { @@ -38,6 +40,10 @@ import { installMockPrebuiltRules, removeServerGeneratedProperties, waitForRuleSuccess, + getRuleSOById, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFields, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -2997,5 +3003,421 @@ export default ({ getService }: FtrProviderContext): void => { expect(rule.timeline_id).to.eql(timelineId); expect(rule.timeline_title).to.eql(timelineTitle); }); + + describe('legacy investigation fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('should export rules with legacy investigation_fields and transform legacy field in response', async () => { + const { body } = await postBulkAction() + .send({ query: '', action: BulkActionType.export }) + .expect(200) + .expect('Content-Type', 'application/ndjson') + .expect('Content-Disposition', 'attachment; filename="rules_export.ndjson"') + .parse(binaryToString); + + const [rule1, rule2, rule3, exportDetailsJson] = body.toString().split(/\n/); + + const ruleToCompareWithLegacyInvestigationField = removeServerGeneratedProperties( + JSON.parse(rule1) + ); + expect(ruleToCompareWithLegacyInvestigationField.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + + const ruleToCompareWithLegacyInvestigationFieldEmptyArray = removeServerGeneratedProperties( + JSON.parse(rule2) + ); + expect(ruleToCompareWithLegacyInvestigationFieldEmptyArray.investigation_fields).to.eql( + undefined + ); + + const ruleWithInvestigationField = removeServerGeneratedProperties(JSON.parse(rule3)); + expect(ruleWithInvestigationField.investigation_fields).to.eql({ + field_names: ['host.name'], + }); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should not include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, JSON.parse(rule1).id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + + const exportDetails = JSON.parse(exportDetailsJson); + expect(exportDetails).to.eql({ + exported_exception_list_count: 0, + exported_exception_list_item_count: 0, + exported_count: 3, + exported_rules_count: 3, + missing_exception_list_item_count: 0, + missing_exception_list_items: [], + missing_exception_lists: [], + missing_exception_lists_count: 0, + missing_rules: [], + missing_rules_count: 0, + excluded_action_connection_count: 0, + excluded_action_connections: [], + exported_action_connector_count: 0, + missing_action_connection_count: 0, + missing_action_connections: [], + }); + }); + + it('should delete rules with investigation fields and transform legacy field in response', async () => { + const { body } = await postBulkAction() + .send({ query: '', action: BulkActionType.delete }) + .expect(200); + + expect(body.attributes.summary).to.eql({ failed: 0, skipped: 0, succeeded: 3, total: 3 }); + + // Check that the deleted rule is returned with the response + const names = body.attributes.results.deleted.map( + (returnedRule: RuleResponse) => returnedRule.name + ); + expect(names.includes('Test investigation fields')).to.eql(true); + expect(names.includes('Test investigation fields empty array')).to.eql(true); + expect(names.includes('Test investigation fields object')).to.eql(true); + + const ruleWithLegacyField = body.attributes.results.deleted.find( + (returnedRule: RuleResponse) => + returnedRule.rule_id === ruleWithLegacyInvestigationField.params.ruleId + ); + + expect(ruleWithLegacyField.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + + // Check that the updates have been persisted + await fetchRule(ruleWithLegacyInvestigationField.params.ruleId).expect(404); + await fetchRule(ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId).expect(404); + await fetchRule('rule-with-investigation-field').expect(404); + }); + + it('should enable rules with legacy investigation fields and transform legacy field in response', async () => { + const { body } = await postBulkAction() + .send({ query: '', action: BulkActionType.enable }) + .expect(200); + + expect(body.attributes.summary).to.eql({ failed: 0, skipped: 0, succeeded: 3, total: 3 }); + + // Check that the updated rule is returned with the response + // and field transformed on response + expect( + body.attributes.results.updated.every( + (returnedRule: RuleResponse) => returnedRule.enabled + ) + ).to.eql(true); + + const ruleWithLegacyField = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => + returnedRule.rule_id === ruleWithLegacyInvestigationField.params.ruleId + ); + expect(ruleWithLegacyField.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + + const ruleWithEmptyArray = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => + returnedRule.rule_id === ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId + ); + expect(ruleWithEmptyArray.investigation_fields).to.eql(undefined); + + const ruleWithIntendedType = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => returnedRule.rule_id === 'rule-with-investigation-field' + ); + expect(ruleWithIntendedType.investigation_fields).to.eql({ field_names: ['host.name'] }); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should not include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyField.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + expect(ruleSO?.alert?.enabled).to.eql(true); + + const { + hits: { + hits: [{ _source: ruleSO2 }], + }, + } = await getRuleSOById(es, ruleWithEmptyArray.id); + expect(ruleSO2?.alert?.params?.investigationFields).to.eql([]); + expect(ruleSO?.alert?.enabled).to.eql(true); + + const { + hits: { + hits: [{ _source: ruleSO3 }], + }, + } = await getRuleSOById(es, ruleWithIntendedType.id); + expect(ruleSO3?.alert?.params?.investigationFields).to.eql({ field_names: ['host.name'] }); + expect(ruleSO?.alert?.enabled).to.eql(true); + }); + + it('should disable rules with legacy investigation fields and transform legacy field in response', async () => { + const { body } = await postBulkAction() + .send({ query: '', action: BulkActionType.disable }) + .expect(200); + + expect(body.attributes.summary).to.eql({ failed: 0, skipped: 0, succeeded: 3, total: 3 }); + + // Check that the updated rule is returned with the response + // and field transformed on response + expect( + body.attributes.results.updated.every( + (returnedRule: RuleResponse) => !returnedRule.enabled + ) + ).to.eql(true); + + const ruleWithLegacyField = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => + returnedRule.rule_id === ruleWithLegacyInvestigationField.params.ruleId + ); + expect(ruleWithLegacyField.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + + const ruleWithEmptyArray = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => + returnedRule.rule_id === ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId + ); + expect(ruleWithEmptyArray.investigation_fields).to.eql(undefined); + + const ruleWithIntendedType = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => returnedRule.rule_id === 'rule-with-investigation-field' + ); + expect(ruleWithIntendedType.investigation_fields).to.eql({ field_names: ['host.name'] }); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should not include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyField.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + + const { + hits: { + hits: [{ _source: ruleSO2 }], + }, + } = await getRuleSOById(es, ruleWithEmptyArray.id); + expect(ruleSO2?.alert?.params?.investigationFields).to.eql([]); + + const { + hits: { + hits: [{ _source: ruleSO3 }], + }, + } = await getRuleSOById(es, ruleWithIntendedType.id); + expect(ruleSO3?.alert?.params?.investigationFields).to.eql({ field_names: ['host.name'] }); + }); + + it('should duplicate rules with legacy investigation fields and transform field in response', async () => { + const { body } = await postBulkAction() + .send({ + query: '', + action: BulkActionType.duplicate, + duplicate: { include_exceptions: false, include_expired_exceptions: false }, + }) + .expect(200); + + expect(body.attributes.summary).to.eql({ failed: 0, skipped: 0, succeeded: 3, total: 3 }); + + // Check that the duplicated rule is returned with the response + const names = body.attributes.results.created.map( + (returnedRule: RuleResponse) => returnedRule.name + ); + expect(names.includes('Test investigation fields [Duplicate]')).to.eql(true); + expect(names.includes('Test investigation fields empty array [Duplicate]')).to.eql(true); + expect(names.includes('Test investigation fields object [Duplicate]')).to.eql(true); + + // Check that the updates have been persisted + const { body: rulesResponse } = await supertest + .get(`${DETECTION_ENGINE_RULES_URL}/_find`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .expect(200); + + expect(rulesResponse.total).to.eql(6); + + const ruleWithLegacyField = body.attributes.results.created.find( + (returnedRule: RuleResponse) => + returnedRule.name === 'Test investigation fields [Duplicate]' + ); + const ruleWithEmptyArray = body.attributes.results.created.find( + (returnedRule: RuleResponse) => + returnedRule.name === 'Test investigation fields empty array [Duplicate]' + ); + const ruleWithIntendedType = body.attributes.results.created.find( + (returnedRule: RuleResponse) => + returnedRule.name === 'Test investigation fields object [Duplicate]' + ); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, duplicated + * rules should NOT have migrated value on write. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyField.id); + + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + + const { + hits: { + hits: [{ _source: ruleSO2 }], + }, + } = await getRuleSOById(es, ruleWithEmptyArray.id); + expect(ruleSO2?.alert?.params?.investigationFields).to.eql([]); + + const { + hits: { + hits: [{ _source: ruleSO3 }], + }, + } = await getRuleSOById(es, ruleWithIntendedType.id); + expect(ruleSO3?.alert?.params?.investigationFields).to.eql({ field_names: ['host.name'] }); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, the original + * rules selected to be duplicated should not be migrated. + */ + const { + hits: { + hits: [{ _source: ruleSOOriginalLegacy }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationField.id); + + expect(ruleSOOriginalLegacy?.alert?.params?.investigationFields).to.eql([ + 'client.address', + 'agent.name', + ]); + + const { + hits: { + hits: [{ _source: ruleSOOriginalLegacyEmptyArray }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationFieldEmptyArray.id); + expect(ruleSOOriginalLegacyEmptyArray?.alert?.params?.investigationFields).to.eql([]); + + const { + hits: { + hits: [{ _source: ruleSOOriginalNoLegacy }], + }, + } = await getRuleSOById(es, ruleWithIntendedType.id); + expect(ruleSOOriginalNoLegacy?.alert?.params?.investigationFields).to.eql({ + field_names: ['host.name'], + }); + }); + + it('should edit rules with legacy investigation fields', async () => { + const { body } = await postBulkAction().send({ + query: '', + action: BulkActionType.edit, + [BulkActionType.edit]: [ + { + type: BulkActionEditType.set_tags, + value: ['reset-tag'], + }, + ], + }); + expect(body.attributes.summary).to.eql({ + failed: 0, + skipped: 0, + succeeded: 3, + total: 3, + }); + + // Check that the updated rule is returned with the response + // and field transformed on response + const ruleWithLegacyField = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => + returnedRule.rule_id === ruleWithLegacyInvestigationField.params.ruleId + ); + expect(ruleWithLegacyField.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + expect(ruleWithLegacyField.tags).to.eql(['reset-tag']); + + const ruleWithEmptyArray = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => + returnedRule.rule_id === ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId + ); + expect(ruleWithEmptyArray.investigation_fields).to.eql(undefined); + expect(ruleWithEmptyArray.tags).to.eql(['reset-tag']); + + const ruleWithIntendedType = body.attributes.results.updated.find( + (returnedRule: RuleResponse) => returnedRule.rule_id === 'rule-with-investigation-field' + ); + expect(ruleWithIntendedType.investigation_fields).to.eql({ field_names: ['host.name'] }); + expect(ruleWithIntendedType.tags).to.eql(['reset-tag']); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should not include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationField.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + + const { + hits: { + hits: [{ _source: ruleSO2 }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationFieldEmptyArray.id); + expect(ruleSO2?.alert?.params?.investigationFields).to.eql([]); + + const { + hits: { + hits: [{ _source: ruleSO3 }], + }, + } = await getRuleSOById(es, ruleWithIntendedType.id); + expect(ruleSO3?.alert?.params?.investigationFields).to.eql({ field_names: ['host.name'] }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/read_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/read_rules.ts index 364731d4864b425..9c5fddaf8588ff1 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/read_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/read_rules.ts @@ -6,7 +6,8 @@ */ import expect from '@kbn/expect'; - +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { DETECTION_ENGINE_RULES_URL, UPDATE_OR_CREATE_LEGACY_ACTIONS, @@ -24,6 +25,10 @@ import { getWebHookAction, removeServerGeneratedProperties, removeServerGeneratedPropertiesIncludingRuleId, + getRuleSOById, + getRuleSavedObjectWithLegacyInvestigationFields, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -267,5 +272,111 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe('investigation_fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('should be able to read a rule with a legacy investigation field', async () => { + const { body } = await supertest + .get( + `${DETECTION_ENGINE_RULES_URL}?rule_id=${ruleWithLegacyInvestigationField.params.ruleId}` + ) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send() + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql({ + field_names: ['client.address', 'agent.name'], + }); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * just be a transform on read, not a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + }); + + it('should be able to read a rule with a legacy investigation field - empty array', async () => { + const { body } = await supertest + .get( + `${DETECTION_ENGINE_RULES_URL}?rule_id=${ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId}` + ) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send() + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql(undefined); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * just be a transform on read, not a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql([]); + }); + + it('does not migrate investigation fields when intended object type', async () => { + const { body } = await supertest + .get(`${DETECTION_ENGINE_RULES_URL}?rule_id=rule-with-investigation-field`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send() + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql({ field_names: ['host.name'] }); + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change should + * just be a transform on read, not a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql({ field_names: ['host.name'] }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/update_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/update_rules.ts index 226432cbdbb39b3..c30edde0abea809 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/update_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/update_rules.ts @@ -6,7 +6,8 @@ */ import expect from '@kbn/expect'; - +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { DETECTION_ENGINE_RULES_URL, NOTIFICATION_DEFAULT_FREQUENCY, @@ -35,6 +36,10 @@ import { getThresholdRuleForSignalTesting, getLegacyActionSO, getSimpleRuleWithoutRuleId, + getRuleSOById, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFields, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, } from '../../utils'; import { getActionsWithFrequencies, @@ -909,5 +914,102 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe('legacy investigation fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('errors if sending legacy investigation fields type', async () => { + const updatedRule = { + ...getSimpleRuleUpdate(ruleWithLegacyInvestigationField.params.ruleId), + investigation_fields: ['foo'], + }; + + const { body } = await supertest + .put(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send(updatedRule) + .expect(400); + + expect(body.message).to.eql( + '[request body]: Invalid value "["foo"]" supplied to "investigation_fields"' + ); + }); + + it('unsets legacy investigation fields when field not specified for update', async () => { + // rule_id of a rule with legacy investigation fields set + const updatedRule = getSimpleRuleUpdate(ruleWithLegacyInvestigationField.params.ruleId); + + const { body } = await supertest + .put(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send(updatedRule) + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql(undefined); + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(undefined); + }); + + it('updates a rule with legacy investigation fields when field specified for update in intended format', async () => { + // rule_id of a rule with legacy investigation fields set + const updatedRule = { + ...getSimpleRuleUpdate(ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId), + investigation_fields: { + field_names: ['foo'], + }, + }; + + const { body } = await supertest + .put(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send(updatedRule) + .expect(200); + + const bodyToCompare = removeServerGeneratedProperties(body); + expect(bodyToCompare.investigation_fields).to.eql({ + field_names: ['foo'], + }); + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, body.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql({ + field_names: ['foo'], + }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/update_rules_bulk.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/update_rules_bulk.ts index 669247af1b08870..ed87c118bedb971 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/update_rules_bulk.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/update_rules_bulk.ts @@ -7,7 +7,8 @@ import expect from '@kbn/expect'; import { RuleResponse } from '@kbn/security-solution-plugin/common/api/detection_engine'; - +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { DETECTION_ENGINE_RULES_URL, DETECTION_ENGINE_RULES_BULK_UPDATE, @@ -32,6 +33,10 @@ import { removeServerGeneratedPropertiesIncludingRuleId, getSimpleRuleWithoutRuleId, getSimpleRuleOutputWithoutRuleId, + getRuleSavedObjectWithLegacyInvestigationFields, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, + getRuleSOById, } from '../../utils'; import { removeUUIDFromActions } from '../../utils/remove_uuid_from_actions'; import { @@ -803,5 +808,124 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe('legacy investigation fields', () => { + let ruleWithLegacyInvestigationField: Rule; + let ruleWithLegacyInvestigationFieldEmptyArray: Rule; + let ruleWithInvestigationFields: RuleResponse; + + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + await createSignalsIndex(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + ruleWithLegacyInvestigationFieldEmptyArray = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() + ); + ruleWithInvestigationFields = await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('errors if trying to update investigation fields using legacy format', async () => { + // update rule + const { body } = await supertest + .put(DETECTION_ENGINE_RULES_BULK_UPDATE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([ + { + ...getSimpleRule(), + name: 'New name', + rule_id: ruleWithLegacyInvestigationField.params.ruleId, + investigation_fields: ['client.foo'], + }, + ]) + .expect(400); + + expect(body.message).to.eql( + '[request body]: Invalid value "["client.foo"]" supplied to "investigation_fields"' + ); + }); + + it('updates a rule with legacy investigation fields and transforms field in response', async () => { + // update rule + const { body } = await supertest + .put(DETECTION_ENGINE_RULES_BULK_UPDATE) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send([ + { + ...getSimpleRule(), + name: 'New name - used to have legacy investigation fields', + rule_id: ruleWithLegacyInvestigationField.params.ruleId, + }, + { + ...getSimpleRule(), + name: 'New name - used to have legacy investigation fields, empty array', + rule_id: ruleWithLegacyInvestigationFieldEmptyArray.params.ruleId, + investigation_fields: { + field_names: ['foo'], + }, + }, + { + ...getSimpleRule(), + name: 'New name - never had legacy investigation fields', + rule_id: 'rule-with-investigation-field', + investigation_fields: { + field_names: ['bar'], + }, + }, + ]) + .expect(200); + + expect(body[0].investigation_fields).to.eql(undefined); + expect(body[0].name).to.eql('New name - used to have legacy investigation fields'); + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationField.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(undefined); + + expect(body[1].investigation_fields).to.eql({ + field_names: ['foo'], + }); + expect(body[1].name).to.eql( + 'New name - used to have legacy investigation fields, empty array' + ); + const { + hits: { + hits: [{ _source: ruleSO2 }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationFieldEmptyArray.id); + expect(ruleSO2?.alert?.params?.investigationFields).to.eql({ + field_names: ['foo'], + }); + + expect(body[2].investigation_fields).to.eql({ + field_names: ['bar'], + }); + expect(body[2].name).to.eql('New name - never had legacy investigation fields'); + const { + hits: { + hits: [{ _source: ruleSO3 }], + }, + } = await getRuleSOById(es, ruleWithInvestigationFields.id); + expect(ruleSO3?.alert?.params?.investigationFields).to.eql({ + field_names: ['bar'], + }); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group4/telemetry/usage_collector/detection_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group4/telemetry/usage_collector/detection_rules.ts index 73d377fa87f2ddb..652d13fd32a53d8 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group4/telemetry/usage_collector/detection_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group4/telemetry/usage_collector/detection_rules.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; + import type { ThreatMatchRuleCreateProps, ThresholdRuleCreateProps, @@ -35,6 +36,9 @@ import { waitForSignalsToBePresent, updateRule, deleteAllEventLogExecutionEvents, + getRuleSavedObjectWithLegacyInvestigationFields, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray, + createRuleThroughAlertingEndpoint, } from '../../../../utils'; // eslint-disable-next-line import/no-default-export @@ -241,16 +245,25 @@ export default ({ getService }: FtrProviderContext) => { }); describe('legacy investigation fields', () => { - before(async () => { - await esArchiver.load( - 'x-pack/test/functional/es_archives/security_solution/legacy_investigation_fields' + beforeEach(async () => { + await deleteAllRules(supertest, log); + await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray() ); + await createRule(supertest, log, { + ...getSimpleRule('rule-with-investigation-field'), + name: 'Test investigation fields object', + investigation_fields: { field_names: ['host.name'] }, + }); }); - after(async () => { - await esArchiver.unload( - 'x-pack/test/functional/es_archives/security_solution/legacy_investigation_fields' - ); + afterEach(async () => { + await deleteAllRules(supertest, log); }); it('should show "legacy_investigation_fields" to be greater than 0 when a rule has "investigation_fields" set to array or empty array', async () => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts index b2033fed23ed494..410af1f3f7df3c5 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts @@ -20,6 +20,8 @@ import { ALERT_LAST_DETECTED, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { orderBy } from 'lodash'; import { v4 as uuidv4 } from 'uuid'; @@ -27,6 +29,7 @@ import { v4 as uuidv4 } from 'uuid'; import { QueryRuleCreateProps, AlertSuppressionMissingFieldsStrategy, + BulkActionType, } from '@kbn/security-solution-plugin/common/api/detection_engine'; import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; import { Ancestor } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/types'; @@ -36,7 +39,11 @@ import { ALERT_ORIGINAL_TIME, ALERT_ORIGINAL_EVENT, } from '@kbn/security-solution-plugin/common/field_maps/field_names'; -import { DETECTION_ENGINE_SIGNALS_STATUS_URL } from '@kbn/security-solution-plugin/common/constants'; +import { + DETECTION_ENGINE_RULES_BULK_ACTION, + DETECTION_ENGINE_RULES_URL, + DETECTION_ENGINE_SIGNALS_STATUS_URL, +} from '@kbn/security-solution-plugin/common/constants'; import { getMaxSignalsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; import { deleteAllExceptions } from '../../../lists_api_integration/utils'; import { @@ -51,6 +58,9 @@ import { getSimpleRule, previewRule, setSignalStatus, + getRuleSOById, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFields, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { dataGeneratorFactory } from '../../utils/data_generator'; @@ -2265,5 +2275,51 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts[1]._source?.agent).to.have.property('name', 'test-3'); }); }); + + describe('legacy investigation_fields', () => { + let ruleWithLegacyInvestigationField: Rule; + + beforeEach(async () => { + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('should generate alerts when rule includes legacy investigation_fields', async () => { + // enable rule + await supertest + .post(DETECTION_ENGINE_RULES_BULK_ACTION) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ query: '', action: BulkActionType.enable }) + .expect(200); + + // Confirming that enabling did not migrate rule, so rule + // run/alerts generated here were from rule with legacy investigation field + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationField.id); + expect(ruleSO?.alert?.params?.investigationFields).to.eql(['client.address', 'agent.name']); + + // fetch rule for format needed to pass into + const { body: ruleBody } = await supertest + .get( + `${DETECTION_ENGINE_RULES_URL}?rule_id=${ruleWithLegacyInvestigationField.params.ruleId}` + ) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .expect(200); + + const alertsAfterEnable = await getOpenSignals(supertest, log, es, ruleBody, 'succeeded'); + expect(alertsAfterEnable.hits.hits.length > 0).eql(true); + }); + }); }); }; diff --git a/x-pack/test/detection_engine_api_integration/utils/create_rule_saved_object.ts b/x-pack/test/detection_engine_api_integration/utils/create_rule_saved_object.ts new file mode 100644 index 000000000000000..93a632201162362 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/create_rule_saved_object.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; + +import { Rule } from '@kbn/alerting-plugin/common'; +import { + BaseRuleParams, + InternalRuleCreate, +} from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; + +/** + * Creates a rule using the alerting APIs directly. + * This allows us to test some legacy types that are not exposed + * on our APIs + * + * @param supertest + */ +export const createRuleThroughAlertingEndpoint = async ( + supertest: SuperTest.SuperTest, + rule: InternalRuleCreate +): Promise> => { + const { body } = await supertest + .post('/api/alerting/rule') + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send(rule) + .expect(200); + + return body; +}; diff --git a/x-pack/test/detection_engine_api_integration/utils/get_rule_so_by_id.ts b/x-pack/test/detection_engine_api_integration/utils/get_rule_so_by_id.ts index 159d1e3010a2a7e..584b49e379ee7dc 100644 --- a/x-pack/test/detection_engine_api_integration/utils/get_rule_so_by_id.ts +++ b/x-pack/test/detection_engine_api_integration/utils/get_rule_so_by_id.ts @@ -10,9 +10,10 @@ import { SavedObjectReference } from '@kbn/core/server'; import { ALERTING_CASES_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import type { SearchResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; interface RuleSO { - alert: Rule; + alert: Rule; references: SavedObjectReference[]; } diff --git a/x-pack/test/detection_engine_api_integration/utils/get_rule_with_legacy_investigation_fields.ts b/x-pack/test/detection_engine_api_integration/utils/get_rule_with_legacy_investigation_fields.ts new file mode 100644 index 000000000000000..55056748a094553 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/get_rule_with_legacy_investigation_fields.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { InternalRuleCreate } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; + +export const getRuleSavedObjectWithLegacyInvestigationFields = (): InternalRuleCreate => + ({ + name: 'Test investigation fields', + tags: ['migration'], + rule_type_id: 'siem.queryRule', + consumer: 'siem', + params: { + author: [], + buildingBlockType: undefined, + falsePositives: [], + description: 'a', + ruleId: '2297be91-894c-4831-830f-b424a0ec84f0', + from: '1900-01-01T00:00:00.000Z', + immutable: false, + license: '', + outputIndex: '', + investigationFields: ['client.address', 'agent.name'], + maxSignals: 100, + meta: undefined, + riskScore: 21, + riskScoreMapping: [], + severity: 'low', + severityMapping: [], + threat: [], + to: 'now', + references: [], + timelineId: undefined, + timelineTitle: undefined, + ruleNameOverride: undefined, + timestampOverride: undefined, + timestampOverrideFallbackDisabled: undefined, + namespace: undefined, + note: undefined, + requiredFields: undefined, + version: 1, + exceptionsList: [], + relatedIntegrations: [], + setup: '', + type: 'query', + language: 'kuery', + index: ['auditbeat-*'], + query: '_id:BhbXBmkBR346wHgn4PeZ or _id:GBbXBmkBR346wHgn5_eR or _id:x10zJ2oE9v5HJNSHhyxi', + filters: [], + savedId: undefined, + responseActions: undefined, + alertSuppression: undefined, + dataViewId: undefined, + }, + schedule: { + interval: '5m', + }, + enabled: false, + actions: [], + throttle: null, + // cast is due to alerting API expecting rule_type_id + // and our internal schema expecting alertTypeId + } as unknown as InternalRuleCreate); + +export const getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray = (): InternalRuleCreate => + ({ + name: 'Test investigation fields empty array', + tags: ['migration'], + rule_type_id: 'siem.queryRule', + consumer: 'siem', + params: { + author: [], + description: 'a', + ruleId: '2297be91-894c-4831-830f-b424a0ec5678', + falsePositives: [], + from: '1900-01-01T00:00:00.000Z', + immutable: false, + license: '', + outputIndex: '', + investigationFields: [], + maxSignals: 100, + riskScore: 21, + riskScoreMapping: [], + severity: 'low', + severityMapping: [], + threat: [], + to: 'now', + references: [], + version: 1, + exceptionsList: [], + type: 'query', + language: 'kuery', + index: ['auditbeat-*'], + query: '_id:BhbXBmkBR346wHgn4PeZ or _id:GBbXBmkBR346wHgn5_eR or _id:x10zJ2oE9v5HJNSHhyxi', + filters: [], + relatedIntegrations: [], + setup: '', + buildingBlockType: undefined, + meta: undefined, + timelineId: undefined, + timelineTitle: undefined, + ruleNameOverride: undefined, + timestampOverride: undefined, + timestampOverrideFallbackDisabled: undefined, + namespace: undefined, + note: undefined, + requiredFields: undefined, + savedId: undefined, + responseActions: undefined, + alertSuppression: undefined, + dataViewId: undefined, + }, + schedule: { + interval: '5m', + }, + enabled: false, + actions: [], + throttle: null, + // cast is due to alerting API expecting rule_type_id + // and our internal schema expecting alertTypeId + } as unknown as InternalRuleCreate); diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts index 4ad3ec1f1a62ec1..0e75e72a2d0ed02 100644 --- a/x-pack/test/detection_engine_api_integration/utils/index.ts +++ b/x-pack/test/detection_engine_api_integration/utils/index.ts @@ -17,6 +17,7 @@ export * from './create_new_action'; export * from './create_rule'; export * from './create_rule_with_auth'; export * from './create_rule_with_exception_entries'; +export * from './create_rule_saved_object'; export * from './create_signals_index'; export * from './delete_all_rules'; export * from './delete_all_event_log_execution_events'; @@ -51,6 +52,7 @@ export * from './get_rule_for_signal_testing'; export * from './get_rule_so_by_id'; export * from './get_rule_for_signal_testing_with_timestamp_override'; export * from './get_rule_with_web_hook_action'; +export * from './get_rule_with_legacy_investigation_fields'; export * from './get_saved_query_rule_for_signal_testing'; export * from './get_security_telemetry_stats'; export * from './get_signal_status'; diff --git a/x-pack/test/functional/es_archives/security_solution/legacy_investigation_fields/data.json b/x-pack/test/functional/es_archives/security_solution/legacy_investigation_fields/data.json deleted file mode 100644 index f16c2d1a9f9d51a..000000000000000 --- a/x-pack/test/functional/es_archives/security_solution/legacy_investigation_fields/data.json +++ /dev/null @@ -1,271 +0,0 @@ -{ - "type": "doc", - "value": { - "index": ".kibana_alerting_cases", - "id": "alert:9095ee90-b075-11ec-bb3f-1f063f8e1234", - "source": { - "alert": { - "name":"Test investigation fields", - "tags":["migration"], - "alertTypeId": "siem.queryRule", - "consumer": "siem", - "revision": 0, - "params": { - "author": [], - "description": "a", - "ruleId": "2297be91-894c-4831-830f-b424a0ec84f0", - "falsePositives": [], - "from": "now-360s", - "immutable": false, - "license": "", - "outputIndex": "", - "investigationFields":["client.address","agent.name"], - "meta": { - "from": "1m", - "kibana_siem_app_url": "https://actions.kb.us-central1.gcp.cloud.es.io:9243/app/security" - }, - "maxSignals": 100, - "riskScore": 21, - "riskScoreMapping": [], - "severity": "low", - "severityMapping": [], - "threat": [], - "to": "now", - "references": [], - "version": 1, - "exceptionsList": [], - "type": "query", - "language": "kuery", - "index": [ - "apm-*-transaction*", - "traces-apm*", - "auditbeat-*", - "endgame-*", - "filebeat-*", - "logs-*", - "packetbeat-*", - "winlogbeat-*" - ], - "query": "*:*", - "filters": [] - }, - "schedule": { - "interval": "5m" - }, - "enabled": false, - "actions": [], - "throttle": null, - "notifyWhen": "onActiveAlert", - "apiKeyOwner": null, - "apiKey": null, - "createdBy": "1527796724", - "updatedBy": "1527796724", - "createdAt": "2022-03-30T22:05:53.511Z", - "updatedAt": "2022-03-30T22:05:53.511Z", - "muteAll": false, - "mutedInstanceIds": [], - "executionStatus": { - "status": "ok", - "lastExecutionDate": "2022-03-31T19:53:37.507Z", - "error": null, - "lastDuration": 2377 - }, - "meta": { - "versionApiKeyLastmodified": "8.10.0" - }, - "scheduledTaskId": null, - "legacyId": "9095ee90-b075-11ec-bb3f-1f063f8e0abc" - }, - "type": "alert", - "references": [], - "namespaces": [ - "default" - ], - "typeMigrationVersion": "8.10.0", - "coreMigrationVersion":"8.10.0", - "updated_at": "2022-03-31T19:53:39.885Z" - } - } -} - -{ - "type": "doc", - "value": { - "index": ".kibana_alerting_cases", - "id": "alert:9095ee90-b075-11ec-bb3f-1f063f8e5678", - "source": { - "alert": { - "name":"Test investigation fields empty array", - "tags":["migration"], - "alertTypeId": "siem.queryRule", - "consumer": "siem", - "revision": 0, - "params": { - "author": [], - "description": "a", - "ruleId": "2297be91-894c-4831-830f-b424a0ec5678", - "falsePositives": [], - "from": "now-360s", - "immutable": false, - "license": "", - "outputIndex": "", - "investigationFields":[], - "meta": { - "from": "1m", - "kibana_siem_app_url": "https://actions.kb.us-central1.gcp.cloud.es.io:9243/app/security" - }, - "maxSignals": 100, - "riskScore": 21, - "riskScoreMapping": [], - "severity": "low", - "severityMapping": [], - "threat": [], - "to": "now", - "references": [], - "version": 1, - "exceptionsList": [], - "type": "query", - "language": "kuery", - "index": [ - "apm-*-transaction*", - "traces-apm*", - "auditbeat-*", - "endgame-*", - "filebeat-*", - "logs-*", - "packetbeat-*", - "winlogbeat-*" - ], - "query": "*:*", - "filters": [] - }, - "schedule": { - "interval": "5m" - }, - "enabled": false, - "actions": [], - "throttle": null, - "notifyWhen": "onActiveAlert", - "apiKeyOwner": null, - "apiKey": null, - "createdBy": "1527796724", - "updatedBy": "1527796724", - "createdAt": "2022-03-30T22:05:53.511Z", - "updatedAt": "2022-03-30T22:05:53.511Z", - "muteAll": false, - "mutedInstanceIds": [], - "executionStatus": { - "status": "ok", - "lastExecutionDate": "2022-03-31T19:53:37.507Z", - "error": null, - "lastDuration": 2377 - }, - "meta": { - "versionApiKeyLastmodified": "8.10.0" - }, - "scheduledTaskId": null, - "legacyId": "9095ee90-b075-11ec-bb3f-1f063f8e0def" - }, - "type": "alert", - "references": [], - "namespaces": [ - "default" - ], - "typeMigrationVersion": "8.10.0", - "coreMigrationVersion":"8.10.0", - "updated_at": "2022-03-31T19:53:39.885Z" - } - } -} - -{ - "type": "doc", - "value": { - "index": ".kibana_alerting_cases", - "id": "alert:9095ee90-b075-11ec-bb3f-1f063f8e9102", - "source": { - "alert": { - "name":"Test investigation fields object", - "tags":["migration"], - "alertTypeId": "siem.queryRule", - "consumer": "siem", - "revision": 0, - "params": { - "author": [], - "description": "a", - "ruleId": "2297be91-894c-4831-830f-b424a0ec9102", - "falsePositives": [], - "from": "now-360s", - "immutable": false, - "license": "", - "outputIndex": "", - "investigationFields": { - "field_names": ["host.name"] - }, - "meta": { - "from": "1m", - "kibana_siem_app_url": "https://actions.kb.us-central1.gcp.cloud.es.io:9243/app/security" - }, - "maxSignals": 100, - "riskScore": 21, - "riskScoreMapping": [], - "severity": "low", - "severityMapping": [], - "threat": [], - "to": "now", - "references": [], - "version": 1, - "exceptionsList": [], - "type": "query", - "language": "kuery", - "index": [ - "apm-*-transaction*", - "traces-apm*", - "auditbeat-*", - "endgame-*", - "filebeat-*", - "logs-*", - "packetbeat-*", - "winlogbeat-*" - ], - "query": "*:*", - "filters": [] - }, - "schedule": { - "interval": "5m" - }, - "enabled": false, - "actions": [], - "throttle": null, - "notifyWhen": "onActiveAlert", - "apiKeyOwner": null, - "apiKey": null, - "createdBy": "1527796724", - "updatedBy": "1527796724", - "createdAt": "2022-03-30T22:05:53.511Z", - "updatedAt": "2022-03-30T22:05:53.511Z", - "muteAll": false, - "mutedInstanceIds": [], - "executionStatus": { - "status": "ok", - "lastExecutionDate": "2022-03-31T19:53:37.507Z", - "error": null, - "lastDuration": 2377 - }, - "meta": { - "versionApiKeyLastmodified": "8.11.0" - }, - "scheduledTaskId": null, - "legacyId": "9095ee90-b075-11ec-bb3f-1f063f8e0ghi" - }, - "type": "alert", - "references": [], - "namespaces": [ - "default" - ], - "typeMigrationVersion": "8.11.0", - "coreMigrationVersion":"8.11.0", - "updated_at": "2022-03-31T19:53:39.885Z" - } - } -} \ No newline at end of file diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_rule_exceptions.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_rule_exceptions.ts index 8d78a0d7e48c47a..7a4b757c80775c1 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_rule_exceptions.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/create_rule_exceptions.ts @@ -6,7 +6,8 @@ */ import expect from '@kbn/expect'; - +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; import { CreateExceptionListSchema, @@ -23,6 +24,9 @@ import { deleteAllRules, createExceptionList, deleteAllAlerts, + getRuleSOById, + createRuleThroughAlertingEndpoint, + getRuleSavedObjectWithLegacyInvestigationFields, } from '../../../utils'; import { deleteAllExceptions, @@ -247,5 +251,50 @@ export default ({ getService }: FtrProviderContext) => { status_code: 500, }); }); + + // TODO: When available this tag should be @skipInServerless + // This use case is not relevant to serverless. + describe('@brokenInServerless legacy investigation_fields', () => { + let ruleWithLegacyInvestigationField: Rule; + + beforeEach(async () => { + await deleteAllRules(supertest, log); + ruleWithLegacyInvestigationField = await createRuleThroughAlertingEndpoint( + supertest, + getRuleSavedObjectWithLegacyInvestigationFields() + ); + }); + + afterEach(async () => { + await deleteAllRules(supertest, log); + }); + + it('creates and associates a `rule_default` exception list to a rule with a legacy investigation_field', async () => { + await supertest + .post(`${DETECTION_ENGINE_RULES_URL}/${ruleWithLegacyInvestigationField.id}/exceptions`) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send({ + items: [getRuleExceptionItemMock()], + }) + .expect(200); + + /** + * Confirm type on SO so that it's clear in the tests whether it's expected that + * the SO itself is migrated to the inteded object type, or if the transformation is + * happening just on the response. In this case, change will + * NOT include a migration on SO. + */ + const { + hits: { + hits: [{ _source: ruleSO }], + }, + } = await getRuleSOById(es, ruleWithLegacyInvestigationField.id); + expect( + ruleSO?.alert.params.exceptionsList.some((list) => list.type === 'rule_default') + ).to.eql(true); + expect(ruleSO?.alert.params.investigationFields).to.eql(['client.address', 'agent.name']); + }); + }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/create_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/create_rules.ts index 297470d452d4e14..7bd58191bb8c4b3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/create_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/create_rules.ts @@ -541,6 +541,44 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe('investigation_fields', () => { + it('should create a rule with investigation_fields', async () => { + const rule = { + ...getSimpleRule(), + investigation_fields: { + field_names: ['host.name'], + }, + }; + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send(rule) + .expect(200); + + expect(body.investigation_fields).to.eql({ + field_names: ['host.name'], + }); + }); + + it('should NOT create a rule with legacy investigation_fields', async () => { + const rule = { + ...getSimpleRule(), + investigation_fields: ['host.name'], + }; + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send(rule) + .expect(400); + + expect(body.message).to.eql( + '[request body]: Invalid value "["host.name"]" supplied to "investigation_fields"' + ); + }); + }); }); describe('@brokenInServerless missing timestamps', () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/create_rule_saved_object.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/create_rule_saved_object.ts new file mode 100644 index 000000000000000..93a632201162362 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/create_rule_saved_object.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; + +import { Rule } from '@kbn/alerting-plugin/common'; +import { + BaseRuleParams, + InternalRuleCreate, +} from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; + +/** + * Creates a rule using the alerting APIs directly. + * This allows us to test some legacy types that are not exposed + * on our APIs + * + * @param supertest + */ +export const createRuleThroughAlertingEndpoint = async ( + supertest: SuperTest.SuperTest, + rule: InternalRuleCreate +): Promise> => { + const { body } = await supertest + .post('/api/alerting/rule') + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send(rule) + .expect(200); + + return body; +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/get_rule_so_by_id.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/get_rule_so_by_id.ts new file mode 100644 index 000000000000000..584b49e379ee7dc --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/get_rule_so_by_id.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import { SavedObjectReference } from '@kbn/core/server'; +import { ALERTING_CASES_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import type { SearchResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Rule } from '@kbn/alerting-plugin/common'; +import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; + +interface RuleSO { + alert: Rule; + references: SavedObjectReference[]; +} + +/** + * Fetch legacy action sidecar SOs from the alerting savedObjects index + * @param es The ElasticSearch service + * @param id SO id + */ +export const getRuleSOById = async (es: Client, id: string): Promise> => + es.search({ + index: ALERTING_CASES_SAVED_OBJECT_INDEX, + q: `type:alert AND _id:"alert:${id}"`, + }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/get_rule_with_legacy_investigation_fields.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/get_rule_with_legacy_investigation_fields.ts new file mode 100644 index 000000000000000..55056748a094553 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/get_rule_with_legacy_investigation_fields.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { InternalRuleCreate } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema'; + +export const getRuleSavedObjectWithLegacyInvestigationFields = (): InternalRuleCreate => + ({ + name: 'Test investigation fields', + tags: ['migration'], + rule_type_id: 'siem.queryRule', + consumer: 'siem', + params: { + author: [], + buildingBlockType: undefined, + falsePositives: [], + description: 'a', + ruleId: '2297be91-894c-4831-830f-b424a0ec84f0', + from: '1900-01-01T00:00:00.000Z', + immutable: false, + license: '', + outputIndex: '', + investigationFields: ['client.address', 'agent.name'], + maxSignals: 100, + meta: undefined, + riskScore: 21, + riskScoreMapping: [], + severity: 'low', + severityMapping: [], + threat: [], + to: 'now', + references: [], + timelineId: undefined, + timelineTitle: undefined, + ruleNameOverride: undefined, + timestampOverride: undefined, + timestampOverrideFallbackDisabled: undefined, + namespace: undefined, + note: undefined, + requiredFields: undefined, + version: 1, + exceptionsList: [], + relatedIntegrations: [], + setup: '', + type: 'query', + language: 'kuery', + index: ['auditbeat-*'], + query: '_id:BhbXBmkBR346wHgn4PeZ or _id:GBbXBmkBR346wHgn5_eR or _id:x10zJ2oE9v5HJNSHhyxi', + filters: [], + savedId: undefined, + responseActions: undefined, + alertSuppression: undefined, + dataViewId: undefined, + }, + schedule: { + interval: '5m', + }, + enabled: false, + actions: [], + throttle: null, + // cast is due to alerting API expecting rule_type_id + // and our internal schema expecting alertTypeId + } as unknown as InternalRuleCreate); + +export const getRuleSavedObjectWithLegacyInvestigationFieldsEmptyArray = (): InternalRuleCreate => + ({ + name: 'Test investigation fields empty array', + tags: ['migration'], + rule_type_id: 'siem.queryRule', + consumer: 'siem', + params: { + author: [], + description: 'a', + ruleId: '2297be91-894c-4831-830f-b424a0ec5678', + falsePositives: [], + from: '1900-01-01T00:00:00.000Z', + immutable: false, + license: '', + outputIndex: '', + investigationFields: [], + maxSignals: 100, + riskScore: 21, + riskScoreMapping: [], + severity: 'low', + severityMapping: [], + threat: [], + to: 'now', + references: [], + version: 1, + exceptionsList: [], + type: 'query', + language: 'kuery', + index: ['auditbeat-*'], + query: '_id:BhbXBmkBR346wHgn4PeZ or _id:GBbXBmkBR346wHgn5_eR or _id:x10zJ2oE9v5HJNSHhyxi', + filters: [], + relatedIntegrations: [], + setup: '', + buildingBlockType: undefined, + meta: undefined, + timelineId: undefined, + timelineTitle: undefined, + ruleNameOverride: undefined, + timestampOverride: undefined, + timestampOverrideFallbackDisabled: undefined, + namespace: undefined, + note: undefined, + requiredFields: undefined, + savedId: undefined, + responseActions: undefined, + alertSuppression: undefined, + dataViewId: undefined, + }, + schedule: { + interval: '5m', + }, + enabled: false, + actions: [], + throttle: null, + // cast is due to alerting API expecting rule_type_id + // and our internal schema expecting alertTypeId + } as unknown as InternalRuleCreate); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts index 571ed891d22fd27..3289ea7d8f7ab9a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts @@ -8,6 +8,9 @@ export * from './rules'; export * from './exception_list_and_item'; export * from './alerts'; export * from './actions'; +export * from './get_rule_so_by_id'; +export * from './create_rule_saved_object'; +export * from './get_rule_with_legacy_investigation_fields'; export * from './count_down_test'; export * from './count_down_es'; diff --git a/x-pack/test/security_solution_api_integration/tsconfig.json b/x-pack/test/security_solution_api_integration/tsconfig.json index cb46d96f11a7f81..7690200b3b2f3d0 100644 --- a/x-pack/test/security_solution_api_integration/tsconfig.json +++ b/x-pack/test/security_solution_api_integration/tsconfig.json @@ -27,6 +27,8 @@ "@kbn/tooling-log", "@kbn/rule-data-utils", "@kbn/securitysolution-list-constants", - "@kbn/core-saved-objects-server" + "@kbn/core-saved-objects-server", + "@kbn/core", + "@kbn/alerting-plugin" ] }