From e9b81f72ca7f8222c1a97fdad965f441f224e0b7 Mon Sep 17 00:00:00 2001 From: nnamdifrankie <56440728+nnamdifrankie@users.noreply.github.com> Date: Wed, 1 Jul 2020 22:49:56 -0400 Subject: [PATCH 01/31] SECURITY-ENDPOINT: add fields for events to metadata document (#70491) SECURITY-ENDPOINT: EMT-492 add fields for events to metadata document --- .../common/endpoint/generate_data.ts | 11 +++++++++-- .../security_solution/common/endpoint/types.ts | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index c075e1041973b3..a6fe12a9b029fa 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -391,6 +391,13 @@ export class EndpointDocGenerator { '@timestamp': ts, event: { created: ts, + id: this.seededUUIDv4(), + kind: 'metric', + category: ['host'], + type: ['info'], + module: 'endpoint', + action: 'endpoint_metadata', + dataset: 'endpoint.metadata', }, ...this.commonInfo, }; @@ -1225,8 +1232,8 @@ export class EndpointDocGenerator { created: ts, id: this.seededUUIDv4(), kind: 'state', - category: 'host', - type: 'change', + category: ['host'], + type: ['change'], module: 'endpoint', action: 'endpoint_policy_response', dataset: 'endpoint.policy', diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index ca5cc449a7ad71..f76da977eef857 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -399,6 +399,13 @@ export type HostMetadata = Immutable<{ '@timestamp': number; event: { created: number; + kind: string; + id: string; + category: string[]; + type: string[]; + module: string; + action: string; + dataset: string; }; elastic: { agent: { @@ -771,8 +778,8 @@ export interface HostPolicyResponse { created: number; kind: string; id: string; - category: string; - type: string; + category: string[]; + type: string[]; module: string; action: string; dataset: string; From 591e10355aa5e9fe590361fd22483257d146b9ba Mon Sep 17 00:00:00 2001 From: Garrett Spong Date: Wed, 1 Jul 2020 22:49:30 -0600 Subject: [PATCH 02/31] [Security] Adds field mapping support to rule creation (#70288) ## Summary Resolves: https://github.com/elastic/kibana/issues/65941, https://github.com/elastic/kibana/issues/66317, and `Add support for "building block" alerts` This PR is `Part I` and adds additional fields to the `rules schema` in supporting the ability to map and override fields when generating alerts. A few bookkeeping fields like `license` and `author` have been added as well. The new fields are as follows: ``` ts export interface TheseAreTheNewFields { author: string[]; building_block_type: string; // 'default' license: string; risk_score_mapping: Array< { field: string; operator: string; // 'equals' value: string; } >; rule_name_override: string; severity_mapping: Array< { field: string; operator: string; // 'equals' value: string; severity: string; // 'low' | 'medium' | 'high' | 'critical' } >; timestamp_override: string; } ``` These new fields are exposed as additional settings on the `About rule` section of the Rule Creation UI. ##### Default collapsed view, no severity or risk score override specified:

##### Severity & risk score override specified:

##### Additional fields in Advanced settings:

Note: This PR adds the fields to the `Rules Schema`, the `signals index mapping`, and creates the UI for adding these fields during Rule Creation/Editing. The follow-up `Part II` will add the business logic for mapping fields during `rule execution`, and also add UI validation/additional tests. ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md) - [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials - Syncing w/ @benskelker - [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios - [x] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist) ### For maintainers - [x] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process) --- .../schemas/common/schemas.ts | 71 ++++++ .../add_prepackaged_rules_schema.mock.ts | 3 + .../request/add_prepackaged_rules_schema.ts | 22 ++ .../add_prepackged_rules_schema.test.ts | 24 ++ .../request/create_rules_schema.mock.ts | 3 + .../request/create_rules_schema.test.ts | 36 +++ .../schemas/request/create_rules_schema.ts | 22 ++ .../request/import_rules_schema.mock.ts | 3 + .../request/import_rules_schema.test.ts | 30 +++ .../schemas/request/import_rules_schema.ts | 22 ++ .../schemas/request/patch_rules_schema.ts | 14 ++ .../request/update_rules_schema.mock.ts | 3 + .../request/update_rules_schema.test.ts | 30 +++ .../schemas/request/update_rules_schema.ts | 22 ++ .../schemas/response/rules_schema.mocks.ts | 3 + .../schemas/response/rules_schema.ts | 16 ++ .../types/default_risk_score_mapping_array.ts | 24 ++ .../types/default_severity_mapping_array.ts | 24 ++ .../detection_engine/schemas/types/index.ts | 2 + .../rules/description_step/index.test.tsx | 2 +- .../rules/description_step/index.tsx | 19 +- .../rules/risk_score_mapping/index.tsx | 190 ++++++++++++++++ .../rules/risk_score_mapping/translations.tsx | 57 +++++ .../rules/severity_mapping/index.tsx | 214 ++++++++++++++++++ .../rules/severity_mapping/translations.tsx | 57 +++++ .../components/rules/step_about_rule/data.tsx | 2 +- .../rules/step_about_rule/default_value.ts | 9 +- .../rules/step_about_rule/index.test.tsx | 18 +- .../rules/step_about_rule/index.tsx | 184 ++++++++++----- .../rules/step_about_rule/schema.tsx | 123 ++++++++-- .../detection_engine/rules/api.test.ts | 2 +- .../containers/detection_engine/rules/mock.ts | 9 + .../detection_engine/rules/types.ts | 18 ++ .../detection_engine/rules/use_rule.test.tsx | 3 + .../rules/use_rule_status.test.tsx | 3 + .../detection_engine/rules/use_rules.test.tsx | 6 + .../rules/all/__mocks__/mock.ts | 19 +- .../rules/create/helpers.test.ts | 24 ++ .../detection_engine/rules/create/helpers.ts | 27 ++- .../detection_engine/rules/create/index.tsx | 3 + .../detection_engine/rules/helpers.test.tsx | 9 +- .../pages/detection_engine/rules/helpers.tsx | 22 +- .../pages/detection_engine/rules/types.ts | 35 ++- .../routes/__mocks__/request_responses.ts | 7 + .../routes/__mocks__/utils.ts | 4 + .../routes/index/signals_mapping.json | 44 ++++ .../rules/add_prepackaged_rules_route.test.ts | 3 + .../routes/rules/create_rules_bulk_route.ts | 16 +- .../routes/rules/create_rules_route.ts | 16 +- .../routes/rules/import_rules_route.ts | 23 +- .../routes/rules/patch_rules_bulk_route.ts | 14 ++ .../routes/rules/patch_rules_route.ts | 14 ++ .../routes/rules/update_rules_bulk_route.ts | 14 ++ .../routes/rules/update_rules_route.ts | 14 ++ .../detection_engine/routes/rules/utils.ts | 7 + .../routes/rules/validate.test.ts | 4 + .../rules/create_rules.mock.ts | 14 ++ .../detection_engine/rules/create_rules.ts | 14 ++ .../create_rules_stream_from_ndjson.test.ts | 30 +++ .../rules/get_export_all.test.ts | 4 + .../rules/get_export_by_object_ids.test.ts | 8 + .../rules/install_prepacked_rules.ts | 14 ++ .../rules/patch_rules.mock.ts | 14 ++ .../lib/detection_engine/rules/patch_rules.ts | 21 ++ .../lib/detection_engine/rules/types.ts | 31 +++ .../rules/update_prepacked_rules.ts | 14 ++ .../rules/update_rules.mock.ts | 14 ++ .../detection_engine/rules/update_rules.ts | 21 ++ .../lib/detection_engine/rules/utils.test.ts | 21 ++ .../lib/detection_engine/rules/utils.ts | 14 ++ .../lib/detection_engine/scripts/post_rule.sh | 2 +- .../rules/queries/query_with_mappings.json | 44 ++++ .../signals/__mocks__/es_results.ts | 7 + .../signals/build_bulk_body.test.ts | 20 ++ .../signals/build_rule.test.ts | 15 ++ .../detection_engine/signals/build_rule.ts | 13 +- .../signals/signal_params_schema.mock.ts | 7 + .../signals/signal_params_schema.ts | 8 + .../server/lib/detection_engine/types.ts | 14 ++ .../detection_engine_api_integration/utils.ts | 9 + 80 files changed, 1868 insertions(+), 114 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_risk_score_mapping_array.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_severity_mapping_array.ts create mode 100644 x-pack/plugins/security_solution/public/alerts/components/rules/risk_score_mapping/index.tsx create mode 100644 x-pack/plugins/security_solution/public/alerts/components/rules/risk_score_mapping/translations.tsx create mode 100644 x-pack/plugins/security_solution/public/alerts/components/rules/severity_mapping/index.tsx create mode 100644 x-pack/plugins/security_solution/public/alerts/components/rules/severity_mapping/translations.tsx create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index f6b732cd1f64e5..6e43bd645fd7bc 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -13,6 +13,18 @@ import { IsoDateString } from '../types/iso_date_string'; import { PositiveIntegerGreaterThanZero } from '../types/positive_integer_greater_than_zero'; import { PositiveInteger } from '../types/positive_integer'; +export const author = t.array(t.string); +export type Author = t.TypeOf; + +export const authorOrUndefined = t.union([author, t.undefined]); +export type AuthorOrUndefined = t.TypeOf; + +export const building_block_type = t.string; +export type BuildingBlockType = t.TypeOf; + +export const buildingBlockTypeOrUndefined = t.union([building_block_type, t.undefined]); +export type BuildingBlockTypeOrUndefined = t.TypeOf; + export const description = t.string; export type Description = t.TypeOf; @@ -111,6 +123,12 @@ export type Language = t.TypeOf; export const languageOrUndefined = t.union([language, t.undefined]); export type LanguageOrUndefined = t.TypeOf; +export const license = t.string; +export type License = t.TypeOf; + +export const licenseOrUndefined = t.union([license, t.undefined]); +export type LicenseOrUndefined = t.TypeOf; + export const objects = t.array(t.type({ rule_id })); export const output_index = t.string; @@ -137,6 +155,12 @@ export type TimelineTitle = t.TypeOf; export const timelineTitleOrUndefined = t.union([timeline_title, t.undefined]); export type TimelineTitleOrUndefined = t.TypeOf; +export const timestamp_override = t.string; +export type TimestampOverride = t.TypeOf; + +export const timestampOverrideOrUndefined = t.union([timestamp_override, t.undefined]); +export type TimestampOverrideOrUndefined = t.TypeOf; + export const throttle = t.string; export type Throttle = t.TypeOf; @@ -179,18 +203,65 @@ export type Name = t.TypeOf; export const nameOrUndefined = t.union([name, t.undefined]); export type NameOrUndefined = t.TypeOf; +export const operator = t.keyof({ + equals: null, +}); +export type Operator = t.TypeOf; +export enum OperatorEnum { + EQUALS = 'equals', +} + export const risk_score = RiskScore; export type RiskScore = t.TypeOf; export const riskScoreOrUndefined = t.union([risk_score, t.undefined]); export type RiskScoreOrUndefined = t.TypeOf; +export const risk_score_mapping_field = t.string; +export const risk_score_mapping_value = t.string; +export const risk_score_mapping_item = t.exact( + t.type({ + field: risk_score_mapping_field, + operator, + value: risk_score_mapping_value, + }) +); + +export const risk_score_mapping = t.array(risk_score_mapping_item); +export type RiskScoreMapping = t.TypeOf; + +export const riskScoreMappingOrUndefined = t.union([risk_score_mapping, t.undefined]); +export type RiskScoreMappingOrUndefined = t.TypeOf; + +export const rule_name_override = t.string; +export type RuleNameOverride = t.TypeOf; + +export const ruleNameOverrideOrUndefined = t.union([rule_name_override, t.undefined]); +export type RuleNameOverrideOrUndefined = t.TypeOf; + export const severity = t.keyof({ low: null, medium: null, high: null, critical: null }); export type Severity = t.TypeOf; export const severityOrUndefined = t.union([severity, t.undefined]); export type SeverityOrUndefined = t.TypeOf; +export const severity_mapping_field = t.string; +export const severity_mapping_value = t.string; +export const severity_mapping_item = t.exact( + t.type({ + field: severity_mapping_field, + operator, + value: severity_mapping_value, + severity, + }) +); + +export const severity_mapping = t.array(severity_mapping_item); +export type SeverityMapping = t.TypeOf; + +export const severityMappingOrUndefined = t.union([severity_mapping, t.undefined]); +export type SeverityMappingOrUndefined = t.TypeOf; + export const status = t.keyof({ open: null, closed: null, 'in-progress': null }); export type Status = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts index 52a210f3a01aa8..b666b95ea1e976 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.mock.ts @@ -23,12 +23,15 @@ export const getAddPrepackagedRulesSchemaMock = (): AddPrepackagedRulesSchema => }); export const getAddPrepackagedRulesSchemaDecodedMock = (): AddPrepackagedRulesSchemaDecoded => ({ + author: [], description: 'some description', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', severity: 'high', + severity_mapping: [], type: 'query', risk_score: 55, + risk_score_mapping: [], language: 'kuery', references: [], actions: [], diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index 43000f6d36f467..bf96be5e688fa0 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -37,6 +37,13 @@ import { query, rule_id, version, + building_block_type, + license, + rule_name_override, + timestamp_override, + Author, + RiskScoreMapping, + SeverityMapping, } from '../common/schemas'; /* eslint-enable @typescript-eslint/camelcase */ @@ -52,6 +59,8 @@ import { DefaultThrottleNull, DefaultListArray, ListArray, + DefaultRiskScoreMappingArray, + DefaultSeverityMappingArray, } from '../types'; /** @@ -79,6 +88,8 @@ export const addPrepackagedRulesSchema = t.intersection([ t.partial({ actions: DefaultActionsArray, // defaults to empty actions array if not set during decode anomaly_threshold, // defaults to undefined if not set during decode + author: DefaultStringArray, // defaults to empty array of strings if not set during decode + building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanFalse, // defaults to false if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode @@ -87,16 +98,21 @@ export const addPrepackagedRulesSchema = t.intersection([ interval: DefaultIntervalString, // defaults to "5m" if not set during decode query, // defaults to undefined if not set during decode language, // defaults to undefined if not set during decode + license, // defaults to "undefined" if not set during decode saved_id, // defaults to "undefined" if not set during decode timeline_id, // defaults to "undefined" if not set during decode timeline_title, // defaults to "undefined" if not set during decode meta, // defaults to "undefined" if not set during decode machine_learning_job_id, // defaults to "undefined" if not set during decode max_signals: DefaultMaxSignalsNumber, // defaults to DEFAULT_MAX_SIGNALS (100) if not set during decode + risk_score_mapping: DefaultRiskScoreMappingArray, // defaults to empty risk score mapping array if not set during decode + rule_name_override, // defaults to "undefined" if not set during decode + severity_mapping: DefaultSeverityMappingArray, // defaults to empty actions array if not set during decode tags: DefaultStringArray, // defaults to empty string array if not set during decode to: DefaultToString, // defaults to "now" if not set during decode threat: DefaultThreatArray, // defaults to empty array if not set during decode throttle: DefaultThrottleNull, // defaults to "null" if not set during decode + timestamp_override, // defaults to "undefined" if not set during decode references: DefaultStringArray, // defaults to empty array of strings if not set during decode note, // defaults to "undefined" if not set during decode exceptions_list: DefaultListArray, // defaults to empty array if not set during decode @@ -109,6 +125,7 @@ export type AddPrepackagedRulesSchema = t.TypeOf & { + author: Author; references: References; actions: Actions; enabled: Enabled; @@ -129,6 +149,8 @@ export type AddPrepackagedRulesSchemaDecoded = Omit< from: From; interval: Interval; max_signals: MaxSignals; + risk_score_mapping: RiskScoreMapping; + severity_mapping: SeverityMapping; tags: Tags; to: To; threat: Threat; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts index 47a98166927b41..0c45a7b1ef6bb5 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackged_rules_schema.test.ts @@ -261,6 +261,9 @@ describe('add prepackaged rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: AddPrepackagedRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -333,6 +336,9 @@ describe('add prepackaged rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: AddPrepackagedRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -430,6 +436,9 @@ describe('add prepackaged rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: AddPrepackagedRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -508,6 +517,9 @@ describe('add prepackaged rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: AddPrepackagedRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -1354,6 +1366,9 @@ describe('add prepackaged rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: AddPrepackagedRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1404,6 +1419,9 @@ describe('add prepackaged rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: AddPrepackagedRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1462,6 +1480,9 @@ describe('add prepackaged rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: AddPrepackagedRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1539,6 +1560,9 @@ describe('add prepackaged rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: AddPrepackagedRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts index 2847bd32df514d..f1e87bdb11e75f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts @@ -30,6 +30,9 @@ export const getCreateMlRulesSchemaMock = (ruleId = 'rule-1') => { }; export const getCreateRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => ({ + author: [], + severity_mapping: [], + risk_score_mapping: [], description: 'Detecting root and admin users', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts index 1648044f5305a3..e529cf3fa555c6 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts @@ -248,6 +248,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -318,6 +321,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -366,6 +372,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -412,6 +421,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -438,6 +450,9 @@ describe('create rules schema', () => { test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, output_index] does validate', () => { const payload: CreateRulesSchema = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -456,6 +471,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -535,6 +553,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -1228,6 +1249,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1399,6 +1423,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], type: 'machine_learning', anomaly_threshold: 50, machine_learning_job_id: 'linux_anomalous_network_activity_ecs', @@ -1459,6 +1486,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1516,6 +1546,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1591,6 +1624,9 @@ describe('create rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: CreateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index d623cff8f1fc31..0debe01e5a4d74 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -10,6 +10,7 @@ import * as t from 'io-ts'; import { description, anomaly_threshold, + building_block_type, filters, RuleId, index, @@ -38,6 +39,12 @@ import { Interval, language, query, + license, + rule_name_override, + timestamp_override, + Author, + RiskScoreMapping, + SeverityMapping, } from '../common/schemas'; /* eslint-enable @typescript-eslint/camelcase */ @@ -55,6 +62,8 @@ import { DefaultListArray, ListArray, DefaultUuid, + DefaultRiskScoreMappingArray, + DefaultSeverityMappingArray, } from '../types'; export const createRulesSchema = t.intersection([ @@ -71,6 +80,8 @@ export const createRulesSchema = t.intersection([ t.partial({ actions: DefaultActionsArray, // defaults to empty actions array if not set during decode anomaly_threshold, // defaults to undefined if not set during decode + author: DefaultStringArray, // defaults to empty array of strings if not set during decode + building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode @@ -80,6 +91,7 @@ export const createRulesSchema = t.intersection([ interval: DefaultIntervalString, // defaults to "5m" if not set during decode query, // defaults to undefined if not set during decode language, // defaults to undefined if not set during decode + license, // defaults to "undefined" if not set during decode // TODO: output_index: This should be removed eventually output_index, // defaults to "undefined" if not set during decode saved_id, // defaults to "undefined" if not set during decode @@ -88,10 +100,14 @@ export const createRulesSchema = t.intersection([ meta, // defaults to "undefined" if not set during decode machine_learning_job_id, // defaults to "undefined" if not set during decode max_signals: DefaultMaxSignalsNumber, // defaults to DEFAULT_MAX_SIGNALS (100) if not set during decode + risk_score_mapping: DefaultRiskScoreMappingArray, // defaults to empty risk score mapping array if not set during decode + rule_name_override, // defaults to "undefined" if not set during decode + severity_mapping: DefaultSeverityMappingArray, // defaults to empty actions array if not set during decode tags: DefaultStringArray, // defaults to empty string array if not set during decode to: DefaultToString, // defaults to "now" if not set during decode threat: DefaultThreatArray, // defaults to empty array if not set during decode throttle: DefaultThrottleNull, // defaults to "null" if not set during decode + timestamp_override, // defaults to "undefined" if not set during decode references: DefaultStringArray, // defaults to empty array of strings if not set during decode note, // defaults to "undefined" if not set during decode version: DefaultVersionNumber, // defaults to 1 if not set during decode @@ -105,6 +121,7 @@ export type CreateRulesSchema = t.TypeOf; // This type is used after a decode since some things are defaults after a decode. export type CreateRulesSchemaDecoded = Omit< CreateRulesSchema, + | 'author' | 'references' | 'actions' | 'enabled' @@ -112,6 +129,8 @@ export type CreateRulesSchemaDecoded = Omit< | 'from' | 'interval' | 'max_signals' + | 'risk_score_mapping' + | 'severity_mapping' | 'tags' | 'to' | 'threat' @@ -120,6 +139,7 @@ export type CreateRulesSchemaDecoded = Omit< | 'exceptions_list' | 'rule_id' > & { + author: Author; references: References; actions: Actions; enabled: Enabled; @@ -127,6 +147,8 @@ export type CreateRulesSchemaDecoded = Omit< from: From; interval: Interval; max_signals: MaxSignals; + risk_score_mapping: RiskScoreMapping; + severity_mapping: SeverityMapping; tags: Tags; to: To; threat: Threat; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts index aaeb90ffc5bcf0..e3b4196c90c6c9 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts @@ -31,12 +31,15 @@ export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): ImportRulesSc }); export const getImportRulesSchemaDecodedMock = (): ImportRulesSchemaDecoded => ({ + author: [], description: 'some description', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', severity: 'high', + severity_mapping: [], type: 'query', risk_score: 55, + risk_score_mapping: [], language: 'kuery', references: [], actions: [], diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts index 12a13ab1a5ed1b..bbf0a8debd6518 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.test.ts @@ -253,6 +253,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -324,6 +327,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -373,6 +379,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -420,6 +429,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -465,6 +477,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -545,6 +560,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -1543,6 +1561,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1593,6 +1614,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1651,6 +1675,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1727,6 +1754,9 @@ describe('import rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: ImportRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index 7d79861aacf38b..f61a1546e3e8a3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -44,6 +44,13 @@ import { updated_at, created_by, updated_by, + building_block_type, + license, + rule_name_override, + timestamp_override, + Author, + RiskScoreMapping, + SeverityMapping, } from '../common/schemas'; /* eslint-enable @typescript-eslint/camelcase */ @@ -62,6 +69,8 @@ import { DefaultStringBooleanFalse, DefaultListArray, ListArray, + DefaultRiskScoreMappingArray, + DefaultSeverityMappingArray, } from '../types'; /** @@ -90,6 +99,8 @@ export const importRulesSchema = t.intersection([ id, // defaults to undefined if not set during decode actions: DefaultActionsArray, // defaults to empty actions array if not set during decode anomaly_threshold, // defaults to undefined if not set during decode + author: DefaultStringArray, // defaults to empty array of strings if not set during decode + building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode @@ -99,6 +110,7 @@ export const importRulesSchema = t.intersection([ interval: DefaultIntervalString, // defaults to "5m" if not set during decode query, // defaults to undefined if not set during decode language, // defaults to undefined if not set during decode + license, // defaults to "undefined" if not set during decode // TODO: output_index: This should be removed eventually output_index, // defaults to "undefined" if not set during decode saved_id, // defaults to "undefined" if not set during decode @@ -107,10 +119,14 @@ export const importRulesSchema = t.intersection([ meta, // defaults to "undefined" if not set during decode machine_learning_job_id, // defaults to "undefined" if not set during decode max_signals: DefaultMaxSignalsNumber, // defaults to DEFAULT_MAX_SIGNALS (100) if not set during decode + risk_score_mapping: DefaultRiskScoreMappingArray, // defaults to empty risk score mapping array if not set during decode + rule_name_override, // defaults to "undefined" if not set during decode + severity_mapping: DefaultSeverityMappingArray, // defaults to empty actions array if not set during decode tags: DefaultStringArray, // defaults to empty string array if not set during decode to: DefaultToString, // defaults to "now" if not set during decode threat: DefaultThreatArray, // defaults to empty array if not set during decode throttle: DefaultThrottleNull, // defaults to "null" if not set during decode + timestamp_override, // defaults to "undefined" if not set during decode references: DefaultStringArray, // defaults to empty array of strings if not set during decode note, // defaults to "undefined" if not set during decode version: DefaultVersionNumber, // defaults to 1 if not set during decode @@ -128,6 +144,7 @@ export type ImportRulesSchema = t.TypeOf; // This type is used after a decode since some things are defaults after a decode. export type ImportRulesSchemaDecoded = Omit< ImportRulesSchema, + | 'author' | 'references' | 'actions' | 'enabled' @@ -135,6 +152,8 @@ export type ImportRulesSchemaDecoded = Omit< | 'from' | 'interval' | 'max_signals' + | 'risk_score_mapping' + | 'severity_mapping' | 'tags' | 'to' | 'threat' @@ -144,6 +163,7 @@ export type ImportRulesSchemaDecoded = Omit< | 'rule_id' | 'immutable' > & { + author: Author; references: References; actions: Actions; enabled: Enabled; @@ -151,6 +171,8 @@ export type ImportRulesSchemaDecoded = Omit< from: From; interval: Interval; max_signals: MaxSignals; + risk_score_mapping: RiskScoreMapping; + severity_mapping: SeverityMapping; tags: Tags; to: To; threat: Threat; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index 29d5467071a3d0..070f3ccfd03b06 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -39,6 +39,13 @@ import { language, query, id, + building_block_type, + author, + license, + rule_name_override, + timestamp_override, + risk_score_mapping, + severity_mapping, } from '../common/schemas'; import { listArrayOrUndefined } from '../types/lists'; /* eslint-enable @typescript-eslint/camelcase */ @@ -48,6 +55,8 @@ import { listArrayOrUndefined } from '../types/lists'; */ export const patchRulesSchema = t.exact( t.partial({ + author, + building_block_type, description, risk_score, name, @@ -65,6 +74,7 @@ export const patchRulesSchema = t.exact( interval, query, language, + license, // TODO: output_index: This should be removed eventually output_index, saved_id, @@ -73,10 +83,14 @@ export const patchRulesSchema = t.exact( meta, machine_learning_job_id, max_signals, + risk_score_mapping, + rule_name_override, + severity_mapping, tags, to, threat, throttle, + timestamp_override, references, note, version, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.mock.ts index b8a99115ba7d5e..b3fbf961883528 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.mock.ts @@ -19,12 +19,15 @@ export const getUpdateRulesSchemaMock = (): UpdateRulesSchema => ({ }); export const getUpdateRulesSchemaDecodedMock = (): UpdateRulesSchemaDecoded => ({ + author: [], description: 'some description', name: 'Query with a rule id', query: 'user.name: root or user.name: admin', severity: 'high', + severity_mapping: [], type: 'query', risk_score: 55, + risk_score_mapping: [], language: 'kuery', references: [], actions: [], diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts index 02f8e7bbeb59b2..c15803eee874e0 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.test.ts @@ -248,6 +248,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -317,6 +320,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', risk_score: 50, description: 'some description', @@ -364,6 +370,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -409,6 +418,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -452,6 +464,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -530,6 +545,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, @@ -1353,6 +1371,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1401,6 +1422,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1457,6 +1481,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', @@ -1531,6 +1558,9 @@ describe('update rules schema', () => { const message = pipe(checked, foldLeftRight); expect(getPaths(left(message.errors))).toEqual([]); const expected: UpdateRulesSchemaDecoded = { + author: [], + severity_mapping: [], + risk_score_mapping: [], rule_id: 'rule-1', description: 'some description', from: 'now-5m', diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts index 73078e617efc6f..98082c2de838a3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts @@ -40,6 +40,13 @@ import { language, query, id, + building_block_type, + license, + rule_name_override, + timestamp_override, + Author, + RiskScoreMapping, + SeverityMapping, } from '../common/schemas'; /* eslint-enable @typescript-eslint/camelcase */ @@ -55,6 +62,8 @@ import { DefaultThrottleNull, DefaultListArray, ListArray, + DefaultRiskScoreMappingArray, + DefaultSeverityMappingArray, } from '../types'; /** @@ -79,6 +88,8 @@ export const updateRulesSchema = t.intersection([ id, // defaults to "undefined" if not set during decode actions: DefaultActionsArray, // defaults to empty actions array if not set during decode anomaly_threshold, // defaults to undefined if not set during decode + author: DefaultStringArray, // defaults to empty array of strings if not set during decode + building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode @@ -88,6 +99,7 @@ export const updateRulesSchema = t.intersection([ interval: DefaultIntervalString, // defaults to "5m" if not set during decode query, // defaults to undefined if not set during decode language, // defaults to undefined if not set during decode + license, // defaults to "undefined" if not set during decode // TODO: output_index: This should be removed eventually output_index, // defaults to "undefined" if not set during decode saved_id, // defaults to "undefined" if not set during decode @@ -96,10 +108,14 @@ export const updateRulesSchema = t.intersection([ meta, // defaults to "undefined" if not set during decode machine_learning_job_id, // defaults to "undefined" if not set during decode max_signals: DefaultMaxSignalsNumber, // defaults to DEFAULT_MAX_SIGNALS (100) if not set during decode + risk_score_mapping: DefaultRiskScoreMappingArray, // defaults to empty risk score mapping array if not set during decode + rule_name_override, // defaults to "undefined" if not set during decode + severity_mapping: DefaultSeverityMappingArray, // defaults to empty actions array if not set during decode tags: DefaultStringArray, // defaults to empty string array if not set during decode to: DefaultToString, // defaults to "now" if not set during decode threat: DefaultThreatArray, // defaults to empty array if not set during decode throttle: DefaultThrottleNull, // defaults to "null" if not set during decode + timestamp_override, // defaults to "undefined" if not set during decode references: DefaultStringArray, // defaults to empty array of strings if not set during decode note, // defaults to "undefined" if not set during decode version, // defaults to "undefined" if not set during decode @@ -113,6 +129,7 @@ export type UpdateRulesSchema = t.TypeOf; // This type is used after a decode since some things are defaults after a decode. export type UpdateRulesSchemaDecoded = Omit< UpdateRulesSchema, + | 'author' | 'references' | 'actions' | 'enabled' @@ -120,6 +137,8 @@ export type UpdateRulesSchemaDecoded = Omit< | 'from' | 'interval' | 'max_signals' + | 'risk_score_mapping' + | 'severity_mapping' | 'tags' | 'to' | 'threat' @@ -127,6 +146,7 @@ export type UpdateRulesSchemaDecoded = Omit< | 'exceptions_list' | 'rule_id' > & { + author: Author; references: References; actions: Actions; enabled: Enabled; @@ -134,6 +154,8 @@ export type UpdateRulesSchemaDecoded = Omit< from: From; interval: Interval; max_signals: MaxSignals; + risk_score_mapping: RiskScoreMapping; + severity_mapping: SeverityMapping; tags: Tags; to: To; threat: Threat; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index e63a7ad981e120..ed9fb8930ea1bb 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -36,6 +36,7 @@ export const getPartialRulesSchemaMock = (): Partial => ({ }); export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => ({ + author: [], id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', created_at: new Date(anchorDate).toISOString(), updated_at: new Date(anchorDate).toISOString(), @@ -49,6 +50,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem query: 'user.name: root or user.name: admin', references: ['test 1', 'test 2'], severity: 'high', + severity_mapping: [], updated_by: 'elastic_kibana', tags: [], to: 'now', @@ -62,6 +64,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem output_index: '.siem-signals-hassanabad-frank-default', max_signals: 100, risk_score: 55, + risk_score_mapping: [], language: 'kuery', rule_id: 'query-rule-id', interval: '5m', diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 9803a80f57857e..c0fec2b2eefc2d 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -55,8 +55,17 @@ import { filters, meta, note, + building_block_type, + license, + rule_name_override, + timestamp_override, } from '../common/schemas'; import { DefaultListArray } from '../types/lists_default_array'; +import { + DefaultStringArray, + DefaultRiskScoreMappingArray, + DefaultSeverityMappingArray, +} from '../types'; /** * This is the required fields for the rules schema response. Put all required properties on @@ -64,6 +73,7 @@ import { DefaultListArray } from '../types/lists_default_array'; * output schema. */ export const requiredRulesSchema = t.type({ + author: DefaultStringArray, description, enabled, false_positives, @@ -75,9 +85,11 @@ export const requiredRulesSchema = t.type({ output_index, max_signals, risk_score, + risk_score_mapping: DefaultRiskScoreMappingArray, name, references, severity, + severity_mapping: DefaultSeverityMappingArray, updated_by, tags, to, @@ -120,9 +132,13 @@ export const dependentRulesSchema = t.partial({ */ export const partialRulesSchema = t.partial({ actions, + building_block_type, + license, throttle, + rule_name_override, status: job_status, status_date, + timestamp_override, last_success_at, last_success_message, last_failure_at, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_risk_score_mapping_array.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_risk_score_mapping_array.ts new file mode 100644 index 00000000000000..ba74045b4e32c9 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_risk_score_mapping_array.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { Either } from 'fp-ts/lib/Either'; +// eslint-disable-next-line @typescript-eslint/camelcase +import { risk_score_mapping, RiskScoreMapping } from '../common/schemas'; + +/** + * Types the DefaultStringArray as: + * - If null or undefined, then a default risk_score_mapping array will be set + */ +export const DefaultRiskScoreMappingArray = new t.Type( + 'DefaultRiskScoreMappingArray', + risk_score_mapping.is, + (input, context): Either => + input == null ? t.success([]) : risk_score_mapping.validate(input, context), + t.identity +); + +export type DefaultRiskScoreMappingArrayC = typeof DefaultRiskScoreMappingArray; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_severity_mapping_array.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_severity_mapping_array.ts new file mode 100644 index 00000000000000..8e68b73148af1a --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/default_severity_mapping_array.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { Either } from 'fp-ts/lib/Either'; +// eslint-disable-next-line @typescript-eslint/camelcase +import { severity_mapping, SeverityMapping } from '../common/schemas'; + +/** + * Types the DefaultStringArray as: + * - If null or undefined, then a default severity_mapping array will be set + */ +export const DefaultSeverityMappingArray = new t.Type( + 'DefaultSeverityMappingArray', + severity_mapping.is, + (input, context): Either => + input == null ? t.success([]) : severity_mapping.validate(input, context), + t.identity +); + +export type DefaultSeverityMappingArrayC = typeof DefaultSeverityMappingArray; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/index.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/index.ts index 368dd4922eec48..aab9a550d25e73 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/index.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/index.ts @@ -15,6 +15,8 @@ export * from './default_language_string'; export * from './default_max_signals_number'; export * from './default_page'; export * from './default_per_page'; +export * from './default_risk_score_mapping_array'; +export * from './default_severity_mapping_array'; export * from './default_string_array'; export * from './default_string_boolean_false'; export * from './default_threat_array'; diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.test.tsx index 2bd90f17daf0c2..0a7e666d65aef1 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.test.tsx @@ -257,7 +257,7 @@ describe('description_step', () => { test('returns expected ListItems array when given valid inputs', () => { const result: ListItems[] = buildListItems(mockAboutStep, schema, mockFilterManager); - expect(result.length).toEqual(9); + expect(result.length).toEqual(11); }); }); diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.tsx index b9642b87610193..8f3a76c6aea577 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/description_step/index.tsx @@ -18,7 +18,11 @@ import { } from '../../../../../../../../src/plugins/data/public'; import { DEFAULT_TIMELINE_TITLE } from '../../../../timelines/components/timeline/translations'; import { useKibana } from '../../../../common/lib/kibana'; -import { IMitreEnterpriseAttack } from '../../../pages/detection_engine/rules/types'; +import { + AboutStepRiskScore, + AboutStepSeverity, + IMitreEnterpriseAttack, +} from '../../../pages/detection_engine/rules/types'; import { FieldValueTimeline } from '../pick_timeline'; import { FormSchema } from '../../../../shared_imports'; import { ListItems } from './types'; @@ -184,9 +188,18 @@ export const getDescriptionItem = ( } else if (Array.isArray(get(field, data))) { const values: string[] = get(field, data); return buildStringArrayDescription(label, field, values); + // TODO: Add custom UI for Risk/Severity Mappings (and fix missing label) + } else if (field === 'riskScore') { + const val: AboutStepRiskScore = get(field, data); + return [ + { + title: label, + description: val.value, + }, + ]; } else if (field === 'severity') { - const val: string = get(field, data); - return buildSeverityDescription(label, val); + const val: AboutStepSeverity = get(field, data); + return buildSeverityDescription(label, val.value); } else if (field === 'timeline') { const timeline = get(field, data) as FieldValueTimeline; return [ diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/risk_score_mapping/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/risk_score_mapping/index.tsx new file mode 100644 index 00000000000000..bdf1ac600faef6 --- /dev/null +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/risk_score_mapping/index.tsx @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + EuiFormRow, + EuiFieldText, + EuiCheckbox, + EuiText, + EuiFlexGroup, + EuiFlexItem, + EuiFormLabel, + EuiIcon, + EuiSpacer, +} from '@elastic/eui'; +import React, { useCallback, useMemo, useState } from 'react'; +import styled from 'styled-components'; +import * as i18n from './translations'; +import { FieldHook } from '../../../../../../../../src/plugins/es_ui_shared/static/forms/hook_form_lib'; +import { CommonUseField } from '../../../../cases/components/create'; +import { AboutStepRiskScore } from '../../../pages/detection_engine/rules/types'; + +const NestedContent = styled.div` + margin-left: 24px; +`; + +const EuiFlexItemIconColumn = styled(EuiFlexItem)` + width: 20px; +`; + +const EuiFlexItemRiskScoreColumn = styled(EuiFlexItem)` + width: 160px; +`; + +interface RiskScoreFieldProps { + dataTestSubj: string; + field: FieldHook; + idAria: string; + indices: string[]; +} + +export const RiskScoreField = ({ dataTestSubj, field, idAria, indices }: RiskScoreFieldProps) => { + const [isRiskScoreMappingSelected, setIsRiskScoreMappingSelected] = useState(false); + + const updateRiskScoreMapping = useCallback( + (event) => { + const values = field.value as AboutStepRiskScore; + field.setValue({ + value: values.value, + mapping: [ + { + field: event.target.value, + operator: 'equals', + value: '', + }, + ], + }); + }, + [field] + ); + + const severityLabel = useMemo(() => { + return ( +
+ + {i18n.RISK_SCORE} + + + {i18n.RISK_SCORE_DESCRIPTION} +
+ ); + }, []); + + const severityMappingLabel = useMemo(() => { + return ( +
+ setIsRiskScoreMappingSelected(!isRiskScoreMappingSelected)} + > + + setIsRiskScoreMappingSelected(e.target.checked)} + /> + + {i18n.RISK_SCORE_MAPPING} + + + + {i18n.RISK_SCORE_MAPPING_DESCRIPTION} + +
+ ); + }, [isRiskScoreMappingSelected, setIsRiskScoreMappingSelected]); + + return ( + + + + + + + + {i18n.RISK_SCORE_MAPPING_DETAILS} + ) : ( + '' + ) + } + error={'errorMessage'} + isInvalid={false} + fullWidth + data-test-subj={dataTestSubj} + describedByIds={idAria ? [idAria] : undefined} + > + + + {isRiskScoreMappingSelected && ( + + + + + {i18n.SOURCE_FIELD} + + + + {i18n.RISK_SCORE} + + + + + + + + + + + + + + {i18n.RISK_SCORE_FIELD} + + + + + )} + + + + + ); +}; diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/risk_score_mapping/translations.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/risk_score_mapping/translations.tsx new file mode 100644 index 00000000000000..a75bf19b5b3c4f --- /dev/null +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/risk_score_mapping/translations.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const RISK_SCORE = i18n.translate( + 'xpack.securitySolution.alerts.riskScoreMapping.riskScoreTitle', + { + defaultMessage: 'Default risk score', + } +); + +export const RISK_SCORE_FIELD = i18n.translate( + 'xpack.securitySolution.alerts.riskScoreMapping.riskScoreFieldTitle', + { + defaultMessage: 'signal.rule.risk_score', + } +); + +export const SOURCE_FIELD = i18n.translate( + 'xpack.securitySolution.alerts.riskScoreMapping.sourceFieldTitle', + { + defaultMessage: 'Source field', + } +); + +export const RISK_SCORE_MAPPING = i18n.translate( + 'xpack.securitySolution.alerts.riskScoreMapping.riskScoreMappingTitle', + { + defaultMessage: 'Risk score override', + } +); + +export const RISK_SCORE_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.alerts.riskScoreMapping.defaultDescriptionLabel', + { + defaultMessage: 'Select a risk score for all alerts generated by this rule.', + } +); + +export const RISK_SCORE_MAPPING_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.alerts.riskScoreMapping.mappingDescriptionLabel', + { + defaultMessage: 'Map a field from the source event (scaled 1-100) to risk score.', + } +); + +export const RISK_SCORE_MAPPING_DETAILS = i18n.translate( + 'xpack.securitySolution.alerts.riskScoreMapping.mappingDetailsLabel', + { + defaultMessage: + 'If value is out of bounds, or field is not present, the default risk score will be used.', + } +); diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/severity_mapping/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/severity_mapping/index.tsx new file mode 100644 index 00000000000000..47c45a6bdf88da --- /dev/null +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/severity_mapping/index.tsx @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + EuiFormRow, + EuiFieldText, + EuiCheckbox, + EuiText, + EuiFlexGroup, + EuiFlexItem, + EuiFormLabel, + EuiIcon, + EuiSpacer, +} from '@elastic/eui'; +import React, { useCallback, useMemo, useState } from 'react'; +import styled from 'styled-components'; +import * as i18n from './translations'; +import { FieldHook } from '../../../../../../../../src/plugins/es_ui_shared/static/forms/hook_form_lib'; +import { SeverityOptionItem } from '../step_about_rule/data'; +import { CommonUseField } from '../../../../cases/components/create'; +import { AboutStepSeverity } from '../../../pages/detection_engine/rules/types'; + +const NestedContent = styled.div` + margin-left: 24px; +`; + +const EuiFlexItemIconColumn = styled(EuiFlexItem)` + width: 20px; +`; + +const EuiFlexItemSeverityColumn = styled(EuiFlexItem)` + width: 80px; +`; + +interface SeverityFieldProps { + dataTestSubj: string; + field: FieldHook; + idAria: string; + indices: string[]; + options: SeverityOptionItem[]; +} + +export const SeverityField = ({ + dataTestSubj, + field, + idAria, + indices, // TODO: To be used with autocomplete fields once https://github.com/elastic/kibana/pull/67013 is merged + options, +}: SeverityFieldProps) => { + const [isSeverityMappingChecked, setIsSeverityMappingChecked] = useState(false); + + const updateSeverityMapping = useCallback( + (index: number, severity: string, mappingField: string, event) => { + const values = field.value as AboutStepSeverity; + field.setValue({ + value: values.value, + mapping: [ + ...values.mapping.slice(0, index), + { + ...values.mapping[index], + [mappingField]: event.target.value, + operator: 'equals', + severity, + }, + ...values.mapping.slice(index + 1), + ], + }); + }, + [field] + ); + + const severityLabel = useMemo(() => { + return ( +
+ + {i18n.SEVERITY} + + + {i18n.SEVERITY_DESCRIPTION} +
+ ); + }, []); + + const severityMappingLabel = useMemo(() => { + return ( +
+ setIsSeverityMappingChecked(!isSeverityMappingChecked)} + > + + setIsSeverityMappingChecked(e.target.checked)} + /> + + {i18n.SEVERITY_MAPPING} + + + + {i18n.SEVERITY_MAPPING_DESCRIPTION} + +
+ ); + }, [isSeverityMappingChecked, setIsSeverityMappingChecked]); + + return ( + + + + + + + + + {i18n.SEVERITY_MAPPING_DETAILS} + ) : ( + '' + ) + } + error={'errorMessage'} + isInvalid={false} + fullWidth + data-test-subj={dataTestSubj} + describedByIds={idAria ? [idAria] : undefined} + > + + + {isSeverityMappingChecked && ( + + + + + {i18n.SOURCE_FIELD} + + + {i18n.SOURCE_VALUE} + + + + {i18n.SEVERITY} + + + + + {options.map((option, index) => ( + + + + + + + + + + + + + + {option.inputDisplay} + + + + ))} + + )} + + + + + ); +}; diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/severity_mapping/translations.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/severity_mapping/translations.tsx new file mode 100644 index 00000000000000..9c9784bac6b63a --- /dev/null +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/severity_mapping/translations.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const SEVERITY = i18n.translate( + 'xpack.securitySolution.alerts.severityMapping.severityTitle', + { + defaultMessage: 'Default severity', + } +); + +export const SOURCE_FIELD = i18n.translate( + 'xpack.securitySolution.alerts.severityMapping.sourceFieldTitle', + { + defaultMessage: 'Source field', + } +); + +export const SOURCE_VALUE = i18n.translate( + 'xpack.securitySolution.alerts.severityMapping.sourceValueTitle', + { + defaultMessage: 'Source value', + } +); + +export const SEVERITY_MAPPING = i18n.translate( + 'xpack.securitySolution.alerts.severityMapping.severityMappingTitle', + { + defaultMessage: 'Severity override', + } +); + +export const SEVERITY_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.alerts.severityMapping.defaultDescriptionLabel', + { + defaultMessage: 'Select a severity level for all alerts generated by this rule.', + } +); + +export const SEVERITY_MAPPING_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.alerts.severityMapping.mappingDescriptionLabel', + { + defaultMessage: 'Map a value from the source event to a specific severity.', + } +); + +export const SEVERITY_MAPPING_DETAILS = i18n.translate( + 'xpack.securitySolution.alerts.severityMapping.mappingDetailsLabel', + { + defaultMessage: + 'For multiple matches the highest severity match will apply. If no match is found, the default severity will be used.', + } +); diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/data.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/data.tsx index 269d2d4509508d..1ef3edf8c720e4 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/data.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/data.tsx @@ -12,7 +12,7 @@ import * as I18n from './translations'; export type SeverityValue = 'low' | 'medium' | 'high' | 'critical'; -interface SeverityOptionItem { +export interface SeverityOptionItem { value: SeverityValue; inputDisplay: React.ReactElement; } diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/default_value.ts b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/default_value.ts index 977769158481e0..060a2183eb06e4 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/default_value.ts +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/default_value.ts @@ -15,14 +15,19 @@ export const threatDefault = [ ]; export const stepAboutDefaultValue: AboutStepRule = { + author: [], name: '', description: '', + isBuildingBlock: false, isNew: true, - severity: 'low', - riskScore: 50, + severity: { value: 'low', mapping: [] }, + riskScore: { value: 50, mapping: [] }, references: [''], falsePositives: [''], + license: '', + ruleNameOverride: '', tags: [], + timestampOverride: '', threat: threatDefault, note: '', }; diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/index.test.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/index.test.tsx index 5a08b0a20d1fce..b21c54a0b61313 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/index.test.tsx @@ -164,13 +164,18 @@ describe('StepAboutRuleComponent', () => { wrapper.find('button[data-test-subj="about-continue"]').first().simulate('click').update(); await wait(); const expected: Omit = { + author: [], + isBuildingBlock: false, + license: '', + ruleNameOverride: '', + timestampOverride: '', description: 'Test description text', falsePositives: [''], name: 'Test name text', note: '', references: [''], - riskScore: 50, - severity: 'low', + riskScore: { value: 50, mapping: [] }, + severity: { value: 'low', mapping: [] }, tags: [], threat: [ { @@ -217,13 +222,18 @@ describe('StepAboutRuleComponent', () => { wrapper.find('[data-test-subj="about-continue"]').first().simulate('click').update(); await wait(); const expected: Omit = { + author: [], + isBuildingBlock: false, + license: '', + ruleNameOverride: '', + timestampOverride: '', description: 'Test description text', falsePositives: [''], name: 'Test name text', note: '', references: [''], - riskScore: 80, - severity: 'low', + riskScore: { value: 80, mapping: [] }, + severity: { value: 'low', mapping: [] }, tags: [], threat: [ { diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/index.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/index.tsx index f23c51e019f248..7f7ee94ed85b7a 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/index.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EuiAccordion, EuiFlexItem, EuiSpacer, EuiButtonEmpty } from '@elastic/eui'; +import { EuiAccordion, EuiFlexItem, EuiSpacer, EuiButtonEmpty, EuiFormRow } from '@elastic/eui'; import React, { FC, memo, useCallback, useEffect, useState } from 'react'; import styled from 'styled-components'; import deepEqual from 'fast-deep-equal'; @@ -13,6 +13,7 @@ import { RuleStepProps, RuleStep, AboutStepRule, + DefineStepRule, } from '../../../pages/detection_engine/rules/types'; import { AddItem } from '../add_item_form'; import { StepRuleDescription } from '../description_step'; @@ -35,11 +36,14 @@ import { StepContentWrapper } from '../step_content_wrapper'; import { NextStep } from '../next_step'; import { MarkdownEditorForm } from '../../../../common/components/markdown_editor/form'; import { setFieldValue } from '../../../pages/detection_engine/rules/helpers'; +import { SeverityField } from '../severity_mapping'; +import { RiskScoreField } from '../risk_score_mapping'; const CommonUseField = getUseField({ component: Field }); interface StepAboutRuleProps extends RuleStepProps { defaultValues?: AboutStepRule | null; + defineRuleData?: DefineStepRule | null; } const ThreeQuartersContainer = styled.div` @@ -77,6 +81,7 @@ const AdvancedSettingsAccordionButton = ( const StepAboutRuleComponent: FC = ({ addPadding = false, defaultValues, + defineRuleData, descriptionColumns = 'singleSplit', isReadOnlyView, isUpdateView = false, @@ -132,64 +137,54 @@ const StepAboutRuleComponent: FC = ({ <>
- - - - - - - - + + + + + - - + @@ -207,13 +202,13 @@ const StepAboutRuleComponent: FC = ({ }} /> - + - + = ({ dataTestSubj: 'detectionEngineStepAboutRuleMitreThreat', }} /> - - - + + + + + + + + + + + + + + + + - + {({ severity }) => { diff --git a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/schema.tsx b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/schema.tsx index 59ecebaeb9e4e0..309557e5c94218 100644 --- a/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/alerts/components/rules/step_about_rule/schema.tsx @@ -22,6 +22,23 @@ import * as I18n from './translations'; const { emptyField } = fieldValidators; export const schema: FormSchema = { + author: { + type: FIELD_TYPES.COMBO_BOX, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldAuthorLabel', + { + defaultMessage: 'Author', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldAuthorHelpText', + { + defaultMessage: + 'Type one or more author for this rule. Press enter after each author to add a new one.', + } + ), + labelAppend: OptionalFieldLabel, + }, name: { type: FIELD_TYPES.TEXT, label: i18n.translate( @@ -64,36 +81,44 @@ export const schema: FormSchema = { }, ], }, - severity: { - type: FIELD_TYPES.SUPER_SELECT, + isBuildingBlock: { + type: FIELD_TYPES.CHECKBOX, label: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldSeverityLabel', + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldBuildingBlockLabel', { - defaultMessage: 'Severity', + defaultMessage: 'Mark all generated alerts as "building block" alerts', } ), - validations: [ - { - validator: emptyField( - i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.severityFieldRequiredError', - { - defaultMessage: 'A severity is required.', - } - ) - ), - }, - ], + labelAppend: OptionalFieldLabel, + }, + severity: { + value: { + type: FIELD_TYPES.SUPER_SELECT, + validations: [ + { + validator: emptyField( + i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.severityFieldRequiredError', + { + defaultMessage: 'A severity is required.', + } + ) + ), + }, + ], + }, + mapping: { + type: FIELD_TYPES.TEXT, + }, }, riskScore: { - type: FIELD_TYPES.RANGE, - serializer: (input: string) => Number(input), - label: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldRiskScoreLabel', - { - defaultMessage: 'Risk score', - } - ), + value: { + type: FIELD_TYPES.RANGE, + serializer: (input: string) => Number(input), + }, + mapping: { + type: FIELD_TYPES.TEXT, + }, }, references: { label: i18n.translate( @@ -135,6 +160,39 @@ export const schema: FormSchema = { ), labelAppend: OptionalFieldLabel, }, + license: { + type: FIELD_TYPES.TEXT, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldLicenseLabel', + { + defaultMessage: 'License', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldLicenseHelpText', + { + defaultMessage: 'Add a license name', + } + ), + labelAppend: OptionalFieldLabel, + }, + ruleNameOverride: { + type: FIELD_TYPES.TEXT, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldRuleNameOverrideLabel', + { + defaultMessage: 'Rule name override', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldRuleNameOverrideHelpText', + { + defaultMessage: + 'Choose a field from the source event to populate the rule name in the alert list.', + } + ), + labelAppend: OptionalFieldLabel, + }, threat: { label: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldMitreThreatLabel', @@ -166,6 +224,23 @@ export const schema: FormSchema = { }, ], }, + timestampOverride: { + type: FIELD_TYPES.TEXT, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTimestampOverrideLabel', + { + defaultMessage: 'Timestamp override', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldTimestampOverrideHelpText', + { + defaultMessage: + 'Choose timestamp field used when executing rule. Pick field with timestamp closest to ingest time (e.g. event.ingested).', + } + ), + labelAppend: OptionalFieldLabel, + }, tags: { type: FIELD_TYPES.COMBO_BOX, label: i18n.translate( diff --git a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/api.test.ts b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/api.test.ts index abba7c02cf8757..46829b9cb8f7b2 100644 --- a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/api.test.ts +++ b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/api.test.ts @@ -291,7 +291,7 @@ describe('Detections Rules API', () => { await duplicateRules({ rules: rulesMock.data }); expect(fetchMock).toHaveBeenCalledWith('/api/detection_engine/rules/_bulk_create', { body: - '[{"actions":[],"description":"Elastic Endpoint detected Credential Dumping. Click the Elastic Endpoint icon in the event.module column or the link in the rule.reference column in the External Alerts tab of the SIEM Detections page for additional information.","enabled":false,"false_positives":[],"from":"now-660s","index":["endgame-*"],"interval":"10m","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":73,"name":"Credential Dumping - Detected - Elastic Endpoint [Duplicate]","query":"event.kind:alert and event.module:endgame and event.action:cred_theft_event and endgame.metadata.type:detection","filters":[],"references":[],"severity":"high","tags":["Elastic","Endpoint"],"to":"now","type":"query","threat":[],"throttle":null,"version":1},{"actions":[],"description":"Elastic Endpoint detected an Adversary Behavior. Click the Elastic Endpoint icon in the event.module column or the link in the rule.reference column in the External Alerts tab of the SIEM Detections page for additional information.","enabled":false,"false_positives":[],"from":"now-660s","index":["endgame-*"],"interval":"10m","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":47,"name":"Adversary Behavior - Detected - Elastic Endpoint [Duplicate]","query":"event.kind:alert and event.module:endgame and event.action:rules_engine_event","filters":[],"references":[],"severity":"medium","tags":["Elastic","Endpoint"],"to":"now","type":"query","threat":[],"throttle":null,"version":1}]', + '[{"actions":[],"author":[],"description":"Elastic Endpoint detected Credential Dumping. Click the Elastic Endpoint icon in the event.module column or the link in the rule.reference column in the External Alerts tab of the SIEM Detections page for additional information.","enabled":false,"false_positives":[],"from":"now-660s","index":["endgame-*"],"interval":"10m","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":73,"risk_score_mapping":[],"name":"Credential Dumping - Detected - Elastic Endpoint [Duplicate]","query":"event.kind:alert and event.module:endgame and event.action:cred_theft_event and endgame.metadata.type:detection","filters":[],"references":[],"severity":"high","severity_mapping":[],"tags":["Elastic","Endpoint"],"to":"now","type":"query","threat":[],"throttle":null,"version":1},{"actions":[],"author":[],"description":"Elastic Endpoint detected an Adversary Behavior. Click the Elastic Endpoint icon in the event.module column or the link in the rule.reference column in the External Alerts tab of the SIEM Detections page for additional information.","enabled":false,"false_positives":[],"from":"now-660s","index":["endgame-*"],"interval":"10m","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":47,"risk_score_mapping":[],"name":"Adversary Behavior - Detected - Elastic Endpoint [Duplicate]","query":"event.kind:alert and event.module:endgame and event.action:rules_engine_event","filters":[],"references":[],"severity":"medium","severity_mapping":[],"tags":["Elastic","Endpoint"],"to":"now","type":"query","threat":[],"throttle":null,"version":1}]', method: 'POST', }); }); diff --git a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/mock.ts b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/mock.ts index 59782e8a36338f..fa11cfabcdf8ba 100644 --- a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/mock.ts +++ b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/mock.ts @@ -36,6 +36,7 @@ export const ruleMock: NewRule = { }; export const savedRuleMock: Rule = { + author: [], actions: [], created_at: 'mm/dd/yyyyTHH:MM:sssz', created_by: 'mockUser', @@ -58,11 +59,13 @@ export const savedRuleMock: Rule = { rule_id: 'bbd3106e-b4b5-4d7c-a1a2-47531d6a2baf', language: 'kuery', risk_score: 75, + risk_score_mapping: [], name: 'Test rule', max_signals: 100, query: "user.email: 'root@elastic.co'", references: [], severity: 'high', + severity_mapping: [], tags: ['APM'], to: 'now', type: 'query', @@ -79,6 +82,7 @@ export const rulesMock: FetchRulesResponse = { data: [ { actions: [], + author: [], created_at: '2020-02-14T19:49:28.178Z', updated_at: '2020-02-14T19:49:28.320Z', created_by: 'elastic', @@ -96,12 +100,14 @@ export const rulesMock: FetchRulesResponse = { output_index: '.siem-signals-default', max_signals: 100, risk_score: 73, + risk_score_mapping: [], name: 'Credential Dumping - Detected - Elastic Endpoint', query: 'event.kind:alert and event.module:endgame and event.action:cred_theft_event and endgame.metadata.type:detection', filters: [], references: [], severity: 'high', + severity_mapping: [], updated_by: 'elastic', tags: ['Elastic', 'Endpoint'], to: 'now', @@ -112,6 +118,7 @@ export const rulesMock: FetchRulesResponse = { }, { actions: [], + author: [], created_at: '2020-02-14T19:49:28.189Z', updated_at: '2020-02-14T19:49:28.326Z', created_by: 'elastic', @@ -129,11 +136,13 @@ export const rulesMock: FetchRulesResponse = { output_index: '.siem-signals-default', max_signals: 100, risk_score: 47, + risk_score_mapping: [], name: 'Adversary Behavior - Detected - Elastic Endpoint', query: 'event.kind:alert and event.module:endgame and event.action:rules_engine_event', filters: [], references: [], severity: 'medium', + severity_mapping: [], updated_by: 'elastic', tags: ['Elastic', 'Endpoint'], to: 'now', diff --git a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/types.ts index ab9b88fb81fa7e..d991cc35b8dfed 100644 --- a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/types.ts @@ -7,6 +7,17 @@ import * as t from 'io-ts'; import { RuleTypeSchema } from '../../../../../common/detection_engine/types'; +/* eslint-disable @typescript-eslint/camelcase */ +import { + author, + building_block_type, + license, + risk_score_mapping, + rule_name_override, + severity_mapping, + timestamp_override, +} from '../../../../../common/detection_engine/schemas/common/schemas'; +/* eslint-enable @typescript-eslint/camelcase */ /** * Params is an "record", since it is a type of AlertActionParams which is action templates. @@ -76,6 +87,7 @@ const MetaRule = t.intersection([ export const RuleSchema = t.intersection([ t.type({ + author, created_at: t.string, created_by: t.string, description: t.string, @@ -89,8 +101,10 @@ export const RuleSchema = t.intersection([ max_signals: t.number, references: t.array(t.string), risk_score: t.number, + risk_score_mapping, rule_id: t.string, severity: t.string, + severity_mapping, tags: t.array(t.string), type: RuleTypeSchema, to: t.string, @@ -101,21 +115,25 @@ export const RuleSchema = t.intersection([ throttle: t.union([t.string, t.null]), }), t.partial({ + building_block_type, anomaly_threshold: t.number, filters: t.array(t.unknown), index: t.array(t.string), language: t.string, + license, last_failure_at: t.string, last_failure_message: t.string, meta: MetaRule, machine_learning_job_id: t.string, output_index: t.string, query: t.string, + rule_name_override, saved_id: t.string, status: t.string, status_date: t.string, timeline_id: t.string, timeline_title: t.string, + timestamp_override, note: t.string, version: t.number, }), diff --git a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rule.test.tsx b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rule.test.tsx index 9bfbade0603032..e3cc6878eabca3 100644 --- a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rule.test.tsx +++ b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rule.test.tsx @@ -32,6 +32,7 @@ describe('useRule', () => { false, { actions: [], + author: [], created_at: 'mm/dd/yyyyTHH:MM:sssz', created_by: 'mockUser', description: 'some desc', @@ -56,8 +57,10 @@ describe('useRule', () => { query: "user.email: 'root@elastic.co'", references: [], risk_score: 75, + risk_score_mapping: [], rule_id: 'bbd3106e-b4b5-4d7c-a1a2-47531d6a2baf', severity: 'high', + severity_mapping: [], tags: ['APM'], threat: [], throttle: null, diff --git a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rule_status.test.tsx b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rule_status.test.tsx index f203eca42cde62..1f2c0c32d590f6 100644 --- a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rule_status.test.tsx +++ b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rule_status.test.tsx @@ -27,6 +27,7 @@ const testRule: Rule = { }, }, ], + author: [], created_at: 'mm/dd/yyyyTHH:MM:sssz', created_by: 'mockUser', description: 'some desc', @@ -51,8 +52,10 @@ const testRule: Rule = { query: "user.email: 'root@elastic.co'", references: [], risk_score: 75, + risk_score_mapping: [], rule_id: 'bbd3106e-b4b5-4d7c-a1a2-47531d6a2baf', severity: 'high', + severity_mapping: [], tags: ['APM'], threat: [], throttle: null, diff --git a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rules.test.tsx b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rules.test.tsx index ad34c39272bbfe..76f2a5b58754ee 100644 --- a/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rules.test.tsx +++ b/x-pack/plugins/security_solution/public/alerts/containers/detection_engine/rules/use_rules.test.tsx @@ -59,6 +59,7 @@ describe('useRules', () => { data: [ { actions: [], + author: [], created_at: '2020-02-14T19:49:28.178Z', created_by: 'elastic', description: @@ -79,8 +80,10 @@ describe('useRules', () => { 'event.kind:alert and event.module:endgame and event.action:cred_theft_event and endgame.metadata.type:detection', references: [], risk_score: 73, + risk_score_mapping: [], rule_id: '571afc56-5ed9-465d-a2a9-045f099f6e7e', severity: 'high', + severity_mapping: [], tags: ['Elastic', 'Endpoint'], threat: [], throttle: null, @@ -92,6 +95,7 @@ describe('useRules', () => { }, { actions: [], + author: [], created_at: '2020-02-14T19:49:28.189Z', created_by: 'elastic', description: @@ -112,8 +116,10 @@ describe('useRules', () => { 'event.kind:alert and event.module:endgame and event.action:rules_engine_event', references: [], risk_score: 47, + risk_score_mapping: [], rule_id: '77a3c3df-8ec4-4da4-b758-878f551dee69', severity: 'medium', + severity_mapping: [], tags: ['Elastic', 'Endpoint'], threat: [], throttle: null, diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/all/__mocks__/mock.ts index 1b43a513d0d297..f1416bfbc41b5a 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -41,6 +41,7 @@ export const mockQueryBar: FieldValueQueryBar = { export const mockRule = (id: string): Rule => ({ actions: [], + author: [], created_at: '2020-01-10T21:11:45.839Z', updated_at: '2020-01-10T21:11:45.839Z', created_by: 'elastic', @@ -58,6 +59,7 @@ export const mockRule = (id: string): Rule => ({ output_index: '.siem-signals-default', max_signals: 100, risk_score: 21, + risk_score_mapping: [], name: 'Home Grown!', query: '', references: [], @@ -66,6 +68,7 @@ export const mockRule = (id: string): Rule => ({ timeline_title: 'Untitled timeline', meta: { from: '0m' }, severity: 'low', + severity_mapping: [], updated_by: 'elastic', tags: [], to: 'now', @@ -78,6 +81,7 @@ export const mockRule = (id: string): Rule => ({ export const mockRuleWithEverything = (id: string): Rule => ({ actions: [], + author: [], created_at: '2020-01-10T21:11:45.839Z', updated_at: '2020-01-10T21:11:45.839Z', created_by: 'elastic', @@ -113,9 +117,12 @@ export const mockRuleWithEverything = (id: string): Rule => ({ interval: '5m', rule_id: 'b5ba41ab-aaf3-4f43-971b-bdf9434ce0ea', language: 'kuery', + license: 'Elastic License', output_index: '.siem-signals-default', max_signals: 100, risk_score: 21, + risk_score_mapping: [], + rule_name_override: 'message', name: 'Query with rule-id', query: 'user.name: root or user.name: admin', references: ['www.test.co'], @@ -124,6 +131,7 @@ export const mockRuleWithEverything = (id: string): Rule => ({ timeline_title: 'Titled timeline', meta: { from: '0m' }, severity: 'low', + severity_mapping: [], updated_by: 'elastic', tags: ['tag1', 'tag2'], to: 'now', @@ -146,16 +154,23 @@ export const mockRuleWithEverything = (id: string): Rule => ({ }, ], throttle: 'no_actions', + timestamp_override: 'event.ingested', note: '# this is some markdown documentation', version: 1, }); +// TODO: update types mapping export const mockAboutStepRule = (isNew = false): AboutStepRule => ({ isNew, + author: ['Elastic'], + isBuildingBlock: false, + timestampOverride: '', + ruleNameOverride: '', + license: 'Elastic License', name: 'Query with rule-id', description: '24/7', - severity: 'low', - riskScore: 21, + severity: { value: 'low', mapping: [] }, + riskScore: { value: 21, mapping: [] }, references: ['www.test.co'], falsePositives: ['test'], tags: ['tag1', 'tag2'], diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.test.ts b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.test.ts index d9cbcfc8979a19..bbfbbaae058d40 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.test.ts @@ -339,13 +339,18 @@ describe('helpers', () => { test('returns formatted object as AboutStepRuleJson', () => { const result: AboutStepRuleJson = formatAboutStepData(mockData); const expected = { + author: ['Elastic'], description: '24/7', false_positives: ['test'], + license: 'Elastic License', name: 'Query with rule-id', note: '# this is some markdown documentation', references: ['www.test.co'], risk_score: 21, + risk_score_mapping: [], + rule_name_override: '', severity: 'low', + severity_mapping: [], tags: ['tag1', 'tag2'], threat: [ { @@ -364,6 +369,7 @@ describe('helpers', () => { ], }, ], + timestamp_override: '', }; expect(result).toEqual(expected); @@ -377,13 +383,18 @@ describe('helpers', () => { }; const result: AboutStepRuleJson = formatAboutStepData(mockStepData); const expected = { + author: ['Elastic'], description: '24/7', false_positives: ['test'], + license: 'Elastic License', name: 'Query with rule-id', note: '# this is some markdown documentation', references: ['www.test.co'], risk_score: 21, + risk_score_mapping: [], + rule_name_override: '', severity: 'low', + severity_mapping: [], tags: ['tag1', 'tag2'], threat: [ { @@ -402,6 +413,7 @@ describe('helpers', () => { ], }, ], + timestamp_override: '', }; expect(result).toEqual(expected); @@ -414,12 +426,17 @@ describe('helpers', () => { }; const result: AboutStepRuleJson = formatAboutStepData(mockStepData); const expected = { + author: ['Elastic'], description: '24/7', false_positives: ['test'], + license: 'Elastic License', name: 'Query with rule-id', references: ['www.test.co'], risk_score: 21, + risk_score_mapping: [], + rule_name_override: '', severity: 'low', + severity_mapping: [], tags: ['tag1', 'tag2'], threat: [ { @@ -438,6 +455,7 @@ describe('helpers', () => { ], }, ], + timestamp_override: '', }; expect(result).toEqual(expected); @@ -481,13 +499,18 @@ describe('helpers', () => { }; const result: AboutStepRuleJson = formatAboutStepData(mockStepData); const expected = { + author: ['Elastic'], + license: 'Elastic License', description: '24/7', false_positives: ['test'], name: 'Query with rule-id', note: '# this is some markdown documentation', references: ['www.test.co'], risk_score: 21, + risk_score_mapping: [], + rule_name_override: '', severity: 'low', + severity_mapping: [], tags: ['tag1', 'tag2'], threat: [ { @@ -496,6 +519,7 @@ describe('helpers', () => { technique: [{ id: '456', name: 'technique1', reference: 'technique reference' }], }, ], + timestamp_override: '', }; expect(result).toEqual(expected); diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts index d5ce57ce5b3a9d..b7cf94bb4f3197 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/helpers.ts @@ -122,11 +122,30 @@ export const formatScheduleStepData = (scheduleData: ScheduleStepRule): Schedule }; export const formatAboutStepData = (aboutStepData: AboutStepRule): AboutStepRuleJson => { - const { falsePositives, references, riskScore, threat, isNew, note, ...rest } = aboutStepData; - return { + const { + author, + falsePositives, + references, + riskScore, + severity, + threat, + isBuildingBlock, + isNew, + note, + ruleNameOverride, + timestampOverride, + ...rest + } = aboutStepData; + const resp = { + author: author.filter((item) => !isEmpty(item)), + ...(isBuildingBlock ? { building_block_type: 'default' } : {}), false_positives: falsePositives.filter((item) => !isEmpty(item)), references: references.filter((item) => !isEmpty(item)), - risk_score: riskScore, + risk_score: riskScore.value, + risk_score_mapping: riskScore.mapping, + rule_name_override: ruleNameOverride, + severity: severity.value, + severity_mapping: severity.mapping, threat: threat .filter((singleThreat) => singleThreat.tactic.name !== 'none') .map((singleThreat) => ({ @@ -137,9 +156,11 @@ export const formatAboutStepData = (aboutStepData: AboutStepRule): AboutStepRule return { id, name, reference }; }), })), + timestamp_override: timestampOverride, ...(!isEmpty(note) ? { note } : {}), ...rest, }; + return resp; }; export const formatActionsStepData = (actionsStepData: ActionsStepRule): ActionsStepRuleJson => { diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/index.tsx b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/index.tsx index de3e23b11aaf8e..4be408039d6f6f 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/index.tsx +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/create/index.tsx @@ -358,6 +358,9 @@ const CreateRulePageComponent: React.FC = () => { { }, }; const aboutRuleStepData = { + author: [], description: '24/7', falsePositives: ['test'], + isBuildingBlock: false, isNew: false, + license: 'Elastic License', name: 'Query with rule-id', note: '# this is some markdown documentation', references: ['www.test.co'], - riskScore: 21, - severity: 'low', + riskScore: { value: 21, mapping: [] }, + ruleNameOverride: 'message', + severity: { value: 'low', mapping: [] }, tags: ['tag1', 'tag2'], threat: [ { @@ -106,6 +110,7 @@ describe('rule helpers', () => { ], }, ], + timestampOverride: 'event.ingested', }; const scheduleRuleStepData = { from: '0s', interval: '5m', isNew: false }; const ruleActionsStepData = { diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx index 2cd211a35e9ba3..2a792f7d35eaa4 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/helpers.tsx @@ -116,6 +116,13 @@ export const getHumanizedDuration = (from: string, interval: string): string => export const getAboutStepsData = (rule: Rule, detailsView: boolean): AboutStepRule => { const { name, description, note } = determineDetailsValue(rule, detailsView); const { + author, + building_block_type: buildingBlockType, + license, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, + severity_mapping: severityMapping, + timestamp_override: timestampOverride, references, severity, false_positives: falsePositives, @@ -126,13 +133,24 @@ export const getAboutStepsData = (rule: Rule, detailsView: boolean): AboutStepRu return { isNew: false, + author, + isBuildingBlock: buildingBlockType !== undefined, + license: license ?? '', + ruleNameOverride: ruleNameOverride ?? '', + timestampOverride: timestampOverride ?? '', name, description, note: note!, references, - severity, + severity: { + value: severity, + mapping: severityMapping, + }, tags, - riskScore, + riskScore: { + value: riskScore, + mapping: riskScoreMapping, + }, falsePositives, threat: threat as IMitreEnterpriseAttack[], }; diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/types.ts index 5f81409010a280..f453b5a95994d0 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/types.ts @@ -10,6 +10,15 @@ import { Filter } from '../../../../../../../../src/plugins/data/common'; import { FormData, FormHook } from '../../../../shared_imports'; import { FieldValueQueryBar } from '../../../components/rules/query_bar'; import { FieldValueTimeline } from '../../../components/rules/pick_timeline'; +import { + Author, + BuildingBlockType, + License, + RiskScoreMapping, + RuleNameOverride, + SeverityMapping, + TimestampOverride, +} from '../../../../../common/detection_engine/schemas/common/schemas'; export interface EuiBasicTableSortTypes { field: string; @@ -52,13 +61,18 @@ interface StepRuleData { isNew: boolean; } export interface AboutStepRule extends StepRuleData { + author: string[]; name: string; description: string; - severity: string; - riskScore: number; + isBuildingBlock: boolean; + severity: AboutStepSeverity; + riskScore: AboutStepRiskScore; references: string[]; falsePositives: string[]; + license: string; + ruleNameOverride: string; tags: string[]; + timestampOverride: string; threat: IMitreEnterpriseAttack[]; note: string; } @@ -68,6 +82,16 @@ export interface AboutStepRuleDetails { description: string; } +export interface AboutStepSeverity { + value: string; + mapping: SeverityMapping; +} + +export interface AboutStepRiskScore { + value: number; + mapping: RiskScoreMapping; +} + export interface DefineStepRule extends StepRuleData { anomalyThreshold: number; index: string[]; @@ -104,14 +128,21 @@ export interface DefineStepRuleJson { } export interface AboutStepRuleJson { + author: Author; + building_block_type?: BuildingBlockType; name: string; description: string; + license: License; severity: string; + severity_mapping: SeverityMapping; risk_score: number; + risk_score_mapping: RiskScoreMapping; references: string[]; false_positives: string[]; + rule_name_override: RuleNameOverride; tags: string[]; threat: IMitreEnterpriseAttack[]; + timestamp_override: TimestampOverride; note?: string; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 581946f2300b41..9ca102b4375114 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -342,6 +342,8 @@ export const getResult = (): RuleAlertType => ({ alertTypeId: 'siem.signals', consumer: 'siem', params: { + author: ['Elastic'], + buildingBlockType: undefined, anomalyThreshold: undefined, description: 'Detecting root and admin users', ruleId: 'rule-1', @@ -352,6 +354,7 @@ export const getResult = (): RuleAlertType => ({ savedId: undefined, query: 'user.name: root or user.name: admin', language: 'kuery', + license: 'Elastic License', machineLearningJobId: undefined, outputIndex: '.siem-signals', timelineId: 'some-timeline-id', @@ -367,8 +370,11 @@ export const getResult = (): RuleAlertType => ({ }, ], riskScore: 50, + riskScoreMapping: [], + ruleNameOverride: undefined, maxSignals: 100, severity: 'high', + severityMapping: [], to: 'now', type: 'query', threat: [ @@ -388,6 +394,7 @@ export const getResult = (): RuleAlertType => ({ ], }, ], + timestampOverride: undefined, references: ['http://www.example.com', 'https://ww.example.com'], note: '# Investigative notes', version: 1, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts index 7b7d3fbdea0bfd..87903d1035903a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts @@ -36,6 +36,7 @@ export const getOutputRuleAlertForRest = (): Omit< RulesSchema, 'machine_learning_job_id' | 'anomaly_threshold' > => ({ + author: ['Elastic'], actions: [], created_by: 'elastic', created_at: '2019-12-13T16:40:33.400Z', @@ -49,14 +50,17 @@ export const getOutputRuleAlertForRest = (): Omit< index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', risk_score: 50, + risk_score_mapping: [], rule_id: 'rule-1', language: 'kuery', + license: 'Elastic License', max_signals: 100, name: 'Detect Root/Admin Users', output_index: '.siem-signals', query: 'user.name: root or user.name: admin', references: ['http://www.example.com', 'https://ww.example.com'], severity: 'high', + severity_mapping: [], updated_by: 'elastic', tags: [], throttle: 'no_actions', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index dc20f0793a6f85..aa4166e93f4a14 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -46,6 +46,12 @@ "rule_id": { "type": "keyword" }, + "author": { + "type": "keyword" + }, + "building_block_type": { + "type": "keyword" + }, "false_positives": { "type": "keyword" }, @@ -64,6 +70,19 @@ "risk_score": { "type": "keyword" }, + "risk_score_mapping": { + "properties": { + "field": { + "type": "keyword" + }, + "operator": { + "type": "keyword" + }, + "value": { + "type": "keyword" + } + } + }, "output_index": { "type": "keyword" }, @@ -85,9 +104,15 @@ "language": { "type": "keyword" }, + "license": { + "type": "keyword" + }, "name": { "type": "keyword" }, + "rule_name_override": { + "type": "keyword" + }, "query": { "type": "keyword" }, @@ -97,6 +122,22 @@ "severity": { "type": "keyword" }, + "severity_mapping": { + "properties": { + "field": { + "type": "keyword" + }, + "operator": { + "type": "keyword" + }, + "value": { + "type": "keyword" + }, + "severity": { + "type": "keyword" + } + } + }, "tags": { "type": "keyword" }, @@ -136,6 +177,9 @@ "note": { "type": "text" }, + "timestamp_override": { + "type": "keyword" + }, "type": { "type": "keyword" }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts index 4b65ee5efdff0a..fc2cc6551450c8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts @@ -21,9 +21,12 @@ jest.mock('../../rules/get_prepackaged_rules', () => { getPrepackagedRules: (): AddPrepackagedRulesSchemaDecoded[] => { return [ { + author: ['Elastic'], tags: [], rule_id: 'rule-1', risk_score: 50, + risk_score_mapping: [], + severity_mapping: [], description: 'some description', from: 'now-5m', to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index 92a7ea17e7eaf9..2942413057e375 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -64,12 +64,15 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => const { actions: actionsRest, anomaly_threshold: anomalyThreshold, + author, + building_block_type: buildingBlockType, description, enabled, false_positives: falsePositives, from, query: queryOrUndefined, language: languageOrUndefined, + license, machine_learning_job_id: machineLearningJobId, output_index: outputIndex, saved_id: savedId, @@ -80,11 +83,15 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, threat, throttle, + timestamp_override: timestampOverride, to, type, references, @@ -139,6 +146,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => const createdRule = await createRules({ alertsClient, anomalyThreshold, + author, + buildingBlockType, description, enabled, falsePositives, @@ -146,6 +155,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => immutable: false, query, language, + license, machineLearningJobId, outputIndex: finalIndex, savedId, @@ -157,13 +167,17 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => index, interval, maxSignals, - riskScore, name, + riskScore, + riskScoreMapping, + ruleNameOverride, severity, + severityMapping, tags, to, type, threat, + timestampOverride, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 78d67e0e9366c6..310a9da56282d0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -47,12 +47,15 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void const { actions: actionsRest, anomaly_threshold: anomalyThreshold, + author, + building_block_type: buildingBlockType, description, enabled, false_positives: falsePositives, from, query: queryOrUndefined, language: languageOrUndefined, + license, output_index: outputIndex, saved_id: savedId, timeline_id: timelineId, @@ -65,11 +68,15 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, threat, throttle, + timestamp_override: timestampOverride, to, type, references, @@ -121,6 +128,8 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void const createdRule = await createRules({ alertsClient, anomalyThreshold, + author, + buildingBlockType, description, enabled, falsePositives, @@ -128,6 +137,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void immutable: false, query, language, + license, outputIndex: finalIndex, savedId, timelineId, @@ -139,13 +149,17 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void index, interval, maxSignals, - riskScore, name, + riskScore, + riskScoreMapping, + ruleNameOverride, severity, + severityMapping, tags, to, type, threat, + timestampOverride, references, note, version: 1, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index a277f97ccf9f0a..43aa1ecd31922f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -134,6 +134,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP } const { anomaly_threshold: anomalyThreshold, + author, + building_block_type: buildingBlockType, description, enabled, false_positives: falsePositives, @@ -141,6 +143,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP immutable, query: queryOrUndefined, language: languageOrUndefined, + license, machine_learning_job_id: machineLearningJobId, output_index: outputIndex, saved_id: savedId, @@ -151,10 +154,14 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, threat, + timestamp_override: timestampOverride, to, type, references, @@ -184,6 +191,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP await createRules({ alertsClient, anomalyThreshold, + author, + buildingBlockType, description, enabled, falsePositives, @@ -191,6 +200,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP immutable, query, language, + license, machineLearningJobId, outputIndex: signalsIndex, savedId, @@ -202,13 +212,17 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP index, interval, maxSignals, - riskScore, name, + riskScore, + riskScoreMapping, + ruleNameOverride, severity, + severityMapping, tags, to, type, threat, + timestampOverride, references, note, version, @@ -219,6 +233,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP } else if (rule != null && request.query.overwrite) { await patchRules({ alertsClient, + author, + buildingBlockType, savedObjectsClient, description, enabled, @@ -226,6 +242,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP from, query, language, + license, outputIndex, savedId, timelineId, @@ -237,9 +254,13 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, + timestampOverride, to, type, threat, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index b2a9fdd103a682..c3d6f920e47a92 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -55,12 +55,15 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => request.body.map(async (payloadRule) => { const { actions: actionsRest, + author, + building_block_type: buildingBlockType, description, enabled, false_positives: falsePositives, from, query, language, + license, output_index: outputIndex, saved_id: savedId, timeline_id: timelineId, @@ -73,12 +76,16 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, to, type, threat, + timestamp_override: timestampOverride, throttle, references, note, @@ -107,12 +114,15 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => const rule = await patchRules({ rule: existingRule, alertsClient, + author, + buildingBlockType, description, enabled, falsePositives, from, query, language, + license, outputIndex, savedId, savedObjectsClient, @@ -124,12 +134,16 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, to, type, threat, + timestampOverride, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index 385eec0fe11802..eb9624e6412e9c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -46,12 +46,15 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { } const { actions: actionsRest, + author, + building_block_type: buildingBlockType, description, enabled, false_positives: falsePositives, from, query, language, + license, output_index: outputIndex, saved_id: savedId, timeline_id: timelineId, @@ -64,12 +67,16 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, to, type, threat, + timestamp_override: timestampOverride, throttle, references, note, @@ -105,12 +112,15 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { const ruleStatusClient = ruleStatusSavedObjectsClientFactory(savedObjectsClient); const rule = await patchRules({ alertsClient, + author, + buildingBlockType, description, enabled, falsePositives, from, query, language, + license, outputIndex, savedId, savedObjectsClient, @@ -123,12 +133,16 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, to, type, threat, + timestampOverride, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index 1e6815a3571548..c1ab1be2dbd0a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -57,12 +57,15 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => const { actions: actionsRest, anomaly_threshold: anomalyThreshold, + author, + building_block_type: buildingBlockType, description, enabled, false_positives: falsePositives, from, query: queryOrUndefined, language: languageOrUndefined, + license, machine_learning_job_id: machineLearningJobId, output_index: outputIndex, saved_id: savedId, @@ -76,13 +79,17 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, to, type, threat, throttle, + timestamp_override: timestampOverride, references, note, version, @@ -117,12 +124,15 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => const rule = await updateRules({ alertsClient, anomalyThreshold, + author, + buildingBlockType, description, enabled, falsePositives, from, query, language, + license, machineLearningJobId, outputIndex: finalIndex, savedId, @@ -137,12 +147,16 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, to, type, threat, + timestampOverride, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index f2b47f195ca5c4..717f388cfc1e9d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -47,12 +47,15 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { const { actions: actionsRest, anomaly_threshold: anomalyThreshold, + author, + building_block_type: buildingBlockType, description, enabled, false_positives: falsePositives, from, query: queryOrUndefined, language: languageOrUndefined, + license, machine_learning_job_id: machineLearningJobId, output_index: outputIndex, saved_id: savedId, @@ -66,13 +69,17 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, to, type, threat, throttle, + timestamp_override: timestampOverride, references, note, version, @@ -107,12 +114,15 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { const rule = await updateRules({ alertsClient, anomalyThreshold, + author, + buildingBlockType, description, enabled, falsePositives, from, query, language, + license, machineLearningJobId, outputIndex: finalIndex, savedId, @@ -127,12 +137,16 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, to, type, threat, + timestampOverride, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index 9320eba26df0bc..9e93dc051a0413 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -105,7 +105,9 @@ export const transformAlertToRule = ( ruleStatus?: SavedObject ): Partial => { return pickBy((value: unknown) => value != null, { + author: alert.params.author ?? [], actions: ruleActions?.actions ?? [], + building_block_type: alert.params.buildingBlockType, created_at: alert.createdAt.toISOString(), updated_at: alert.updatedAt.toISOString(), created_by: alert.createdBy ?? 'elastic', @@ -121,10 +123,13 @@ export const transformAlertToRule = ( interval: alert.schedule.interval, rule_id: alert.params.ruleId, language: alert.params.language, + license: alert.params.license, output_index: alert.params.outputIndex, max_signals: alert.params.maxSignals, machine_learning_job_id: alert.params.machineLearningJobId, risk_score: alert.params.riskScore, + risk_score_mapping: alert.params.riskScoreMapping ?? [], + rule_name_override: alert.params.ruleNameOverride, name: alert.name, query: alert.params.query, references: alert.params.references, @@ -133,12 +138,14 @@ export const transformAlertToRule = ( timeline_title: alert.params.timelineTitle, meta: alert.params.meta, severity: alert.params.severity, + severity_mapping: alert.params.severityMapping ?? [], updated_by: alert.updatedBy ?? 'elastic', tags: transformTags(alert.tags), to: alert.params.to, type: alert.params.type, threat: alert.params.threat ?? [], throttle: ruleActions?.ruleThrottle || 'no_actions', + timestamp_override: alert.params.timestampOverride, note: alert.params.note, version: alert.params.version, status: ruleStatus?.attributes.status ?? undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts index 00656967126280..4dafafe3153ef9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/validate.test.ts @@ -18,6 +18,7 @@ import { getListArrayMock } from '../../../../../common/detection_engine/schemas export const ruleOutput: RulesSchema = { actions: [], + author: ['Elastic'], created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', created_by: 'elastic', @@ -30,13 +31,16 @@ export const ruleOutput: RulesSchema = { interval: '5m', rule_id: 'rule-1', language: 'kuery', + license: 'Elastic License', output_index: '.siem-signals', max_signals: 100, risk_score: 50, + risk_score_mapping: [], name: 'Detect Root/Admin Users', query: 'user.name: root or user.name: admin', references: ['http://www.example.com', 'https://ww.example.com'], severity: 'high', + severity_mapping: [], updated_by: 'elastic', tags: [], to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index d00bffb96ad057..a7e24a1ac16096 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -8,6 +8,8 @@ import { CreateRulesOptions } from './types'; import { alertsClientMock } from '../../../../../alerts/server/mocks'; export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ + author: ['Elastic'], + buildingBlockType: undefined, alertsClient: alertsClientMock.create(), anomalyThreshold: undefined, description: 'some description', @@ -16,6 +18,7 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ from: 'now-6m', query: 'user.name: root or user.name: admin', language: 'kuery', + license: 'Elastic License', savedId: 'savedId-123', timelineId: 'timelineid-123', timelineTitle: 'timeline-title-123', @@ -28,11 +31,15 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ interval: '5m', maxSignals: 100, riskScore: 80, + riskScoreMapping: [], + ruleNameOverride: undefined, outputIndex: 'output-1', name: 'Query with a rule id', severity: 'high', + severityMapping: [], tags: [], threat: [], + timestampOverride: undefined, to: 'now', type: 'query', references: ['http://www.example.com'], @@ -43,6 +50,8 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ }); export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ + author: ['Elastic'], + buildingBlockType: undefined, alertsClient: alertsClientMock.create(), anomalyThreshold: 55, description: 'some description', @@ -51,6 +60,7 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ from: 'now-6m', query: undefined, language: undefined, + license: 'Elastic License', savedId: 'savedId-123', timelineId: 'timelineid-123', timelineTitle: 'timeline-title-123', @@ -63,11 +73,15 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ interval: '5m', maxSignals: 100, riskScore: 80, + riskScoreMapping: [], + ruleNameOverride: undefined, outputIndex: 'output-1', name: 'Machine Learning Job', severity: 'high', + severityMapping: [], tags: [], threat: [], + timestampOverride: undefined, to: 'now', type: 'machine_learning', references: ['http://www.example.com'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 83e9b0de16f064..b4e246718efd77 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -14,12 +14,15 @@ import { hasListsFeature } from '../feature_flags'; export const createRules = async ({ alertsClient, anomalyThreshold, + author, + buildingBlockType, description, enabled, falsePositives, from, query, language, + license, savedId, timelineId, timelineTitle, @@ -32,11 +35,15 @@ export const createRules = async ({ interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, outputIndex, name, severity, + severityMapping, tags, threat, + timestampOverride, to, type, references, @@ -55,6 +62,8 @@ export const createRules = async ({ consumer: APP_ID, params: { anomalyThreshold, + author, + buildingBlockType, description, ruleId, index, @@ -63,6 +72,7 @@ export const createRules = async ({ immutable, query, language, + license, outputIndex, savedId, timelineId, @@ -72,8 +82,12 @@ export const createRules = async ({ filters, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, severity, + severityMapping, threat, + timestampOverride, to, type, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.test.ts index c4d7df61061bdf..f2061ce1d36de5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.test.ts @@ -49,16 +49,19 @@ describe('create_rules_stream_from_ndjson', () => { ]); expect(result).toEqual([ { + author: [], actions: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -73,16 +76,19 @@ describe('create_rules_stream_from_ndjson', () => { version: 1, }, { + author: [], actions: [], rule_id: 'rule-2', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -135,16 +141,19 @@ describe('create_rules_stream_from_ndjson', () => { ]); expect(result).toEqual([ { + author: [], actions: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -159,16 +168,19 @@ describe('create_rules_stream_from_ndjson', () => { version: 1, }, { + author: [], actions: [], rule_id: 'rule-2', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -204,16 +216,19 @@ describe('create_rules_stream_from_ndjson', () => { ]); expect(result).toEqual([ { + author: [], actions: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -228,16 +243,19 @@ describe('create_rules_stream_from_ndjson', () => { version: 1, }, { + author: [], actions: [], rule_id: 'rule-2', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -273,16 +291,19 @@ describe('create_rules_stream_from_ndjson', () => { ]); const resultOrError = result as Error[]; expect(resultOrError[0]).toEqual({ + author: [], actions: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -298,16 +319,19 @@ describe('create_rules_stream_from_ndjson', () => { }); expect(resultOrError[1].message).toEqual('Unexpected token , in JSON at position 1'); expect(resultOrError[2]).toEqual({ + author: [], actions: [], rule_id: 'rule-2', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -342,16 +366,19 @@ describe('create_rules_stream_from_ndjson', () => { ]); const resultOrError = result as BadRequestError[]; expect(resultOrError[0]).toEqual({ + author: [], actions: [], rule_id: 'rule-1', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, @@ -369,16 +396,19 @@ describe('create_rules_stream_from_ndjson', () => { 'Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "name",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "rule_id"' ); expect(resultOrError[2]).toEqual({ + author: [], actions: [], rule_id: 'rule-2', output_index: '.siem-signals', risk_score: 50, + risk_score_mapping: [], description: 'some description', from: 'now-5m', to: 'now', index: ['index-1'], name: 'some-name', severity: 'low', + severity_mapping: [], interval: '5m', type: 'query', enabled: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.test.ts index 7d4bbfdced4324..c8ea000dd0dcdf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.test.ts @@ -30,6 +30,7 @@ describe('getExportAll', () => { const exports = await getExportAll(alertsClient); expect(exports).toEqual({ rulesNdjson: `${JSON.stringify({ + author: ['Elastic'], actions: [], created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', @@ -45,9 +46,11 @@ describe('getExportAll', () => { interval: '5m', rule_id: 'rule-1', language: 'kuery', + license: 'Elastic License', output_index: '.siem-signals', max_signals: 100, risk_score: 50, + risk_score_mapping: [], name: 'Detect Root/Admin Users', query: 'user.name: root or user.name: admin', references: ['http://www.example.com', 'https://ww.example.com'], @@ -55,6 +58,7 @@ describe('getExportAll', () => { timeline_title: 'some-timeline-title', meta: { someMeta: 'someField' }, severity: 'high', + severity_mapping: [], updated_by: 'elastic', tags: [], to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts index 043e563a4c8b5c..d5dffff00b8966 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_by_object_ids.test.ts @@ -38,6 +38,7 @@ describe('get_export_by_object_ids', () => { const exports = await getExportByObjectIds(alertsClient, objects); expect(exports).toEqual({ rulesNdjson: `${JSON.stringify({ + author: ['Elastic'], actions: [], created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', @@ -53,9 +54,11 @@ describe('get_export_by_object_ids', () => { interval: '5m', rule_id: 'rule-1', language: 'kuery', + license: 'Elastic License', output_index: '.siem-signals', max_signals: 100, risk_score: 50, + risk_score_mapping: [], name: 'Detect Root/Admin Users', query: 'user.name: root or user.name: admin', references: ['http://www.example.com', 'https://ww.example.com'], @@ -63,6 +66,7 @@ describe('get_export_by_object_ids', () => { timeline_title: 'some-timeline-title', meta: { someMeta: 'someField' }, severity: 'high', + severity_mapping: [], updated_by: 'elastic', tags: [], to: 'now', @@ -139,6 +143,7 @@ describe('get_export_by_object_ids', () => { rules: [ { actions: [], + author: ['Elastic'], created_at: '2019-12-13T16:40:33.400Z', updated_at: '2019-12-13T16:40:33.400Z', created_by: 'elastic', @@ -153,9 +158,11 @@ describe('get_export_by_object_ids', () => { interval: '5m', rule_id: 'rule-1', language: 'kuery', + license: 'Elastic License', output_index: '.siem-signals', max_signals: 100, risk_score: 50, + risk_score_mapping: [], name: 'Detect Root/Admin Users', query: 'user.name: root or user.name: admin', references: ['http://www.example.com', 'https://ww.example.com'], @@ -163,6 +170,7 @@ describe('get_export_by_object_ids', () => { timeline_title: 'some-timeline-title', meta: { someMeta: 'someField' }, severity: 'high', + severity_mapping: [], updated_by: 'elastic', tags: [], to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index a51acf99b570cc..8a86a0f103371e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -18,12 +18,15 @@ export const installPrepackagedRules = ( rules.reduce>>((acc, rule) => { const { anomaly_threshold: anomalyThreshold, + author, + building_block_type: buildingBlockType, description, enabled, false_positives: falsePositives, from, query, language, + license, machine_learning_job_id: machineLearningJobId, saved_id: savedId, timeline_id: timelineId, @@ -35,12 +38,16 @@ export const installPrepackagedRules = ( interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, to, type, threat, + timestamp_override: timestampOverride, references, note, version, @@ -54,6 +61,8 @@ export const installPrepackagedRules = ( createRules({ alertsClient, anomalyThreshold, + author, + buildingBlockType, description, enabled, falsePositives, @@ -61,6 +70,7 @@ export const installPrepackagedRules = ( immutable: true, // At the moment we force all prepackaged rules to be immutable query, language, + license, machineLearningJobId, outputIndex, savedId, @@ -73,12 +83,16 @@ export const installPrepackagedRules = ( interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, to, type, threat, + timestampOverride, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index e711d8d2ac2877..f3102a5ad2cf37 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -113,6 +113,8 @@ const rule: SanitizedAlert = { }; export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ + author: ['Elastic'], + buildingBlockType: undefined, alertsClient: alertsClientMock.create(), savedObjectsClient: savedObjectsClientMock.create(), anomalyThreshold: undefined, @@ -122,6 +124,7 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ from: 'now-6m', query: 'user.name: root or user.name: admin', language: 'kuery', + license: 'Elastic License', savedId: 'savedId-123', timelineId: 'timelineid-123', timelineTitle: 'timeline-title-123', @@ -132,11 +135,15 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ interval: '5m', maxSignals: 100, riskScore: 80, + riskScoreMapping: [], + ruleNameOverride: undefined, outputIndex: 'output-1', name: 'Query with a rule id', severity: 'high', + severityMapping: [], tags: [], threat: [], + timestampOverride: undefined, to: 'now', type: 'query', references: ['http://www.example.com'], @@ -148,6 +155,8 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ }); export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ + author: ['Elastic'], + buildingBlockType: undefined, alertsClient: alertsClientMock.create(), savedObjectsClient: savedObjectsClientMock.create(), anomalyThreshold: 55, @@ -157,6 +166,7 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ from: 'now-6m', query: undefined, language: undefined, + license: 'Elastic License', savedId: 'savedId-123', timelineId: 'timelineid-123', timelineTitle: 'timeline-title-123', @@ -167,11 +177,15 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ interval: '5m', maxSignals: 100, riskScore: 80, + riskScoreMapping: [], + ruleNameOverride: undefined, outputIndex: 'output-1', name: 'Machine Learning Job', severity: 'high', + severityMapping: [], tags: [], threat: [], + timestampOverride: undefined, to: 'now', type: 'machine_learning', references: ['http://www.example.com'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index 0c103b7176db4d..577d8d426b63d4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -14,12 +14,15 @@ import { ruleStatusSavedObjectsClientFactory } from '../signals/rule_status_save export const patchRules = async ({ alertsClient, + author, + buildingBlockType, savedObjectsClient, description, falsePositives, enabled, query, language, + license, outputIndex, savedId, timelineId, @@ -31,11 +34,15 @@ export const patchRules = async ({ interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, rule, name, severity, + severityMapping, tags, threat, + timestampOverride, to, type, references, @@ -51,10 +58,13 @@ export const patchRules = async ({ } const calculatedVersion = calculateVersion(rule.params.immutable, rule.params.version, { + author, + buildingBlockType, description, falsePositives, query, language, + license, outputIndex, savedId, timelineId, @@ -66,10 +76,14 @@ export const patchRules = async ({ interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, threat, + timestampOverride, to, type, references, @@ -85,11 +99,14 @@ export const patchRules = async ({ ...rule.params, }, { + author, + buildingBlockType, description, falsePositives, from, query, language, + license, outputIndex, savedId, timelineId, @@ -99,8 +116,12 @@ export const patchRules = async ({ index, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, severity, + severityMapping, threat, + timestampOverride, to, type, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index fc95f0cfeb78e9..7b793ffbdb3629 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -73,6 +73,16 @@ import { LastSuccessMessage, LastFailureAt, LastFailureMessage, + Author, + AuthorOrUndefined, + LicenseOrUndefined, + RiskScoreMapping, + RiskScoreMappingOrUndefined, + SeverityMapping, + SeverityMappingOrUndefined, + TimestampOverrideOrUndefined, + BuildingBlockTypeOrUndefined, + RuleNameOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { AlertsClient, PartialAlert } from '../../../../../alerts/server'; import { Alert, SanitizedAlert } from '../../../../../alerts/common'; @@ -165,6 +175,8 @@ export const isRuleStatusFindTypes = ( export interface CreateRulesOptions { alertsClient: AlertsClient; anomalyThreshold: AnomalyThresholdOrUndefined; + author: Author; + buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; enabled: Enabled; falsePositives: FalsePositives; @@ -181,13 +193,18 @@ export interface CreateRulesOptions { immutable: Immutable; index: IndexOrUndefined; interval: Interval; + license: LicenseOrUndefined; maxSignals: MaxSignals; riskScore: RiskScore; + riskScoreMapping: RiskScoreMapping; + ruleNameOverride: RuleNameOverrideOrUndefined; outputIndex: OutputIndex; name: Name; severity: Severity; + severityMapping: SeverityMapping; tags: Tags; threat: Threat; + timestampOverride: TimestampOverrideOrUndefined; to: To; type: Type; references: References; @@ -202,6 +219,8 @@ export interface UpdateRulesOptions { savedObjectsClient: SavedObjectsClientContract; alertsClient: AlertsClient; anomalyThreshold: AnomalyThresholdOrUndefined; + author: Author; + buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; enabled: Enabled; falsePositives: FalsePositives; @@ -217,13 +236,18 @@ export interface UpdateRulesOptions { ruleId: RuleIdOrUndefined; index: IndexOrUndefined; interval: Interval; + license: LicenseOrUndefined; maxSignals: MaxSignals; riskScore: RiskScore; + riskScoreMapping: RiskScoreMapping; + ruleNameOverride: RuleNameOverrideOrUndefined; outputIndex: OutputIndex; name: Name; severity: Severity; + severityMapping: SeverityMapping; tags: Tags; threat: Threat; + timestampOverride: TimestampOverrideOrUndefined; to: To; type: Type; references: References; @@ -237,6 +261,8 @@ export interface PatchRulesOptions { savedObjectsClient: SavedObjectsClientContract; alertsClient: AlertsClient; anomalyThreshold: AnomalyThresholdOrUndefined; + author: AuthorOrUndefined; + buildingBlockType: BuildingBlockTypeOrUndefined; description: DescriptionOrUndefined; enabled: EnabledOrUndefined; falsePositives: FalsePositivesOrUndefined; @@ -251,13 +277,18 @@ export interface PatchRulesOptions { filters: PartialFilter[]; index: IndexOrUndefined; interval: IntervalOrUndefined; + license: LicenseOrUndefined; maxSignals: MaxSignalsOrUndefined; riskScore: RiskScoreOrUndefined; + riskScoreMapping: RiskScoreMappingOrUndefined; + ruleNameOverride: RuleNameOverrideOrUndefined; outputIndex: OutputIndexOrUndefined; name: NameOrUndefined; severity: SeverityOrUndefined; + severityMapping: SeverityMappingOrUndefined; tags: TagsOrUndefined; threat: ThreatOrUndefined; + timestampOverride: TimestampOverrideOrUndefined; to: ToOrUndefined; type: TypeOrUndefined; references: ReferencesOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts index c4792eaa97ee14..6466cc596d8915 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts @@ -20,11 +20,14 @@ export const updatePrepackagedRules = async ( await Promise.all( rules.map(async (rule) => { const { + author, + building_block_type: buildingBlockType, description, false_positives: falsePositives, from, query, language, + license, saved_id: savedId, meta, filters: filtersObject, @@ -33,12 +36,16 @@ export const updatePrepackagedRules = async ( interval, max_signals: maxSignals, risk_score: riskScore, + risk_score_mapping: riskScoreMapping, + rule_name_override: ruleNameOverride, name, severity, + severity_mapping: severityMapping, tags, to, type, threat, + timestamp_override: timestampOverride, references, version, note, @@ -58,11 +65,14 @@ export const updatePrepackagedRules = async ( // or enable rules on the user when they were not expecting it if a rule updates return patchRules({ alertsClient, + author, + buildingBlockType, description, falsePositives, from, query, language, + license, outputIndex, rule: existingRule, savedId, @@ -73,9 +83,13 @@ export const updatePrepackagedRules = async ( interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, + timestampOverride, to, type, threat, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts index 7812c66a74d1f9..fdc0a61274e759 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts @@ -9,6 +9,8 @@ import { alertsClientMock } from '../../../../../alerts/server/mocks'; import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ + author: ['Elastic'], + buildingBlockType: undefined, id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', alertsClient: alertsClientMock.create(), savedObjectsClient: savedObjectsClientMock.create(), @@ -19,6 +21,7 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ from: 'now-6m', query: 'user.name: root or user.name: admin', language: 'kuery', + license: 'Elastic License', savedId: 'savedId-123', timelineId: 'timelineid-123', timelineTitle: 'timeline-title-123', @@ -30,11 +33,15 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ interval: '5m', maxSignals: 100, riskScore: 80, + riskScoreMapping: [], + ruleNameOverride: undefined, outputIndex: 'output-1', name: 'Query with a rule id', severity: 'high', + severityMapping: [], tags: [], threat: [], + timestampOverride: undefined, to: 'now', type: 'query', references: ['http://www.example.com'], @@ -45,6 +52,8 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ }); export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ + author: ['Elastic'], + buildingBlockType: undefined, id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', alertsClient: alertsClientMock.create(), savedObjectsClient: savedObjectsClientMock.create(), @@ -55,6 +64,7 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ from: 'now-6m', query: undefined, language: undefined, + license: 'Elastic License', savedId: 'savedId-123', timelineId: 'timelineid-123', timelineTitle: 'timeline-title-123', @@ -66,11 +76,15 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ interval: '5m', maxSignals: 100, riskScore: 80, + riskScoreMapping: [], + ruleNameOverride: undefined, outputIndex: 'output-1', name: 'Machine Learning Job', severity: 'high', + severityMapping: [], tags: [], threat: [], + timestampOverride: undefined, to: 'now', type: 'machine_learning', references: ['http://www.example.com'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts index b3f327857dbb3d..5cc68db25afc88 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts @@ -15,12 +15,15 @@ import { ruleStatusSavedObjectsClientFactory } from '../signals/rule_status_save export const updateRules = async ({ alertsClient, + author, + buildingBlockType, savedObjectsClient, description, falsePositives, enabled, query, language, + license, outputIndex, savedId, timelineId, @@ -34,10 +37,14 @@ export const updateRules = async ({ interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, threat, + timestampOverride, to, type, references, @@ -54,10 +61,13 @@ export const updateRules = async ({ } const calculatedVersion = calculateVersion(rule.params.immutable, rule.params.version, { + author, + buildingBlockType, description, falsePositives, query, language, + license, outputIndex, savedId, timelineId, @@ -69,10 +79,14 @@ export const updateRules = async ({ interval, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, name, severity, + severityMapping, tags, threat, + timestampOverride, to, type, references, @@ -95,6 +109,8 @@ export const updateRules = async ({ actions: actions.map(transformRuleToAlertAction), throttle: null, params: { + author, + buildingBlockType, description, ruleId: rule.params.ruleId, falsePositives, @@ -102,6 +118,7 @@ export const updateRules = async ({ immutable: rule.params.immutable, query, language, + license, outputIndex, savedId, timelineId, @@ -111,8 +128,12 @@ export const updateRules = async ({ index, maxSignals, riskScore, + riskScoreMapping, + ruleNameOverride, severity, + severityMapping, threat, + timestampOverride, to, type, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts index 0f65b2a78ec4ce..aa0512678b073c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts @@ -28,10 +28,13 @@ describe('utils', () => { test('returning the same version number if given an immutable but no updated version number', () => { expect( calculateVersion(true, 1, { + author: [], + buildingBlockType: undefined, description: 'some description change', falsePositives: undefined, query: undefined, language: undefined, + license: undefined, outputIndex: undefined, savedId: undefined, timelineId: undefined, @@ -43,11 +46,15 @@ describe('utils', () => { interval: undefined, maxSignals: undefined, riskScore: undefined, + riskScoreMapping: undefined, + ruleNameOverride: undefined, name: undefined, severity: undefined, + severityMapping: undefined, tags: undefined, threat: undefined, to: undefined, + timestampOverride: undefined, type: undefined, references: undefined, version: undefined, @@ -62,10 +69,13 @@ describe('utils', () => { test('returning an updated version number if given an immutable and an updated version number', () => { expect( calculateVersion(true, 2, { + author: [], + buildingBlockType: undefined, description: 'some description change', falsePositives: undefined, query: undefined, language: undefined, + license: undefined, outputIndex: undefined, savedId: undefined, timelineId: undefined, @@ -77,11 +87,15 @@ describe('utils', () => { interval: undefined, maxSignals: undefined, riskScore: undefined, + riskScoreMapping: undefined, + ruleNameOverride: undefined, name: undefined, severity: undefined, + severityMapping: undefined, tags: undefined, threat: undefined, to: undefined, + timestampOverride: undefined, type: undefined, references: undefined, version: undefined, @@ -96,10 +110,13 @@ describe('utils', () => { test('returning an updated version number if not given an immutable but but an updated description', () => { expect( calculateVersion(false, 1, { + author: [], + buildingBlockType: undefined, description: 'some description change', falsePositives: undefined, query: undefined, language: undefined, + license: undefined, outputIndex: undefined, savedId: undefined, timelineId: undefined, @@ -111,11 +128,15 @@ describe('utils', () => { interval: undefined, maxSignals: undefined, riskScore: undefined, + riskScoreMapping: undefined, + ruleNameOverride: undefined, name: undefined, severity: undefined, + severityMapping: undefined, tags: undefined, threat: undefined, to: undefined, + timestampOverride: undefined, type: undefined, references: undefined, version: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts index 5c620a5df61f8e..861d02a8203e6b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts @@ -31,6 +31,13 @@ import { ThreatOrUndefined, TypeOrUndefined, ReferencesOrUndefined, + AuthorOrUndefined, + BuildingBlockTypeOrUndefined, + LicenseOrUndefined, + RiskScoreMappingOrUndefined, + RuleNameOverrideOrUndefined, + SeverityMappingOrUndefined, + TimestampOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { PartialFilter } from '../types'; import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types'; @@ -49,11 +56,14 @@ export const calculateInterval = ( }; export interface UpdateProperties { + author: AuthorOrUndefined; + buildingBlockType: BuildingBlockTypeOrUndefined; description: DescriptionOrUndefined; falsePositives: FalsePositivesOrUndefined; from: FromOrUndefined; query: QueryOrUndefined; language: LanguageOrUndefined; + license: LicenseOrUndefined; savedId: SavedIdOrUndefined; timelineId: TimelineIdOrUndefined; timelineTitle: TimelineTitleOrUndefined; @@ -64,11 +74,15 @@ export interface UpdateProperties { interval: IntervalOrUndefined; maxSignals: MaxSignalsOrUndefined; riskScore: RiskScoreOrUndefined; + riskScoreMapping: RiskScoreMappingOrUndefined; + ruleNameOverride: RuleNameOverrideOrUndefined; outputIndex: OutputIndexOrUndefined; name: NameOrUndefined; severity: SeverityOrUndefined; + severityMapping: SeverityMappingOrUndefined; tags: TagsOrUndefined; threat: ThreatOrUndefined; + timestampOverride: TimestampOverrideOrUndefined; to: ToOrUndefined; type: TypeOrUndefined; references: ReferencesOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh index 432045634ba7b6..ac551781d2042a 100755 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/post_rule.sh @@ -24,7 +24,7 @@ do { -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/rules \ -d @${RULE} \ - | jq .; + | jq -S .; } & done diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json new file mode 100644 index 00000000000000..f0d7cb4ec914b4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_mappings.json @@ -0,0 +1,44 @@ +{ + "description": "Makes external events actionable within Elastic Security! 🎬", + "enabled": false, + "index": [ + "apm-*-transaction*", + "auditbeat-*", + "endgame-*", + "filebeat-*", + "packetbeat-*", + "winlogbeat-*" + ], + "language": "kuery", + "risk_score": 50, + "severity": "high", + "name": "External alerts", + "query": "event.type: \"alert\"", + "type": "query", + "author": ["Elastic"], + "building_block_type": "default", + "license": "Elastic License", + "risk_score_mapping": [ + { + "field": "event.risk_score", + "operator": "equals", + "value": "0" + } + ], + "rule_name_override": "event.message", + "severity_mapping":[ + { + "field": "event.severity", + "operator": "equals", + "value": "low", + "severity": "low" + }, + { + "field": "event.severity", + "operator": "equals", + "value": "medium", + "severity": "medium" + } + ], + "timestamp_override": "event.ingested" +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 50f6e7d9e9c10f..7492422968062b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -20,6 +20,8 @@ export const sampleRuleAlertParams = ( maxSignals?: number | undefined, riskScore?: number | undefined ): RuleTypeParams => ({ + author: ['Elastic'], + buildingBlockType: 'default', ruleId: 'rule-1', description: 'Detecting root and admin users', falsePositives: [], @@ -29,11 +31,15 @@ export const sampleRuleAlertParams = ( from: 'now-6m', to: 'now', severity: 'high', + severityMapping: [], query: 'user.name: root or user.name: admin', language: 'kuery', + license: 'Elastic License', outputIndex: '.siem-signals', references: ['http://google.com'], riskScore: riskScore ? riskScore : 50, + riskScoreMapping: [], + ruleNameOverride: undefined, maxSignals: maxSignals ? maxSignals : 10000, note: '', anomalyThreshold: undefined, @@ -42,6 +48,7 @@ export const sampleRuleAlertParams = ( savedId: undefined, timelineId: undefined, timelineTitle: undefined, + timestampOverride: undefined, meta: undefined, threat: undefined, version: 1, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index ad439328188364..e840ae96cf3c18 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -64,11 +64,14 @@ describe('buildBulkBody', () => { status: 'open', rule: { actions: [], + author: ['Elastic'], + building_block_type: 'default', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', rule_id: 'rule-1', false_positives: [], max_signals: 10000, risk_score: 50, + risk_score_mapping: [], output_index: '.siem-signals', description: 'Detecting root and admin users', from: 'now-6m', @@ -76,10 +79,12 @@ describe('buildBulkBody', () => { index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', language: 'kuery', + license: 'Elastic License', name: 'rule-name', query: 'user.name: root or user.name: admin', references: ['http://google.com'], severity: 'high', + severity_mapping: [], tags: ['some fake tag 1', 'some fake tag 2'], threat: [], throttle: 'no_actions', @@ -160,11 +165,14 @@ describe('buildBulkBody', () => { status: 'open', rule: { actions: [], + author: ['Elastic'], + building_block_type: 'default', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', rule_id: 'rule-1', false_positives: [], max_signals: 10000, risk_score: 50, + risk_score_mapping: [], output_index: '.siem-signals', description: 'Detecting root and admin users', from: 'now-6m', @@ -172,10 +180,12 @@ describe('buildBulkBody', () => { index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', language: 'kuery', + license: 'Elastic License', name: 'rule-name', query: 'user.name: root or user.name: admin', references: ['http://google.com'], severity: 'high', + severity_mapping: [], tags: ['some fake tag 1', 'some fake tag 2'], type: 'query', to: 'now', @@ -254,11 +264,14 @@ describe('buildBulkBody', () => { status: 'open', rule: { actions: [], + author: ['Elastic'], + building_block_type: 'default', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', rule_id: 'rule-1', false_positives: [], max_signals: 10000, risk_score: 50, + risk_score_mapping: [], output_index: '.siem-signals', description: 'Detecting root and admin users', from: 'now-6m', @@ -266,10 +279,12 @@ describe('buildBulkBody', () => { index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', language: 'kuery', + license: 'Elastic License', name: 'rule-name', query: 'user.name: root or user.name: admin', references: ['http://google.com'], severity: 'high', + severity_mapping: [], threat: [], tags: ['some fake tag 1', 'some fake tag 2'], type: 'query', @@ -341,11 +356,14 @@ describe('buildBulkBody', () => { status: 'open', rule: { actions: [], + author: ['Elastic'], + building_block_type: 'default', id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', rule_id: 'rule-1', false_positives: [], max_signals: 10000, risk_score: 50, + risk_score_mapping: [], output_index: '.siem-signals', description: 'Detecting root and admin users', from: 'now-6m', @@ -353,10 +371,12 @@ describe('buildBulkBody', () => { index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', language: 'kuery', + license: 'Elastic License', name: 'rule-name', query: 'user.name: root or user.name: admin', references: ['http://google.com'], severity: 'high', + severity_mapping: [], tags: ['some fake tag 1', 'some fake tag 2'], threat: [], type: 'query', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index 9aef5a370b86a4..ed632ee2576dc8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -43,6 +43,8 @@ describe('buildRule', () => { }); const expected: Partial = { actions: [], + author: ['Elastic'], + building_block_type: 'default', created_by: 'elastic', description: 'Detecting root and admin users', enabled: false, @@ -53,14 +55,17 @@ describe('buildRule', () => { index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: 'some interval', language: 'kuery', + license: 'Elastic License', max_signals: 10000, name: 'some-name', output_index: '.siem-signals', query: 'user.name: root or user.name: admin', references: ['http://google.com'], risk_score: 50, + risk_score_mapping: [], rule_id: 'rule-1', severity: 'high', + severity_mapping: [], tags: ['some fake tag 1', 'some fake tag 2'], threat: [], to: 'now', @@ -106,6 +111,8 @@ describe('buildRule', () => { }); const expected: Partial = { actions: [], + author: ['Elastic'], + building_block_type: 'default', created_by: 'elastic', description: 'Detecting root and admin users', enabled: true, @@ -116,14 +123,17 @@ describe('buildRule', () => { index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: 'some interval', language: 'kuery', + license: 'Elastic License', max_signals: 10000, name: 'some-name', output_index: '.siem-signals', query: 'user.name: root or user.name: admin', references: ['http://google.com'], risk_score: 50, + risk_score_mapping: [], rule_id: 'rule-1', severity: 'high', + severity_mapping: [], tags: ['some fake tag 1', 'some fake tag 2'], threat: [], to: 'now', @@ -158,6 +168,8 @@ describe('buildRule', () => { }); const expected: Partial = { actions: [], + author: ['Elastic'], + building_block_type: 'default', created_by: 'elastic', description: 'Detecting root and admin users', enabled: true, @@ -168,6 +180,7 @@ describe('buildRule', () => { index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: 'some interval', language: 'kuery', + license: 'Elastic License', max_signals: 10000, name: 'some-name', note: '', @@ -175,8 +188,10 @@ describe('buildRule', () => { query: 'user.name: root or user.name: admin', references: ['http://google.com'], risk_score: 50, + risk_score_mapping: [], rule_id: 'rule-1', severity: 'high', + severity_mapping: [], tags: ['some fake tag 1', 'some fake tag 2'], threat: [], to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index bde9c970b0c8c3..fc8b26450c8522 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -42,13 +42,16 @@ export const buildRule = ({ id, rule_id: ruleParams.ruleId ?? '(unknown rule_id)', actions, + author: ruleParams.author ?? [], + building_block_type: ruleParams.buildingBlockType, false_positives: ruleParams.falsePositives, saved_id: ruleParams.savedId, timeline_id: ruleParams.timelineId, timeline_title: ruleParams.timelineTitle, meta: ruleParams.meta, max_signals: ruleParams.maxSignals, - risk_score: ruleParams.riskScore, + risk_score: ruleParams.riskScore, // TODO: Risk Score Override via risk_score_mapping + risk_score_mapping: ruleParams.riskScoreMapping ?? [], output_index: ruleParams.outputIndex, description: ruleParams.description, note: ruleParams.note, @@ -57,10 +60,13 @@ export const buildRule = ({ index: ruleParams.index, interval, language: ruleParams.language, - name, + license: ruleParams.license, + name, // TODO: Rule Name Override via rule_name_override query: ruleParams.query, references: ruleParams.references, - severity: ruleParams.severity, + rule_name_override: ruleParams.ruleNameOverride, + severity: ruleParams.severity, // TODO: Severity Override via severity_mapping + severity_mapping: ruleParams.severityMapping ?? [], tags, type: ruleParams.type, to: ruleParams.to, @@ -69,6 +75,7 @@ export const buildRule = ({ created_by: createdBy, updated_by: updatedBy, threat: ruleParams.threat ?? [], + timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override throttle, version: ruleParams.version, created_at: createdAt, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts index d60509b28f7da6..0c56ed300cb483 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts @@ -19,6 +19,8 @@ export const getSignalParamsSchemaMock = (): Partial => ({ }); export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ + author: [], + buildingBlockType: null, description: 'Detecting root and admin users', falsePositives: [], filters: null, @@ -26,6 +28,7 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ immutable: false, index: null, language: 'kuery', + license: null, maxSignals: 100, meta: null, note: null, @@ -33,12 +36,16 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ query: 'user.name: root or user.name: admin', references: [], riskScore: 55, + riskScoreMapping: null, + ruleNameOverride: null, ruleId: 'rule-1', savedId: null, severity: 'high', + severityMapping: null, threat: null, timelineId: null, timelineTitle: null, + timestampOverride: null, to: 'now', type: 'query', version: 1, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index 5f95f635a6bd85..2583cf2c8da912 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -10,6 +10,8 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../common/constants'; const signalSchema = schema.object({ anomalyThreshold: schema.maybe(schema.number()), + author: schema.arrayOf(schema.string(), { defaultValue: [] }), + buildingBlockType: schema.nullable(schema.string()), description: schema.string(), note: schema.nullable(schema.string()), falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }), @@ -18,6 +20,7 @@ const signalSchema = schema.object({ immutable: schema.boolean({ defaultValue: false }), index: schema.nullable(schema.arrayOf(schema.string())), language: schema.nullable(schema.string()), + license: schema.nullable(schema.string()), outputIndex: schema.nullable(schema.string()), savedId: schema.nullable(schema.string()), timelineId: schema.nullable(schema.string()), @@ -28,8 +31,13 @@ const signalSchema = schema.object({ filters: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), maxSignals: schema.number({ defaultValue: DEFAULT_MAX_SIGNALS }), riskScore: schema.number(), + // TODO: Specify types explicitly since they're known? + riskScoreMapping: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + ruleNameOverride: schema.nullable(schema.string()), severity: schema.string(), + severityMapping: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threat: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), + timestampOverride: schema.nullable(schema.string()), to: schema.string(), type: schema.string(), references: schema.arrayOf(schema.string(), { defaultValue: [] }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index 0fb743c9c3ed67..365222d62d3224 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -28,6 +28,13 @@ import { Version, MetaOrUndefined, RuleId, + AuthorOrUndefined, + BuildingBlockTypeOrUndefined, + LicenseOrUndefined, + RiskScoreMappingOrUndefined, + RuleNameOverrideOrUndefined, + SeverityMappingOrUndefined, + TimestampOverrideOrUndefined, } from '../../../common/detection_engine/schemas/common/schemas'; import { LegacyCallAPIOptions } from '../../../../../../src/core/server'; import { Filter } from '../../../../../../src/plugins/data/server'; @@ -38,6 +45,8 @@ export type PartialFilter = Partial; export interface RuleTypeParams { anomalyThreshold: AnomalyThresholdOrUndefined; + author: AuthorOrUndefined; + buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; note: NoteOrUndefined; falsePositives: FalsePositives; @@ -46,6 +55,7 @@ export interface RuleTypeParams { immutable: Immutable; index: IndexOrUndefined; language: LanguageOrUndefined; + license: LicenseOrUndefined; outputIndex: OutputIndex; savedId: SavedIdOrUndefined; timelineId: TimelineIdOrUndefined; @@ -56,8 +66,12 @@ export interface RuleTypeParams { filters: PartialFilter[] | undefined; maxSignals: MaxSignals; riskScore: RiskScore; + riskScoreMapping: RiskScoreMappingOrUndefined; + ruleNameOverride: RuleNameOverrideOrUndefined; severity: Severity; + severityMapping: SeverityMappingOrUndefined; threat: ThreatOrUndefined; + timestampOverride: TimestampOverrideOrUndefined; to: To; type: RuleType; references: References; diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 816df9c133ea1e..6ad9cf4cd5baf4 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -179,6 +179,7 @@ export const binaryToString = (res: any, callback: any): void => { */ export const getSimpleRuleOutput = (ruleId = 'rule-1'): Partial => ({ actions: [], + author: [], created_by: 'elastic', description: 'Simple Rule Query', enabled: true, @@ -192,10 +193,12 @@ export const getSimpleRuleOutput = (ruleId = 'rule-1'): Partial => output_index: '.siem-signals-default', max_signals: 100, risk_score: 1, + risk_score_mapping: [], name: 'Simple Rule Query', query: 'user.name: root or user.name: admin', references: [], severity: 'high', + severity_mapping: [], updated_by: 'elastic', tags: [], to: 'now', @@ -307,6 +310,7 @@ export const ruleToNdjson = (rule: Partial): Buffer => { */ export const getComplexRule = (ruleId = 'rule-1'): Partial => ({ actions: [], + author: [], name: 'Complex Rule Query', description: 'Complex Rule Query', false_positives: [ @@ -314,6 +318,7 @@ export const getComplexRule = (ruleId = 'rule-1'): Partial => ({ 'some text string about why another condition could be a false positive', ], risk_score: 1, + risk_score_mapping: [], rule_id: ruleId, filters: [ { @@ -340,6 +345,7 @@ export const getComplexRule = (ruleId = 'rule-1'): Partial => ({ to: 'now', from: 'now-6m', severity: 'high', + severity_mapping: [], language: 'kuery', type: 'query', threat: [ @@ -391,6 +397,7 @@ export const getComplexRule = (ruleId = 'rule-1'): Partial => ({ */ export const getComplexRuleOutput = (ruleId = 'rule-1'): Partial => ({ actions: [], + author: [], created_by: 'elastic', name: 'Complex Rule Query', description: 'Complex Rule Query', @@ -399,6 +406,7 @@ export const getComplexRuleOutput = (ruleId = 'rule-1'): Partial => 'some text string about why another condition could be a false positive', ], risk_score: 1, + risk_score_mapping: [], rule_id: ruleId, filters: [ { @@ -426,6 +434,7 @@ export const getComplexRuleOutput = (ruleId = 'rule-1'): Partial => to: 'now', from: 'now-6m', severity: 'high', + severity_mapping: [], language: 'kuery', type: 'query', threat: [ From 0f7afd4402e50f2a42f8951c94ea2fae7c422906 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 2 Jul 2020 01:00:27 -0400 Subject: [PATCH 03/31] [SIEM][Security Solution][Endpoint] Endpoint Artifact Manifest Management + Artifact Download and Distribution (#67707) * stub out task for the exceptions list packager * Hits list code and pages * refactor * Begin adding saved object and type definitions * Transforms to endpoint exceptions * Get internal SO client * update messaging * cleanup * Integrating with task manager * Integrated with task manager properly * Begin adding schemas * Add multiple OS and schema version support * filter by OS * Fixing sort * Move to security_solutions * siem -> securitySolution * Progress on downloads, cleanup * Add config, update artifact creation, add TODOs * Fixing buffer serialization problem * Adding cleanup to task * Handle HEAD req * proper header * More robust task management * single -> agnostic * Fix OS filtering * Scaffolding digital signatures / tests * Adds rotue for creating endpoint user * Cleanup * persisting user * Adding route to fetch created user * Addings tests for translating exceptions * Adding test for download API * Download tweaks + artifact generation fixes * reorganize * fix imports * Fixing test * Changes id of SO * integration tests setup * Add first integration tests * Cache layer * more schema validation * Set up for manifest update * minor change * remove setup code * add manifest schema * refactoring * manifest rewrite (partial) * finish scaffolding new manifest logic * syntax errors * more refactoring * Move to endpoint directory * minor cleanup * clean up old artifacts * Use diff appropriately * Fix download * schedule task on interval * Split up into client/manager * more mocks * config interval * Fixing download tests and adding cache tests * lint * mo money, mo progress * Converting to io-ts * More tests and mocks * even more tests and mocks * Merging both refactors * Adding more tests for the convertion layer * fix conflicts * Adding lzma types * Bug fixes * lint * resolve some type errors * Adding back in cache * Fixing download test * Changing cache to be sized * Fix manifest manager initialization * Hook up datasource service * Fix download tests * Incremental progress * Adds integration with ingest manager for auth * Update test fixture * Add manifest dispatch * Refactoring to use the same SO Client from ingest * bug fixes * build renovate config * Fix endpoint_app_context_services tests * Only index the fields that are necessary for searching * Integ test progress * mock and test city * Add task tests * Tests for artifact_client and manifest_client * Add manifest_manager tests * minor refactor * Finish manifest_manager tests * Type errors * Update integ test * Type errors, final cleanup * Fix integration test and add test for invalid api key * minor fixup * Remove compression * Update task interval * Removing .text suffix from translated list * Fixes hashes for unit tests * clean up yarn.lock * Remove lzma-native from package.json * missed updating one of the tests Co-authored-by: Alex Kahan --- x-pack/plugins/lists/server/mocks.ts | 2 +- .../common/endpoint/generate_data.ts | 7 + .../common/endpoint/schema/common.ts | 22 ++ .../common/endpoint/schema/manifest.ts | 27 ++ .../common/endpoint/types.ts | 4 + x-pack/plugins/security_solution/kibana.json | 1 + .../policy/store/policy_details/index.test.ts | 7 + .../endpoint_app_context_services.test.ts | 14 +- .../endpoint/endpoint_app_context_services.ts | 33 +- .../server/endpoint/ingest_integration.ts | 82 +++-- .../endpoint/lib/artifacts/cache.test.ts | 43 +++ .../server/endpoint/lib/artifacts/cache.ts | 37 +++ .../server/endpoint/lib/artifacts/common.ts | 17 + .../server/endpoint/lib/artifacts/index.ts | 12 + .../endpoint/lib/artifacts/lists.test.ts | 196 ++++++++++++ .../server/endpoint/lib/artifacts/lists.ts | 157 +++++++++ .../endpoint/lib/artifacts/manifest.test.ts | 150 +++++++++ .../server/endpoint/lib/artifacts/manifest.ts | 130 ++++++++ .../lib/artifacts/manifest_entry.test.ts | 64 ++++ .../endpoint/lib/artifacts/manifest_entry.ts | 48 +++ .../lib/artifacts/saved_object_mappings.ts | 67 ++++ .../endpoint/lib/artifacts/task.mock.ts | 11 + .../endpoint/lib/artifacts/task.test.ts | 73 +++++ .../server/endpoint/lib/artifacts/task.ts | 107 +++++++ .../server/endpoint/mocks.ts | 48 ++- .../artifacts/download_exception_list.test.ts | 301 ++++++++++++++++++ .../artifacts/download_exception_list.ts | 107 +++++++ .../server/endpoint/routes/artifacts/index.ts | 7 + .../endpoint/schemas/artifacts/common.ts | 19 ++ .../endpoint/schemas/artifacts/index.ts | 11 + .../endpoint/schemas/artifacts/lists.mock.ts | 32 ++ .../endpoint/schemas/artifacts/lists.ts | 65 ++++ .../request/download_artifact_schema.ts | 19 ++ .../schemas/artifacts/request/index.ts | 7 + .../response/download_artifact_schema.ts | 25 ++ .../schemas/artifacts/response/index.ts | 7 + .../schemas/artifacts/saved_objects.mock.ts | 40 +++ .../schemas/artifacts/saved_objects.ts | 31 ++ .../server/endpoint/schemas/index.ts | 7 + .../artifacts/artifact_client.mock.ts | 18 ++ .../artifacts/artifact_client.test.ts | 49 +++ .../services/artifacts/artifact_client.ts | 42 +++ .../endpoint/services/artifacts/index.ts | 8 + .../artifacts/manifest_client.mock.ts | 18 ++ .../artifacts/manifest_client.test.ts | 69 ++++ .../services/artifacts/manifest_client.ts | 85 +++++ .../artifacts/manifest_manager/index.ts | 7 + .../manifest_manager/manifest_manager.mock.ts | 111 +++++++ .../manifest_manager/manifest_manager.test.ts | 82 +++++ .../manifest_manager/manifest_manager.ts | 270 ++++++++++++++++ .../server/endpoint/services/index.ts | 7 + .../server/endpoint/types.ts | 2 +- .../security_solution/server/plugin.ts | 52 ++- .../security_solution/server/saved_objects.ts | 14 +- .../apis/endpoint/artifacts/index.ts | 93 ++++++ .../endpoint/artifacts/api_feature/data.json | 179 +++++++++++ 56 files changed, 3101 insertions(+), 42 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/endpoint/schema/common.ts create mode 100644 x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/schemas/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/services/index.ts create mode 100644 x-pack/test/api_integration/apis/endpoint/artifacts/index.ts create mode 100644 x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json diff --git a/x-pack/plugins/lists/server/mocks.ts b/x-pack/plugins/lists/server/mocks.ts index aad4a25a900a1a..ba565216fe431e 100644 --- a/x-pack/plugins/lists/server/mocks.ts +++ b/x-pack/plugins/lists/server/mocks.ts @@ -18,6 +18,6 @@ const createSetupMock = (): jest.Mocked => { export const listMock = { createSetup: createSetupMock, - getExceptionList: getExceptionListClientMock, + getExceptionListClient: getExceptionListClientMock, getListClient: getListClientMock, }; diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index a6fe12a9b029fa..6720f3523d5c73 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1034,6 +1034,13 @@ export class EndpointDocGenerator { enabled: true, streams: [], config: { + artifact_manifest: { + value: { + manifest_version: 'v0', + schema_version: '1.0.0', + artifacts: {}, + }, + }, policy: { value: policyFactory(), }, diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts new file mode 100644 index 00000000000000..7f8c938d54feb5 --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const identifier = t.string; + +export const manifestVersion = t.string; + +export const manifestSchemaVersion = t.keyof({ + '1.0.0': null, +}); +export type ManifestSchemaVersion = t.TypeOf; + +export const sha256 = t.string; + +export const size = t.number; + +export const url = t.string; diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts new file mode 100644 index 00000000000000..470e9b13ef78ae --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { identifier, manifestSchemaVersion, manifestVersion, sha256, size, url } from './common'; + +export const manifestEntrySchema = t.exact( + t.type({ + url, + sha256, + size, + }) +); + +export const manifestSchema = t.exact( + t.type({ + manifest_version: manifestVersion, + schema_version: manifestSchemaVersion, + artifacts: t.record(identifier, manifestEntrySchema), + }) +); + +export type ManifestEntrySchema = t.TypeOf; +export type ManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index f76da977eef857..4efe89b2429ad6 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -5,6 +5,7 @@ */ import { PackageConfig, NewPackageConfig } from '../../../ingest_manager/common'; +import { ManifestSchema } from './schema/manifest'; /** * Object that allows you to maintain stateful information in the location object across navigation events @@ -691,6 +692,9 @@ export type NewPolicyData = NewPackageConfig & { enabled: boolean; streams: []; config: { + artifact_manifest: { + value: ManifestSchema; + }; policy: { value: PolicyConfig; }; diff --git a/x-pack/plugins/security_solution/kibana.json b/x-pack/plugins/security_solution/kibana.json index 8ce8820a8e57db..f6f2d5171312cc 100644 --- a/x-pack/plugins/security_solution/kibana.json +++ b/x-pack/plugins/security_solution/kibana.json @@ -12,6 +12,7 @@ "features", "home", "ingestManager", + "taskManager", "inspector", "licensing", "maps", diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 469b71854dfcc4..0bd623b27f4fbc 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -41,6 +41,13 @@ describe('policy details: ', () => { enabled: true, streams: [], config: { + artifact_manifest: { + value: { + manifest_version: 'v0', + schema_version: '1.0.0', + artifacts: {}, + }, + }, policy: { value: policyConfigFactory(), }, diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index 8cf2ada9907d33..2daf259941cbfb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -3,11 +3,23 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +import { httpServerMock } from '../../../../../src/core/server/mocks'; import { EndpointAppContextService } from './endpoint_app_context_services'; describe('test endpoint app context services', () => { - it('should throw error if start is not called', async () => { + it('should throw error on getAgentService if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); expect(() => endpointAppContextService.getAgentService()).toThrow(Error); }); + it('should return undefined on getManifestManager if start is not called', async () => { + const endpointAppContextService = new EndpointAppContextService(); + expect(endpointAppContextService.getManifestManager()).toEqual(undefined); + }); + it('should throw error on getScopedSavedObjectsClient if start is not called', async () => { + const endpointAppContextService = new EndpointAppContextService(); + expect(() => + endpointAppContextService.getScopedSavedObjectsClient(httpServerMock.createKibanaRequest()) + ).toThrow(Error); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 1fce5d355f5a73..97a82049634c40 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -3,14 +3,22 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { + SavedObjectsServiceStart, + KibanaRequest, + SavedObjectsClientContract, +} from 'src/core/server'; import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; -import { handlePackageConfigCreate } from './ingest_integration'; +import { getPackageConfigCreateCallback } from './ingest_integration'; +import { ManifestManager } from './services/artifacts'; export type EndpointAppContextServiceStartContract = Pick< IngestManagerStartContract, 'agentService' > & { + manifestManager?: ManifestManager | undefined; registerIngestCallback: IngestManagerStartContract['registerExternalCallback']; + savedObjectsStart: SavedObjectsServiceStart; }; /** @@ -19,10 +27,20 @@ export type EndpointAppContextServiceStartContract = Pick< */ export class EndpointAppContextService { private agentService: AgentService | undefined; + private manifestManager: ManifestManager | undefined; + private savedObjectsStart: SavedObjectsServiceStart | undefined; public start(dependencies: EndpointAppContextServiceStartContract) { this.agentService = dependencies.agentService; - dependencies.registerIngestCallback('packageConfigCreate', handlePackageConfigCreate); + this.manifestManager = dependencies.manifestManager; + this.savedObjectsStart = dependencies.savedObjectsStart; + + if (this.manifestManager !== undefined) { + dependencies.registerIngestCallback( + 'packageConfigCreate', + getPackageConfigCreateCallback(this.manifestManager) + ); + } } public stop() {} @@ -33,4 +51,15 @@ export class EndpointAppContextService { } return this.agentService; } + + public getManifestManager(): ManifestManager | undefined { + return this.manifestManager; + } + + public getScopedSavedObjectsClient(req: KibanaRequest): SavedObjectsClientContract { + if (!this.savedObjectsStart) { + throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); + } + return this.savedObjectsStart.getScopedClient(req, { excludedWrappers: ['security'] }); + } } diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index bb0b950dad0196..67a331f4ba6771 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -4,46 +4,62 @@ * you may not use this file except in compliance with the Elastic License. */ +import { NewPackageConfig } from '../../../ingest_manager/common/types/models'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; -import { NewPackageConfig } from '../../../ingest_manager/common/types/models'; +import { ManifestManager } from './services/artifacts'; /** - * Callback to handle creation of package configs in Ingest Manager - * @param newPackageConfig + * Callback to handle creation of PackageConfigs in Ingest Manager */ -export const handlePackageConfigCreate = async ( - newPackageConfig: NewPackageConfig -): Promise => { - // We only care about Endpoint package configs - if (newPackageConfig.package?.name !== 'endpoint') { - return newPackageConfig; - } +export const getPackageConfigCreateCallback = ( + manifestManager: ManifestManager +): ((newPackageConfig: NewPackageConfig) => Promise) => { + const handlePackageConfigCreate = async ( + newPackageConfig: NewPackageConfig + ): Promise => { + // We only care about Endpoint package configs + if (newPackageConfig.package?.name !== 'endpoint') { + return newPackageConfig; + } - // We cast the type here so that any changes to the Endpoint specific data - // follow the types/schema expected - let updatedPackageConfig = newPackageConfig as NewPolicyData; + // We cast the type here so that any changes to the Endpoint specific data + // follow the types/schema expected + let updatedPackageConfig = newPackageConfig as NewPolicyData; - // Until we get the Default Policy Configuration in the Endpoint package, - // we will add it here manually at creation time. - // @ts-ignore - if (newPackageConfig.inputs.length === 0) { - updatedPackageConfig = { - ...newPackageConfig, - inputs: [ - { - type: 'endpoint', - enabled: true, - streams: [], - config: { - policy: { - value: policyConfigFactory(), + const wrappedManifest = await manifestManager.refresh({ initialize: true }); + if (wrappedManifest !== null) { + // Until we get the Default Policy Configuration in the Endpoint package, + // we will add it here manually at creation time. + // @ts-ignore + if (newPackageConfig.inputs.length === 0) { + updatedPackageConfig = { + ...newPackageConfig, + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + artifact_manifest: { + value: wrappedManifest.manifest.toEndpointFormat(), + }, + policy: { + value: policyConfigFactory(), + }, + }, }, - }, - }, - ], - }; - } + ], + }; + } + } + + try { + return updatedPackageConfig; + } finally { + await manifestManager.commit(wrappedManifest); + } + }; - return updatedPackageConfig; + return handlePackageConfigCreate; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts new file mode 100644 index 00000000000000..5a0fb913455529 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ExceptionsCache } from './cache'; + +describe('ExceptionsCache tests', () => { + let cache: ExceptionsCache; + + beforeEach(() => { + jest.clearAllMocks(); + cache = new ExceptionsCache(3); + }); + + test('it should cache', async () => { + cache.set('test', 'body'); + const cacheResp = cache.get('test'); + expect(cacheResp).toEqual('body'); + }); + + test('it should handle cache miss', async () => { + cache.set('test', 'body'); + const cacheResp = cache.get('not test'); + expect(cacheResp).toEqual(undefined); + }); + + test('it should handle cache eviction', async () => { + cache.set('1', 'a'); + cache.set('2', 'b'); + cache.set('3', 'c'); + const cacheResp = cache.get('1'); + expect(cacheResp).toEqual('a'); + + cache.set('4', 'd'); + const secondResp = cache.get('1'); + expect(secondResp).toEqual(undefined); + expect(cache.get('2')).toEqual('b'); + expect(cache.get('3')).toEqual('c'); + expect(cache.get('4')).toEqual('d'); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts new file mode 100644 index 00000000000000..b7a4c2feb6bf84 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +const DEFAULT_MAX_SIZE = 10; + +/** + * FIFO cache implementation for artifact downloads. + */ +export class ExceptionsCache { + private cache: Map; + private queue: string[]; + private maxSize: number; + + constructor(maxSize: number) { + this.cache = new Map(); + this.queue = []; + this.maxSize = maxSize || DEFAULT_MAX_SIZE; + } + + set(id: string, body: string) { + if (this.queue.length + 1 > this.maxSize) { + const entry = this.queue.shift(); + if (entry !== undefined) { + this.cache.delete(entry); + } + } + this.queue.push(id); + this.cache.set(id, body); + } + + get(id: string): string | undefined { + return this.cache.get(id); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts new file mode 100644 index 00000000000000..4c3153ca0ef116 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const ArtifactConstants = { + GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', + SAVED_OBJECT_TYPE: 'endpoint:exceptions-artifact', + SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], + SCHEMA_VERSION: '1.0.0', +}; + +export const ManifestConstants = { + SAVED_OBJECT_TYPE: 'endpoint:exceptions-manifest', + SCHEMA_VERSION: '1.0.0', +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts new file mode 100644 index 00000000000000..ee7d44459aa385 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './cache'; +export * from './common'; +export * from './lists'; +export * from './manifest'; +export * from './manifest_entry'; +export * from './task'; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts new file mode 100644 index 00000000000000..738890fb4038f1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ExceptionListClient } from '../../../../../lists/server'; +import { listMock } from '../../../../../lists/server/mocks'; +import { getFoundExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; +import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { EntriesArray, EntryList } from '../../../../../lists/common/schemas/types/entries'; +import { getFullEndpointExceptionList } from './lists'; + +describe('buildEventTypeSignal', () => { + let mockExceptionClient: ExceptionListClient; + + beforeEach(() => { + jest.clearAllMocks(); + mockExceptionClient = listMock.getExceptionListClient(); + }); + + test('it should convert the exception lists response to the proper endpoint format', async () => { + const expectedEndpointExceptions = { + exceptions_list: [ + { + entries: [ + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should convert simple fields', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + { field: 'host.hostname', operator: 'included', type: 'match', value: 'estc' }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_cased', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + { + field: 'host.hostname', + operator: 'included', + type: 'exact_cased', + value: 'estc', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should convert fields case sensitive', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + { + field: 'host.hostname.text', + operator: 'included', + type: 'match_any', + value: ['estc', 'kibana'], + }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_caseless', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + { + field: 'host.hostname', + operator: 'included', + type: 'exact_caseless_any', + value: ['estc', 'kibana'], + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should ignore unsupported entries', async () => { + // Lists and exists are not supported by the Endpoint + const testEntries: EntriesArray = [ + { field: 'server.domain', operator: 'included', type: 'match', value: 'DOMAIN' }, + { + field: 'server.domain', + operator: 'included', + type: 'list', + list: { + id: 'lists_not_supported', + type: 'keyword', + }, + } as EntryList, + { field: 'server.ip', operator: 'included', type: 'exists' }, + ]; + + const expectedEndpointExceptions = { + exceptions_list: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_cased', + value: 'DOMAIN', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp).toEqual(expectedEndpointExceptions); + }); + + test('it should convert the exception lists response to the proper endpoint format while paging', async () => { + // The first call returns one exception + const first = getFoundExceptionListItemSchemaMock(); + + // The second call returns two exceptions + const second = getFoundExceptionListItemSchemaMock(); + second.data.push(getExceptionListItemSchemaMock()); + + // The third call returns no exceptions, paging stops + const third = getFoundExceptionListItemSchemaMock(); + third.data = []; + mockExceptionClient.findExceptionListItem = jest + .fn() + .mockReturnValueOnce(first) + .mockReturnValueOnce(second) + .mockReturnValueOnce(third); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp.exceptions_list.length).toEqual(6); + }); + + test('it should handle no exceptions', async () => { + const exceptionsResponse = getFoundExceptionListItemSchemaMock(); + exceptionsResponse.data = []; + exceptionsResponse.total = 0; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + expect(resp.exceptions_list.length).toEqual(0); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts new file mode 100644 index 00000000000000..7fd057afdbd55f --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; +import { validate } from '../../../../common/validate'; + +import { + Entry, + EntryNested, + EntryMatch, + EntryMatchAny, +} from '../../../../../lists/common/schemas/types/entries'; +import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; +import { ExceptionListClient } from '../../../../../lists/server'; +import { + InternalArtifactSchema, + TranslatedEntry, + TranslatedEntryMatch, + TranslatedEntryMatchAny, + TranslatedEntryNested, + WrappedTranslatedExceptionList, + wrappedExceptionList, +} from '../../schemas'; +import { ArtifactConstants } from './common'; + +export async function buildArtifact( + exceptions: WrappedTranslatedExceptionList, + os: string, + schemaVersion: string +): Promise { + const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions)); + const sha256 = createHash('sha256').update(exceptionsBuffer.toString()).digest('hex'); + + return { + identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, + sha256, + encoding: 'application/json', + created: Date.now(), + body: exceptionsBuffer.toString('base64'), + size: exceptionsBuffer.byteLength, + }; +} + +export async function getFullEndpointExceptionList( + eClient: ExceptionListClient, + os: string, + schemaVersion: string +): Promise { + const exceptions: WrappedTranslatedExceptionList = { exceptions_list: [] }; + let numResponses = 0; + let page = 1; + + do { + const response = await eClient.findExceptionListItem({ + listId: 'endpoint_list', + namespaceType: 'agnostic', + filter: `exception-list-agnostic.attributes._tags:\"os:${os}\"`, + perPage: 100, + page, + sortField: 'created_at', + sortOrder: 'desc', + }); + + if (response?.data !== undefined) { + numResponses = response.data.length; + + exceptions.exceptions_list = exceptions.exceptions_list.concat( + translateToEndpointExceptions(response, schemaVersion) + ); + + page++; + } else { + break; + } + } while (numResponses > 0); + + const [validated, errors] = validate(exceptions, wrappedExceptionList); + if (errors != null) { + throw new Error(errors); + } + return validated as WrappedTranslatedExceptionList; +} + +/** + * Translates Exception list items to Exceptions the endpoint can understand + * @param exc + */ +export function translateToEndpointExceptions( + exc: FoundExceptionListItemSchema, + schemaVersion: string +): TranslatedEntry[] { + const translatedList: TranslatedEntry[] = []; + + if (schemaVersion === '1.0.0') { + exc.data.forEach((list) => { + list.entries.forEach((entry) => { + const tEntry = translateEntry(schemaVersion, entry); + if (tEntry !== undefined) { + translatedList.push(tEntry); + } + }); + }); + } else { + throw new Error('unsupported schemaVersion'); + } + return translatedList; +} + +function translateEntry( + schemaVersion: string, + entry: Entry | EntryNested +): TranslatedEntry | undefined { + let translatedEntry; + switch (entry.type) { + case 'nested': { + const e = (entry as unknown) as EntryNested; + const nestedEntries: TranslatedEntry[] = []; + for (const nestedEntry of e.entries) { + const translation = translateEntry(schemaVersion, nestedEntry); + if (translation !== undefined) { + nestedEntries.push(translation); + } + } + translatedEntry = { + entries: nestedEntries, + field: e.field, + type: 'nested', + } as TranslatedEntryNested; + break; + } + case 'match': { + const e = (entry as unknown) as EntryMatch; + translatedEntry = { + field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field, + operator: e.operator, + type: e.field.endsWith('.text') ? 'exact_caseless' : 'exact_cased', + value: e.value, + } as TranslatedEntryMatch; + break; + } + case 'match_any': + { + const e = (entry as unknown) as EntryMatchAny; + translatedEntry = { + field: e.field.endsWith('.text') ? e.field.substring(0, e.field.length - 5) : e.field, + operator: e.operator, + type: e.field.endsWith('.text') ? 'exact_caseless_any' : 'exact_cased_any', + value: e.value, + } as TranslatedEntryMatchAny; + } + break; + } + return translatedEntry || undefined; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts new file mode 100644 index 00000000000000..0434e3d8ffcb26 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; +import { InternalArtifactSchema } from '../../schemas'; +import { + getInternalArtifactMock, + getInternalArtifactMockWithDiffs, +} from '../../schemas/artifacts/saved_objects.mock'; +import { Manifest } from './manifest'; + +describe('manifest', () => { + describe('Manifest object sanity checks', () => { + const artifacts: InternalArtifactSchema[] = []; + const now = new Date(); + let manifest1: Manifest; + let manifest2: Manifest; + + beforeAll(async () => { + const artifactLinux = await getInternalArtifactMock('linux', '1.0.0'); + const artifactMacos = await getInternalArtifactMock('macos', '1.0.0'); + const artifactWindows = await getInternalArtifactMock('windows', '1.0.0'); + artifacts.push(artifactLinux); + artifacts.push(artifactMacos); + artifacts.push(artifactWindows); + + manifest1 = new Manifest(now, '1.0.0', 'v0'); + manifest1.addEntry(artifactLinux); + manifest1.addEntry(artifactMacos); + manifest1.addEntry(artifactWindows); + manifest1.setVersion('abcd'); + + const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', '1.0.0'); + manifest2 = new Manifest(new Date(), '1.0.0', 'v0'); + manifest2.addEntry(newArtifactLinux); + manifest2.addEntry(artifactMacos); + manifest2.addEntry(artifactWindows); + }); + + test('Can create manifest with valid schema version', () => { + const manifest = new Manifest(new Date(), '1.0.0', 'v0'); + expect(manifest).toBeInstanceOf(Manifest); + }); + + test('Cannot create manifest with invalid schema version', () => { + expect(() => { + new Manifest(new Date(), 'abcd' as ManifestSchemaVersion, 'v0'); + }).toThrow(); + }); + + test('Manifest transforms correctly to expected endpoint format', async () => { + expect(manifest1.toEndpointFormat()).toStrictEqual({ + artifacts: { + 'endpoint-exceptionlist-linux-1.0.0': { + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + size: 268, + url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + }, + 'endpoint-exceptionlist-macos-1.0.0': { + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + size: 268, + url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + }, + 'endpoint-exceptionlist-windows-1.0.0': { + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + size: 268, + url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + }, + }, + manifest_version: 'abcd', + schema_version: '1.0.0', + }); + }); + + test('Manifest transforms correctly to expected saved object format', async () => { + expect(manifest1.toSavedObject()).toStrictEqual({ + created: now.getTime(), + ids: [ + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + ], + }); + }); + + test('Manifest returns diffs since supplied manifest', async () => { + const diffs = manifest2.diff(manifest1); + expect(diffs).toEqual([ + { + id: + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + type: 'delete', + }, + { + id: + 'endpoint-exceptionlist-linux-1.0.0-69328f83418f4957470640ed6cc605be6abb5fe80e0e388fd74f9764ad7ed5d1', + type: 'add', + }, + ]); + }); + + test('Manifest returns data for given artifact', async () => { + const artifact = artifacts[0]; + const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.sha256}`); + expect(returned).toEqual(artifact); + }); + + test('Manifest returns entries map', async () => { + const entries = manifest1.getEntries(); + const keys = Object.keys(entries); + expect(keys).toEqual([ + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + ]); + }); + + test('Manifest returns true if contains artifact', async () => { + const found = manifest1.contains( + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + expect(found).toEqual(true); + }); + + test('Manifest can be created from list of artifacts', async () => { + const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); + expect( + manifest.contains( + 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ) + ).toEqual(true); + expect( + manifest.contains( + 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ) + ).toEqual(true); + expect( + manifest.contains( + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ) + ).toEqual(true); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts new file mode 100644 index 00000000000000..c343568226e229 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { validate } from '../../../../common/validate'; +import { InternalArtifactSchema, InternalManifestSchema } from '../../schemas/artifacts'; +import { + manifestSchemaVersion, + ManifestSchemaVersion, +} from '../../../../common/endpoint/schema/common'; +import { ManifestSchema, manifestSchema } from '../../../../common/endpoint/schema/manifest'; +import { ManifestEntry } from './manifest_entry'; + +export interface ManifestDiff { + type: string; + id: string; +} + +export class Manifest { + private created: Date; + private entries: Record; + private schemaVersion: ManifestSchemaVersion; + + // For concurrency control + private version: string; + + constructor(created: Date, schemaVersion: string, version: string) { + this.created = created; + this.entries = {}; + this.version = version; + + const [validated, errors] = validate( + (schemaVersion as unknown) as object, + manifestSchemaVersion + ); + + if (errors != null || validated === null) { + throw new Error(`Invalid manifest version: ${schemaVersion}`); + } + + this.schemaVersion = validated; + } + + public static fromArtifacts( + artifacts: InternalArtifactSchema[], + schemaVersion: string, + version: string + ): Manifest { + const manifest = new Manifest(new Date(), schemaVersion, version); + artifacts.forEach((artifact) => { + manifest.addEntry(artifact); + }); + return manifest; + } + + public getSchemaVersion(): ManifestSchemaVersion { + return this.schemaVersion; + } + + public getVersion(): string { + return this.version; + } + + public setVersion(version: string) { + this.version = version; + } + + public addEntry(artifact: InternalArtifactSchema) { + const entry = new ManifestEntry(artifact); + this.entries[entry.getDocId()] = entry; + } + + public contains(artifactId: string): boolean { + return artifactId in this.entries; + } + + public getEntries(): Record { + return this.entries; + } + + public getArtifact(artifactId: string): InternalArtifactSchema { + return this.entries[artifactId].getArtifact(); + } + + public diff(manifest: Manifest): ManifestDiff[] { + const diffs: ManifestDiff[] = []; + + for (const id in manifest.getEntries()) { + if (!this.contains(id)) { + diffs.push({ type: 'delete', id }); + } + } + + for (const id in this.entries) { + if (!manifest.contains(id)) { + diffs.push({ type: 'add', id }); + } + } + + return diffs; + } + + public toEndpointFormat(): ManifestSchema { + const manifestObj: ManifestSchema = { + manifest_version: this.version ?? 'v0', + schema_version: this.schemaVersion, + artifacts: {}, + }; + + for (const entry of Object.values(this.entries)) { + manifestObj.artifacts[entry.getIdentifier()] = entry.getRecord(); + } + + const [validated, errors] = validate(manifestObj, manifestSchema); + if (errors != null) { + throw new Error(errors); + } + + return validated as ManifestSchema; + } + + public toSavedObject(): InternalManifestSchema { + return { + created: this.created.getTime(), + ids: Object.keys(this.entries), + }; + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts new file mode 100644 index 00000000000000..34bd2b0f388e1c --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InternalArtifactSchema } from '../../schemas'; +import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; +import { ManifestEntry } from './manifest_entry'; + +describe('manifest_entry', () => { + describe('ManifestEntry object sanity checks', () => { + let artifact: InternalArtifactSchema; + let manifestEntry: ManifestEntry; + + beforeAll(async () => { + artifact = await getInternalArtifactMock('windows', '1.0.0'); + manifestEntry = new ManifestEntry(artifact); + }); + + test('Can create manifest entry', () => { + expect(manifestEntry).toBeInstanceOf(ManifestEntry); + }); + + test('Correct doc_id is returned', () => { + expect(manifestEntry.getDocId()).toEqual( + 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + }); + + test('Correct identifier is returned', () => { + expect(manifestEntry.getIdentifier()).toEqual('endpoint-exceptionlist-windows-1.0.0'); + }); + + test('Correct sha256 is returned', () => { + expect(manifestEntry.getSha256()).toEqual( + '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + }); + + test('Correct size is returned', () => { + expect(manifestEntry.getSize()).toEqual(268); + }); + + test('Correct url is returned', () => { + expect(manifestEntry.getUrl()).toEqual( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + ); + }); + + test('Correct artifact is returned', () => { + expect(manifestEntry.getArtifact()).toEqual(artifact); + }); + + test('Correct record is returned', () => { + expect(manifestEntry.getRecord()).toEqual({ + sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + size: 268, + url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts new file mode 100644 index 00000000000000..00fd446bf14b51 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { InternalArtifactSchema } from '../../schemas/artifacts'; +import { ManifestEntrySchema } from '../../../../common/endpoint/schema/manifest'; + +export class ManifestEntry { + private artifact: InternalArtifactSchema; + + constructor(artifact: InternalArtifactSchema) { + this.artifact = artifact; + } + + public getDocId(): string { + return `${this.getIdentifier()}-${this.getSha256()}`; + } + + public getIdentifier(): string { + return this.artifact.identifier; + } + + public getSha256(): string { + return this.artifact.sha256; + } + + public getSize(): number { + return this.artifact.size; + } + + public getUrl(): string { + return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getSha256()}`; + } + + public getArtifact(): InternalArtifactSchema { + return this.artifact; + } + + public getRecord(): ManifestEntrySchema { + return { + sha256: this.getSha256(), + size: this.getSize(), + url: this.getUrl(), + }; + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts new file mode 100644 index 00000000000000..d38026fbcbbd90 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsType } from '../../../../../../../src/core/server'; + +import { ArtifactConstants, ManifestConstants } from './common'; + +export const exceptionsArtifactSavedObjectType = ArtifactConstants.SAVED_OBJECT_TYPE; +export const manifestSavedObjectType = ManifestConstants.SAVED_OBJECT_TYPE; + +export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] = { + properties: { + identifier: { + type: 'keyword', + }, + sha256: { + type: 'keyword', + }, + encoding: { + type: 'keyword', + index: false, + }, + created: { + type: 'date', + index: false, + }, + body: { + type: 'binary', + index: false, + }, + size: { + type: 'long', + index: false, + }, + }, +}; + +export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { + properties: { + created: { + type: 'date', + index: false, + }, + // array of doc ids + ids: { + type: 'keyword', + index: false, + }, + }, +}; + +export const exceptionsArtifactType: SavedObjectsType = { + name: exceptionsArtifactSavedObjectType, + hidden: false, // TODO: should these be hidden? + namespaceType: 'agnostic', + mappings: exceptionsArtifactSavedObjectMappings, +}; + +export const manifestType: SavedObjectsType = { + name: manifestSavedObjectType, + hidden: false, // TODO: should these be hidden? + namespaceType: 'agnostic', + mappings: manifestSavedObjectMappings, +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts new file mode 100644 index 00000000000000..4391d89f3b2b28 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.mock.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ManifestTask } from './task'; + +export class MockManifestTask extends ManifestTask { + public runTask = jest.fn(); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts new file mode 100644 index 00000000000000..daa8a7dd83ee03 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { taskManagerMock } from '../../../../../task_manager/server/mocks'; +import { TaskStatus } from '../../../../../task_manager/server'; + +import { createMockEndpointAppContext } from '../../mocks'; + +import { ManifestTaskConstants, ManifestTask } from './task'; +import { MockManifestTask } from './task.mock'; + +describe('task', () => { + describe('Periodic task sanity checks', () => { + test('can create task', () => { + const manifestTask = new ManifestTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: taskManagerMock.createSetup(), + }); + expect(manifestTask).toBeInstanceOf(ManifestTask); + }); + + test('task should be registered', () => { + const mockTaskManager = taskManagerMock.createSetup(); + new ManifestTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: mockTaskManager, + }); + expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); + }); + + test('task should be scheduled', async () => { + const mockTaskManagerSetup = taskManagerMock.createSetup(); + const manifestTask = new ManifestTask({ + endpointAppContext: createMockEndpointAppContext(), + taskManager: mockTaskManagerSetup, + }); + const mockTaskManagerStart = taskManagerMock.createStart(); + manifestTask.start({ taskManager: mockTaskManagerStart }); + expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); + }); + + test('task should run', async () => { + const mockContext = createMockEndpointAppContext(); + const mockTaskManager = taskManagerMock.createSetup(); + const mockManifestTask = new MockManifestTask({ + endpointAppContext: mockContext, + taskManager: mockTaskManager, + }); + const mockTaskInstance = { + id: ManifestTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: ManifestTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ManifestTaskConstants.TYPE] + .createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(mockManifestTask.runTask).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts new file mode 100644 index 00000000000000..08d02e70dac168 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'src/core/server'; +import { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../../task_manager/server'; +import { EndpointAppContext } from '../../types'; + +export const ManifestTaskConstants = { + TIMEOUT: '1m', + TYPE: 'securitySolution:endpoint:exceptions-packager', + VERSION: '1.0.0', +}; + +export interface ManifestTaskSetupContract { + endpointAppContext: EndpointAppContext; + taskManager: TaskManagerSetupContract; +} + +export interface ManifestTaskStartContract { + taskManager: TaskManagerStartContract; +} + +export class ManifestTask { + private endpointAppContext: EndpointAppContext; + private logger: Logger; + + constructor(setupContract: ManifestTaskSetupContract) { + this.endpointAppContext = setupContract.endpointAppContext; + this.logger = this.endpointAppContext.logFactory.get(this.getTaskId()); + + setupContract.taskManager.registerTaskDefinitions({ + [ManifestTaskConstants.TYPE]: { + title: 'Security Solution Endpoint Exceptions Handler', + type: ManifestTaskConstants.TYPE, + timeout: ManifestTaskConstants.TIMEOUT, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + run: async () => { + await this.runTask(taskInstance.id); + }, + cancel: async () => {}, + }; + }, + }, + }); + } + + public start = async (startContract: ManifestTaskStartContract) => { + try { + await startContract.taskManager.ensureScheduled({ + id: this.getTaskId(), + taskType: ManifestTaskConstants.TYPE, + scope: ['securitySolution'], + schedule: { + interval: '60s', + }, + state: {}, + params: { version: ManifestTaskConstants.VERSION }, + }); + } catch (e) { + this.logger.debug(`Error scheduling task, received ${e.message}`); + } + }; + + private getTaskId = (): string => { + return `${ManifestTaskConstants.TYPE}:${ManifestTaskConstants.VERSION}`; + }; + + public runTask = async (taskId: string) => { + // Check that this task is current + if (taskId !== this.getTaskId()) { + // old task, return + this.logger.debug(`Outdated task running: ${taskId}`); + return; + } + + const manifestManager = this.endpointAppContext.service.getManifestManager(); + + if (manifestManager === undefined) { + this.logger.debug('Manifest Manager not available.'); + return; + } + + manifestManager + .refresh() + .then((wrappedManifest) => { + if (wrappedManifest) { + return manifestManager.dispatch(wrappedManifest); + } + }) + .then((wrappedManifest) => { + if (wrappedManifest) { + return manifestManager.commit(wrappedManifest); + } + }) + .catch((err) => { + this.logger.error(err); + }); + }; +} diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index ffd919db87fc99..55d7baec36dc6d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -5,23 +5,67 @@ */ import { ILegacyScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; +import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks'; import { xpackMocks } from '../../../../mocks'; import { AgentService, IngestManagerStartContract, ExternalCallback, } from '../../../ingest_manager/server'; -import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services'; import { createPackageConfigServiceMock } from '../../../ingest_manager/server/mocks'; +import { ConfigType } from '../config'; +import { createMockConfig } from '../lib/detection_engine/routes/__mocks__'; +import { + EndpointAppContextService, + EndpointAppContextServiceStartContract, +} from './endpoint_app_context_services'; +import { + ManifestManagerMock, + getManifestManagerMock, +} from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { EndpointAppContext } from './types'; + +/** + * Creates a mocked EndpointAppContext. + */ +export const createMockEndpointAppContext = ( + mockManifestManager?: ManifestManagerMock +): EndpointAppContext => { + return { + logFactory: loggingSystemMock.create(), + // @ts-ignore + config: createMockConfig() as ConfigType, + service: createMockEndpointAppContextService(mockManifestManager), + }; +}; + +/** + * Creates a mocked EndpointAppContextService + */ +export const createMockEndpointAppContextService = ( + mockManifestManager?: ManifestManagerMock +): jest.Mocked => { + return { + start: jest.fn(), + stop: jest.fn(), + getAgentService: jest.fn(), + // @ts-ignore + getManifestManager: mockManifestManager ?? jest.fn(), + getScopedSavedObjectsClient: jest.fn(), + }; +}; /** - * Crates a mocked input contract for the `EndpointAppContextService#start()` method + * Creates a mocked input contract for the `EndpointAppContextService#start()` method */ export const createMockEndpointAppContextServiceStartContract = (): jest.Mocked< EndpointAppContextServiceStartContract > => { return { agentService: createMockAgentService(), + savedObjectsStart: savedObjectsServiceMock.createStartContract(), + // @ts-ignore + manifestManager: getManifestManagerMock(), registerIngestCallback: jest.fn< ReturnType, Parameters diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts new file mode 100644 index 00000000000000..540976134d8ae1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -0,0 +1,301 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { + ILegacyClusterClient, + IRouter, + SavedObjectsClientContract, + ILegacyScopedClusterClient, + RouteConfig, + RequestHandler, + KibanaResponseFactory, + RequestHandlerContext, + SavedObject, +} from 'kibana/server'; +import { + elasticsearchServiceMock, + savedObjectsClientMock, + httpServiceMock, + httpServerMock, + loggingSystemMock, +} from 'src/core/server/mocks'; +import { ExceptionsCache } from '../../lib/artifacts/cache'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { registerDownloadExceptionListRoute } from './download_exception_list'; +import { EndpointAppContextService } from '../../endpoint_app_context_services'; +import { createMockEndpointAppContextServiceStartContract } from '../../mocks'; +import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; +import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; + +const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; +const expectedEndpointExceptions: WrappedTranslatedExceptionList = { + exceptions_list: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], +}; +const mockIngestSOResponse = { + page: 1, + per_page: 100, + total: 1, + saved_objects: [ + { + id: 'agent1', + type: 'agent', + references: [], + score: 0, + attributes: { + active: true, + access_api_key_id: 'pedTuHIBTEDt93wW0Fhr', + }, + }, + ], +}; +const AuthHeader = 'ApiKey cGVkVHVISUJURUR0OTN3VzBGaHI6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw=='; + +describe('test alerts route', () => { + let routerMock: jest.Mocked; + let mockClusterClient: jest.Mocked; + let mockScopedClient: jest.Mocked; + let mockSavedObjectClient: jest.Mocked; + let mockResponse: jest.Mocked; + // @ts-ignore + let routeConfig: RouteConfig; + let routeHandler: RequestHandler; + let endpointAppContextService: EndpointAppContextService; + let cache: ExceptionsCache; + let ingestSavedObjectClient: jest.Mocked; + + beforeEach(() => { + mockClusterClient = elasticsearchServiceMock.createClusterClient(); + mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); + mockSavedObjectClient = savedObjectsClientMock.create(); + mockResponse = httpServerMock.createResponseFactory(); + mockClusterClient.asScoped.mockReturnValue(mockScopedClient); + routerMock = httpServiceMock.createRouter(); + endpointAppContextService = new EndpointAppContextService(); + cache = new ExceptionsCache(5); + const startContract = createMockEndpointAppContextServiceStartContract(); + + // The authentication with the Fleet Plugin needs a separate scoped SO Client + ingestSavedObjectClient = savedObjectsClientMock.create(); + ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); + // @ts-ignore + startContract.savedObjectsStart.getScopedClient.mockReturnValue(ingestSavedObjectClient); + endpointAppContextService.start(startContract); + + registerDownloadExceptionListRoute( + routerMock, + { + logFactory: loggingSystemMock.create(), + service: endpointAppContextService, + config: () => Promise.resolve(createMockConfig()), + }, + cache + ); + }); + + it('should serve the artifact to download', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/123456`, + method: 'get', + params: { sha256: '123456' }, + headers: { + authorization: AuthHeader, + }, + }); + + // Mock the SavedObjectsClient get response for fetching the artifact + const mockArtifact = { + id: '2468', + type: 'test', + references: [], + attributes: { + identifier: mockArtifactName, + schemaVersion: '1.0.0', + sha256: '123456', + encoding: 'application/json', + created: Date.now(), + body: Buffer.from(JSON.stringify(expectedEndpointExceptions)).toString('base64'), + size: 100, + }, + }; + const soFindResp: SavedObject = { + ...mockArtifact, + }; + ingestSavedObjectClient.get.mockImplementationOnce(() => Promise.resolve(soFindResp)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + + const expectedHeaders = { + 'content-encoding': 'application/json', + 'content-disposition': `attachment; filename=${mockArtifactName}.json`, + }; + + expect(mockResponse.ok).toBeCalled(); + expect(mockResponse.ok.mock.calls[0][0]?.headers).toEqual(expectedHeaders); + const artifact = mockResponse.ok.mock.calls[0][0]?.body; + expect(artifact).toEqual(Buffer.from(mockArtifact.attributes.body, 'base64').toString()); + }); + + it('should handle fetching a non-existent artifact', async () => { + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/123456`, + method: 'get', + params: { sha256: '789' }, + headers: { + authorization: AuthHeader, + }, + }); + + ingestSavedObjectClient.get.mockImplementationOnce(() => + // eslint-disable-next-line prefer-promise-reject-errors + Promise.reject({ output: { statusCode: 404 } }) + ); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.notFound).toBeCalled(); + }); + + it('should utilize the cache', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, identifier: mockArtifactName }, + headers: { + authorization: AuthHeader, + }, + }); + + // Add to the download cache + const mockArtifact = expectedEndpointExceptions; + const cacheKey = `${mockArtifactName}-${mockSha}`; + cache.set(cacheKey, JSON.stringify(mockArtifact)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.ok).toBeCalled(); + // The saved objects client should be bypassed as the cache will contain the download + expect(ingestSavedObjectClient.get.mock.calls.length).toEqual(0); + }); + + it('should respond with a 401 if a valid API Token is not supplied', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, identifier: mockArtifactName }, + }); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.unauthorized).toBeCalled(); + }); + + it('should respond with a 404 if an agent cannot be linked to the API token', async () => { + const mockSha = '123456789'; + const mockRequest = httpServerMock.createKibanaRequest({ + path: `/api/endpoint/artifacts/download/${mockArtifactName}/${mockSha}`, + method: 'get', + params: { sha256: mockSha, identifier: mockArtifactName }, + headers: { + authorization: AuthHeader, + }, + }); + + // Mock the SavedObjectsClient find response for verifying the API token with no results + mockIngestSOResponse.saved_objects = []; + mockIngestSOResponse.total = 0; + ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); + + [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => + path.startsWith('/api/endpoint/artifacts/download') + )!; + + await routeHandler( + ({ + core: { + savedObjects: { + client: mockSavedObjectClient, + }, + }, + } as unknown) as RequestHandlerContext, + mockRequest, + mockResponse + ); + expect(mockResponse.notFound).toBeCalled(); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts new file mode 100644 index 00000000000000..337393e768a8f2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + IRouter, + SavedObjectsClientContract, + HttpResponseOptions, + IKibanaResponse, + SavedObject, +} from 'src/core/server'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { authenticateAgentWithAccessToken } from '../../../../../ingest_manager/server/services/agents/authenticate'; +import { validate } from '../../../../common/validate'; +import { buildRouteValidation } from '../../../utils/build_validation/route_validation'; +import { ArtifactConstants, ExceptionsCache } from '../../lib/artifacts'; +import { + DownloadArtifactRequestParamsSchema, + downloadArtifactRequestParamsSchema, + downloadArtifactResponseSchema, + InternalArtifactSchema, +} from '../../schemas/artifacts'; +import { EndpointAppContext } from '../../types'; + +const allowlistBaseRoute: string = '/api/endpoint/artifacts'; + +/** + * Registers the exception list route to enable sensors to download an allowlist artifact + */ +export function registerDownloadExceptionListRoute( + router: IRouter, + endpointContext: EndpointAppContext, + cache: ExceptionsCache +) { + router.get( + { + path: `${allowlistBaseRoute}/download/{identifier}/{sha256}`, + validate: { + params: buildRouteValidation< + typeof downloadArtifactRequestParamsSchema, + DownloadArtifactRequestParamsSchema + >(downloadArtifactRequestParamsSchema), + }, + options: { tags: [] }, + }, + // @ts-ignore + async (context, req, res) => { + let scopedSOClient: SavedObjectsClientContract; + const logger = endpointContext.logFactory.get('download_exception_list'); + + // The ApiKey must be associated with an enrolled Fleet agent + try { + scopedSOClient = endpointContext.service.getScopedSavedObjectsClient(req); + await authenticateAgentWithAccessToken(scopedSOClient, req); + } catch (err) { + if (err.output.statusCode === 401) { + return res.unauthorized(); + } else { + return res.notFound(); + } + } + + const buildAndValidateResponse = (artName: string, body: string): IKibanaResponse => { + const artifact: HttpResponseOptions = { + body, + headers: { + 'content-encoding': 'application/json', + 'content-disposition': `attachment; filename=${artName}.json`, + }, + }; + + const [validated, errors] = validate(artifact, downloadArtifactResponseSchema); + if (errors !== null || validated === null) { + return res.internalError({ body: errors! }); + } else { + return res.ok((validated as unknown) as HttpResponseOptions); + } + }; + + const id = `${req.params.identifier}-${req.params.sha256}`; + const cacheResp = cache.get(id); + + if (cacheResp) { + logger.debug(`Cache HIT artifact ${id}`); + return buildAndValidateResponse(req.params.identifier, cacheResp); + } else { + logger.debug(`Cache MISS artifact ${id}`); + return scopedSOClient + .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) + .then((artifact: SavedObject) => { + const body = Buffer.from(artifact.attributes.body, 'base64').toString(); + cache.set(id, body); + return buildAndValidateResponse(artifact.attributes.identifier, body); + }) + .catch((err) => { + if (err?.output?.statusCode === 404) { + return res.notFound({ body: `No artifact found for ${id}` }); + } else { + return res.internalError({ body: err }); + } + }); + } + } + ); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts new file mode 100644 index 00000000000000..945646c73c46c9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_exception_list'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts new file mode 100644 index 00000000000000..3c066e150288ae --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +export const body = t.string; + +export const created = t.number; // TODO: Make this into an ISO Date string check + +export const encoding = t.keyof({ + 'application/json': null, +}); + +export const schemaVersion = t.keyof({ + '1.0.0': null, +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts new file mode 100644 index 00000000000000..908fbb698adefd --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './common'; +export * from './lists'; +export * from './request'; +export * from './response'; +export * from './saved_objects'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts new file mode 100644 index 00000000000000..7354b5fd0ec4d7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { WrappedTranslatedExceptionList } from './lists'; + +export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList => { + return { + exceptions_list: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts new file mode 100644 index 00000000000000..21d1105a313e78 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { operator } from '../../../../../lists/common/schemas'; + +export const translatedEntryMatchAny = t.exact( + t.type({ + field: t.string, + operator, + type: t.keyof({ + exact_cased_any: null, + exact_caseless_any: null, + }), + value: t.array(t.string), + }) +); +export type TranslatedEntryMatchAny = t.TypeOf; + +export const translatedEntryMatch = t.exact( + t.type({ + field: t.string, + operator, + type: t.keyof({ + exact_cased: null, + exact_caseless: null, + }), + value: t.string, + }) +); +export type TranslatedEntryMatch = t.TypeOf; + +export const translatedEntryNested = t.exact( + t.type({ + field: t.string, + type: t.keyof({ nested: null }), + entries: t.array(t.union([translatedEntryMatch, translatedEntryMatchAny])), + }) +); +export type TranslatedEntryNested = t.TypeOf; + +export const translatedEntry = t.union([ + translatedEntryNested, + translatedEntryMatch, + translatedEntryMatchAny, +]); +export type TranslatedEntry = t.TypeOf; + +export const translatedExceptionList = t.exact( + t.type({ + type: t.string, + entries: t.array(translatedEntry), + }) +); +export type TranslatedExceptionList = t.TypeOf; + +export const wrappedExceptionList = t.exact( + t.type({ + exceptions_list: t.array(translatedEntry), + }) +); +export type WrappedTranslatedExceptionList = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts new file mode 100644 index 00000000000000..7a194fdc7b5f44 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/download_artifact_schema.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { identifier, sha256 } from '../../../../../common/endpoint/schema/common'; + +export const downloadArtifactRequestParamsSchema = t.exact( + t.type({ + identifier, + sha256, + }) +); + +export type DownloadArtifactRequestParamsSchema = t.TypeOf< + typeof downloadArtifactRequestParamsSchema +>; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts new file mode 100644 index 00000000000000..13e4165eb5f16e --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/request/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_artifact_schema'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts new file mode 100644 index 00000000000000..537f7707889e44 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { encoding } from '../common'; + +const body = t.string; +const headers = t.exact( + t.type({ + 'content-encoding': encoding, + 'content-disposition': t.string, + }) +); + +export const downloadArtifactResponseSchema = t.exact( + t.type({ + body, + headers, + }) +); + +export type DownloadArtifactResponseSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts new file mode 100644 index 00000000000000..13e4165eb5f16e --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './download_artifact_schema'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts new file mode 100644 index 00000000000000..1a9cc55ca57250 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ArtifactConstants, buildArtifact } from '../../lib/artifacts'; +import { getTranslatedExceptionListMock } from './lists.mock'; +import { InternalArtifactSchema, InternalManifestSchema } from './saved_objects'; + +export const getInternalArtifactMock = async ( + os: string, + schemaVersion: string +): Promise => { + return buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); +}; + +export const getInternalArtifactMockWithDiffs = async ( + os: string, + schemaVersion: string +): Promise => { + const mock = getTranslatedExceptionListMock(); + mock.exceptions_list.pop(); + return buildArtifact(mock, os, schemaVersion); +}; + +export const getInternalArtifactsMock = async ( + os: string, + schemaVersion: string +): Promise => { + // @ts-ignore + return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.map(async () => { + await buildArtifact(getTranslatedExceptionListMock(), os, schemaVersion); + }); +}; + +export const getInternalManifestMock = (): InternalManifestSchema => ({ + created: Date.now(), + ids: [], +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts new file mode 100644 index 00000000000000..2e71ef98387f1f --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { identifier, sha256, size } from '../../../../common/endpoint/schema/common'; +import { body, created, encoding } from './common'; + +export const internalArtifactSchema = t.exact( + t.type({ + identifier, + sha256, + encoding, + created, + body, + size, + }) +); + +export type InternalArtifactSchema = t.TypeOf; + +export const internalManifestSchema = t.exact( + t.type({ + created, + ids: t.array(identifier), + }) +); + +export type InternalManifestSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts new file mode 100644 index 00000000000000..a3b6e68e4ada26 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './artifacts'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts new file mode 100644 index 00000000000000..6392c59b2377ca --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { SavedObjectsClientContract } from 'src/core/server'; +import { ArtifactClient } from './artifact_client'; + +export const getArtifactClientMock = ( + savedObjectsClient?: SavedObjectsClientContract +): ArtifactClient => { + if (savedObjectsClient !== undefined) { + return new ArtifactClient(savedObjectsClient); + } + return new ArtifactClient(savedObjectsClientMock.create()); +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts new file mode 100644 index 00000000000000..08e29b5c6b82b4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; +import { getArtifactClientMock } from './artifact_client.mock'; +import { ArtifactClient } from './artifact_client'; + +describe('artifact_client', () => { + describe('ArtifactClient sanity checks', () => { + test('can create ArtifactClient', () => { + const artifactClient = new ArtifactClient(savedObjectsClientMock.create()); + expect(artifactClient).toBeInstanceOf(ArtifactClient); + }); + + test('can get artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + await artifactClient.getArtifact('abcd'); + expect(savedObjectsClient.get).toHaveBeenCalled(); + }); + + test('can create artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + const artifact = await getInternalArtifactMock('linux', '1.0.0'); + await artifactClient.createArtifact(artifact); + expect(savedObjectsClient.create).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifact, + { id: artifactClient.getArtifactId(artifact) } + ); + }); + + test('can delete artifact', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const artifactClient = getArtifactClientMock(savedObjectsClient); + await artifactClient.deleteArtifact('abcd'); + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + 'abcd' + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts new file mode 100644 index 00000000000000..4a3dcaae1bd3d7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import { ArtifactConstants } from '../../lib/artifacts'; +import { InternalArtifactSchema } from '../../schemas/artifacts'; + +export class ArtifactClient { + private savedObjectsClient: SavedObjectsClientContract; + + constructor(savedObjectsClient: SavedObjectsClientContract) { + this.savedObjectsClient = savedObjectsClient; + } + + public getArtifactId(artifact: InternalArtifactSchema) { + return `${artifact.identifier}-${artifact.sha256}`; + } + + public async getArtifact(id: string): Promise> { + return this.savedObjectsClient.get( + ArtifactConstants.SAVED_OBJECT_TYPE, + id + ); + } + + public async createArtifact( + artifact: InternalArtifactSchema + ): Promise> { + return this.savedObjectsClient.create( + ArtifactConstants.SAVED_OBJECT_TYPE, + artifact, + { id: this.getArtifactId(artifact) } + ); + } + + public async deleteArtifact(id: string) { + return this.savedObjectsClient.delete(ArtifactConstants.SAVED_OBJECT_TYPE, id); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts new file mode 100644 index 00000000000000..44a4d7e77dbcb7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './artifact_client'; +export * from './manifest_manager'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts new file mode 100644 index 00000000000000..bfeacbcedf2cb9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsClientContract } from 'src/core/server'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ManifestClient } from './manifest_client'; + +export const getManifestClientMock = ( + savedObjectsClient?: SavedObjectsClientContract +): ManifestClient => { + if (savedObjectsClient !== undefined) { + return new ManifestClient(savedObjectsClient, '1.0.0'); + } + return new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts new file mode 100644 index 00000000000000..5780c6279ee6ae --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; +import { ManifestConstants } from '../../lib/artifacts'; +import { getInternalManifestMock } from '../../schemas/artifacts/saved_objects.mock'; +import { getManifestClientMock } from './manifest_client.mock'; +import { ManifestClient } from './manifest_client'; + +describe('manifest_client', () => { + describe('ManifestClient sanity checks', () => { + test('can create ManifestClient', () => { + const manifestClient = new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); + expect(manifestClient).toBeInstanceOf(ManifestClient); + }); + + test('cannot create ManifestClient with invalid schema version', () => { + expect(() => { + new ManifestClient(savedObjectsClientMock.create(), 'invalid' as ManifestSchemaVersion); + }).toThrow(); + }); + + test('can get manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + await manifestClient.getManifest(); + expect(savedObjectsClient.get).toHaveBeenCalled(); + }); + + test('can create manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + const manifest = getInternalManifestMock(); + await manifestClient.createManifest(manifest); + expect(savedObjectsClient.create).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifest, + { id: manifestClient.getManifestId() } + ); + }); + + test('can update manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + const manifest = getInternalManifestMock(); + await manifestClient.updateManifest(manifest, { version: 'abcd' }); + expect(savedObjectsClient.update).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifestClient.getManifestId(), + manifest, + { version: 'abcd' } + ); + }); + + test('can delete manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestClient = getManifestClientMock(savedObjectsClient); + await manifestClient.deleteManifest(); + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ManifestConstants.SAVED_OBJECT_TYPE, + manifestClient.getManifestId() + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts new file mode 100644 index 00000000000000..45182841e56fc5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + SavedObject, + SavedObjectsClientContract, + SavedObjectsUpdateResponse, +} from 'src/core/server'; +import { + manifestSchemaVersion, + ManifestSchemaVersion, +} from '../../../../common/endpoint/schema/common'; +import { validate } from '../../../../common/validate'; +import { ManifestConstants } from '../../lib/artifacts'; +import { InternalManifestSchema } from '../../schemas/artifacts'; + +interface UpdateManifestOpts { + version: string; +} + +export class ManifestClient { + private schemaVersion: ManifestSchemaVersion; + private savedObjectsClient: SavedObjectsClientContract; + + constructor( + savedObjectsClient: SavedObjectsClientContract, + schemaVersion: ManifestSchemaVersion + ) { + this.savedObjectsClient = savedObjectsClient; + + const [validated, errors] = validate( + (schemaVersion as unknown) as object, + manifestSchemaVersion + ); + + if (errors != null || validated === null) { + throw new Error(`Invalid manifest version: ${schemaVersion}`); + } + + this.schemaVersion = validated; + } + + public getManifestId(): string { + return `endpoint-manifest-${this.schemaVersion}`; + } + + public async getManifest(): Promise> { + return this.savedObjectsClient.get( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId() + ); + } + + public async createManifest( + manifest: InternalManifestSchema + ): Promise> { + return this.savedObjectsClient.create( + ManifestConstants.SAVED_OBJECT_TYPE, + manifest, + { id: this.getManifestId() } + ); + } + + public async updateManifest( + manifest: InternalManifestSchema, + opts?: UpdateManifestOpts + ): Promise> { + return this.savedObjectsClient.update( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId(), + manifest, + opts + ); + } + + public async deleteManifest() { + return this.savedObjectsClient.delete( + ManifestConstants.SAVED_OBJECT_TYPE, + this.getManifestId() + ); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts new file mode 100644 index 00000000000000..03d5d27b3ff788 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './manifest_manager'; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts new file mode 100644 index 00000000000000..cd70b11aef305a --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// eslint-disable-next-line max-classes-per-file +import { savedObjectsClientMock, loggingSystemMock } from 'src/core/server/mocks'; +import { Logger } from 'src/core/server'; +import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; +import { listMock } from '../../../../../../lists/server/mocks'; +import { + ExceptionsCache, + Manifest, + buildArtifact, + getFullEndpointExceptionList, +} from '../../../lib/artifacts'; +import { InternalArtifactSchema } from '../../../schemas/artifacts'; +import { getArtifactClientMock } from '../artifact_client.mock'; +import { getManifestClientMock } from '../manifest_client.mock'; +import { ManifestManager } from './manifest_manager'; + +function getMockPackageConfig() { + return { + id: 'c6d16e42-c32d-4dce-8a88-113cfe276ad1', + inputs: [ + { + config: {}, + }, + ], + revision: 1, + version: 'abcd', // TODO: not yet implemented in ingest_manager (https://github.com/elastic/kibana/issues/69992) + updated_at: '2020-06-25T16:03:38.159292', + updated_by: 'kibana', + created_at: '2020-06-25T16:03:38.159292', + created_by: 'kibana', + }; +} + +class PackageConfigServiceMock { + public create = jest.fn().mockResolvedValue(getMockPackageConfig()); + public get = jest.fn().mockResolvedValue(getMockPackageConfig()); + public getByIds = jest.fn().mockResolvedValue([getMockPackageConfig()]); + public list = jest.fn().mockResolvedValue({ + items: [getMockPackageConfig()], + total: 1, + page: 1, + perPage: 20, + }); + public update = jest.fn().mockResolvedValue(getMockPackageConfig()); +} + +export function getPackageConfigServiceMock() { + return new PackageConfigServiceMock(); +} + +async function mockBuildExceptionListArtifacts( + os: string, + schemaVersion: string +): Promise { + const mockExceptionClient = listMock.getExceptionListClient(); + const first = getFoundExceptionListItemSchemaMock(); + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + const exceptions = await getFullEndpointExceptionList(mockExceptionClient, os, schemaVersion); + return [await buildArtifact(exceptions, os, schemaVersion)]; +} + +// @ts-ignore +export class ManifestManagerMock extends ManifestManager { + // @ts-ignore + private buildExceptionListArtifacts = async () => { + return mockBuildExceptionListArtifacts('linux', '1.0.0'); + }; + + // @ts-ignore + private getLastDispatchedManifest = jest + .fn() + .mockResolvedValue(new Manifest(new Date(), '1.0.0', 'v0')); + + // @ts-ignore + private getManifestClient = jest + .fn() + .mockReturnValue(getManifestClientMock(this.savedObjectsClient)); +} + +export const getManifestManagerMock = (opts?: { + packageConfigService?: PackageConfigServiceMock; + savedObjectsClient?: ReturnType; +}): ManifestManagerMock => { + let packageConfigService = getPackageConfigServiceMock(); + if (opts?.packageConfigService !== undefined) { + packageConfigService = opts.packageConfigService; + } + + let savedObjectsClient = savedObjectsClientMock.create(); + if (opts?.savedObjectsClient !== undefined) { + savedObjectsClient = opts.savedObjectsClient; + } + + const manifestManager = new ManifestManagerMock({ + artifactClient: getArtifactClientMock(savedObjectsClient), + cache: new ExceptionsCache(5), + // @ts-ignore + packageConfigService, + exceptionListClient: listMock.getExceptionListClient(), + logger: loggingSystemMock.create().get() as jest.Mocked, + savedObjectsClient, + }); + + return manifestManager; +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts new file mode 100644 index 00000000000000..bbb6fdfd508109 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { ArtifactConstants, ManifestConstants, Manifest } from '../../../lib/artifacts'; +import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_manager.mock'; + +describe('manifest_manager', () => { + describe('ManifestManager sanity checks', () => { + test('ManifestManager can refresh manifest', async () => { + const manifestManager = getManifestManagerMock(); + const manifestWrapper = await manifestManager.refresh(); + expect(manifestWrapper!.diffs).toEqual([ + { + id: + 'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49', + type: 'add', + }, + ]); + expect(manifestWrapper!.manifest).toBeInstanceOf(Manifest); + }); + + test('ManifestManager can dispatch manifest', async () => { + const packageConfigService = getPackageConfigServiceMock(); + const manifestManager = getManifestManagerMock({ packageConfigService }); + const manifestWrapperRefresh = await manifestManager.refresh(); + const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); + expect(manifestWrapperRefresh).toEqual(manifestWrapperDispatch); + const entries = manifestWrapperDispatch!.manifest.getEntries(); + const artifact = Object.values(entries)[0].getArtifact(); + expect( + packageConfigService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value + ).toEqual({ + manifest_version: 'v0', + schema_version: '1.0.0', + artifacts: { + [artifact.identifier]: { + sha256: artifact.sha256, + size: artifact.size, + url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.sha256}`, + }, + }, + }); + }); + + test('ManifestManager can commit manifest', async () => { + const savedObjectsClient: ReturnType = savedObjectsClientMock.create(); + const manifestManager = getManifestManagerMock({ + savedObjectsClient, + }); + + const manifestWrapperRefresh = await manifestManager.refresh(); + const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); + const diff = { + id: 'abcd', + type: 'delete', + }; + manifestWrapperDispatch!.diffs.push(diff); + + await manifestManager.commit(manifestWrapperDispatch); + + // created new artifact + expect(savedObjectsClient.create.mock.calls[0][0]).toEqual( + ArtifactConstants.SAVED_OBJECT_TYPE + ); + + // deleted old artifact + expect(savedObjectsClient.delete).toHaveBeenCalledWith( + ArtifactConstants.SAVED_OBJECT_TYPE, + 'abcd' + ); + + // committed new manifest + expect(savedObjectsClient.create.mock.calls[1][0]).toEqual( + ManifestConstants.SAVED_OBJECT_TYPE + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts new file mode 100644 index 00000000000000..33b0d5db575c6c --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -0,0 +1,270 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger, SavedObjectsClientContract, SavedObject } from 'src/core/server'; +import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; +import { ExceptionListClient } from '../../../../../../lists/server'; +import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; +import { + ArtifactConstants, + ManifestConstants, + Manifest, + buildArtifact, + getFullEndpointExceptionList, + ExceptionsCache, + ManifestDiff, +} from '../../../lib/artifacts'; +import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; +import { ArtifactClient } from '../artifact_client'; +import { ManifestClient } from '../manifest_client'; + +export interface ManifestManagerContext { + savedObjectsClient: SavedObjectsClientContract; + artifactClient: ArtifactClient; + exceptionListClient: ExceptionListClient; + packageConfigService: PackageConfigServiceInterface; + logger: Logger; + cache: ExceptionsCache; +} + +export interface ManifestRefreshOpts { + initialize?: boolean; +} + +export interface WrappedManifest { + manifest: Manifest; + diffs: ManifestDiff[]; +} + +export class ManifestManager { + protected artifactClient: ArtifactClient; + protected exceptionListClient: ExceptionListClient; + protected packageConfigService: PackageConfigServiceInterface; + protected savedObjectsClient: SavedObjectsClientContract; + protected logger: Logger; + protected cache: ExceptionsCache; + + constructor(context: ManifestManagerContext) { + this.artifactClient = context.artifactClient; + this.exceptionListClient = context.exceptionListClient; + this.packageConfigService = context.packageConfigService; + this.savedObjectsClient = context.savedObjectsClient; + this.logger = context.logger; + this.cache = context.cache; + } + + private getManifestClient(schemaVersion: string): ManifestClient { + return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion); + } + + private async buildExceptionListArtifacts( + schemaVersion: string + ): Promise { + const artifacts: InternalArtifactSchema[] = []; + + for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { + const exceptionList = await getFullEndpointExceptionList( + this.exceptionListClient, + os, + schemaVersion + ); + const artifact = await buildArtifact(exceptionList, os, schemaVersion); + + artifacts.push(artifact); + } + + return artifacts; + } + + private async getLastDispatchedManifest(schemaVersion: string): Promise { + return this.getManifestClient(schemaVersion) + .getManifest() + .then(async (manifestSo: SavedObject) => { + if (manifestSo.version === undefined) { + throw new Error('No version returned for manifest.'); + } + const manifest = new Manifest( + new Date(manifestSo.attributes.created), + schemaVersion, + manifestSo.version + ); + + for (const id of manifestSo.attributes.ids) { + const artifactSo = await this.artifactClient.getArtifact(id); + manifest.addEntry(artifactSo.attributes); + } + + return manifest; + }) + .catch((err) => { + if (err.output.statusCode !== 404) { + throw err; + } + return null; + }); + } + + public async refresh(opts?: ManifestRefreshOpts): Promise { + let oldManifest: Manifest | null; + + // Get the last-dispatched manifest + oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); + + if (oldManifest === null && opts !== undefined && opts.initialize) { + oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION, 'v0'); // create empty manifest + } else if (oldManifest == null) { + this.logger.debug('Manifest does not exist yet. Waiting...'); + return null; + } + + // Build new exception list artifacts + const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + + // Build new manifest + const newManifest = Manifest.fromArtifacts( + artifacts, + ManifestConstants.SCHEMA_VERSION, + oldManifest.getVersion() + ); + + // Get diffs + const diffs = newManifest.diff(oldManifest); + + // Create new artifacts + for (const diff of diffs) { + if (diff.type === 'add') { + const artifact = newManifest.getArtifact(diff.id); + try { + await this.artifactClient.createArtifact(artifact); + // Cache the body of the artifact + this.cache.set(diff.id, artifact.body); + } catch (err) { + if (err.status === 409) { + // This artifact already existed... + this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); + } else { + throw err; + } + } + } + } + + return { + manifest: newManifest, + diffs, + }; + } + + /** + * Dispatches the manifest by writing it to the endpoint packageConfig. + * + * @return {WrappedManifest | null} WrappedManifest if all dispatched, else null + */ + public async dispatch(wrappedManifest: WrappedManifest | null): Promise { + if (wrappedManifest === null) { + this.logger.debug('wrappedManifest was null, aborting dispatch'); + return null; + } + + function showDiffs(diffs: ManifestDiff[]) { + return diffs.map((diff) => { + const op = diff.type === 'add' ? '(+)' : '(-)'; + return `${op}${diff.id}`; + }); + } + + if (wrappedManifest.diffs.length > 0) { + this.logger.info(`Dispatching new manifest with diffs: ${showDiffs(wrappedManifest.diffs)}`); + + let paging = true; + let success = true; + + while (paging) { + const { items, total, page } = await this.packageConfigService.list( + this.savedObjectsClient, + { + page: 1, + perPage: 20, + kuery: 'ingest-package-configs.package.name:endpoint', + } + ); + + for (const packageConfig of items) { + const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; + + if ( + newPackageConfig.inputs.length > 0 && + newPackageConfig.inputs[0].config !== undefined + ) { + const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { + value: {}, + }; + artifactManifest.value = wrappedManifest.manifest.toEndpointFormat(); + newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; + + await this.packageConfigService + .update(this.savedObjectsClient, id, newPackageConfig) + .then((response) => { + this.logger.debug(`Updated package config ${id}`); + }) + .catch((err) => { + success = false; + this.logger.debug(`Error updating package config ${id}`); + this.logger.error(err); + }); + } else { + success = false; + this.logger.debug(`Package config ${id} has no config.`); + } + } + + paging = page * items.length < total; + } + + return success ? wrappedManifest : null; + } else { + this.logger.debug('No manifest diffs [no-op]'); + } + + return null; + } + + public async commit(wrappedManifest: WrappedManifest | null) { + if (wrappedManifest === null) { + this.logger.debug('wrappedManifest was null, aborting commit'); + return; + } + + const manifestClient = this.getManifestClient(wrappedManifest.manifest.getSchemaVersion()); + + // Commit the new manifest + if (wrappedManifest.manifest.getVersion() === 'v0') { + await manifestClient.createManifest(wrappedManifest.manifest.toSavedObject()); + } else { + const version = wrappedManifest.manifest.getVersion(); + if (version === 'v0') { + throw new Error('Updating existing manifest with baseline version. Bad state.'); + } + await manifestClient.updateManifest(wrappedManifest.manifest.toSavedObject(), { + version, + }); + } + + this.logger.info(`Commited manifest ${wrappedManifest.manifest.getVersion()}`); + + // Clean up old artifacts + for (const diff of wrappedManifest.diffs) { + try { + if (diff.type === 'delete') { + await this.artifactClient.deleteArtifact(diff.id); + this.logger.info(`Cleaned up artifact ${diff.id}`); + } + } catch (err) { + this.logger.error(err); + } + } + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/services/index.ts b/x-pack/plugins/security_solution/server/endpoint/services/index.ts new file mode 100644 index 00000000000000..a3b6e68e4ada26 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/services/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './artifacts'; diff --git a/x-pack/plugins/security_solution/server/endpoint/types.ts b/x-pack/plugins/security_solution/server/endpoint/types.ts index fbcc5bc833d732..3c6630db8ebd89 100644 --- a/x-pack/plugins/security_solution/server/endpoint/types.ts +++ b/x-pack/plugins/security_solution/server/endpoint/types.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import { LoggerFactory } from 'kibana/server'; -import { EndpointAppContextService } from './endpoint_app_context_services'; import { ConfigType } from '../config'; +import { EndpointAppContextService } from './endpoint_app_context_services'; /** * The context for Endpoint apps. diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 9bb1bea0949e0f..a97f1eee56342c 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -11,9 +11,10 @@ import { i18n } from '@kbn/i18n'; import { CoreSetup, CoreStart, + Logger, Plugin as IPlugin, PluginInitializerContext, - Logger, + SavedObjectsClient, } from '../../../../src/core/server'; import { PluginSetupContract as AlertingSetup } from '../../alerts/server'; import { SecurityPluginSetup as SecuritySetup } from '../../security/server'; @@ -24,6 +25,7 @@ import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from ' import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; import { IngestManagerStartContract } from '../../ingest_manager/server'; +import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; import { initServer } from './init_server'; import { compose } from './lib/compose/kibana'; import { initRoutes } from './routes'; @@ -32,6 +34,7 @@ import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule import { rulesNotificationAlertType } from './lib/detection_engine/notifications/rules_notification_alert_type'; import { isNotificationAlertExecutor } from './lib/detection_engine/notifications/types'; import { hasListsFeature, listsEnvFeatureFlagName } from './lib/detection_engine/feature_flags'; +import { ManifestTask, ExceptionsCache } from './endpoint/lib/artifacts'; import { initSavedObjects, savedObjectTypes } from './saved_objects'; import { AppClientFactory } from './client'; import { createConfig$, ConfigType } from './config'; @@ -40,8 +43,10 @@ import { APP_ID, APP_ICON, SERVER_APP_ID } from '../common/constants'; import { registerEndpointRoutes } from './endpoint/routes/metadata'; import { registerResolverRoutes } from './endpoint/routes/resolver'; import { registerPolicyRoutes } from './endpoint/routes/policy'; +import { ArtifactClient, ManifestManager } from './endpoint/services'; import { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; import { EndpointAppContext } from './endpoint/types'; +import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts'; export interface SetupPlugins { alerts: AlertingSetup; @@ -50,12 +55,14 @@ export interface SetupPlugins { licensing: LicensingPluginSetup; security?: SecuritySetup; spaces?: SpacesSetup; + taskManager: TaskManagerSetupContract; ml?: MlSetup; lists?: ListPluginSetup; } export interface StartPlugins { ingestManager: IngestManagerStartContract; + taskManager: TaskManagerStartContract; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -70,11 +77,17 @@ export class Plugin implements IPlugin type.name); diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts new file mode 100644 index 00000000000000..2721592ba33503 --- /dev/null +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const supertestWithoutAuth = getSupertestWithoutAuth(providerContext); + let agentAccessAPIKey: string; + + describe('artifact download', () => { + setupIngest(providerContext); + before(async () => { + await esArchiver.load('endpoint/artifacts/api_feature', { useCreate: true }); + + const { body: enrollmentApiKeysResponse } = await supertest + .get(`/api/ingest_manager/fleet/enrollment-api-keys`) + .expect(200); + expect(enrollmentApiKeysResponse.list).length(2); + + const { body: enrollmentApiKeyResponse } = await supertest + .get( + `/api/ingest_manager/fleet/enrollment-api-keys/${enrollmentApiKeysResponse.list[0].id}` + ) + .expect(200); + expect(enrollmentApiKeyResponse.item).to.have.key('api_key'); + const enrollmentAPIToken = enrollmentApiKeyResponse.item.api_key; + + // 2. Enroll agent + const { body: enrollmentResponse } = await supertestWithoutAuth + .post(`/api/ingest_manager/fleet/agents/enroll`) + .set('kbn-xsrf', 'xxx') + .set('Authorization', `ApiKey ${enrollmentAPIToken}`) + .send({ + type: 'PERMANENT', + metadata: { + local: { + elastic: { + agent: { + version: '7.0.0', + }, + }, + }, + user_provided: {}, + }, + }) + .expect(200); + expect(enrollmentResponse.success).to.eql(true); + + agentAccessAPIKey = enrollmentResponse.item.access_api_key; + }); + after(() => esArchiver.unload('endpoint/artifacts/api_feature')); + + it('should fail to find artifact with invalid hash', async () => { + await supertestWithoutAuth + .get('/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/abcd') + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(404); + }); + + it('should download an artifact with correct hash', async () => { + await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(200); + }); + + it('should fail on invalid api key', async () => { + await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey iNvAlId`) + .send() + .expect(401); + }); + }); +} diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json new file mode 100644 index 00000000000000..a886b60e7e0dcc --- /dev/null +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -0,0 +1,179 @@ +{ + "type": "doc", + "value": { + "id": "endpoint:exceptions-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "index": ".kibana", + "source": { + "references": [ + ], + "endpoint:exceptions-artifact": { + "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==", + "created": 1593016187465, + "encoding": "application/json", + "identifier": "endpoint-exceptionlist-linux-1.0.0", + "sha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "size": 22 + }, + "type": "endpoint:exceptions-artifact", + "updated_at": "2020-06-24T16:29:47.584Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "endpoint:exceptions-manifest:endpoint-manifest-1.0.0", + "index": ".kibana", + "source": { + "references": [ + ], + "endpoint:exceptions-manifest": { + "created": 1593183699663, + "ids": [ + "endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "endpoint-exceptionlist-macos-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "endpoint-exceptionlist-windows-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d" + ] + }, + "type": "endpoint:exceptions-manifest", + "updated_at": "2020-06-26T15:01:39.704Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "exception-list-agnostic:13a7ef40-b63b-11ea-ace9-591c8e572c76", + "index": ".kibana", + "source": { + "exception-list-agnostic": { + "_tags": [ + "endpoint", + "process", + "malware", + "os:linux" + ], + "created_at": "2020-06-24T16:52:23.689Z", + "created_by": "akahan", + "description": "This is a sample agnostic endpoint type exception", + "list_id": "endpoint_list", + "list_type": "list", + "name": "Sample Endpoint Exception List", + "tags": [ + "user added string for a tag", + "malware" + ], + "tie_breaker_id": "e3b20e6e-c023-4575-a033-47990115969c", + "type": "endpoint", + "updated_by": "akahan" + }, + "references": [ + ], + "type": "exception-list-agnostic", + "updated_at": "2020-06-24T16:52:23.732Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "exception-list-agnostic:679b95a0-b714-11ea-a4c9-0963ae39bc3d", + "index": ".kibana", + "source": { + "exception-list-agnostic": { + "_tags": [ + "os:windows" + ], + "comments": [ + ], + "created_at": "2020-06-25T18:48:05.326Z", + "created_by": "akahan", + "description": "This is a sample endpoint type exception", + "entries": [ + { + "field": "actingProcess.file.signer", + "operator": "included", + "type": "match", + "value": "Elastic, N.V." + }, + { + "field": "event.category", + "operator": "included", + "type": "match_any", + "value": [ + "process", + "malware" + ] + } + ], + "item_id": "61142b8f-5876-4709-9952-95160cd58f2f", + "list_id": "endpoint_list", + "list_type": "item", + "name": "Sample Endpoint Exception List", + "tags": [ + "user added string for a tag", + "malware" + ], + "tie_breaker_id": "b36176d2-bc75-4641-a8e3-e811c6bc30d8", + "type": "endpoint", + "updated_by": "akahan" + }, + "references": [ + ], + "type": "exception-list-agnostic", + "updated_at": "2020-06-25T18:48:05.369Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "fleet-agents:a34d87c1-726e-4c30-b2ff-1b4b95f59d2a", + "index": ".kibana", + "source": { + "fleet-agents": { + "access_api_key_id": "8ZnT7HIBwLFvkUEPQaT3", + "active": true, + "config_id": "2dd2a110-b6f6-11ea-a66d-63cf082a3b58", + "enrolled_at": "2020-06-25T18:52:47.290Z", + "local_metadata": { + "os": "macos" + }, + "type": "PERMANENT", + "user_provided_metadata": { + "region": "us-east" + } + }, + "references": [ + ], + "type": "fleet-agents", + "updated_at": "2020-06-25T18:52:48.464Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "fleet-enrollment-api-keys:8178eb66-392f-4b76-9dc9-704ed1a5c56e", + "index": ".kibana", + "source": { + "fleet-enrollment-api-keys": { + "active": true, + "api_key": "8ZnT7HIBwLFvkUEPQaT3", + "api_key_id": "8ZnT7HIBwLFvkUEPQaT3", + "config_id": "2dd2a110-b6f6-11ea-a66d-63cf082a3b58", + "created_at": "2020-06-25T17:25:30.065Z", + "name": "Default (93aa98c8-d650-422e-aa7b-663dae3dff83)" + }, + "references": [ + ], + "type": "fleet-enrollment-api-keys", + "updated_at": "2020-06-25T17:25:30.114Z" + } + } +} \ No newline at end of file From 257c115f6694e38cffd933269dfd6e2f151cbc1b Mon Sep 17 00:00:00 2001 From: MadameSheema Date: Thu, 2 Jul 2020 07:49:01 +0200 Subject: [PATCH 04/31] [SIEM] Reenabling Cypress tests (#70397) * reenabling cypress * skips Overview tests * skips search bar test * skips URL test --- test/scripts/jenkins_security_solution_cypress.sh | 15 +++++---------- .../cypress/integration/overview.spec.ts | 2 +- .../cypress/integration/search_bar.spec.ts | 2 +- .../cypress/integration/url_state.spec.ts | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/test/scripts/jenkins_security_solution_cypress.sh b/test/scripts/jenkins_security_solution_cypress.sh index 8aa3425be0beb2..204911a3eedaa6 100644 --- a/test/scripts/jenkins_security_solution_cypress.sh +++ b/test/scripts/jenkins_security_solution_cypress.sh @@ -11,16 +11,11 @@ export KIBANA_INSTALL_DIR="$destDir" echo " -> Running security solution cypress tests" cd "$XPACK_DIR" -# Failures across multiple suites, skipping all -# https://github.com/elastic/kibana/issues/69847 -# https://github.com/elastic/kibana/issues/69848 -# https://github.com/elastic/kibana/issues/69849 - -# checks-reporter-with-killswitch "Security solution Cypress Tests" \ -# node scripts/functional_tests \ -# --debug --bail \ -# --kibana-install-dir "$KIBANA_INSTALL_DIR" \ -# --config test/security_solution_cypress/config.ts +checks-reporter-with-killswitch "Security solution Cypress Tests" \ + node scripts/functional_tests \ + --debug --bail \ + --kibana-install-dir "$KIBANA_INSTALL_DIR" \ + --config test/security_solution_cypress/config.ts echo "" echo "" diff --git a/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts b/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts index b799d487acd086..6fb3840d897640 100644 --- a/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/overview.spec.ts @@ -11,7 +11,7 @@ import { loginAndWaitForPage } from '../tasks/login'; import { OVERVIEW_URL } from '../urls/navigation'; -describe('Overview Page', () => { +describe.skip('Overview Page', () => { before(() => { cy.stubSecurityApi('overview'); loginAndWaitForPage(OVERVIEW_URL); diff --git a/x-pack/plugins/security_solution/cypress/integration/search_bar.spec.ts b/x-pack/plugins/security_solution/cypress/integration/search_bar.spec.ts index ce053d1ac76166..9104f494e3e6bc 100644 --- a/x-pack/plugins/security_solution/cypress/integration/search_bar.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/search_bar.spec.ts @@ -12,7 +12,7 @@ import { hostIpFilter } from '../objects/filter'; import { HOSTS_URL } from '../urls/navigation'; import { waitForAllHostsToBeLoaded } from '../tasks/hosts/all_hosts'; -describe('SearchBar', () => { +describe.skip('SearchBar', () => { before(() => { loginAndWaitForPage(HOSTS_URL); waitForAllHostsToBeLoaded(); diff --git a/x-pack/plugins/security_solution/cypress/integration/url_state.spec.ts b/x-pack/plugins/security_solution/cypress/integration/url_state.spec.ts index 6c456c2f5e1003..a3a927cbea7d46 100644 --- a/x-pack/plugins/security_solution/cypress/integration/url_state.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/url_state.spec.ts @@ -234,7 +234,7 @@ describe('url state', () => { cy.get(KQL_INPUT).should('have.attr', 'value', 'source.ip: "10.142.0.9"'); }); - it('sets and reads the url state for timeline by id', () => { + it.skip('sets and reads the url state for timeline by id', () => { loginAndWaitForPage(HOSTS_URL); openTimeline(); executeTimelineKQL('host.name: *'); From eca4cc5d3e2b02342977e839844a426cd2ea1b82 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 1 Jul 2020 23:01:21 -0700 Subject: [PATCH 05/31] Skip failing endgame tests (#70548) Co-authored-by: spalger --- x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts b/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts index ff42e322f6857e..cafb1c2a2ea696 100644 --- a/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts +++ b/x-pack/test/endpoint_api_integration_no_ingest/apis/index.ts @@ -7,7 +7,8 @@ import { FtrProviderContext } from '../ftr_provider_context'; export default function endpointAPIIntegrationTests({ loadTestFile }: FtrProviderContext) { - describe('Endpoint plugin', function () { + // Failing ES snapshot promotion: https://github.com/elastic/kibana/issues/70535 + describe.skip('Endpoint plugin', function () { this.tags('ciGroup7'); loadTestFile(require.resolve('./metadata')); }); From 8fe5d154c177f140d3ad703d135c2c5e621600bb Mon Sep 17 00:00:00 2001 From: Marta Bondyra Date: Thu, 2 Jul 2020 08:05:08 +0200 Subject: [PATCH 06/31] [Lens] fix dimension label performance issues (#69978) --- .../dimension_panel/dimension_panel.test.tsx | 8 +++++ .../dimension_panel/popover_editor.tsx | 35 +++++++++++++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index a1c084f83e4475..e4dbc641845289 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -27,6 +27,14 @@ import { OperationMetadata } from '../../types'; jest.mock('../loader'); jest.mock('../state_helpers'); +jest.mock('lodash', () => { + const original = jest.requireActual('lodash'); + + return { + ...original, + debounce: (fn: unknown) => fn, + }; +}); const expectedIndexPatterns = { 1: { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/popover_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/popover_editor.tsx index eb2475756417e7..34a4384ec0d40e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/popover_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/popover_editor.tsx @@ -6,7 +6,7 @@ import './popover_editor.scss'; import _ from 'lodash'; -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useEffect } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiFlexItem, @@ -56,6 +56,31 @@ function asOperationOptions(operationTypes: OperationType[], compatibleWithCurre })); } +const LabelInput = ({ value, onChange }: { value: string; onChange: (value: string) => void }) => { + const [inputValue, setInputValue] = useState(value); + + useEffect(() => { + setInputValue(value); + }, [value, setInputValue]); + + const onChangeDebounced = useMemo(() => _.debounce(onChange, 256), [onChange]); + + const handleInputChange = (e: React.ChangeEvent) => { + const val = String(e.target.value); + setInputValue(val); + onChangeDebounced(val); + }; + + return ( + + ); +}; + export function PopoverEditor(props: PopoverEditorProps) { const { selectedColumn, @@ -320,11 +345,9 @@ export function PopoverEditor(props: PopoverEditorProps) { })} display="rowCompressed" > - { + onChange={(value) => { setState({ ...state, layers: { @@ -335,7 +358,7 @@ export function PopoverEditor(props: PopoverEditorProps) { ...state.layers[layerId].columns, [columnId]: { ...selectedColumn, - label: e.target.value, + label: value, customLabel: true, }, }, From 45f0322fbcabf47883f7e571a41bd3846dfb82a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Thu, 2 Jul 2020 08:08:01 +0100 Subject: [PATCH 07/31] Reduce SavedObjects mappings for Application Usage (#70475) --- .../application_usage/saved_objects_types.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts index a0de79da565e62..8d6a2d110efe0f 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/saved_objects_types.ts @@ -35,10 +35,12 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe hidden: false, namespaceType: 'agnostic', mappings: { + dynamic: false, properties: { - appId: { type: 'keyword' }, - numberOfClicks: { type: 'long' }, - minutesOnScreen: { type: 'float' }, + // Disabled the mapping of these fields since they are not searched and we need to reduce the amount of indexed fields (#43673) + // appId: { type: 'keyword' }, + // numberOfClicks: { type: 'long' }, + // minutesOnScreen: { type: 'float' }, }, }, }); @@ -48,11 +50,13 @@ export function registerMappings(registerType: SavedObjectsServiceSetup['registe hidden: false, namespaceType: 'agnostic', mappings: { + dynamic: false, properties: { timestamp: { type: 'date' }, - appId: { type: 'keyword' }, - numberOfClicks: { type: 'long' }, - minutesOnScreen: { type: 'float' }, + // Disabled the mapping of these fields since they are not searched and we need to reduce the amount of indexed fields (#43673) + // appId: { type: 'keyword' }, + // numberOfClicks: { type: 'long' }, + // minutesOnScreen: { type: 'float' }, }, }, }); From 6607bf7b49c710b0870367ceef1444c24b2c0d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Thu, 2 Jul 2020 08:08:35 +0100 Subject: [PATCH 08/31] [Telemetry] Report data shippers (#64935) Co-authored-by: Christiane (Tina) Heiligers Co-authored-by: Elastic Machine --- src/plugins/telemetry/server/index.ts | 4 + .../__tests__/get_local_stats.js | 19 +- .../get_data_telemetry/constants.ts | 136 ++++++++++ .../get_data_telemetry.test.ts | 251 +++++++++++++++++ .../get_data_telemetry/get_data_telemetry.ts | 253 +++++++++++++++++ .../get_data_telemetry/index.ts | 27 ++ .../telemetry_collection/get_local_stats.ts | 7 +- .../server/telemetry_collection/index.ts | 6 + .../apis/telemetry/telemetry_local.js | 20 ++ .../lib/elasticsearch/indices/get_indices.js | 40 ++- .../server/lib/elasticsearch/indices/index.js | 2 +- .../get_stats_with_xpack.test.ts.snap | 254 +++++++++--------- .../get_stats_with_xpack.test.ts | 12 +- .../apis/telemetry/fixtures/basiccluster.json | 8 +- .../apis/telemetry/fixtures/multicluster.json | 72 ++--- 15 files changed, 903 insertions(+), 208 deletions(-) create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/constants.ts create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts diff --git a/src/plugins/telemetry/server/index.ts b/src/plugins/telemetry/server/index.ts index d048c8f5e94279..42259d2e5187ce 100644 --- a/src/plugins/telemetry/server/index.ts +++ b/src/plugins/telemetry/server/index.ts @@ -47,4 +47,8 @@ export { getLocalLicense, getLocalStats, TelemetryLocalStats, + DATA_TELEMETRY_ID, + DataTelemetryIndex, + DataTelemetryPayload, + buildDataTelemetryPayload, } from './telemetry_collection'; diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js index e78b92498e6e78..8541745faea3b6 100644 --- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js +++ b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js @@ -179,23 +179,36 @@ describe('get_local_stats', () => { describe('handleLocalStats', () => { it('returns expected object without xpack and kibana data', () => { - const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context); + const result = handleLocalStats( + clusterInfo, + clusterStatsWithNodesUsage, + void 0, + void 0, + context + ); expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); expect(result.version).to.be('2.3.4'); expect(result.collection).to.be('local'); expect(result.license).to.be(undefined); - expect(result.stack_stats).to.eql({ kibana: undefined }); + expect(result.stack_stats).to.eql({ kibana: undefined, data: undefined }); }); it('returns expected object with xpack', () => { - const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context); + const result = handleLocalStats( + clusterInfo, + clusterStatsWithNodesUsage, + void 0, + void 0, + context + ); const { stack_stats: stack, ...cluster } = result; expect(cluster.collection).to.be(combinedStatsResult.collection); expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid); expect(cluster.cluster_name).to.be(combinedStatsResult.cluster_name); expect(stack.kibana).to.be(undefined); // not mocked for this test + expect(stack.data).to.be(undefined); // not mocked for this test expect(cluster.version).to.eql(combinedStatsResult.version); expect(cluster.cluster_stats).to.eql(combinedStatsResult.cluster_stats); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/constants.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/constants.ts new file mode 100644 index 00000000000000..2d0864b1cb75f8 --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/constants.ts @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const DATA_TELEMETRY_ID = 'data'; + +export const DATA_KNOWN_TYPES = ['logs', 'traces', 'metrics'] as const; + +export type DataTelemetryType = typeof DATA_KNOWN_TYPES[number]; + +export type DataPatternName = typeof DATA_DATASETS_INDEX_PATTERNS[number]['patternName']; + +// TODO: Ideally this list should be updated from an external public URL (similar to the newsfeed) +// But it's good to have a minimum list shipped with the build. +export const DATA_DATASETS_INDEX_PATTERNS = [ + // Enterprise Search - Elastic + { pattern: '.ent-search-*', patternName: 'enterprise-search' }, + { pattern: '.app-search-*', patternName: 'app-search' }, + // Enterprise Search - 3rd party + { pattern: '*magento2*', patternName: 'magento2' }, + { pattern: '*magento*', patternName: 'magento' }, + { pattern: '*shopify*', patternName: 'shopify' }, + { pattern: '*wordpress*', patternName: 'wordpress' }, + // { pattern: '*wp*', patternName: 'wordpress' }, // TODO: Too vague? + { pattern: '*drupal*', patternName: 'drupal' }, + { pattern: '*joomla*', patternName: 'joomla' }, + { pattern: '*search*', patternName: 'search' }, // TODO: Too vague? + // { pattern: '*wix*', patternName: 'wix' }, // TODO: Too vague? + { pattern: '*sharepoint*', patternName: 'sharepoint' }, + { pattern: '*squarespace*', patternName: 'squarespace' }, + // { pattern: '*aem*', patternName: 'aem' }, // TODO: Too vague? + { pattern: '*sitecore*', patternName: 'sitecore' }, + { pattern: '*weebly*', patternName: 'weebly' }, + { pattern: '*acquia*', patternName: 'acquia' }, + + // Observability - Elastic + { pattern: 'filebeat-*', patternName: 'filebeat', shipper: 'filebeat' }, + { pattern: 'metricbeat-*', patternName: 'metricbeat', shipper: 'metricbeat' }, + { pattern: 'apm-*', patternName: 'apm', shipper: 'apm' }, + { pattern: 'functionbeat-*', patternName: 'functionbeat', shipper: 'functionbeat' }, + { pattern: 'heartbeat-*', patternName: 'heartbeat', shipper: 'heartbeat' }, + { pattern: 'logstash-*', patternName: 'logstash', shipper: 'logstash' }, + // Observability - 3rd party + { pattern: 'fluentd*', patternName: 'fluentd' }, + { pattern: 'telegraf*', patternName: 'telegraf' }, + { pattern: 'prometheusbeat*', patternName: 'prometheusbeat' }, + { pattern: 'fluentbit*', patternName: 'fluentbit' }, + { pattern: '*nginx*', patternName: 'nginx' }, + { pattern: '*apache*', patternName: 'apache' }, // Already in Security (keeping it in here for documentation) + // { pattern: '*logs*', patternName: 'third-party-logs' }, Disabled for now + + // Security - Elastic + { pattern: 'logstash-*', patternName: 'logstash', shipper: 'logstash' }, + { pattern: 'endgame-*', patternName: 'endgame', shipper: 'endgame' }, + { pattern: 'logs-endpoint.*', patternName: 'logs-endpoint', shipper: 'endpoint' }, // It should be caught by the `mappings` logic, but just in case + { pattern: 'metrics-endpoint.*', patternName: 'metrics-endpoint', shipper: 'endpoint' }, // It should be caught by the `mappings` logic, but just in case + { pattern: '.siem-signals-*', patternName: 'siem-signals' }, + { pattern: 'auditbeat-*', patternName: 'auditbeat', shipper: 'auditbeat' }, + { pattern: 'winlogbeat-*', patternName: 'winlogbeat', shipper: 'winlogbeat' }, + { pattern: 'packetbeat-*', patternName: 'packetbeat', shipper: 'packetbeat' }, + { pattern: 'filebeat-*', patternName: 'filebeat', shipper: 'filebeat' }, + // Security - 3rd party + { pattern: '*apache*', patternName: 'apache' }, // Already in Observability (keeping it in here for documentation) + { pattern: '*tomcat*', patternName: 'tomcat' }, + { pattern: '*artifactory*', patternName: 'artifactory' }, + { pattern: '*aruba*', patternName: 'aruba' }, + { pattern: '*barracuda*', patternName: 'barracuda' }, + { pattern: '*bluecoat*', patternName: 'bluecoat' }, + { pattern: 'arcsight-*', patternName: 'arcsight', shipper: 'arcsight' }, + // { pattern: '*cef*', patternName: 'cef' }, // Disabled because it's too vague + { pattern: '*checkpoint*', patternName: 'checkpoint' }, + { pattern: '*cisco*', patternName: 'cisco' }, + { pattern: '*citrix*', patternName: 'citrix' }, + { pattern: '*cyberark*', patternName: 'cyberark' }, + { pattern: '*cylance*', patternName: 'cylance' }, + { pattern: '*fireeye*', patternName: 'fireeye' }, + { pattern: '*fortinet*', patternName: 'fortinet' }, + { pattern: '*infoblox*', patternName: 'infoblox' }, + { pattern: '*kaspersky*', patternName: 'kaspersky' }, + { pattern: '*mcafee*', patternName: 'mcafee' }, + // paloaltonetworks + { pattern: '*paloaltonetworks*', patternName: 'paloaltonetworks' }, + { pattern: 'pan-*', patternName: 'paloaltonetworks' }, + { pattern: 'pan_*', patternName: 'paloaltonetworks' }, + { pattern: 'pan.*', patternName: 'paloaltonetworks' }, + + // rsa + { pattern: 'rsa.*', patternName: 'rsa' }, + { pattern: 'rsa-*', patternName: 'rsa' }, + { pattern: 'rsa_*', patternName: 'rsa' }, + + // snort + { pattern: 'snort-*', patternName: 'snort' }, + { pattern: 'logstash-snort*', patternName: 'snort' }, + + { pattern: '*sonicwall*', patternName: 'sonicwall' }, + { pattern: '*sophos*', patternName: 'sophos' }, + + // squid + { pattern: 'squid-*', patternName: 'squid' }, + { pattern: 'squid_*', patternName: 'squid' }, + { pattern: 'squid.*', patternName: 'squid' }, + + { pattern: '*symantec*', patternName: 'symantec' }, + { pattern: '*tippingpoint*', patternName: 'tippingpoint' }, + { pattern: '*trendmicro*', patternName: 'trendmicro' }, + { pattern: '*tripwire*', patternName: 'tripwire' }, + { pattern: '*zscaler*', patternName: 'zscaler' }, + { pattern: '*zeek*', patternName: 'zeek' }, + { pattern: '*sigma_doc*', patternName: 'sigma_doc' }, + // { pattern: '*bro*', patternName: 'bro' }, // Disabled because it's too vague + { pattern: 'ecs-corelight*', patternName: 'ecs-corelight' }, + { pattern: '*suricata*', patternName: 'suricata' }, + // { pattern: '*fsf*', patternName: 'fsf' }, // Disabled because it's too vague + { pattern: '*wazuh*', patternName: 'wazuh' }, +] as const; + +// Get the unique list of index patterns (some are duplicated for documentation purposes) +export const DATA_DATASETS_INDEX_PATTERNS_UNIQUE = DATA_DATASETS_INDEX_PATTERNS.filter( + (entry, index, array) => !array.slice(0, index).find(({ pattern }) => entry.pattern === pattern) +); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts new file mode 100644 index 00000000000000..8bffc5d012a741 --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts @@ -0,0 +1,251 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { buildDataTelemetryPayload, getDataTelemetry } from './get_data_telemetry'; +import { DATA_DATASETS_INDEX_PATTERNS, DATA_DATASETS_INDEX_PATTERNS_UNIQUE } from './constants'; + +describe('get_data_telemetry', () => { + describe('DATA_DATASETS_INDEX_PATTERNS', () => { + DATA_DATASETS_INDEX_PATTERNS.forEach((entry, index, array) => { + describe(`Pattern ${entry.pattern}`, () => { + test('there should only be one in DATA_DATASETS_INDEX_PATTERNS_UNIQUE', () => { + expect( + DATA_DATASETS_INDEX_PATTERNS_UNIQUE.filter(({ pattern }) => pattern === entry.pattern) + ).toHaveLength(1); + }); + + // This test is to make us sure that we don't update one of the duplicated entries and forget about any other repeated ones + test('when a document is duplicated, the duplicates should be identical', () => { + array.slice(0, index).forEach((previousEntry) => { + if (entry.pattern === previousEntry.pattern) { + expect(entry).toStrictEqual(previousEntry); + } + }); + }); + }); + }); + }); + + describe('buildDataTelemetryPayload', () => { + test('return the base object when no indices provided', () => { + expect(buildDataTelemetryPayload([])).toStrictEqual([]); + }); + + test('return the base object when no matching indices provided', () => { + expect( + buildDataTelemetryPayload([ + { name: 'no__way__this__can_match_anything', sizeInBytes: 10 }, + { name: '.kibana-event-log-8.0.0' }, + ]) + ).toStrictEqual([]); + }); + + test('matches some indices and puts them in their own category', () => { + expect( + buildDataTelemetryPayload([ + // APM Indices have known shipper (so we can infer the datasetType from mapping constant) + { name: 'apm-7.7.0-error-000001', shipper: 'apm', isECS: true }, + { name: 'apm-7.7.0-metric-000001', shipper: 'apm', isECS: true }, + { name: 'apm-7.7.0-onboarding-2020.05.17', shipper: 'apm', isECS: true }, + { name: 'apm-7.7.0-profile-000001', shipper: 'apm', isECS: true }, + { name: 'apm-7.7.0-span-000001', shipper: 'apm', isECS: true }, + { name: 'apm-7.7.0-transaction-000001', shipper: 'apm', isECS: true }, + // Packetbeat indices with known shipper (we can infer datasetType from mapping constant) + { name: 'packetbeat-7.7.0-2020.06.11-000001', shipper: 'packetbeat', isECS: true }, + // Matching patterns from the list => known datasetName but the rest is unknown + { name: 'filebeat-12314', docCount: 100, sizeInBytes: 10 }, + { name: 'metricbeat-1234', docCount: 100, sizeInBytes: 10, isECS: false }, + { name: '.app-search-1234', docCount: 0 }, + { name: 'logs-endpoint.1234', docCount: 0 }, // Matching pattern with a dot in the name + // New Indexing strategy: everything can be inferred from the constant_keyword values + { + name: 'logs-nginx.access-default-000001', + datasetName: 'nginx.access', + datasetType: 'logs', + shipper: 'filebeat', + isECS: true, + docCount: 1000, + sizeInBytes: 1000, + }, + { + name: 'logs-nginx.access-default-000002', + datasetName: 'nginx.access', + datasetType: 'logs', + shipper: 'filebeat', + isECS: true, + docCount: 1000, + sizeInBytes: 60, + }, + ]) + ).toStrictEqual([ + { + shipper: 'apm', + index_count: 6, + ecs_index_count: 6, + }, + { + shipper: 'packetbeat', + index_count: 1, + ecs_index_count: 1, + }, + { + pattern_name: 'filebeat', + shipper: 'filebeat', + index_count: 1, + doc_count: 100, + size_in_bytes: 10, + }, + { + pattern_name: 'metricbeat', + shipper: 'metricbeat', + index_count: 1, + ecs_index_count: 0, + doc_count: 100, + size_in_bytes: 10, + }, + { + pattern_name: 'app-search', + index_count: 1, + doc_count: 0, + }, + { + pattern_name: 'logs-endpoint', + shipper: 'endpoint', + index_count: 1, + doc_count: 0, + }, + { + dataset: { name: 'nginx.access', type: 'logs' }, + shipper: 'filebeat', + index_count: 2, + ecs_index_count: 2, + doc_count: 2000, + size_in_bytes: 1060, + }, + ]); + }); + }); + + describe('getDataTelemetry', () => { + test('it returns the base payload (all 0s) because no indices are found', async () => { + const callCluster = mockCallCluster(); + await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]); + }); + + test('can only see the index mappings, but not the stats', async () => { + const callCluster = mockCallCluster(['filebeat-12314']); + await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + { + pattern_name: 'filebeat', + shipper: 'filebeat', + index_count: 1, + ecs_index_count: 0, + }, + ]); + }); + + test('can see the mappings and the stats', async () => { + const callCluster = mockCallCluster( + ['filebeat-12314'], + { isECS: true }, + { + indices: { + 'filebeat-12314': { total: { docs: { count: 100 }, store: { size_in_bytes: 10 } } }, + }, + } + ); + await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + { + pattern_name: 'filebeat', + shipper: 'filebeat', + index_count: 1, + ecs_index_count: 1, + doc_count: 100, + size_in_bytes: 10, + }, + ]); + }); + + test('find an index that does not match any index pattern but has mappings metadata', async () => { + const callCluster = mockCallCluster( + ['cannot_match_anything'], + { isECS: true, datasetType: 'traces', shipper: 'my-beat' }, + { + indices: { + cannot_match_anything: { + total: { docs: { count: 100 }, store: { size_in_bytes: 10 } }, + }, + }, + } + ); + await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([ + { + dataset: { name: undefined, type: 'traces' }, + shipper: 'my-beat', + index_count: 1, + ecs_index_count: 1, + doc_count: 100, + size_in_bytes: 10, + }, + ]); + }); + + test('return empty array when there is an error', async () => { + const callCluster = jest.fn().mockRejectedValue(new Error('Something went terribly wrong')); + await expect(getDataTelemetry(callCluster)).resolves.toStrictEqual([]); + }); + }); +}); + +function mockCallCluster( + indicesMappings: string[] = [], + { isECS = false, datasetName = '', datasetType = '', shipper = '' } = {}, + indexStats: any = {} +) { + return jest.fn().mockImplementation(async (method: string, opts: any) => { + if (method === 'indices.getMapping') { + return Object.fromEntries( + indicesMappings.map((index) => [ + index, + { + mappings: { + ...(shipper && { _meta: { beat: shipper } }), + properties: { + ...(isECS && { ecs: { properties: { version: { type: 'keyword' } } } }), + ...((datasetType || datasetName) && { + dataset: { + properties: { + ...(datasetName && { + name: { type: 'constant_keyword', value: datasetName }, + }), + ...(datasetType && { + type: { type: 'constant_keyword', value: datasetType }, + }), + }, + }, + }), + }, + }, + }, + ]) + ); + } + return indexStats; + }); +} diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts new file mode 100644 index 00000000000000..cf906bc5c86cfc --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts @@ -0,0 +1,253 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { LegacyAPICaller } from 'kibana/server'; +import { + DATA_DATASETS_INDEX_PATTERNS_UNIQUE, + DataPatternName, + DataTelemetryType, +} from './constants'; + +export interface DataTelemetryBasePayload { + index_count: number; + ecs_index_count?: number; + doc_count?: number; + size_in_bytes?: number; +} + +export interface DataTelemetryDocument extends DataTelemetryBasePayload { + dataset?: { + name?: string; + type?: DataTelemetryType | 'unknown' | string; // The union of types is to help autocompletion with some known `dataset.type`s + }; + shipper?: string; + pattern_name?: DataPatternName; +} + +export type DataTelemetryPayload = DataTelemetryDocument[]; + +export interface DataTelemetryIndex { + name: string; + datasetName?: string; // To be obtained from `mappings.dataset.name` if it's a constant keyword + datasetType?: string; // To be obtained from `mappings.dataset.type` if it's a constant keyword + shipper?: string; // To be obtained from `_meta.beat` if it's set + isECS?: boolean; // Optional because it can't be obtained via Monitoring. + + // The fields below are optional because we might not be able to obtain them if the user does not + // have access to the index. + docCount?: number; + sizeInBytes?: number; +} + +type AtLeastOne }> = Partial & U[keyof U]; + +type DataDescriptor = AtLeastOne<{ + datasetName: string; + datasetType: string; + shipper: string; + patternName: DataPatternName; // When found from the list of the index patterns +}>; + +function findMatchingDescriptors({ + name, + shipper, + datasetName, + datasetType, +}: DataTelemetryIndex): DataDescriptor[] { + // If we already have the data from the indices' mappings... + if ([shipper, datasetName, datasetType].some(Boolean)) { + return [ + { + ...(shipper && { shipper }), + ...(datasetName && { datasetName }), + ...(datasetType && { datasetType }), + } as AtLeastOne<{ datasetName: string; datasetType: string; shipper: string }>, // Using casting here because TS doesn't infer at least one exists from the if clause + ]; + } + + // Otherwise, try with the list of known index patterns + return DATA_DATASETS_INDEX_PATTERNS_UNIQUE.filter(({ pattern }) => { + if (!pattern.startsWith('.') && name.startsWith('.')) { + // avoid system indices caught by very fuzzy index patterns (i.e.: *log* would catch `.kibana-log-...`) + return false; + } + return new RegExp(`^${pattern.replace(/\./g, '\\.').replace(/\*/g, '.*')}$`).test(name); + }); +} + +function increaseCounters( + previousValue: DataTelemetryBasePayload = { index_count: 0 }, + { isECS, docCount, sizeInBytes }: DataTelemetryIndex +) { + return { + ...previousValue, + index_count: previousValue.index_count + 1, + ...(typeof isECS === 'boolean' + ? { + ecs_index_count: (previousValue.ecs_index_count || 0) + (isECS ? 1 : 0), + } + : {}), + ...(typeof docCount === 'number' + ? { doc_count: (previousValue.doc_count || 0) + docCount } + : {}), + ...(typeof sizeInBytes === 'number' + ? { size_in_bytes: (previousValue.size_in_bytes || 0) + sizeInBytes } + : {}), + }; +} + +export function buildDataTelemetryPayload(indices: DataTelemetryIndex[]): DataTelemetryPayload { + const startingDotPatternsUntilTheFirstAsterisk = DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map( + ({ pattern }) => pattern.replace(/^\.(.+)\*.*$/g, '.$1') + ).filter(Boolean); + + // Filter out the system indices unless they are required by the patterns + const indexCandidates = indices.filter( + ({ name }) => + !( + name.startsWith('.') && + !startingDotPatternsUntilTheFirstAsterisk.find((pattern) => name.startsWith(pattern)) + ) + ); + + const acc = new Map(); + + for (const indexCandidate of indexCandidates) { + const matchingDescriptors = findMatchingDescriptors(indexCandidate); + for (const { datasetName, datasetType, shipper, patternName } of matchingDescriptors) { + const key = `${datasetName}-${datasetType}-${shipper}-${patternName}`; + acc.set(key, { + ...((datasetName || datasetType) && { dataset: { name: datasetName, type: datasetType } }), + ...(shipper && { shipper }), + ...(patternName && { pattern_name: patternName }), + ...increaseCounters(acc.get(key), indexCandidate), + }); + } + } + + return [...acc.values()]; +} + +interface IndexStats { + indices: { + [indexName: string]: { + total: { + docs: { + count: number; + deleted: number; + }; + store: { + size_in_bytes: number; + }; + }; + }; + }; +} + +interface IndexMappings { + [indexName: string]: { + mappings: { + _meta?: { + beat?: string; + }; + properties: { + dataset?: { + properties: { + name?: { + type: string; + value?: string; + }; + type?: { + type: string; + value?: string; + }; + }; + }; + ecs?: { + properties: { + version?: { + type: string; + }; + }; + }; + }; + }; + }; +} + +export async function getDataTelemetry(callCluster: LegacyAPICaller) { + try { + const index = [ + ...DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map(({ pattern }) => pattern), + '*-*-*-*', // Include new indexing strategy indices {type}-{dataset}-{namespace}-{rollover_counter} + ]; + const [indexMappings, indexStats]: [IndexMappings, IndexStats] = await Promise.all([ + // GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value + callCluster('indices.getMapping', { + index: '*', // Request all indices because filter_path already filters out the indices without any of those fields + filterPath: [ + // _meta.beat tells the shipper + '*.mappings._meta.beat', + // Does it have `ecs.version` in the mappings? => It follows the ECS conventions + '*.mappings.properties.ecs.properties.version.type', + + // Disable the fields below because they are still pending to be confirmed: + // https://github.com/elastic/ecs/pull/845 + // TODO: Re-enable when the final fields are confirmed + // // If `dataset.type` is a `constant_keyword`, it can be reported as a type + // '*.mappings.properties.dataset.properties.type.value', + // // If `dataset.name` is a `constant_keyword`, it can be reported as the dataset + // '*.mappings.properties.dataset.properties.name.value', + ], + }), + // GET /_stats/docs,store?level=indices&filter_path=indices.*.total + callCluster('indices.stats', { + index, + level: 'indices', + metric: ['docs', 'store'], + filterPath: ['indices.*.total'], + }), + ]); + + const indexNames = Object.keys({ ...indexMappings, ...indexStats?.indices }); + const indices = indexNames.map((name) => { + const isECS = !!indexMappings[name]?.mappings?.properties.ecs?.properties.version?.type; + const shipper = indexMappings[name]?.mappings?._meta?.beat; + const datasetName = indexMappings[name]?.mappings?.properties.dataset?.properties.name?.value; + const datasetType = indexMappings[name]?.mappings?.properties.dataset?.properties.type?.value; + + const stats = (indexStats?.indices || {})[name]; + if (stats) { + return { + name, + datasetName, + datasetType, + shipper, + isECS, + docCount: stats.total?.docs?.count, + sizeInBytes: stats.total?.store?.size_in_bytes, + }; + } + return { name, datasetName, datasetType, shipper, isECS }; + }); + return buildDataTelemetryPayload(indices); + } catch (e) { + return []; + } +} diff --git a/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts new file mode 100644 index 00000000000000..d056d1c9f299f3 --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/index.ts @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { DATA_TELEMETRY_ID } from './constants'; + +export { + DataTelemetryIndex, + DataTelemetryPayload, + getDataTelemetry, + buildDataTelemetryPayload, +} from './get_data_telemetry'; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index b42edde2f55ca2..4d4031bb428baf 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -25,6 +25,7 @@ import { getClusterInfo, ESClusterInfo } from './get_cluster_info'; import { getClusterStats } from './get_cluster_stats'; import { getKibana, handleKibanaStats, KibanaUsageStats } from './get_kibana'; import { getNodesUsage } from './get_nodes_usage'; +import { getDataTelemetry, DATA_TELEMETRY_ID, DataTelemetryPayload } from './get_data_telemetry'; /** * Handle the separate local calls by combining them into a single object response that looks like the @@ -39,6 +40,7 @@ export function handleLocalStats( { cluster_name, cluster_uuid, version }: ESClusterInfo, { _nodes, cluster_name: clusterName, ...clusterStats }: any, kibana: KibanaUsageStats, + dataTelemetry: DataTelemetryPayload, context: StatsCollectionContext ) { return { @@ -49,6 +51,7 @@ export function handleLocalStats( cluster_stats: clusterStats, collection: 'local', stack_stats: { + [DATA_TELEMETRY_ID]: dataTelemetry, kibana: handleKibanaStats(context, kibana), }, }; @@ -68,11 +71,12 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( return await Promise.all( clustersDetails.map(async (clustersDetail) => { - const [clusterInfo, clusterStats, nodesUsage, kibana] = await Promise.all([ + const [clusterInfo, clusterStats, nodesUsage, kibana, dataTelemetry] = await Promise.all([ getClusterInfo(callCluster), // cluster info getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_) getNodesUsage(callCluster), // nodes_usage info getKibana(usageCollection, callCluster), + getDataTelemetry(callCluster), ]); return handleLocalStats( clusterInfo, @@ -81,6 +85,7 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( nodes: { ...clusterStats.nodes, usage: nodesUsage }, }, kibana, + dataTelemetry, context ); }) diff --git a/src/plugins/telemetry/server/telemetry_collection/index.ts b/src/plugins/telemetry/server/telemetry_collection/index.ts index 377ddab7b877ce..40cbf0e4caa1d9 100644 --- a/src/plugins/telemetry/server/telemetry_collection/index.ts +++ b/src/plugins/telemetry/server/telemetry_collection/index.ts @@ -17,6 +17,12 @@ * under the License. */ +export { + DATA_TELEMETRY_ID, + DataTelemetryIndex, + DataTelemetryPayload, + buildDataTelemetryPayload, +} from './get_data_telemetry'; export { getLocalStats, TelemetryLocalStats } from './get_local_stats'; export { getLocalLicense } from './get_local_license'; export { getClusterUuids } from './get_cluster_stats'; diff --git a/test/api_integration/apis/telemetry/telemetry_local.js b/test/api_integration/apis/telemetry/telemetry_local.js index e74cd180185ab3..88e6b3a29052e2 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.js +++ b/test/api_integration/apis/telemetry/telemetry_local.js @@ -37,8 +37,17 @@ function flatKeys(source) { export default function ({ getService }) { const supertest = getService('supertest'); + const es = getService('es'); describe('/api/telemetry/v2/clusters/_stats', () => { + before('create some telemetry-data tracked indices', async () => { + return es.indices.create({ index: 'filebeat-telemetry_tests_logs' }); + }); + + after('cleanup telemetry-data tracked indices', () => { + return es.indices.delete({ index: 'filebeat-telemetry_tests_logs' }); + }); + it('should pull local stats and validate data types', async () => { const timeRange = { min: '2018-07-23T22:07:00Z', @@ -71,6 +80,17 @@ export default function ({ getService }) { expect(stats.stack_stats.kibana.plugins.csp.strict).to.be(true); expect(stats.stack_stats.kibana.plugins.csp.warnLegacyBrowsers).to.be(true); expect(stats.stack_stats.kibana.plugins.csp.rulesChangedFromDefault).to.be(false); + + // Testing stack_stats.data + expect(stats.stack_stats.data).to.be.an('object'); + expect(stats.stack_stats.data).to.be.an('array'); + expect(stats.stack_stats.data[0]).to.be.an('object'); + expect(stats.stack_stats.data[0].pattern_name).to.be('filebeat'); + expect(stats.stack_stats.data[0].shipper).to.be('filebeat'); + expect(stats.stack_stats.data[0].index_count).to.be(1); + expect(stats.stack_stats.data[0].doc_count).to.be(0); + expect(stats.stack_stats.data[0].ecs_index_count).to.be(0); + expect(stats.stack_stats.data[0].size_in_bytes).to.be.greaterThan(0); }); it('should pull local stats and validate fields', async () => { diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js b/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js index c087d20a97db1f..ba6d0cb926f063 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/get_indices.js @@ -77,11 +77,11 @@ export function handleResponse(resp, min, max, shardStats) { }); } -export function getIndices(req, esIndexPattern, showSystemIndices = false, shardStats) { - checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getIndices'); - - const { min, max } = req.payload.timeRange; - +export function buildGetIndicesQuery( + esIndexPattern, + clusterUuid, + { start, end, size, showSystemIndices = false } +) { const filters = []; if (!showSystemIndices) { filters.push({ @@ -90,14 +90,11 @@ export function getIndices(req, esIndexPattern, showSystemIndices = false, shard }, }); } - - const clusterUuid = req.params.clusterUuid; const metricFields = ElasticsearchMetric.getMetricFields(); - const config = req.server.config(); - const params = { + + return { index: esIndexPattern, - // TODO: composite aggregation - size: config.get('monitoring.ui.max_bucket_size'), + size, ignoreUnavailable: true, filterPath: [ // only filter path can filter for inner_hits @@ -118,8 +115,8 @@ export function getIndices(req, esIndexPattern, showSystemIndices = false, shard body: { query: createQuery({ type: 'index_stats', - start: min, - end: max, + start, + end, clusterUuid, metric: metricFields, filters, @@ -135,9 +132,24 @@ export function getIndices(req, esIndexPattern, showSystemIndices = false, shard sort: [{ timestamp: { order: 'desc' } }], }, }; +} + +export function getIndices(req, esIndexPattern, showSystemIndices = false, shardStats) { + checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getIndices'); + + const { min: start, max: end } = req.payload.timeRange; + + const clusterUuid = req.params.clusterUuid; + const config = req.server.config(); + const params = buildGetIndicesQuery(esIndexPattern, clusterUuid, { + start, + end, + showSystemIndices, + size: config.get('monitoring.ui.max_bucket_size'), + }); const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); return callWithRequest(req, 'search', params).then((resp) => - handleResponse(resp, min, max, shardStats) + handleResponse(resp, start, end, shardStats) ); } diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/index.js b/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/index.js index 0ac2610bbba629..b07e3511d4804e 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/index.js +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/indices/index.js @@ -4,5 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ -export { getIndices } from './get_indices'; +export { getIndices, buildGetIndicesQuery } from './get_indices'; export { getIndexSummary } from './get_index_summary'; diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap index ed82dc65eb4108..b9bb206b8056f4 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap @@ -1,158 +1,158 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`Telemetry Collection: Get Aggregated Stats OSS-like telemetry (no license nor X-Pack telemetry) 1`] = ` -Array [ - Object { - "cluster_name": "test", - "cluster_stats": Object { - "nodes": Object { - "usage": Object { - "nodes": Array [ - Object { - "aggregations": Object { - "terms": Object { - "bytes": 2, - }, - }, - "node_id": "some_node_id", - "rest_actions": Object { - "nodes_usage_action": 1, +Object { + "cluster_name": "test", + "cluster_stats": Object { + "nodes": Object { + "usage": Object { + "nodes": Array [ + Object { + "aggregations": Object { + "terms": Object { + "bytes": 2, }, - "since": 1588616945163, - "timestamp": 1588617023177, }, - ], - }, - }, - }, - "cluster_uuid": "test", - "collection": "local", - "stack_stats": Object { - "kibana": Object { - "count": 1, - "great": "googlymoogly", - "indices": 1, - "os": Object { - "platformReleases": Array [ - Object { - "count": 1, - "platformRelease": "iv", - }, - ], - "platforms": Array [ - Object { - "count": 1, - "platform": "rocky", + "node_id": "some_node_id", + "rest_actions": Object { + "nodes_usage_action": 1, }, - ], - }, - "plugins": Object { - "clouds": Object { - "chances": 95, + "since": 1588616945163, + "timestamp": 1588617023177, }, - "localization": Object { - "integrities": Object {}, - "labelsCount": 0, - "locale": "en", - }, - "rain": Object { - "chances": 2, - }, - "snow": Object { - "chances": 0, - }, - "sun": Object { - "chances": 5, + ], + }, + }, + }, + "cluster_uuid": "test", + "collection": "local", + "stack_stats": Object { + "data": Array [], + "kibana": Object { + "count": 1, + "great": "googlymoogly", + "indices": 1, + "os": Object { + "platformReleases": Array [ + Object { + "count": 1, + "platformRelease": "iv", }, - }, - "versions": Array [ + ], + "platforms": Array [ Object { "count": 1, - "version": "8675309", + "platform": "rocky", }, ], }, + "plugins": Object { + "clouds": Object { + "chances": 95, + }, + "localization": Object { + "integrities": Object {}, + "labelsCount": 0, + "locale": "en", + }, + "rain": Object { + "chances": 2, + }, + "snow": Object { + "chances": 0, + }, + "sun": Object { + "chances": 5, + }, + }, + "versions": Array [ + Object { + "count": 1, + "version": "8675309", + }, + ], }, - "version": "8.0.0", }, -] + "timestamp": Any, + "version": "8.0.0", +} `; exports[`Telemetry Collection: Get Aggregated Stats X-Pack telemetry (license + X-Pack) 1`] = ` -Array [ - Object { - "cluster_name": "test", - "cluster_stats": Object { - "nodes": Object { - "usage": Object { - "nodes": Array [ - Object { - "aggregations": Object { - "terms": Object { - "bytes": 2, - }, - }, - "node_id": "some_node_id", - "rest_actions": Object { - "nodes_usage_action": 1, +Object { + "cluster_name": "test", + "cluster_stats": Object { + "nodes": Object { + "usage": Object { + "nodes": Array [ + Object { + "aggregations": Object { + "terms": Object { + "bytes": 2, }, - "since": 1588616945163, - "timestamp": 1588617023177, }, - ], - }, - }, - }, - "cluster_uuid": "test", - "collection": "local", - "stack_stats": Object { - "kibana": Object { - "count": 1, - "great": "googlymoogly", - "indices": 1, - "os": Object { - "platformReleases": Array [ - Object { - "count": 1, - "platformRelease": "iv", - }, - ], - "platforms": Array [ - Object { - "count": 1, - "platform": "rocky", + "node_id": "some_node_id", + "rest_actions": Object { + "nodes_usage_action": 1, }, - ], - }, - "plugins": Object { - "clouds": Object { - "chances": 95, + "since": 1588616945163, + "timestamp": 1588617023177, }, - "localization": Object { - "integrities": Object {}, - "labelsCount": 0, - "locale": "en", - }, - "rain": Object { - "chances": 2, - }, - "snow": Object { - "chances": 0, - }, - "sun": Object { - "chances": 5, + ], + }, + }, + }, + "cluster_uuid": "test", + "collection": "local", + "stack_stats": Object { + "data": Array [], + "kibana": Object { + "count": 1, + "great": "googlymoogly", + "indices": 1, + "os": Object { + "platformReleases": Array [ + Object { + "count": 1, + "platformRelease": "iv", }, - }, - "versions": Array [ + ], + "platforms": Array [ Object { "count": 1, - "version": "8675309", + "platform": "rocky", }, ], }, - "xpack": Object {}, + "plugins": Object { + "clouds": Object { + "chances": 95, + }, + "localization": Object { + "integrities": Object {}, + "labelsCount": 0, + "locale": "en", + }, + "rain": Object { + "chances": 2, + }, + "snow": Object { + "chances": 0, + }, + "sun": Object { + "chances": 5, + }, + }, + "versions": Array [ + Object { + "count": 1, + "version": "8675309", + }, + ], }, - "version": "8.0.0", + "xpack": Object {}, }, -] + "timestamp": Any, + "version": "8.0.0", +} `; diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts index a8311933f05317..24382fb89d3373 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts @@ -85,7 +85,11 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { } as any, context ); - expect(stats.map(({ timestamp, ...rest }) => rest)).toMatchSnapshot(); + stats.forEach((entry) => { + expect(entry).toMatchSnapshot({ + timestamp: expect.any(String), + }); + }); }); test('X-Pack telemetry (license + X-Pack)', async () => { @@ -123,6 +127,10 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { } as any, context ); - expect(stats.map(({ timestamp, ...rest }) => rest)).toMatchSnapshot(); + stats.forEach((entry) => { + expect(entry).toMatchSnapshot({ + timestamp: expect.any(String), + }); + }); }); }); diff --git a/x-pack/test/api_integration/apis/telemetry/fixtures/basiccluster.json b/x-pack/test/api_integration/apis/telemetry/fixtures/basiccluster.json index a0097f53ac93b9..74d91a6215c795 100644 --- a/x-pack/test/api_integration/apis/telemetry/fixtures/basiccluster.json +++ b/x-pack/test/api_integration/apis/telemetry/fixtures/basiccluster.json @@ -153,9 +153,7 @@ "min": 223 } }, - "versions": [ - "6.3.1" - ] + "versions": ["6.3.1"] }, "status": "yellow", "timestamp": 1532386499084 @@ -297,9 +295,7 @@ }, "audit": { "enabled": false, - "outputs": [ - "logfile" - ] + "outputs": ["logfile"] }, "available": false, "enabled": true, diff --git a/x-pack/test/api_integration/apis/telemetry/fixtures/multicluster.json b/x-pack/test/api_integration/apis/telemetry/fixtures/multicluster.json index 6cc9c55157b282..7d408e39247ee4 100644 --- a/x-pack/test/api_integration/apis/telemetry/fixtures/multicluster.json +++ b/x-pack/test/api_integration/apis/telemetry/fixtures/multicluster.json @@ -92,9 +92,7 @@ "master": 1, "ingest": 1 }, - "versions": [ - "7.0.0-alpha1" - ], + "versions": ["7.0.0-alpha1"], "os": { "available_processors": 4, "allocated_processors": 1, @@ -214,9 +212,7 @@ } }, "audit": { - "outputs": [ - "logfile" - ], + "outputs": ["logfile"], "enabled": false }, "ipfilter": { @@ -383,9 +379,7 @@ "master": 1, "ingest": 1 }, - "versions": [ - "7.0.0-alpha1" - ], + "versions": ["7.0.0-alpha1"], "os": { "available_processors": 4, "allocated_processors": 1, @@ -461,34 +455,22 @@ "enabled": true, "realms": { "file": { - "name": [ - "default_file" - ], + "name": ["default_file"], "available": true, - "size": [ - 0 - ], + "size": [0], "enabled": true, - "order": [ - 2147483647 - ] + "order": [2147483647] }, "ldap": { "available": true, "enabled": false }, "native": { - "name": [ - "default_native" - ], + "name": ["default_native"], "available": true, - "size": [ - 2 - ], + "size": [2], "enabled": true, - "order": [ - 2147483647 - ] + "order": [2147483647] }, "active_directory": { "available": true, @@ -523,9 +505,7 @@ } }, "audit": { - "outputs": [ - "logfile" - ], + "outputs": ["logfile"], "enabled": false }, "ipfilter": { @@ -700,9 +680,7 @@ "master": 2, "ingest": 2 }, - "versions": [ - "7.0.0-alpha1" - ], + "versions": ["7.0.0-alpha1"], "os": { "available_processors": 8, "allocated_processors": 2, @@ -778,34 +756,22 @@ "enabled": true, "realms": { "file": { - "name": [ - "default_file" - ], + "name": ["default_file"], "available": true, - "size": [ - 0 - ], + "size": [0], "enabled": true, - "order": [ - 2147483647 - ] + "order": [2147483647] }, "ldap": { "available": true, "enabled": false }, "native": { - "name": [ - "default_native" - ], + "name": ["default_native"], "available": true, - "size": [ - 1 - ], + "size": [1], "enabled": true, - "order": [ - 2147483647 - ] + "order": [2147483647] }, "active_directory": { "available": true, @@ -840,9 +806,7 @@ } }, "audit": { - "outputs": [ - "logfile" - ], + "outputs": ["logfile"], "enabled": false }, "ipfilter": { From f86afd4b070b219595dc444b43e86f24364ae30d Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Thu, 2 Jul 2020 11:02:35 +0300 Subject: [PATCH 09/31] [Visualizations] Each visType returns its supported triggers (#70177) * Refactor hardcoded supportedTriggers function with a callback function of visType * use typescript to check if function exists * Remove brush event from heatmap Co-authored-by: Elastic Machine --- .../vis_type_table/public/table_vis_type.ts | 4 +++ .../public/tag_cloud_type.ts | 4 +++ src/plugins/vis_type_vislib/public/area.ts | 4 +++ src/plugins/vis_type_vislib/public/heatmap.ts | 4 +++ .../vis_type_vislib/public/histogram.ts | 4 +++ .../vis_type_vislib/public/horizontal_bar.ts | 4 +++ src/plugins/vis_type_vislib/public/line.ts | 4 +++ src/plugins/vis_type_vislib/public/pie.ts | 4 +++ .../public/embeddable/visualize_embeddable.ts | 25 +------------------ .../public/vis_types/base_vis_type.ts | 4 +++ .../public/vis_types/types_service.ts | 2 ++ .../vis_types/vis_type_alias_registry.ts | 3 +++ 12 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/plugins/vis_type_table/public/table_vis_type.ts b/src/plugins/vis_type_table/public/table_vis_type.ts index c3bc72497007ea..80d53021b7866d 100644 --- a/src/plugins/vis_type_table/public/table_vis_type.ts +++ b/src/plugins/vis_type_table/public/table_vis_type.ts @@ -26,6 +26,7 @@ import { tableVisResponseHandler } from './table_vis_response_handler'; import tableVisTemplate from './table_vis.html'; import { TableOptions } from './components/table_vis_options_lazy'; import { getTableVisualizationControllerClass } from './vis_controller'; +import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; export function getTableVisTypeDefinition(core: CoreSetup, context: PluginInitializerContext) { return { @@ -39,6 +40,9 @@ export function getTableVisTypeDefinition(core: CoreSetup, context: PluginInitia defaultMessage: 'Display values in a table', }), visualization: getTableVisualizationControllerClass(core, context), + getSupportedTriggers: () => { + return [VIS_EVENT_TO_TRIGGER.filter]; + }, visConfig: { defaults: { perPage: 10, diff --git a/src/plugins/vis_type_tagcloud/public/tag_cloud_type.ts b/src/plugins/vis_type_tagcloud/public/tag_cloud_type.ts index 5a8cc3004a3154..023489c6d2e876 100644 --- a/src/plugins/vis_type_tagcloud/public/tag_cloud_type.ts +++ b/src/plugins/vis_type_tagcloud/public/tag_cloud_type.ts @@ -22,6 +22,7 @@ import { i18n } from '@kbn/i18n'; import { Schemas } from '../../vis_default_editor/public'; import { TagCloudOptions } from './components/tag_cloud_options'; +import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; // @ts-ignore import { createTagCloudVisualization } from './components/tag_cloud_visualization'; @@ -31,6 +32,9 @@ export const createTagCloudVisTypeDefinition = (deps: TagCloudVisDependencies) = name: 'tagcloud', title: i18n.translate('visTypeTagCloud.vis.tagCloudTitle', { defaultMessage: 'Tag Cloud' }), icon: 'visTagCloud', + getSupportedTriggers: () => { + return [VIS_EVENT_TO_TRIGGER.filter]; + }, description: i18n.translate('visTypeTagCloud.vis.tagCloudDescription', { defaultMessage: 'A group of words, sized according to their importance', }), diff --git a/src/plugins/vis_type_vislib/public/area.ts b/src/plugins/vis_type_vislib/public/area.ts index c42962ad50a4b0..ec90fbd1746a15 100644 --- a/src/plugins/vis_type_vislib/public/area.ts +++ b/src/plugins/vis_type_vislib/public/area.ts @@ -40,6 +40,7 @@ import { getAreaOptionTabs, countLabel } from './utils/common_config'; import { createVislibVisController } from './vis_controller'; import { VisTypeVislibDependencies } from './plugin'; import { Rotates } from '../../charts/public'; +import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; export const createAreaVisTypeDefinition = (deps: VisTypeVislibDependencies) => ({ name: 'area', @@ -49,6 +50,9 @@ export const createAreaVisTypeDefinition = (deps: VisTypeVislibDependencies) => defaultMessage: 'Emphasize the quantity beneath a line chart', }), visualization: createVislibVisController(deps), + getSupportedTriggers: () => { + return [VIS_EVENT_TO_TRIGGER.filter, VIS_EVENT_TO_TRIGGER.brush]; + }, visConfig: { defaults: { type: 'area', diff --git a/src/plugins/vis_type_vislib/public/heatmap.ts b/src/plugins/vis_type_vislib/public/heatmap.ts index ced7a38568ffd0..bd3d02029cb23a 100644 --- a/src/plugins/vis_type_vislib/public/heatmap.ts +++ b/src/plugins/vis_type_vislib/public/heatmap.ts @@ -28,6 +28,7 @@ import { TimeMarker } from './vislib/visualizations/time_marker'; import { CommonVislibParams, ValueAxis } from './types'; import { VisTypeVislibDependencies } from './plugin'; import { ColorSchemas, ColorSchemaParams } from '../../charts/public'; +import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; export interface HeatmapVisParams extends CommonVislibParams, ColorSchemaParams { type: 'heatmap'; @@ -48,6 +49,9 @@ export const createHeatmapVisTypeDefinition = (deps: VisTypeVislibDependencies) description: i18n.translate('visTypeVislib.heatmap.heatmapDescription', { defaultMessage: 'Shade cells within a matrix', }), + getSupportedTriggers: () => { + return [VIS_EVENT_TO_TRIGGER.filter]; + }, visualization: createVislibVisController(deps), visConfig: { defaults: { diff --git a/src/plugins/vis_type_vislib/public/histogram.ts b/src/plugins/vis_type_vislib/public/histogram.ts index 52242ad11e8f58..8aeeb4ec533abc 100644 --- a/src/plugins/vis_type_vislib/public/histogram.ts +++ b/src/plugins/vis_type_vislib/public/histogram.ts @@ -39,6 +39,7 @@ import { getAreaOptionTabs, countLabel } from './utils/common_config'; import { createVislibVisController } from './vis_controller'; import { VisTypeVislibDependencies } from './plugin'; import { Rotates } from '../../charts/public'; +import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; export const createHistogramVisTypeDefinition = (deps: VisTypeVislibDependencies) => ({ name: 'histogram', @@ -50,6 +51,9 @@ export const createHistogramVisTypeDefinition = (deps: VisTypeVislibDependencies defaultMessage: 'Assign a continuous variable to each axis', }), visualization: createVislibVisController(deps), + getSupportedTriggers: () => { + return [VIS_EVENT_TO_TRIGGER.filter, VIS_EVENT_TO_TRIGGER.brush]; + }, visConfig: { defaults: { type: 'histogram', diff --git a/src/plugins/vis_type_vislib/public/horizontal_bar.ts b/src/plugins/vis_type_vislib/public/horizontal_bar.ts index a58c15f136431e..702581828e60d0 100644 --- a/src/plugins/vis_type_vislib/public/horizontal_bar.ts +++ b/src/plugins/vis_type_vislib/public/horizontal_bar.ts @@ -37,6 +37,7 @@ import { getAreaOptionTabs, countLabel } from './utils/common_config'; import { createVislibVisController } from './vis_controller'; import { VisTypeVislibDependencies } from './plugin'; import { Rotates } from '../../charts/public'; +import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; export const createHorizontalBarVisTypeDefinition = (deps: VisTypeVislibDependencies) => ({ name: 'horizontal_bar', @@ -48,6 +49,9 @@ export const createHorizontalBarVisTypeDefinition = (deps: VisTypeVislibDependen defaultMessage: 'Assign a continuous variable to each axis', }), visualization: createVislibVisController(deps), + getSupportedTriggers: () => { + return [VIS_EVENT_TO_TRIGGER.filter, VIS_EVENT_TO_TRIGGER.brush]; + }, visConfig: { defaults: { type: 'histogram', diff --git a/src/plugins/vis_type_vislib/public/line.ts b/src/plugins/vis_type_vislib/public/line.ts index a94fd3f3945ab7..6e9190229114b5 100644 --- a/src/plugins/vis_type_vislib/public/line.ts +++ b/src/plugins/vis_type_vislib/public/line.ts @@ -38,6 +38,7 @@ import { getAreaOptionTabs, countLabel } from './utils/common_config'; import { createVislibVisController } from './vis_controller'; import { VisTypeVislibDependencies } from './plugin'; import { Rotates } from '../../charts/public'; +import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; export const createLineVisTypeDefinition = (deps: VisTypeVislibDependencies) => ({ name: 'line', @@ -47,6 +48,9 @@ export const createLineVisTypeDefinition = (deps: VisTypeVislibDependencies) => defaultMessage: 'Emphasize trends', }), visualization: createVislibVisController(deps), + getSupportedTriggers: () => { + return [VIS_EVENT_TO_TRIGGER.filter, VIS_EVENT_TO_TRIGGER.brush]; + }, visConfig: { defaults: { type: 'line', diff --git a/src/plugins/vis_type_vislib/public/pie.ts b/src/plugins/vis_type_vislib/public/pie.ts index a68bc5893406f5..1e81dbdde3f685 100644 --- a/src/plugins/vis_type_vislib/public/pie.ts +++ b/src/plugins/vis_type_vislib/public/pie.ts @@ -26,6 +26,7 @@ import { getPositions, Positions } from './utils/collections'; import { createVislibVisController } from './vis_controller'; import { CommonVislibParams } from './types'; import { VisTypeVislibDependencies } from './plugin'; +import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; export interface PieVisParams extends CommonVislibParams { type: 'pie'; @@ -47,6 +48,9 @@ export const createPieVisTypeDefinition = (deps: VisTypeVislibDependencies) => ( defaultMessage: 'Compare parts of a whole', }), visualization: createVislibVisController(deps), + getSupportedTriggers: () => { + return [VIS_EVENT_TO_TRIGGER.filter]; + }, visConfig: { defaults: { type: 'pie', diff --git a/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts b/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts index 26fdd665192a62..2f9cda32fccdc9 100644 --- a/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts +++ b/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts @@ -377,29 +377,6 @@ export class VisualizeEmbeddable extends Embeddable Array; icon?: string; image?: string; stage?: 'experimental' | 'beta' | 'production'; @@ -44,6 +46,7 @@ export class BaseVisType { name: string; title: string; description: string; + getSupportedTriggers?: () => Array; icon?: string; image?: string; stage: 'experimental' | 'beta' | 'production'; @@ -77,6 +80,7 @@ export class BaseVisType { this.name = opts.name; this.description = opts.description || ''; + this.getSupportedTriggers = opts.getSupportedTriggers; this.title = opts.title; this.icon = opts.icon; this.image = opts.image; diff --git a/src/plugins/visualizations/public/vis_types/types_service.ts b/src/plugins/visualizations/public/vis_types/types_service.ts index 321f96180fd68e..14c2a9c50ab0eb 100644 --- a/src/plugins/visualizations/public/vis_types/types_service.ts +++ b/src/plugins/visualizations/public/vis_types/types_service.ts @@ -23,11 +23,13 @@ import { visTypeAliasRegistry, VisTypeAlias } from './vis_type_alias_registry'; import { BaseVisType } from './base_vis_type'; // @ts-ignore import { ReactVisType } from './react_vis_type'; +import { TriggerContextMapping } from '../../../ui_actions/public'; export interface VisType { name: string; title: string; description?: string; + getSupportedTriggers?: () => Array; visualization: any; isAccessible?: boolean; requestHandler: string | unknown; diff --git a/src/plugins/visualizations/public/vis_types/vis_type_alias_registry.ts b/src/plugins/visualizations/public/vis_types/vis_type_alias_registry.ts index bc80d549c81e6f..f6d27b54c7c640 100644 --- a/src/plugins/visualizations/public/vis_types/vis_type_alias_registry.ts +++ b/src/plugins/visualizations/public/vis_types/vis_type_alias_registry.ts @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +import { TriggerContextMapping } from '../../../ui_actions/public'; export interface VisualizationListItem { editUrl: string; @@ -26,6 +27,7 @@ export interface VisualizationListItem { savedObjectType: string; title: string; description?: string; + getSupportedTriggers?: () => Array; typeTitle: string; image?: string; } @@ -53,6 +55,7 @@ export interface VisTypeAlias { icon: string; promotion?: VisTypeAliasPromotion; description: string; + getSupportedTriggers?: () => Array; stage: 'experimental' | 'beta' | 'production'; appExtensions?: { From dc2737b8686f6f96b357fcea91d3e1060683fb9a Mon Sep 17 00:00:00 2001 From: Daniil Suleiman <31325372+sulemanof@users.noreply.github.com> Date: Thu, 2 Jul 2020 11:38:46 +0300 Subject: [PATCH 10/31] Filter out error when calculating a label (#69934) Co-authored-by: Elastic Machine --- .../public/search/tabify/get_columns.test.ts | 16 ++++++++++++++++ .../data/public/search/tabify/get_columns.ts | 9 ++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/public/search/tabify/get_columns.test.ts b/src/plugins/data/public/search/tabify/get_columns.test.ts index 0c5551d95690f2..35f0181f633026 100644 --- a/src/plugins/data/public/search/tabify/get_columns.test.ts +++ b/src/plugins/data/public/search/tabify/get_columns.test.ts @@ -161,4 +161,20 @@ describe('get columns', () => { 'Sum of @timestamp', ]); }); + + test('should not fail if there is no field for date histogram agg', () => { + const columns = tabifyGetColumns( + createAggConfigs([ + { + type: 'date_histogram', + schema: 'segment', + params: {}, + }, + { type: 'sum', schema: 'metric', params: { field: '@timestamp' } }, + ]).aggs, + false + ); + + expect(columns.map((c) => c.name)).toEqual(['', 'Sum of @timestamp']); + }); }); diff --git a/src/plugins/data/public/search/tabify/get_columns.ts b/src/plugins/data/public/search/tabify/get_columns.ts index 8c538288d2feaf..8e907d4b0cb883 100644 --- a/src/plugins/data/public/search/tabify/get_columns.ts +++ b/src/plugins/data/public/search/tabify/get_columns.ts @@ -22,10 +22,17 @@ import { IAggConfig } from '../aggs'; import { TabbedAggColumn } from './types'; const getColumn = (agg: IAggConfig, i: number): TabbedAggColumn => { + let name = ''; + try { + name = agg.makeLabel(); + } catch (e) { + // skip the case when makeLabel throws an error (e.x. no appropriate field for an aggregation) + } + return { aggConfig: agg, id: `col-${i}-${agg.id}`, - name: agg.makeLabel(), + name, }; }; From 6aeda644c8a31fdd58ae43f1f0ea3f1e569476b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Thu, 2 Jul 2020 10:01:10 +0100 Subject: [PATCH 11/31] [APM] Show transaction rate per minute on Observability Overview page (#70336) * changing transaction count to transaction rate per second * sanity check coordinates before calculate the mean * sanity check coordinates before calculate the mean * removing extend_bounds to return empty when no data is available --- .../rest/observability.dashboard.test.ts | 42 ++++++++++++++++++- .../services/rest/observability_dashboard.ts | 9 +++- .../get_transaction_coordinates.ts | 5 ++- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/apm/public/services/rest/observability.dashboard.test.ts b/x-pack/plugins/apm/public/services/rest/observability.dashboard.test.ts index dbb5d6029d0f16..a14d827eeaec5f 100644 --- a/x-pack/plugins/apm/public/services/rest/observability.dashboard.test.ts +++ b/x-pack/plugins/apm/public/services/rest/observability.dashboard.test.ts @@ -58,7 +58,7 @@ describe('Observability dashboard data', () => { transactions: { type: 'number', label: 'Transactions', - value: 6, + value: 2, color: '#6092c0', }, }, @@ -115,5 +115,45 @@ describe('Observability dashboard data', () => { }, }); }); + it('returns transaction stat as 0 when y is undefined', async () => { + callApmApiMock.mockImplementation(() => + Promise.resolve({ + serviceCount: 0, + transactionCoordinates: [{ x: 1 }, { x: 2 }, { x: 3 }], + }) + ); + const response = await fetchLandingPageData( + { + startTime: '1', + endTime: '2', + bucketSize: '3', + }, + { theme } + ); + expect(response).toEqual({ + title: 'APM', + appLink: '/app/apm', + stats: { + services: { + type: 'number', + label: 'Services', + value: 0, + }, + transactions: { + type: 'number', + label: 'Transactions', + value: 0, + color: '#6092c0', + }, + }, + series: { + transactions: { + label: 'Transactions', + coordinates: [{ x: 1 }, { x: 2 }, { x: 3 }], + color: '#6092c0', + }, + }, + }); + }); }); }); diff --git a/x-pack/plugins/apm/public/services/rest/observability_dashboard.ts b/x-pack/plugins/apm/public/services/rest/observability_dashboard.ts index 2107565c5facf9..589199221d7a9e 100644 --- a/x-pack/plugins/apm/public/services/rest/observability_dashboard.ts +++ b/x-pack/plugins/apm/public/services/rest/observability_dashboard.ts @@ -5,7 +5,7 @@ */ import { i18n } from '@kbn/i18n'; -import { sum } from 'lodash'; +import mean from 'lodash.mean'; import { Theme } from '@kbn/ui-shared-deps/theme'; import { ApmFetchDataResponse, @@ -48,7 +48,12 @@ export const fetchLandingPageData = async ( 'xpack.apm.observabilityDashboard.stats.transactions', { defaultMessage: 'Transactions' } ), - value: sum(transactionCoordinates.map((coordinates) => coordinates.y)), + value: + mean( + transactionCoordinates + .map(({ y }) => y) + .filter((y) => y && isFinite(y)) + ) || 0, color: theme.euiColorVis1, }, }, diff --git a/x-pack/plugins/apm/server/lib/observability_dashboard/get_transaction_coordinates.ts b/x-pack/plugins/apm/server/lib/observability_dashboard/get_transaction_coordinates.ts index e78a3c1cec24a5..0d1a4274c16dc9 100644 --- a/x-pack/plugins/apm/server/lib/observability_dashboard/get_transaction_coordinates.ts +++ b/x-pack/plugins/apm/server/lib/observability_dashboard/get_transaction_coordinates.ts @@ -41,17 +41,18 @@ export async function getTransactionCoordinates({ field: '@timestamp', fixed_interval: bucketSize, min_doc_count: 0, - extended_bounds: { min: start, max: end }, }, }, }, }, }); + const deltaAsMinutes = (end - start) / 1000 / 60; + return ( aggregations?.distribution.buckets.map((bucket) => ({ x: bucket.key, - y: bucket.doc_count, + y: bucket.doc_count / deltaAsMinutes, })) || [] ); } From 83beede50cb57f012411cc31952692d8cd888d52 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Thu, 2 Jul 2020 11:02:52 +0200 Subject: [PATCH 12/31] [Ingest Pipelines] Error messages (#70167) * improved error messages * traverse recursive error struct * add check for object with keys * update button position and copy * size adjustments * Refactor i18n texts and change wording Also added missing translation and refactored maximum errors in collapsed state to external constant * use io-ts, add CIT and unit tests * refactor error utilities to separate file Co-authored-by: Elastic Machine --- .../__jest__/client_integration/fixtures.ts | 117 ++++++++++++++++++ .../helpers/pipeline_form.helpers.ts | 2 + .../ingest_pipelines_create.test.tsx | 21 ++++ .../pipeline_form/pipeline_form.tsx | 13 +- .../pipeline_form/pipeline_form_error.tsx | 34 ----- .../pipeline_form_error/error_utils.test.ts | 67 ++++++++++ .../pipeline_form_error/error_utils.ts | 85 +++++++++++++ .../pipeline_form_error/i18n_texts.ts | 38 ++++++ .../pipeline_form_error/index.ts | 7 ++ .../pipeline_form_error.tsx | 99 +++++++++++++++ .../server/routes/api/create.ts | 8 +- .../server/routes/api/shared/index.ts | 7 ++ .../routes/api/shared/is_object_with_keys.ts | 9 ++ .../server/routes/api/update.ts | 8 +- 14 files changed, 473 insertions(+), 42 deletions(-) create mode 100644 x-pack/plugins/ingest_pipelines/__jest__/client_integration/fixtures.ts delete mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/error_utils.test.ts create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/error_utils.ts create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/i18n_texts.ts create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/index.ts create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/pipeline_form_error.tsx create mode 100644 x-pack/plugins/ingest_pipelines/server/routes/api/shared/index.ts create mode 100644 x-pack/plugins/ingest_pipelines/server/routes/api/shared/is_object_with_keys.ts diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/fixtures.ts b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/fixtures.ts new file mode 100644 index 00000000000000..8dddb2421f03d4 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/fixtures.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const nestedProcessorsErrorFixture = { + attributes: { + error: { + root_cause: [ + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + suppressed: [ + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + suppressed: [ + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'csv', + }, + ], + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + ], + }, + ], + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + suppressed: [ + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + suppressed: [ + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'csv', + }, + ], + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + { + type: 'parse_exception', + reason: '[field] required property is missing', + property_name: 'field', + processor_type: 'circle', + }, + ], + }, + status: 400, + }, +}; diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipeline_form.helpers.ts b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipeline_form.helpers.ts index 8a14ed13f20221..85848b3d2f73cb 100644 --- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipeline_form.helpers.ts +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipeline_form.helpers.ts @@ -42,6 +42,8 @@ export type PipelineFormTestSubjects = | 'submitButton' | 'pageTitle' | 'savePipelineError' + | 'savePipelineError.showErrorsButton' + | 'savePipelineError.hideErrorsButton' | 'pipelineForm' | 'versionToggle' | 'versionField' diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_create.test.tsx b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_create.test.tsx index 2cfccbdc6d5783..813057813f1398 100644 --- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_create.test.tsx +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_create.test.tsx @@ -9,6 +9,8 @@ import { act } from 'react-dom/test-utils'; import { setupEnvironment, pageHelpers, nextTick } from './helpers'; import { PipelinesCreateTestBed } from './helpers/pipelines_create.helpers'; +import { nestedProcessorsErrorFixture } from './fixtures'; + const { setup } = pageHelpers.pipelinesCreate; jest.mock('@elastic/eui', () => { @@ -163,6 +165,25 @@ describe('', () => { expect(exists('savePipelineError')).toBe(true); expect(find('savePipelineError').text()).toContain(error.message); }); + + test('displays nested pipeline errors as a flat list', async () => { + const { actions, find, exists, waitFor } = testBed; + httpRequestsMockHelpers.setCreatePipelineResponse(undefined, { + body: nestedProcessorsErrorFixture, + }); + + await act(async () => { + actions.clickSubmitButton(); + await waitFor('savePipelineError'); + }); + + expect(exists('savePipelineError')).toBe(true); + expect(exists('savePipelineError.showErrorsButton')).toBe(true); + find('savePipelineError.showErrorsButton').simulate('click'); + expect(exists('savePipelineError.hideErrorsButton')).toBe(true); + expect(exists('savePipelineError.showErrorsButton')).toBe(false); + expect(find('savePipelineError').find('li').length).toBe(8); + }); }); describe('test pipeline', () => { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form.tsx index 05c9f0a08b0c72..a68e667f4ab432 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form.tsx @@ -11,17 +11,18 @@ import { EuiButton, EuiButtonEmpty, EuiFlexGroup, EuiFlexItem, EuiSpacer } from import { useForm, Form, FormConfig } from '../../../shared_imports'; import { Pipeline } from '../../../../common/types'; -import { PipelineRequestFlyout } from './pipeline_request_flyout'; -import { PipelineTestFlyout } from './pipeline_test_flyout'; -import { PipelineFormFields } from './pipeline_form_fields'; -import { PipelineFormError } from './pipeline_form_error'; -import { pipelineFormSchema } from './schema'; import { OnUpdateHandlerArg, OnUpdateHandler, SerializeResult, } from '../pipeline_processors_editor'; +import { PipelineRequestFlyout } from './pipeline_request_flyout'; +import { PipelineTestFlyout } from './pipeline_test_flyout'; +import { PipelineFormFields } from './pipeline_form_fields'; +import { PipelineFormError } from './pipeline_form_error'; +import { pipelineFormSchema } from './schema'; + export interface PipelineFormProps { onSave: (pipeline: Pipeline) => void; onCancel: () => void; @@ -116,7 +117,7 @@ export const PipelineForm: React.FunctionComponent = ({ error={form.getErrors()} > {/* Request error */} - {saveError && } + {saveError && } {/* All form fields */} = ({ errorMessage }) => { - return ( - <> - - } - color="danger" - iconType="alert" - data-test-subj="savePipelineError" - > -

{errorMessage}

- - - - ); -}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/error_utils.test.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/error_utils.test.ts new file mode 100644 index 00000000000000..1739365eb197df --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/error_utils.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { toKnownError } from './error_utils'; +import { nestedProcessorsErrorFixture } from '../../../../../__jest__/client_integration/fixtures'; + +describe('toKnownError', () => { + test('undefined, null, numbers, arrays and bad objects', () => { + expect(toKnownError(undefined)).toEqual({ errors: [{ reason: 'An unknown error occurred.' }] }); + expect(toKnownError(null)).toEqual({ errors: [{ reason: 'An unknown error occurred.' }] }); + expect(toKnownError(123)).toEqual({ errors: [{ reason: 'An unknown error occurred.' }] }); + expect(toKnownError([])).toEqual({ errors: [{ reason: 'An unknown error occurred.' }] }); + expect(toKnownError({})).toEqual({ errors: [{ reason: 'An unknown error occurred.' }] }); + expect(toKnownError({ attributes: {} })).toEqual({ + errors: [{ reason: 'An unknown error occurred.' }], + }); + }); + + test('non-processors errors', () => { + expect(toKnownError(new Error('my error'))).toEqual({ errors: [{ reason: 'my error' }] }); + expect(toKnownError({ message: 'my error' })).toEqual({ errors: [{ reason: 'my error' }] }); + }); + + test('processors errors', () => { + expect(toKnownError(nestedProcessorsErrorFixture)).toMatchInlineSnapshot(` + Object { + "errors": Array [ + Object { + "processorType": "circle", + "reason": "[field] required property is missing", + }, + Object { + "processorType": "circle", + "reason": "[field] required property is missing", + }, + Object { + "processorType": "circle", + "reason": "[field] required property is missing", + }, + Object { + "processorType": "csv", + "reason": "[field] required property is missing", + }, + Object { + "processorType": "circle", + "reason": "[field] required property is missing", + }, + Object { + "processorType": "circle", + "reason": "[field] required property is missing", + }, + Object { + "processorType": "circle", + "reason": "[field] required property is missing", + }, + Object { + "processorType": "circle", + "reason": "[field] required property is missing", + }, + ], + } + `); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/error_utils.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/error_utils.ts new file mode 100644 index 00000000000000..7f32f962f657c3 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/error_utils.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; +import { flow } from 'fp-ts/lib/function'; +import { isRight } from 'fp-ts/lib/Either'; + +import { i18nTexts } from './i18n_texts'; + +export interface PipelineError { + reason: string; + processorType?: string; +} +interface PipelineErrors { + errors: PipelineError[]; +} + +interface ErrorNode { + reason: string; + processor_type?: string; + suppressed?: ErrorNode[]; +} + +// This is a runtime type (RT) for an error node which is a recursive type +const errorNodeRT = t.recursion('ErrorNode', (ErrorNode) => + t.intersection([ + t.interface({ + reason: t.string, + }), + t.partial({ + processor_type: t.string, + suppressed: t.array(ErrorNode), + }), + ]) +); + +// This is a runtime type for the attributes object we expect to receive from the server +// for processor errors +const errorAttributesObjectRT = t.interface({ + attributes: t.interface({ + error: t.interface({ + root_cause: t.array(errorNodeRT), + }), + }), +}); + +const isProcessorsError = flow(errorAttributesObjectRT.decode, isRight); + +type ErrorAttributesObject = t.TypeOf; + +const flattenErrorsTree = (node: ErrorNode): PipelineError[] => { + const result: PipelineError[] = []; + const recurse = (_node: ErrorNode) => { + result.push({ reason: _node.reason, processorType: _node.processor_type }); + if (_node.suppressed && Array.isArray(_node.suppressed)) { + _node.suppressed.forEach(recurse); + } + }; + recurse(node); + return result; +}; + +export const toKnownError = (error: unknown): PipelineErrors => { + if (typeof error === 'object' && error != null && isProcessorsError(error)) { + const errorAttributes = error as ErrorAttributesObject; + const rootCause = errorAttributes.attributes.error.root_cause[0]; + return { errors: flattenErrorsTree(rootCause) }; + } + + if (typeof error === 'string') { + return { errors: [{ reason: error }] }; + } + + if ( + error instanceof Error || + (typeof error === 'object' && error != null && (error as any).message) + ) { + return { errors: [{ reason: (error as any).message }] }; + } + + return { errors: [{ reason: i18nTexts.errors.unknownError }] }; +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/i18n_texts.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/i18n_texts.ts new file mode 100644 index 00000000000000..e354541db8e7b2 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/i18n_texts.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const i18nTexts = { + title: i18n.translate('xpack.ingestPipelines.form.savePipelineError', { + defaultMessage: 'Unable to create pipeline', + }), + errors: { + processor: (processorType: string) => + i18n.translate('xpack.ingestPipelines.form.savePipelineError.processorLabel', { + defaultMessage: '{type} processor', + values: { type: processorType }, + }), + showErrors: (hiddenErrorsCount: number) => + i18n.translate('xpack.ingestPipelines.form.savePipelineError.showAllButton', { + defaultMessage: + 'Show {hiddenErrorsCount, plural, one {# more error} other {# more errors}}', + values: { + hiddenErrorsCount, + }, + }), + hideErrors: (hiddenErrorsCount: number) => + i18n.translate('xpack.ingestPipelines.form.savePip10mbelineError.showFewerButton', { + defaultMessage: 'Hide {hiddenErrorsCount, plural, one {# error} other {# errors}}', + values: { + hiddenErrorsCount, + }, + }), + unknownError: i18n.translate('xpack.ingestPipelines.form.unknownError', { + defaultMessage: 'An unknown error occurred.', + }), + }, +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/index.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/index.ts new file mode 100644 index 00000000000000..656691f6394985 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { PipelineFormError } from './pipeline_form_error'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/pipeline_form_error.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/pipeline_form_error.tsx new file mode 100644 index 00000000000000..23fb9a16484347 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_form/pipeline_form_error/pipeline_form_error.tsx @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState, useEffect } from 'react'; + +import { EuiSpacer, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiButtonEmpty } from '@elastic/eui'; +import { useKibana } from '../../../../shared_imports'; + +import { i18nTexts } from './i18n_texts'; +import { toKnownError, PipelineError } from './error_utils'; + +interface Props { + error: unknown; +} + +const numberOfErrorsToDisplay = 5; + +export const PipelineFormError: React.FunctionComponent = ({ error }) => { + const { services } = useKibana(); + const [isShowingAllErrors, setIsShowingAllErrors] = useState(false); + const safeErrorResult = toKnownError(error); + const hasMoreErrors = safeErrorResult.errors.length > numberOfErrorsToDisplay; + const hiddenErrorsCount = safeErrorResult.errors.length - numberOfErrorsToDisplay; + const results = isShowingAllErrors + ? safeErrorResult.errors + : safeErrorResult.errors.slice(0, numberOfErrorsToDisplay); + + const renderErrorListItem = ({ processorType, reason }: PipelineError) => { + return ( + <> + {processorType ? <>{i18nTexts.errors.processor(processorType) + ':'}  : undefined} + {reason} + + ); + }; + + useEffect(() => { + services.notifications.toasts.addDanger({ title: i18nTexts.title }); + }, [services, error]); + return ( + <> + + {results.length > 1 ? ( +
    + {results.map((e, idx) => ( +
  • {renderErrorListItem(e)}
    • + ))} +
+ ) : ( + renderErrorListItem(results[0]) + )} + {hasMoreErrors ? ( + + + {isShowingAllErrors ? ( + setIsShowingAllErrors(false)} + color="danger" + iconSide="right" + iconType="arrowUp" + data-test-subj="hideErrorsButton" + > + {i18nTexts.errors.hideErrors(hiddenErrorsCount)} + + ) : ( + setIsShowingAllErrors(true)} + color="danger" + iconSide="right" + iconType="arrowDown" + data-test-subj="showErrorsButton" + > + {i18nTexts.errors.showErrors(hiddenErrorsCount)} + + )} + + + ) : undefined} + + + + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/create.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/create.ts index c1ab3852ee7845..c2328bcc9d0ab5 100644 --- a/x-pack/plugins/ingest_pipelines/server/routes/api/create.ts +++ b/x-pack/plugins/ingest_pipelines/server/routes/api/create.ts @@ -10,6 +10,7 @@ import { Pipeline } from '../../../common/types'; import { API_BASE_PATH } from '../../../common/constants'; import { RouteDependencies } from '../../types'; import { pipelineSchema } from './pipeline_schema'; +import { isObjectWithKeys } from './shared'; const bodySchema = schema.object({ name: schema.string(), @@ -70,7 +71,12 @@ export const registerCreateRoute = ({ if (isEsError(error)) { return res.customError({ statusCode: error.statusCode, - body: error, + body: isObjectWithKeys(error.body) + ? { + message: error.message, + attributes: error.body, + } + : error, }); } diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/shared/index.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/shared/index.ts new file mode 100644 index 00000000000000..1fa794a4fb9961 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/server/routes/api/shared/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { isObjectWithKeys } from './is_object_with_keys'; diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/shared/is_object_with_keys.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/shared/is_object_with_keys.ts new file mode 100644 index 00000000000000..0617bde26cfb61 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/server/routes/api/shared/is_object_with_keys.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const isObjectWithKeys = (value: unknown) => { + return typeof value === 'object' && !!value && Object.keys(value).length > 0; +}; diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/update.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/update.ts index 214b293a43c6c8..cd0e3568f0f601 100644 --- a/x-pack/plugins/ingest_pipelines/server/routes/api/update.ts +++ b/x-pack/plugins/ingest_pipelines/server/routes/api/update.ts @@ -8,6 +8,7 @@ import { schema } from '@kbn/config-schema'; import { API_BASE_PATH } from '../../../common/constants'; import { RouteDependencies } from '../../types'; import { pipelineSchema } from './pipeline_schema'; +import { isObjectWithKeys } from './shared'; const bodySchema = schema.object(pipelineSchema); @@ -52,7 +53,12 @@ export const registerUpdateRoute = ({ if (isEsError(error)) { return res.customError({ statusCode: error.statusCode, - body: error, + body: isObjectWithKeys(error.body) + ? { + message: error.message, + attributes: error.body, + } + : error, }); } From a0e263038c6c7cf077386a1a3da18d6fc6b92def Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 2 Jul 2020 11:24:39 +0200 Subject: [PATCH 13/31] Use dynamic: false for config saved object mappings (#70436) --- src/core/server/ui_settings/saved_objects/ui_settings.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/core/server/ui_settings/saved_objects/ui_settings.ts b/src/core/server/ui_settings/saved_objects/ui_settings.ts index 26704f46a509c4..452d1954b6e235 100644 --- a/src/core/server/ui_settings/saved_objects/ui_settings.ts +++ b/src/core/server/ui_settings/saved_objects/ui_settings.ts @@ -25,10 +25,7 @@ export const uiSettingsType: SavedObjectsType = { hidden: false, namespaceType: 'single', mappings: { - // we don't want to allow `true` in the public `SavedObjectsTypeMappingDefinition` type, however - // this is needed for the config that is kinda a special type. To avoid adding additional internal types - // just for this, we hardcast to any here. - dynamic: true as any, + dynamic: false, properties: { buildNum: { type: 'keyword', From a8347fad1c9c6ef47436cb947154f935615ea1d5 Mon Sep 17 00:00:00 2001 From: Daniil Suleiman <31325372+sulemanof@users.noreply.github.com> Date: Thu, 2 Jul 2020 13:24:00 +0300 Subject: [PATCH 14/31] [Visualize] Add missing advanced settings and custom label for pipeline aggs (#69688) * Show advanced settings in pipeline aggs * Add functional tests * Make sub metric in sibling pipeline to have custom label Co-authored-by: Elastic Machine --- .../public/components/agg_group.tsx | 2 +- .../public/components/agg_params_helper.ts | 6 +- .../public/components/controls/sub_metric.tsx | 7 +- test/functional/apps/visualize/_line_chart.js | 74 +++++++++++++++++++ 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/plugins/vis_default_editor/public/components/agg_group.tsx b/src/plugins/vis_default_editor/public/components/agg_group.tsx index 3030601236687d..4cde33b8fbc314 100644 --- a/src/plugins/vis_default_editor/public/components/agg_group.tsx +++ b/src/plugins/vis_default_editor/public/components/agg_group.tsx @@ -152,7 +152,7 @@ function DefaultEditorAggGroup({ {bucketsError && ( <> - {bucketsError} + {bucketsError} )} diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts index 45abbf8d2b2dd3..39abddb3de853b 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts @@ -111,7 +111,11 @@ function getAggParamsToRender({ const aggType = agg.type.type; const aggName = agg.type.name; const aggParams = get(aggParamsMap, [aggType, aggName], {}); - paramEditor = get(aggParams, param.name) || get(aggParamsMap, ['common', param.type]); + paramEditor = get(aggParams, param.name); + } + + if (!paramEditor) { + paramEditor = get(aggParamsMap, ['common', param.type]); } // show params with an editor component diff --git a/src/plugins/vis_default_editor/public/components/controls/sub_metric.tsx b/src/plugins/vis_default_editor/public/components/controls/sub_metric.tsx index 361eeba9abdbf8..fc79ba703c2b4a 100644 --- a/src/plugins/vis_default_editor/public/components/controls/sub_metric.tsx +++ b/src/plugins/vis_default_editor/public/components/controls/sub_metric.tsx @@ -45,9 +45,10 @@ function SubMetricParamEditor({ defaultMessage: 'Bucket', }); const type = aggParam.name; + const isCustomMetric = type === 'customMetric'; - const aggTitle = type === 'customMetric' ? metricTitle : bucketTitle; - const aggGroup = type === 'customMetric' ? AggGroupNames.Metrics : AggGroupNames.Buckets; + const aggTitle = isCustomMetric ? metricTitle : bucketTitle; + const aggGroup = isCustomMetric ? AggGroupNames.Metrics : AggGroupNames.Buckets; useMount(() => { if (agg.params[type]) { @@ -87,7 +88,7 @@ function SubMetricParamEditor({ setValidity={setValidity} setTouched={setTouched} schemas={schemas} - hideCustomLabel={true} + hideCustomLabel={!isCustomMetric} /> ); diff --git a/test/functional/apps/visualize/_line_chart.js b/test/functional/apps/visualize/_line_chart.js index 5c510617fbb017..a492f3858b524f 100644 --- a/test/functional/apps/visualize/_line_chart.js +++ b/test/functional/apps/visualize/_line_chart.js @@ -279,5 +279,79 @@ export default function ({ getService, getPageObjects }) { expect(labels).to.eql(expectedLabels); }); }); + + describe('pipeline aggregations', () => { + before(async () => { + log.debug('navigateToApp visualize'); + await PageObjects.visualize.navigateToNewVisualization(); + log.debug('clickLineChart'); + await PageObjects.visualize.clickLineChart(); + await PageObjects.visualize.clickNewSearch(); + await PageObjects.timePicker.setDefaultAbsoluteRange(); + }); + + describe('parent pipeline', () => { + it('should have an error if bucket is not selected', async () => { + await PageObjects.visEditor.clickMetricEditor(); + log.debug('Metrics agg = Serial diff'); + await PageObjects.visEditor.selectAggregation('Serial diff', 'metrics'); + await testSubjects.existOrFail('bucketsError'); + }); + + it('should apply with selected bucket', async () => { + log.debug('Bucket = X-axis'); + await PageObjects.visEditor.clickBucket('X-axis'); + log.debug('Aggregation = Date Histogram'); + await PageObjects.visEditor.selectAggregation('Date Histogram'); + await PageObjects.visEditor.clickGo(); + const title = await PageObjects.visChart.getYAxisTitle(); + expect(title).to.be('Serial Diff of Count'); + }); + + it('should change y-axis label to custom', async () => { + log.debug('set custom label of y-axis to "Custom"'); + await PageObjects.visEditor.setCustomLabel('Custom', 1); + await PageObjects.visEditor.clickGo(); + const title = await PageObjects.visChart.getYAxisTitle(); + expect(title).to.be('Custom'); + }); + + it('should have advanced accordion and json input', async () => { + await testSubjects.click('advancedParams-1'); + await testSubjects.existOrFail('advancedParams-1 > codeEditorContainer'); + }); + }); + + describe('sibling pipeline', () => { + it('should apply with selected bucket', async () => { + log.debug('Metrics agg = Average Bucket'); + await PageObjects.visEditor.selectAggregation('Average Bucket', 'metrics'); + await PageObjects.visEditor.clickGo(); + const title = await PageObjects.visChart.getYAxisTitle(); + expect(title).to.be('Overall Average of Count'); + }); + + it('should change sub metric custom label and calculate y-axis title', async () => { + log.debug('set custom label of sub metric to "Cats"'); + await PageObjects.visEditor.setCustomLabel('Cats', '1-metric'); + await PageObjects.visEditor.clickGo(); + const title = await PageObjects.visChart.getYAxisTitle(); + expect(title).to.be('Overall Average of Cats'); + }); + + it('should outer custom label', async () => { + log.debug('set custom label to "Custom"'); + await PageObjects.visEditor.setCustomLabel('Custom', 1); + await PageObjects.visEditor.clickGo(); + const title = await PageObjects.visChart.getYAxisTitle(); + expect(title).to.be('Custom'); + }); + + it('should have advanced accordion and json input', async () => { + await testSubjects.click('advancedParams-1'); + await testSubjects.existOrFail('advancedParams-1 > codeEditorContainer'); + }); + }); + }); }); } From 1cfc9356bd36c54b8005089decc36970c11c101d Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Thu, 2 Jul 2020 13:17:33 +0200 Subject: [PATCH 15/31] add getVisibleTypes API to SO type registry (#70559) * add getVisibleTypes API * doc nit * fix mocking in tests --- ...ver.savedobjecttyperegistry.getalltypes.md | 4 ++- ...savedobjecttyperegistry.getvisibletypes.md | 19 ++++++++++++ ...gin-core-server.savedobjecttyperegistry.md | 3 +- .../saved_objects_type_registry.mock.ts | 2 ++ .../saved_objects_type_registry.test.ts | 29 ++++++++++++++++++- .../saved_objects_type_registry.ts | 13 ++++++++- src/core/server/server.api.md | 1 + .../saved_objects/saved_objects_mixin.js | 2 +- x-pack/plugins/features/server/plugin.test.ts | 8 +---- x-pack/plugins/features/server/plugin.ts | 5 +--- 10 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.getvisibletypes.md diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.getalltypes.md b/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.getalltypes.md index 1e0e89767c4e6a..c839dd16d9a475 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.getalltypes.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.getalltypes.md @@ -4,7 +4,9 @@ ## SavedObjectTypeRegistry.getAllTypes() method -Return all [types](./kibana-plugin-core-server.savedobjectstype.md) currently registered. +Return all [types](./kibana-plugin-core-server.savedobjectstype.md) currently registered, including the hidden ones. + +To only get the visible types (which is the most common use case), use `getVisibleTypes` instead. Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.getvisibletypes.md b/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.getvisibletypes.md new file mode 100644 index 00000000000000..a773c6a0a674fb --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.getvisibletypes.md @@ -0,0 +1,19 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectTypeRegistry](./kibana-plugin-core-server.savedobjecttyperegistry.md) > [getVisibleTypes](./kibana-plugin-core-server.savedobjecttyperegistry.getvisibletypes.md) + +## SavedObjectTypeRegistry.getVisibleTypes() method + +Returns all visible [types](./kibana-plugin-core-server.savedobjectstype.md). + +A visible type is a type that doesn't explicitly define `hidden=true` during registration. + +Signature: + +```typescript +getVisibleTypes(): SavedObjectsType[]; +``` +Returns: + +`SavedObjectsType[]` + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.md b/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.md index 69a94e4ad8c882..55ad7ca137de0a 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjecttyperegistry.md @@ -16,10 +16,11 @@ export declare class SavedObjectTypeRegistry | Method | Modifiers | Description | | --- | --- | --- | -| [getAllTypes()](./kibana-plugin-core-server.savedobjecttyperegistry.getalltypes.md) | | Return all [types](./kibana-plugin-core-server.savedobjectstype.md) currently registered. | +| [getAllTypes()](./kibana-plugin-core-server.savedobjecttyperegistry.getalltypes.md) | | Return all [types](./kibana-plugin-core-server.savedobjectstype.md) currently registered, including the hidden ones.To only get the visible types (which is the most common use case), use getVisibleTypes instead. | | [getImportableAndExportableTypes()](./kibana-plugin-core-server.savedobjecttyperegistry.getimportableandexportabletypes.md) | | Return all [types](./kibana-plugin-core-server.savedobjectstype.md) currently registered that are importable/exportable. | | [getIndex(type)](./kibana-plugin-core-server.savedobjecttyperegistry.getindex.md) | | Returns the indexPattern property for given type, or undefined if the type is not registered. | | [getType(type)](./kibana-plugin-core-server.savedobjecttyperegistry.gettype.md) | | Return the [type](./kibana-plugin-core-server.savedobjectstype.md) definition for given type name. | +| [getVisibleTypes()](./kibana-plugin-core-server.savedobjecttyperegistry.getvisibletypes.md) | | Returns all visible [types](./kibana-plugin-core-server.savedobjectstype.md).A visible type is a type that doesn't explicitly define hidden=true during registration. | | [isHidden(type)](./kibana-plugin-core-server.savedobjecttyperegistry.ishidden.md) | | Returns the hidden property for given type, or false if the type is not registered. | | [isImportableAndExportable(type)](./kibana-plugin-core-server.savedobjecttyperegistry.isimportableandexportable.md) | | Returns the management.importableAndExportable property for given type, or false if the type is not registered or does not define a management section. | | [isMultiNamespace(type)](./kibana-plugin-core-server.savedobjecttyperegistry.ismultinamespace.md) | | Returns whether the type is multi-namespace (shareable); resolves to false if the type is not registered | diff --git a/src/core/server/saved_objects/saved_objects_type_registry.mock.ts b/src/core/server/saved_objects/saved_objects_type_registry.mock.ts index 5636dcadb444e2..44490228490cc3 100644 --- a/src/core/server/saved_objects/saved_objects_type_registry.mock.ts +++ b/src/core/server/saved_objects/saved_objects_type_registry.mock.ts @@ -25,6 +25,7 @@ const createRegistryMock = (): jest.Mocked< const mock = { registerType: jest.fn(), getType: jest.fn(), + getVisibleTypes: jest.fn(), getAllTypes: jest.fn(), getImportableAndExportableTypes: jest.fn(), isNamespaceAgnostic: jest.fn(), @@ -35,6 +36,7 @@ const createRegistryMock = (): jest.Mocked< isImportableAndExportable: jest.fn(), }; + mock.getVisibleTypes.mockReturnValue([]); mock.getAllTypes.mockReturnValue([]); mock.getImportableAndExportableTypes.mockReturnValue([]); mock.getIndex.mockReturnValue('.kibana-test'); diff --git a/src/core/server/saved_objects/saved_objects_type_registry.test.ts b/src/core/server/saved_objects/saved_objects_type_registry.test.ts index e0f4d6fa28e507..25c94324c8f01e 100644 --- a/src/core/server/saved_objects/saved_objects_type_registry.test.ts +++ b/src/core/server/saved_objects/saved_objects_type_registry.test.ts @@ -99,10 +99,37 @@ describe('SavedObjectTypeRegistry', () => { }); }); + describe('#getVisibleTypes', () => { + it('returns only visible registered types', () => { + const typeA = createType({ name: 'typeA', hidden: false }); + const typeB = createType({ name: 'typeB', hidden: true }); + const typeC = createType({ name: 'typeC', hidden: false }); + registry.registerType(typeA); + registry.registerType(typeB); + registry.registerType(typeC); + + const registered = registry.getVisibleTypes(); + expect(registered.length).toEqual(2); + expect(registered).toContainEqual(typeA); + expect(registered).toContainEqual(typeC); + }); + + it('does not mutate the registered types when altering the list', () => { + registry.registerType(createType({ name: 'typeA', hidden: false })); + registry.registerType(createType({ name: 'typeB', hidden: true })); + registry.registerType(createType({ name: 'typeC', hidden: false })); + + const types = registry.getVisibleTypes(); + types.splice(0, 2); + + expect(registry.getVisibleTypes().length).toEqual(2); + }); + }); + describe('#getAllTypes', () => { it('returns all registered types', () => { const typeA = createType({ name: 'typeA' }); - const typeB = createType({ name: 'typeB' }); + const typeB = createType({ name: 'typeB', hidden: true }); const typeC = createType({ name: 'typeC' }); registry.registerType(typeA); registry.registerType(typeB); diff --git a/src/core/server/saved_objects/saved_objects_type_registry.ts b/src/core/server/saved_objects/saved_objects_type_registry.ts index 99262d7a31e215..d0035294226ea9 100644 --- a/src/core/server/saved_objects/saved_objects_type_registry.ts +++ b/src/core/server/saved_objects/saved_objects_type_registry.ts @@ -54,7 +54,18 @@ export class SavedObjectTypeRegistry { } /** - * Return all {@link SavedObjectsType | types} currently registered. + * Returns all visible {@link SavedObjectsType | types}. + * + * A visible type is a type that doesn't explicitly define `hidden=true` during registration. + */ + public getVisibleTypes() { + return [...this.types.values()].filter((type) => !this.isHidden(type.name)); + } + + /** + * Return all {@link SavedObjectsType | types} currently registered, including the hidden ones. + * + * To only get the visible types (which is the most common use case), use `getVisibleTypes` instead. */ public getAllTypes() { return [...this.types.values()]; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 9cc5a8a386b0b6..1cabaa57e519ce 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -2468,6 +2468,7 @@ export class SavedObjectTypeRegistry { getImportableAndExportableTypes(): SavedObjectsType[]; getIndex(type: string): string | undefined; getType(type: string): SavedObjectsType | undefined; + getVisibleTypes(): SavedObjectsType[]; isHidden(type: string): boolean; isImportableAndExportable(type: string): boolean; isMultiNamespace(type: string): boolean; diff --git a/src/legacy/server/saved_objects/saved_objects_mixin.js b/src/legacy/server/saved_objects/saved_objects_mixin.js index 63839b9d0f1d77..185c8807ae8b5f 100644 --- a/src/legacy/server/saved_objects/saved_objects_mixin.js +++ b/src/legacy/server/saved_objects/saved_objects_mixin.js @@ -34,8 +34,8 @@ export function savedObjectsMixin(kbnServer, server) { const typeRegistry = kbnServer.newPlatform.start.core.savedObjects.getTypeRegistry(); const mappings = migrator.getActiveMappings(); const allTypes = typeRegistry.getAllTypes().map((t) => t.name); + const visibleTypes = typeRegistry.getVisibleTypes().map((t) => t.name); const schema = new SavedObjectsSchema(convertTypesToLegacySchema(typeRegistry.getAllTypes())); - const visibleTypes = allTypes.filter((type) => !schema.isHiddenType(type)); server.decorate('server', 'kibanaMigrator', migrator); diff --git a/x-pack/plugins/features/server/plugin.test.ts b/x-pack/plugins/features/server/plugin.test.ts index 79fd012337b00c..3d85c2e9eb751c 100644 --- a/x-pack/plugins/features/server/plugin.test.ts +++ b/x-pack/plugins/features/server/plugin.test.ts @@ -10,19 +10,13 @@ const initContext = coreMock.createPluginInitializerContext(); const coreSetup = coreMock.createSetup(); const coreStart = coreMock.createStart(); const typeRegistry = savedObjectsServiceMock.createTypeRegistryMock(); -typeRegistry.getAllTypes.mockReturnValue([ +typeRegistry.getVisibleTypes.mockReturnValue([ { name: 'foo', hidden: false, mappings: { properties: {} }, namespaceType: 'single' as 'single', }, - { - name: 'bar', - hidden: true, - mappings: { properties: {} }, - namespaceType: 'agnostic' as 'agnostic', - }, ]); coreStart.savedObjects.getTypeRegistry.mockReturnValue(typeRegistry); diff --git a/x-pack/plugins/features/server/plugin.ts b/x-pack/plugins/features/server/plugin.ts index 149c1acfb50863..5783b20eae6484 100644 --- a/x-pack/plugins/features/server/plugin.ts +++ b/x-pack/plugins/features/server/plugin.ts @@ -80,10 +80,7 @@ export class Plugin { private registerOssFeatures(savedObjects: SavedObjectsServiceStart) { const registry = savedObjects.getTypeRegistry(); - const savedObjectTypes = registry - .getAllTypes() - .filter((t) => !t.hidden) - .map((t) => t.name); + const savedObjectTypes = registry.getVisibleTypes().map((t) => t.name); this.logger.debug( `Registering OSS features with SO types: ${savedObjectTypes.join(', ')}. "includeTimelion": ${ From 7d63cafd5d15ce5e85ba551472f6565f361b39d8 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Thu, 2 Jul 2020 12:31:51 +0100 Subject: [PATCH 16/31] chore(NA): disable alerts_detection_rules cypress suites (#70577) --- .../cypress/integration/alerts_detection_rules_custom.spec.ts | 3 ++- .../cypress/integration/alerts_detection_rules_export.spec.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts index 9e9732a403f8fc..2a1a2d2c8e1947 100644 --- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_custom.spec.ts @@ -64,7 +64,8 @@ import { loginAndWaitForPageWithoutDateRange } from '../tasks/login'; import { ALERTS_URL } from '../urls/navigation'; -describe('Detection rules, custom', () => { +// // Skipped as was causing failures on master +describe.skip('Detection rules, custom', () => { before(() => { esArchiverLoad('custom_rule_with_timeline'); }); diff --git a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts index 25fc1fc3a7c110..06e9228de4f490 100644 --- a/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules_export.spec.ts @@ -17,7 +17,8 @@ import { ALERTS_URL } from '../urls/navigation'; const EXPECTED_EXPORTED_RULE_FILE_PATH = 'cypress/test_files/expected_rules_export.ndjson'; -describe('Export rules', () => { +// Skipped as was causing failures on master +describe.skip('Export rules', () => { before(() => { esArchiverLoad('custom_rules'); cy.server(); From 7b74094e0fa33bd4803dbe105b2a4ea3843ef170 Mon Sep 17 00:00:00 2001 From: Eric Davis Date: Thu, 2 Jul 2020 07:55:52 -0400 Subject: [PATCH 17/31] Update docs for api authentication usage (#66819) Co-authored-by: Kaarina Tungseth --- docs/api/using-api.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/using-api.asciidoc b/docs/api/using-api.asciidoc index aba65f2e921c28..e58d9c39ee8c4a 100644 --- a/docs/api/using-api.asciidoc +++ b/docs/api/using-api.asciidoc @@ -10,7 +10,7 @@ NOTE: The {kib} Console supports only Elasticsearch APIs. You are unable to inte [float] [[api-authentication]] === Authentication -{kib} supports token-based authentication with the same username and password that you use to log into the {kib} Console. +{kib} supports token-based authentication with the same username and password that you use to log into the {kib} Console. In a given HTTP tool, and when available, you can select to use its 'Basic Authentication' option, which is where the username and password are stored in order to be passed as part of the call. [float] [[api-calls]] From c081caa6341143eb094a9ef00963eaada7d1c72c Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Thu, 2 Jul 2020 08:47:37 -0400 Subject: [PATCH 18/31] [Security_Solution][Endpoint] Leveraging msearch and ancestry array for resolver (#70134) * Refactor generator for ancestry support * Adding optional ancestry array * Refactor the pagination since the totals are not used anymore * Updating the queries to not use aggregations for determining the totals * Refactoring the children helper to handle pagination without totals * Pinning the seed for the resolver tree generator service * Splitting the fetcher into multiple classes for msearch * Updating tests and api for ancestry array and msearch * Adding more comments and fixing type errors * Fixing resolver test import * Fixing tests and type errors * Fixing type errors and tests * Removing useAncestry field * Fixing test * Removing useAncestry field from tests * An empty array will be returned because that's how ES will do it too --- .../common/endpoint/models/event.ts | 18 + .../common/endpoint/schema/resolver.ts | 2 - .../common/endpoint/types.ts | 25 +- .../endpoint/routes/resolver/children.ts | 4 +- .../routes/resolver/queries/alerts.ts | 19 +- .../endpoint/routes/resolver/queries/base.ts | 29 +- .../routes/resolver/queries/children.test.ts | 2 +- .../routes/resolver/queries/children.ts | 30 +- .../routes/resolver/queries/events.ts | 19 +- .../routes/resolver/queries/lifecycle.ts | 2 +- .../routes/resolver/queries/multi_searcher.ts | 18 +- .../endpoint/routes/resolver/queries/stats.ts | 6 +- .../server/endpoint/routes/resolver/tree.ts | 3 +- .../resolver/utils/alerts_query_handler.ts | 83 +++++ .../resolver/utils/ancestry_query_handler.ts | 130 +++++++ .../resolver/utils/children_helper.test.ts | 238 ++++++++---- .../routes/resolver/utils/children_helper.ts | 161 ++++++-- .../utils/children_lifecycle_query_handler.ts | 71 ++++ .../utils/children_start_query_handler.ts | 109 ++++++ .../resolver/utils/events_query_handler.ts | 81 ++++ .../endpoint/routes/resolver/utils/fetch.ts | 350 +++++++++--------- .../resolver/utils/lifecycle_query_handler.ts | 72 ++++ .../endpoint/routes/resolver/utils/node.ts | 15 +- .../routes/resolver/utils/pagination.test.ts | 36 +- .../routes/resolver/utils/pagination.ts | 90 +---- .../api_integration/apis/endpoint/resolver.ts | 72 ++-- .../test/api_integration/services/resolver.ts | 4 +- 27 files changed, 1204 insertions(+), 485 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/models/event.ts b/x-pack/plugins/security_solution/common/endpoint/models/event.ts index 98f4b4336a1c8e..86cccff9572110 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/event.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/event.ts @@ -60,6 +60,24 @@ export function ancestryArray(event: ResolverEvent): string[] | undefined { return event.process.Ext.ancestry; } +export function getAncestryAsArray(event: ResolverEvent | undefined): string[] { + if (!event) { + return []; + } + + const ancestors = ancestryArray(event); + if (ancestors) { + return ancestors; + } + + const parentID = parentEntityId(event); + if (parentID) { + return [parentID]; + } + + return []; +} + /** * @param event The event to get the category for */ diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts index 398e2710b32531..42cbc2327fc288 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts @@ -13,7 +13,6 @@ export const validateTree = { params: schema.object({ id: schema.string() }), query: schema.object({ children: schema.number({ defaultValue: 10, min: 0, max: 100 }), - generations: schema.number({ defaultValue: 3, min: 0, max: 3 }), ancestors: schema.number({ defaultValue: 3, min: 0, max: 5 }), events: schema.number({ defaultValue: 100, min: 0, max: 1000 }), alerts: schema.number({ defaultValue: 100, min: 0, max: 1000 }), @@ -66,7 +65,6 @@ export const validateChildren = { params: schema.object({ id: schema.string() }), query: schema.object({ children: schema.number({ defaultValue: 10, min: 1, max: 100 }), - generations: schema.number({ defaultValue: 3, min: 1, max: 3 }), afterChild: schema.maybe(schema.string()), legacyEndpointID: schema.maybe(schema.string()), }), diff --git a/x-pack/plugins/security_solution/common/endpoint/types.ts b/x-pack/plugins/security_solution/common/endpoint/types.ts index 4efe89b2429ad6..42b1337a91464e 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types.ts @@ -77,12 +77,18 @@ export interface ResolverNodeStats { */ export interface ResolverChildNode extends ResolverLifecycleNode { /** - * A child node's pagination cursor can be null for a couple reasons: - * 1. At the time of querying it could have no children in ES, in which case it will be marked as - * null because we know it does not have children during this query. - * 2. If the max level was reached we do not know if this node has children or not so we'll mark it as null + * nextChild can have 3 different states: + * + * undefined: This indicates that you should not use this node for additional queries. It does not mean that node does + * not have any more direct children. The node could have more direct children but to determine that, use the + * ResolverChildren node's nextChild. + * + * null: Indicates that we have received all the children of the node. There may be more descendants though. + * + * string: Indicates this is a leaf node and it can be used to continue querying for additional descendants + * using this node's entity_id */ - nextChild: string | null; + nextChild?: string | null; } /** @@ -92,7 +98,14 @@ export interface ResolverChildNode extends ResolverLifecycleNode { export interface ResolverChildren { childNodes: ResolverChildNode[]; /** - * This is the children cursor for the origin of a tree. + * nextChild can have 2 different states: + * + * null: Indicates that we have received all the descendants that can be retrieved using this node. To retrieve more + * nodes in the tree use a cursor provided in one of the returned children. If no other cursor exists then the tree + * is complete. + * + * string: Indicates this node has more descendants that can be retrieved, pass this cursor in while using this node's + * entity_id for the request. */ nextChild: string | null; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts index 74448a324a4ecb..9b8cd9fd3edab1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts @@ -18,14 +18,14 @@ export function handleChildren( return async (context, req, res) => { const { params: { id }, - query: { children, generations, afterChild, legacyEndpointID: endpointID }, + query: { children, afterChild, legacyEndpointID: endpointID }, } = req; try { const client = context.core.elasticsearch.legacy.client; const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID); return res.ok({ - body: await fetcher.children(children, generations, afterChild), + body: await fetcher.children(children, afterChild), }); } catch (err) { log.warn(err); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts index 95bc612c58a1b6..feb4a404b2359d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving alerts for a node. */ -export class AlertsQuery extends ResolverQuery { +export class AlertsQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -38,11 +38,7 @@ export class AlertsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_pid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -60,14 +56,11 @@ export class AlertsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts index 35f8cad01e6720..1b6a8f2f833874 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts @@ -14,10 +14,12 @@ import { MSearchQuery } from './multi_searcher'; /** * ResolverQuery provides the base structure for queries to retrieve events when building a resolver graph. * - * @param T the structured return type of a resolver query. This represents the type that is returned when translating - * Elasticsearch's SearchResponse response. + * @param T the structured return type of a resolver query. This represents the final return type of the query after handling + * any aggregations. + * @param R the is the type after transforming ES's response. Making this definable let's us set whether it is a resolver event + * or something else. */ -export abstract class ResolverQuery implements MSearchQuery { +export abstract class ResolverQuery implements MSearchQuery { /** * * @param indexPattern the index pattern to use in the query for finding indices with documents in ES. @@ -50,7 +52,7 @@ export abstract class ResolverQuery implements MSearchQuery { }; } - protected static getResults(response: SearchResponse): ResolverEvent[] { + protected getResults(response: SearchResponse): R[] { return response.hits.hits.map((hit) => hit._source); } @@ -68,19 +70,26 @@ export abstract class ResolverQuery implements MSearchQuery { } /** - * Searches ES for the specified ids. + * Searches ES for the specified ids and format the response. * * @param client a client for searching ES * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid) */ - async search(client: ILegacyScopedClusterClient, ids: string | string[]): Promise { - const res: SearchResponse = await client.callAsCurrentUser( - 'search', - this.buildSearch(ids) - ); + async searchAndFormat(client: ILegacyScopedClusterClient, ids: string | string[]): Promise { + const res: SearchResponse = await this.search(client, ids); return this.formatResponse(res); } + /** + * Searches ES for the specified ids but do not format the response. + * + * @param client a client for searching ES + * @param ids a single more multiple unique node ids (e.g. entity_id or unique_pid) + */ + async search(client: ILegacyScopedClusterClient, ids: string | string[]) { + return client.callAsCurrentUser('search', this.buildSearch(ids)); + } + /** * Builds a query to search the legacy data format. * diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts index a4d4cd546ef608..8175764b3a0a2f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.test.ts @@ -25,7 +25,7 @@ describe('Children query', () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any const msearch: any = query.buildMSearch(['1234', '5678']); expect(msearch[0].index).toBe('index-pattern'); - expect(msearch[1].query.bool.filter[0]).toStrictEqual({ + expect(msearch[1].query.bool.filter[0].bool.should[0]).toStrictEqual({ terms: { 'process.parent.entity_id': ['1234', '5678'] }, }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts index b7b1a16926a15d..7fd3808662baa7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving descendants of a node. */ -export class ChildrenQuery extends ResolverQuery { +export class ChildrenQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -53,11 +53,7 @@ export class ChildrenQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_ppid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -67,7 +63,16 @@ export class ChildrenQuery extends ResolverQuery { bool: { filter: [ { - terms: { 'process.parent.entity_id': entityIDs }, + bool: { + should: [ + { + terms: { 'process.parent.entity_id': entityIDs }, + }, + { + terms: { 'process.Ext.ancestry': entityIDs }, + }, + ], + }, }, { term: { 'event.category': 'process' }, @@ -81,14 +86,11 @@ export class ChildrenQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.parent.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts index ec65e30d1d5d42..abc86826e77dd7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts @@ -6,13 +6,13 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { ResolverQuery } from './base'; -import { PaginationBuilder, PaginatedResults } from '../utils/pagination'; +import { PaginationBuilder } from '../utils/pagination'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; /** * Builds a query for retrieving related events for a node. */ -export class EventsQuery extends ResolverQuery { +export class EventsQuery extends ResolverQuery { constructor( private readonly pagination: PaginationBuilder, indexPattern: string | string[], @@ -45,11 +45,7 @@ export class EventsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields( - uniquePIDs.length, - 'endgame.serial_event_id', - 'endgame.unique_pid' - ), + ...this.pagination.buildQueryFields('endgame.serial_event_id'), }; } @@ -74,14 +70,11 @@ export class EventsQuery extends ResolverQuery { ], }, }, - ...this.pagination.buildQueryFields(entityIDs.length, 'event.id', 'process.entity_id'), + ...this.pagination.buildQueryFields('event.id'), }; } - formatResponse(response: SearchResponse): PaginatedResults { - return { - results: ResolverQuery.getResults(response), - totals: PaginationBuilder.getTotals(response.aggregations), - }; + formatResponse(response: SearchResponse): ResolverEvent[] { + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts index 93910293b00aff..0b5728958e91f5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/lifecycle.ts @@ -60,6 +60,6 @@ export class LifecycleQuery extends ResolverQuery { } formatResponse(response: SearchResponse): ResolverEvent[] { - return ResolverQuery.getResults(response); + return this.getResults(response); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts index f873ab3019f64c..02dbd92d9252b4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/multi_searcher.ts @@ -5,7 +5,7 @@ */ import { ILegacyScopedClusterClient } from 'kibana/server'; -import { MSearchResponse } from 'elasticsearch'; +import { MSearchResponse, SearchResponse } from 'elasticsearch'; import { ResolverEvent } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; @@ -34,6 +34,10 @@ export interface QueryInfo { * one or many unique identifiers to be searched for in this query */ ids: string | string[]; + /** + * a function to handle the response + */ + handler: (response: SearchResponse) => void; } /** @@ -57,10 +61,10 @@ export class MultiSearcher { throw new Error('No queries provided to MultiSearcher'); } - let searchQuery: JsonObject[] = []; - queries.forEach( - (info) => (searchQuery = [...searchQuery, ...info.query.buildMSearch(info.ids)]) - ); + const searchQuery: JsonObject[] = []; + for (const info of queries) { + searchQuery.push(...info.query.buildMSearch(info.ids)); + } const res: MSearchResponse = await this.client.callAsCurrentUser('msearch', { body: searchQuery, }); @@ -72,6 +76,8 @@ export class MultiSearcher { if (res.responses.length !== queries.length) { throw new Error(`Responses length was: ${res.responses.length} expected ${queries.length}`); } - return res.responses; + for (let i = 0; i < queries.length; i++) { + queries[i].handler(res.responses[i]); + } } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts index a728054bef2197..b8fa409e2ca21a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/stats.ts @@ -7,13 +7,17 @@ import { SearchResponse } from 'elasticsearch'; import { ResolverQuery } from './base'; import { ResolverEvent, EventStats } from '../../../../../common/endpoint/types'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; -import { AggBucket } from '../utils/pagination'; export interface StatsResult { alerts: Record; events: Record; } +interface AggBucket { + key: string; + doc_count: number; +} + interface CategoriesAgg extends AggBucket { /** * The reason categories is optional here is because if no data was returned in the query the categories aggregation diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts index 181fb8c3df3f92..33011078ee8233 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts @@ -21,7 +21,6 @@ export function handleTree( params: { id }, query: { children, - generations, ancestors, events, alerts, @@ -37,7 +36,7 @@ export function handleTree( const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID); const [childrenNodes, ancestry, relatedEvents, relatedAlerts] = await Promise.all([ - fetcher.children(children, generations, afterChild), + fetcher.children(children, afterChild), fetcher.ancestors(ancestors), fetcher.events(events, afterEvent), fetcher.alerts(alerts, afterAlert), diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts new file mode 100644 index 00000000000000..ae17cf4c3a562f --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/alerts_query_handler.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverRelatedAlerts, ResolverEvent } from '../../../../../common/endpoint/types'; +import { createRelatedAlerts } from './node'; +import { AlertsQuery } from '../queries/alerts'; +import { PaginationBuilder } from './pagination'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; + +/** + * Requests related alerts for the given node. + */ +export class RelatedAlertsQueryHandler implements SingleQueryHandler { + private relatedAlerts: ResolverRelatedAlerts | undefined; + private readonly query: AlertsQuery; + constructor( + private readonly limit: number, + private readonly entityID: string, + after: string | undefined, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new AlertsQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.relatedAlerts = createRelatedAlerts( + this.entityID, + results, + PaginationBuilder.buildCursorRequestLimit(this.limit, results) + ); + }; + + /** + * Builds a QueryInfo object that defines the related alerts to search for and how to handle the response. + * + * This will return undefined onces the results have been retrieved from ES. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results after an msearch. + */ + getResults() { + return this.relatedAlerts; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client + */ + async search(client: ILegacyScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createRelatedAlerts(this.entityID); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts new file mode 100644 index 00000000000000..9bf16dac791d74 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/ancestry_query_handler.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { + parentEntityId, + entityId, + getAncestryAsArray, +} from '../../../../../common/endpoint/models/event'; +import { + ResolverAncestry, + ResolverEvent, + ResolverLifecycleNode, +} from '../../../../../common/endpoint/types'; +import { createAncestry, createLifecycle } from './node'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { QueryHandler } from './fetch'; + +/** + * Retrieve the ancestry portion of a resolver tree. + */ +export class AncestryQueryHandler implements QueryHandler { + private readonly ancestry: ResolverAncestry = createAncestry(); + private ancestorsToFind: string[]; + private readonly query: LifecycleQuery; + + constructor( + private levels: number, + indexPattern: string, + legacyEndpointID: string | undefined, + originNode: ResolverLifecycleNode | undefined + ) { + this.ancestorsToFind = getAncestryAsArray(originNode?.lifecycle[0]).slice(0, levels); + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + + // add the origin node to the response if it exists + if (originNode) { + this.ancestry.ancestors.push(originNode); + this.ancestry.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; + } + } + + private toMapOfNodes(results: ResolverEvent[]) { + return results.reduce((nodes: Map, event: ResolverEvent) => { + const nodeId = entityId(event); + let node = nodes.get(nodeId); + if (!node) { + node = createLifecycle(nodeId, []); + } + + node.lifecycle.push(event); + return nodes.set(nodeId, node); + }, new Map()); + } + + private setNoMore() { + this.ancestry.nextAncestor = null; + this.ancestorsToFind = []; + this.levels = 0; + } + + private handleResponse = (searchResp: SearchResponse) => { + const results = this.query.formatResponse(searchResp); + if (results.length === 0) { + this.setNoMore(); + return; + } + + // bucket the start and end events together for a single node + const ancestryNodes = this.toMapOfNodes(results); + + // the order of this array is going to be weird, it will look like this + // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] + this.ancestry.ancestors.push(...ancestryNodes.values()); + this.ancestry.nextAncestor = parentEntityId(results[0]) || null; + this.levels = this.levels - ancestryNodes.size; + // the results come back in ascending order on timestamp so the first entry in the + // results should be the further ancestor (most distant grandparent) + this.ancestorsToFind = getAncestryAsArray(results[0]).slice(0, this.levels); + }; + + /** + * Returns whether there are more results to retrieve based on the limit that is passed in and the results that + * have already been received from ES. + */ + hasMore(): boolean { + return this.levels > 0 && this.ancestorsToFind.length > 0; + } + + /** + * Get a query info for retrieving the next set of results. + */ + nextQuery(): QueryInfo | undefined { + if (this.hasMore()) { + return { + query: this.query, + ids: this.ancestorsToFind, + handler: this.handleResponse, + }; + } + } + + /** + * Return the results after using msearch to find them. + */ + getResults() { + return this.ancestry; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client. + */ + async search(client: ILegacyScopedClusterClient) { + while (this.hasMore()) { + const info = this.nextQuery(); + if (!info) { + break; + } + this.handleResponse(await this.query.search(client, info.ids)); + } + return this.getResults(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts index 1d55cb7cfd7358..ca5b5aef0f6518 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.test.ts @@ -3,95 +3,195 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import _ from 'lodash'; - -import { EndpointDocGenerator } from '../../../../../common/endpoint/generate_data'; +import { + EndpointDocGenerator, + Tree, + Event, + TreeNode, +} from '../../../../../common/endpoint/generate_data'; import { ChildrenNodesHelper } from './children_helper'; -import { eventId, entityId, parentEntityId } from '../../../../../common/endpoint/models/event'; -import { ResolverEvent, ResolverChildren } from '../../../../../common/endpoint/types'; - -function findParents(events: ResolverEvent[]): ResolverEvent[] { - const cache = _.groupBy(events, entityId); +import { eventId, isProcessStart } from '../../../../../common/endpoint/models/event'; - const parents: ResolverEvent[] = []; - Object.values(cache).forEach((lifecycle) => { - const parentNode = cache[parentEntityId(lifecycle[0])!]; - if (parentNode) { - parents.push(parentNode[0]); +function getStartEvents(events: Event[]): Event[] { + const startEvents: Event[] = []; + for (const event of events) { + if (isProcessStart(event)) { + startEvents.push(event); } - }); - return parents; + } + return startEvents; } -function findNode(tree: ResolverChildren, id: string) { - return tree.childNodes.find((node) => { - return node.entityID === id; - }); +function getAllChildrenEvents(tree: Tree) { + const children: Event[] = []; + for (const child of tree.children.values()) { + children.push(...child.lifecycle); + } + return children; +} + +function getStartEventsFromLevels(levels: Array>) { + const startEvents: Event[] = []; + for (const level of levels) { + for (const node of level.values()) { + startEvents.push(...getStartEvents(node.lifecycle)); + } + } + + return startEvents; } describe('Children helper', () => { const generator = new EndpointDocGenerator(); - const root = generator.generateEvent(); + + let tree: Tree; + let helper: ChildrenNodesHelper; + let childrenEvents: Event[]; + let childrenStartEvents: Event[]; + beforeEach(() => { + tree = generator.generateTree({ + children: 3, + alwaysGenMaxChildrenPerNode: true, + generations: 3, + percentTerminated: 100, + ancestryArraySize: 2, + }); + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size); + childrenEvents = getAllChildrenEvents(tree); + childrenStartEvents = getStartEvents(childrenEvents); + }); + + it('returns the correct entity_ids', () => { + helper.addLifecycleEvents(childrenEvents); + expect(helper.getEntityIDs()).toEqual(Array.from(tree.children.keys())); + }); + + it('returns the correct number of nodes', () => { + helper.addLifecycleEvents(childrenEvents); + expect(helper.getNumNodes()).toEqual(tree.children.size); + }); + + it('marks the query nodes as null', () => { + // +1 indicates that we haven't received all the results so it should create a pagination cursor for the + // queried node (aka the origin that we're passing in) + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size + 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + helper.addStartEvents(nextQuery!, []); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('returns undefined when the limit is reached', () => { + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size - 1); + + expect(helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents)).toBeUndefined(); + }); + + it('handles multiple additions of start events', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + let nextQuery = helper.addStartEvents(new Set([tree.origin.id]), level1And2); + expect(nextQuery?.size).toEqual(tree.childrenLevels[1].size); + for (const node of tree.childrenLevels[1].values()) { + expect(nextQuery?.has(node.id)).toBeTruthy(); + } + + const level3 = getStartEventsFromLevels(tree.childrenLevels.slice(2, 3)); + nextQuery = helper.addStartEvents(nextQuery!, level3); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('handles an empty set', () => { + helper = new ChildrenNodesHelper(tree.origin.id, 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), []); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + expect(nodes.childNodes.length).toEqual(0); + }); + + it('handles an empty set after multiple additions', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + let nextQuery = helper.addStartEvents(new Set([tree.origin.id]), level1And2); + + nextQuery = helper.addStartEvents(nextQuery!, []); + expect(nextQuery).toBeUndefined(); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + expect(node.nextChild).toBeNull(); + } + }); + + it('non leaf nodes are set to undefined by default', () => { + // + 1 indicates that we got everything that ES had + helper = new ChildrenNodesHelper(tree.origin.id, childrenStartEvents.length + 1); + const level1And2 = getStartEventsFromLevels(tree.childrenLevels.slice(0, 2)); + helper.addStartEvents(new Set([tree.origin.id]), level1And2); + const nodes = helper.getNodes(); + expect(nodes.nextChild).toBeNull(); + for (const node of nodes.childNodes) { + if (tree.childrenLevels[0].has(node.entityID)) { + expect(node.nextChild).toBeNull(); + } else { + expect(node.nextChild).toBeUndefined(); + } + } + }); + + it('returns the leaf nodes', () => { + helper = new ChildrenNodesHelper(tree.origin.id, tree.children.size + 1); + + const nextQuery = helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + // we're using an ancestry array of 2 so the leaf nodes are at the second level + expect(nextQuery?.size).toEqual(tree.childrenLevels[1].size); + + for (const node of tree.childrenLevels[1].values()) { + expect(nextQuery?.has(node.id)).toBeTruthy(); + } + }); it('builds the children response structure', () => { - const children = Array.from( - generator.descendantsTreeGenerator(root, { - generations: 3, - children: 3, - relatedEvents: 0, - relatedAlerts: 0, - percentTerminated: 100, - alwaysGenMaxChildrenPerNode: true, - }) - ); - - // because we requested the generator to always return the max children, there will always be at least 2 parents - const parents = findParents(children); - - // this represents the aggregation returned from elastic search - // each node in the tree should have 3 children, so if these values are greater than 3 there should be - // pagination cursors created for those children - const totals = { - [root.process.entity_id]: 100, - [entityId(parents[0])]: 10, - [entityId(parents[1])]: 0, - }; - - const helper = new ChildrenNodesHelper(root.process.entity_id); - helper.addChildren(totals, children); - const tree = helper.getNodes(); - expect(tree.nextChild).not.toBeNull(); - - let parent = findNode(tree, entityId(parents[0])); - expect(parent?.nextChild).not.toBeNull(); - parent = findNode(tree, entityId(parents[1])); - expect(parent?.nextChild).toBeNull(); - - tree.childNodes.forEach((node) => { + helper.addStartEvents(new Set([tree.origin.id]), childrenStartEvents); + helper.addLifecycleEvents(childrenEvents); + const childrenNodes = helper.getNodes(); + + // since we got all the nodes all the nextChild cursors should be null + for (const node of childrenNodes.childNodes) { + expect(node.nextChild).toBeUndefined(); + } + expect(childrenNodes.nextChild).not.toBeNull(); + + childrenNodes.childNodes.forEach((node) => { node.lifecycle.forEach((event) => { - expect(children.find((child) => child.event.id === eventId(event))).toEqual(event); + expect(childrenEvents.find((child) => child.event.id === eventId(event))).toEqual(event); }); }); }); it('builds the children response structure twice', () => { - const children = Array.from( - generator.descendantsTreeGenerator(root, { - generations: 3, - children: 3, - relatedEvents: 0, - relatedAlerts: 0, - percentTerminated: 100, - }) - ); - const helper = new ChildrenNodesHelper(root.process.entity_id); - helper.addChildren({}, children); + helper.addLifecycleEvents(childrenEvents); helper.getNodes(); - const tree = helper.getNodes(); - tree.childNodes.forEach((node) => { + const childrenNodes = helper.getNodes(); + childrenNodes.childNodes.forEach((node) => { node.lifecycle.forEach((event) => { - expect(children.find((child) => child.event.id === eventId(event))).toEqual(event); + expect(childrenEvents.find((child) => child.event.id === eventId(event))).toEqual(event); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts index e60e5087c30a9e..01e356682ac478 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_helper.ts @@ -8,35 +8,36 @@ import { entityId, parentEntityId, isProcessStart, + getAncestryAsArray, } from '../../../../../common/endpoint/models/event'; import { ResolverChildNode, ResolverEvent, ResolverChildren, } from '../../../../../common/endpoint/types'; -import { PaginationBuilder } from './pagination'; import { createChild } from './node'; +import { PaginationBuilder } from './pagination'; /** * This class helps construct the children structure when building a resolver tree. */ export class ChildrenNodesHelper { - private readonly cache: Map = new Map(); + private readonly entityToNodeCache: Map = new Map(); - constructor(private readonly rootID: string) { - this.cache.set(rootID, createChild(rootID)); + constructor(private readonly rootID: string, private readonly limit: number) { + this.entityToNodeCache.set(rootID, createChild(rootID)); } /** * Constructs a ResolverChildren response based on the children that were previously add. */ getNodes(): ResolverChildren { - const cacheCopy: Map = new Map(this.cache); + const cacheCopy: Map = new Map(this.entityToNodeCache); const rootNode = cacheCopy.get(this.rootID); let rootNextChild = null; if (rootNode) { - rootNextChild = rootNode.nextChild; + rootNextChild = rootNode.nextChild ?? null; } cacheCopy.delete(this.rootID); @@ -47,51 +48,131 @@ export class ChildrenNodesHelper { } /** - * Add children to the cache. - * - * @param totals a map of unique node IDs to total number of child nodes - * @param results events from a children query + * Get the entity_ids of the nodes that are cached. + */ + getEntityIDs(): string[] { + const cacheCopy: Map = new Map(this.entityToNodeCache); + cacheCopy.delete(this.rootID); + return Array.from(cacheCopy.keys()); + } + + /** + * Get the number of nodes that have been cached. */ - addChildren(totals: Record, results: ResolverEvent[]) { - const startEventsCache: Map = new Map(); + getNumNodes(): number { + // -1 because the root node is in the cache too + return this.entityToNodeCache.size - 1; + } - results.forEach((event) => { + /** + * Add lifecycle events (start, end, etc) to the cache. + * + * @param lifecycle an array of resolver lifecycle events for different process nodes returned from ES. + */ + addLifecycleEvents(lifecycle: ResolverEvent[]) { + for (const event of lifecycle) { const entityID = entityId(event); - const parentID = parentEntityId(event); - if (!entityID || !parentID) { - return; + if (entityID) { + const cachedChild = this.getOrCreateChildNode(entityID); + cachedChild.lifecycle.push(event); } + } + } - let cachedChild = this.cache.get(entityID); - if (!cachedChild) { - cachedChild = createChild(entityID); - this.cache.set(entityID, cachedChild); - } - cachedChild.lifecycle.push(event); + /** + * Add the start events for the nodes received from ES. Pagination cursors will be constructed based on the + * request limit and results returned. + * + * @param queriedNodes the entity_ids of the nodes that returned these start events + * @param startEvents an array of start events returned by ES + */ + addStartEvents(queriedNodes: Set, startEvents: ResolverEvent[]): Set | undefined { + let largestAncestryArray = 0; + const nodesToQueryNext: Map> = new Map(); + const nonLeafNodes: Set = new Set(); - if (isProcessStart(event)) { - let startEvents = startEventsCache.get(parentID); - if (startEvents === undefined) { - startEvents = []; - startEventsCache.set(parentID, startEvents); + const isDistantGrandchild = (event: ResolverEvent) => { + const ancestry = getAncestryAsArray(event); + return ancestry.length > 0 && queriedNodes.has(ancestry[ancestry.length - 1]); + }; + + for (const event of startEvents) { + const parentID = parentEntityId(event); + const entityID = entityId(event); + if (parentID && entityID && isProcessStart(event)) { + // don't actually add the start event to the node, because that'll be done in + // a different call + const childNode = this.getOrCreateChildNode(entityID); + + const ancestry = getAncestryAsArray(event); + // This is to handle the following unlikely but possible scenario: + // if an alert was generated by the kernel process (parent process of all other processes) then + // the direct children of that process would only have an ancestry array of [parent_kernel], a single value in the array. + // The children of those children would have two values in their array [direct parent, parent_kernel] + // we need to determine which nodes are the most distant grandchildren of the queriedNodes because those should + // be used for the next query if more nodes should be retrieved. To generally determine the most distant grandchildren + // we can use the last entry in the ancestry array because of its ordering. The problem with that is in the scenario above + // the direct children of parent_kernel will also meet that criteria even though they are not actually the most + // distant grandchildren. To get around that issue we'll bucket all the nodes by the size of their ancestry array + // and then only return the nodes in the largest bucket because those should be the most distant grandchildren + // from the queried nodes that were passed in. + if (ancestry.length > largestAncestryArray) { + largestAncestryArray = ancestry.length; + } + + // a grandchild must have an array of > 0 and have it's last parent be in the set of previously queried nodes + // this is one of the furthest descendants from the queried nodes + if (isDistantGrandchild(event)) { + let levelOfNodes = nodesToQueryNext.get(ancestry.length); + if (!levelOfNodes) { + levelOfNodes = new Set(); + nodesToQueryNext.set(ancestry.length, levelOfNodes); + } + levelOfNodes.add(entityID); + } else { + nonLeafNodes.add(childNode); } - startEvents.push(event); } - }); + } + + // we may not have received all the possible nodes so mark pagination for the query nodes + // we won't know if the non leaf nodes (non query nodes) have additional children so don't mark them + if (this.limit <= this.getNumNodes()) { + this.setPaginationForNodes(queriedNodes, startEvents); + return; + } + + // the non leaf nodes have received all their children so mark them as finished + for (const nonLeaf of nonLeafNodes.values()) { + nonLeaf.nextChild = null; + } - this.addChildrenPagination(startEventsCache, totals); + // we've received all the descendants of the previously queried node that we can get using it's ancestry array + // so mark those nodes as complete + for (const nodeEntityID of queriedNodes.values()) { + const node = this.entityToNodeCache.get(nodeEntityID); + if (node) { + node.nextChild = null; + } + } + return nodesToQueryNext.get(largestAncestryArray); } - private addChildrenPagination( - startEventsCache: Map, - totals: Record - ) { - Object.entries(totals).forEach(([parentID, total]) => { - const parentNode = this.cache.get(parentID); - const childrenStartEvents = startEventsCache.get(parentID); - if (parentNode && childrenStartEvents) { - parentNode.nextChild = PaginationBuilder.buildCursor(total, childrenStartEvents); + private setPaginationForNodes(nodes: Set, startEvents: ResolverEvent[]) { + for (const nodeEntityID of nodes.values()) { + const cachedNode = this.entityToNodeCache.get(nodeEntityID); + if (cachedNode) { + cachedNode.nextChild = PaginationBuilder.buildCursor(startEvents); } - }); + } + } + + private getOrCreateChildNode(entityID: string) { + let cachedChild = this.entityToNodeCache.get(entityID); + if (!cachedChild) { + cachedChild = createChild(entityID); + this.entityToNodeCache.set(entityID, cachedChild); + } + return cachedChild; } } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts new file mode 100644 index 00000000000000..8aaf809405d631 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_lifecycle_query_handler.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverEvent, ResolverChildren } from '../../../../../common/endpoint/types'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; +import { ChildrenNodesHelper } from './children_helper'; +import { createChildren } from './node'; + +/** + * Returns the children of a resolver tree. + */ +export class ChildrenLifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: ResolverChildren | undefined; + private readonly query: LifecycleQuery; + constructor( + private readonly childrenHelper: ChildrenNodesHelper, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + } + + private handleResponse = (response: SearchResponse) => { + this.childrenHelper.addLifecycleEvents(this.query.formatResponse(response)); + this.lifecycle = this.childrenHelper.getNodes(); + }; + + /** + * Get the query for msearch. Once the results are set this will return undefined. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.childrenHelper.getEntityIDs(), + handler: this.handleResponse, + }; + } + + /** + * Return the results from the search. + */ + getResults(): ResolverChildren | undefined { + return this.lifecycle; + } + + /** + * Perform a regular search and return the results. + * + * @param client the elasticsearch client + */ + async search(client: ILegacyScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.childrenHelper.getEntityIDs())); + return this.getResults() || createChildren(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts new file mode 100644 index 00000000000000..1c741847207931 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/children_start_query_handler.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverEvent } from '../../../../../common/endpoint/types'; +import { ChildrenQuery } from '../queries/children'; +import { QueryInfo } from '../queries/multi_searcher'; +import { QueryHandler } from './fetch'; +import { ChildrenNodesHelper } from './children_helper'; +import { PaginationBuilder } from './pagination'; + +/** + * Retrieve the start lifecycle events for the children of a resolver tree. + * + * If using msearch you should loop over hasMore() because the results are limited to the size of the ancestry array. + */ +export class ChildrenStartQueryHandler implements QueryHandler { + private readonly childrenHelper: ChildrenNodesHelper; + private limitLeft: number; + private query: ChildrenQuery; + private nodesToQuery: Set; + + constructor( + private readonly limit: number, + entityID: string, + after: string | undefined, + private readonly indexPattern: string, + private readonly legacyEndpointID: string | undefined + ) { + this.query = new ChildrenQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + this.childrenHelper = new ChildrenNodesHelper(entityID, this.limit); + this.limitLeft = this.limit; + this.nodesToQuery = new Set([entityID]); + } + + private setNoMore() { + this.nodesToQuery = new Set(); + this.limitLeft = 0; + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.nodesToQuery = this.childrenHelper.addStartEvents(this.nodesToQuery, results) ?? new Set(); + + if (results.length === 0) { + this.setNoMore(); + return; + } + + this.limitLeft = this.limit - this.childrenHelper.getNumNodes(); + this.query = new ChildrenQuery( + PaginationBuilder.createBuilder(this.limitLeft), + this.indexPattern, + this.legacyEndpointID + ); + }; + + /** + * Check if there are more results to retrieve based on the limit that was passed in. + */ + hasMore(): boolean { + return this.limitLeft > 0 && this.nodesToQuery.size > 0; + } + + /** + * Get a query to retrieve the next set of results. + */ + nextQuery(): QueryInfo | undefined { + if (this.hasMore()) { + return { + query: this.query, + // This should never be undefined because the check above + ids: Array.from(this.nodesToQuery.values()), + handler: this.handleResponse, + }; + } + } + + /** + * Get the cached results from the ES responses. + */ + getResults(): ChildrenNodesHelper { + return this.childrenHelper; + } + + /** + * Perform a regular search and return the helper. + * + * @param client the elasticsearch client + */ + async search(client: ILegacyScopedClusterClient) { + while (this.hasMore()) { + const info = this.nextQuery(); + if (!info) { + break; + } + this.handleResponse(await this.query.search(client, info.ids)); + } + return this.getResults(); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts new file mode 100644 index 00000000000000..849dbc25fe4db9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/events_query_handler.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverRelatedEvents, ResolverEvent } from '../../../../../common/endpoint/types'; +import { createRelatedEvents } from './node'; +import { EventsQuery } from '../queries/events'; +import { PaginationBuilder } from './pagination'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; + +/** + * This retrieves the related events for the origin node of a resolver tree. + */ +export class RelatedEventsQueryHandler implements SingleQueryHandler { + private relatedEvents: ResolverRelatedEvents | undefined; + private readonly query: EventsQuery; + constructor( + private readonly limit: number, + private readonly entityID: string, + after: string | undefined, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new EventsQuery( + PaginationBuilder.createBuilder(limit, after), + indexPattern, + legacyEndpointID + ); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + this.relatedEvents = createRelatedEvents( + this.entityID, + results, + PaginationBuilder.buildCursorRequestLimit(this.limit, results) + ); + }; + + /** + * Get a query to use in a msearch. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results after an msearch. + */ + getResults() { + return this.relatedEvents; + } + + /** + * Perform a normal search and return the related events results. + * + * @param client the elasticsearch client + */ + async search(client: ILegacyScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createRelatedEvents(this.entityID); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts index 1a532c54c7d5d1..feb165c308a91c 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts @@ -11,22 +11,61 @@ import { ResolverAncestry, ResolverRelatedAlerts, ResolverLifecycleNode, - ResolverEvent, } from '../../../../../common/endpoint/types'; -import { - entityId, - ancestryArray, - parentEntityId, -} from '../../../../../common/endpoint/models/event'; -import { PaginationBuilder } from './pagination'; import { Tree } from './tree'; import { LifecycleQuery } from '../queries/lifecycle'; -import { ChildrenQuery } from '../queries/children'; -import { EventsQuery } from '../queries/events'; import { StatsQuery } from '../queries/stats'; -import { createAncestry, createRelatedEvents, createLifecycle, createRelatedAlerts } from './node'; -import { ChildrenNodesHelper } from './children_helper'; -import { AlertsQuery } from '../queries/alerts'; +import { createLifecycle } from './node'; +import { MultiSearcher, QueryInfo } from '../queries/multi_searcher'; +import { AncestryQueryHandler } from './ancestry_query_handler'; +import { RelatedEventsQueryHandler } from './events_query_handler'; +import { RelatedAlertsQueryHandler } from './alerts_query_handler'; +import { ChildrenStartQueryHandler } from './children_start_query_handler'; +import { ChildrenLifecycleQueryHandler } from './children_lifecycle_query_handler'; +import { LifecycleQueryHandler } from './lifecycle_query_handler'; + +/** + * The query parameters passed in from the request. These define the limits for the ES requests for retrieving the + * resolver tree. + */ +export interface TreeOptions { + children: number; + ancestors: number; + events: number; + alerts: number; + afterAlert?: string; + afterEvent?: string; + afterChild?: string; +} + +interface QueryBuilder { + nextQuery(): QueryInfo | undefined; +} + +/** + * This interface defines the contract for a query handler that will only be used once in an msearch call. + */ +export interface SingleQueryHandler extends QueryBuilder { + /** + * This method returns the results if the query has been used in an msearch call or undefined if not. + */ + getResults(): T | undefined; + /** + * Do a regular search instead of msearch. + * @param client the elasticsearch client + */ + search(client: ILegacyScopedClusterClient): Promise; +} + +/** + * This interface defines the contract for a query handler that can be used multiple times by msearch. + */ +export interface QueryHandler extends SingleQueryHandler { + /** + * Returns whether additional msearch are required to retrieve the rest of the expected data from ES. + */ + hasMore(): boolean; +} /** * Handles retrieving nodes of a resolver tree. @@ -52,46 +91,138 @@ export class Fetcher { private readonly endpointID?: string ) {} + /** + * This method retrieves the resolver tree starting from the `id` during construction of the class. + * + * @param options the options for retrieving the structure of the tree. + */ + public async tree(options: TreeOptions) { + const addQueryToList = (queryHandler: QueryBuilder, queries: QueryInfo[]) => { + const queryInfo = queryHandler.nextQuery(); + if (queryInfo !== undefined) { + queries.push(queryInfo); + } + }; + + const originHandler = new LifecycleQueryHandler( + this.id, + this.eventsIndexPattern, + this.endpointID + ); + + const eventsHandler = new RelatedEventsQueryHandler( + options.events, + this.id, + options.afterEvent, + this.eventsIndexPattern, + this.endpointID + ); + + const alertsHandler = new RelatedAlertsQueryHandler( + options.alerts, + this.id, + options.afterAlert, + this.alertsIndexPattern, + this.endpointID + ); + + // we need to get the start events first because the API request defines how many nodes to return and we don't want + // to count or limit ourselves based on the other lifecycle events (end, etc) + const childrenHandler = new ChildrenStartQueryHandler( + options.children, + this.id, + options.afterChild, + this.eventsIndexPattern, + this.endpointID + ); + + const msearch = new MultiSearcher(this.client); + + let queries: QueryInfo[] = []; + addQueryToList(eventsHandler, queries); + addQueryToList(alertsHandler, queries); + addQueryToList(childrenHandler, queries); + addQueryToList(originHandler, queries); + + // get the related events, related alerts, the first pass of children start events, and the origin node + // the origin node is needed so we can get the ancestry array for the additional ancestor calls + await msearch.search(queries); + + const ancestryHandler = new AncestryQueryHandler( + options.ancestors, + this.eventsIndexPattern, + this.endpointID, + originHandler.getResults() + ); + + // get the remaining ancestors and children start events + while (ancestryHandler.hasMore() || childrenHandler.hasMore()) { + queries = []; + addQueryToList(ancestryHandler, queries); + addQueryToList(childrenHandler, queries); + await msearch.search(queries); + } + + const childrenTotalsHelper = childrenHandler.getResults(); + + const childrenLifecycleHandler = new ChildrenLifecycleQueryHandler( + childrenTotalsHelper, + this.eventsIndexPattern, + this.endpointID + ); + + // now that we have all the start events get the full lifecycle nodes + childrenLifecycleHandler.search(this.client); + + const tree = new Tree(this.id, { + ancestry: ancestryHandler.getResults(), + relatedEvents: eventsHandler.getResults(), + relatedAlerts: alertsHandler.getResults(), + children: childrenLifecycleHandler.getResults(), + }); + + // add the stats to the tree + return this.stats(tree); + } + /** * Retrieves the ancestor nodes for the resolver tree. * * @param limit upper limit of ancestors to retrieve */ public async ancestors(limit: number): Promise { - const ancestryInfo = createAncestry(); const originNode = await this.getNode(this.id); - if (originNode) { - ancestryInfo.ancestors.push(originNode); - // If the request is only for the origin node then set next to its parent - ancestryInfo.nextAncestor = parentEntityId(originNode.lifecycle[0]) || null; - await this.doAncestors( - // limit the ancestors we're looking for to the number of levels - // the array could be up to length 20 but that could change - Fetcher.getAncestryAsArray(originNode.lifecycle[0]).slice(0, limit), - limit, - ancestryInfo - ); - } - return ancestryInfo; + const ancestryHandler = new AncestryQueryHandler( + limit, + this.eventsIndexPattern, + this.endpointID, + originNode + ); + return ancestryHandler.search(this.client); } /** * Retrieves the children nodes for the resolver tree. * * @param limit the number of children to retrieve for a single level - * @param generations number of levels to return * @param after a cursor to use as the starting point for retrieving children */ - public async children( - limit: number, - generations: number, - after?: string - ): Promise { - const helper = new ChildrenNodesHelper(this.id); - - await this.doChildren(helper, [this.id], limit, generations, after); + public async children(limit: number, after?: string): Promise { + const childrenHandler = new ChildrenStartQueryHandler( + limit, + this.id, + after, + this.eventsIndexPattern, + this.endpointID + ); + const helper = await childrenHandler.search(this.client); + const childrenLifecycleHandler = new ChildrenLifecycleQueryHandler( + helper, + this.eventsIndexPattern, + this.endpointID + ); - return helper.getNodes(); + return childrenLifecycleHandler.search(this.client); } /** @@ -101,7 +232,15 @@ export class Fetcher { * @param after a cursor to use as the starting point for retrieving related events */ public async events(limit: number, after?: string): Promise { - return this.doEvents(limit, after); + const eventsHandler = new RelatedEventsQueryHandler( + limit, + this.id, + after, + this.eventsIndexPattern, + this.endpointID + ); + + return eventsHandler.search(this.client); } /** @@ -111,26 +250,15 @@ export class Fetcher { * @param after a cursor to use as the starting point for retrieving alerts */ public async alerts(limit: number, after?: string): Promise { - const query = new AlertsQuery( - PaginationBuilder.createBuilder(limit, after), + const alertsHandler = new RelatedAlertsQueryHandler( + limit, + this.id, + after, this.alertsIndexPattern, this.endpointID ); - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedAlerts(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } - - return createRelatedAlerts( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); + return alertsHandler.search(this.client); } /** @@ -145,7 +273,7 @@ export class Fetcher { private async getNode(entityID: string): Promise { const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - const results = await query.search(this.client, entityID); + const results = await query.searchAndFormat(this.client, entityID); if (results.length === 0) { return; } @@ -153,125 +281,13 @@ export class Fetcher { return createLifecycle(entityID, results); } - private static getAncestryAsArray(event: ResolverEvent): string[] { - const ancestors = ancestryArray(event); - if (ancestors) { - return ancestors; - } - - const parentID = parentEntityId(event); - if (parentID) { - return [parentID]; - } - - return []; - } - - private async doAncestors( - ancestors: string[], - levels: number, - ancestorInfo: ResolverAncestry - ): Promise { - if (levels <= 0) { - return; - } - - const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - const results = await query.search(this.client, ancestors); - - if (results.length === 0) { - ancestorInfo.nextAncestor = null; - return; - } - - // bucket the start and end events together for a single node - const ancestryNodes = results.reduce( - (nodes: Map, ancestorEvent: ResolverEvent) => { - const nodeId = entityId(ancestorEvent); - let node = nodes.get(nodeId); - if (!node) { - node = createLifecycle(nodeId, []); - } - - node.lifecycle.push(ancestorEvent); - return nodes.set(nodeId, node); - }, - new Map() - ); - - // the order of this array is going to be weird, it will look like this - // [furthest grandparent...closer grandparent, next recursive call furthest grandparent...closer grandparent] - ancestorInfo.ancestors.push(...ancestryNodes.values()); - ancestorInfo.nextAncestor = parentEntityId(results[0]) || null; - const levelsLeft = levels - ancestryNodes.size; - // the results come back in ascending order on timestamp so the first entry in the - // results should be the further ancestor (most distant grandparent) - const next = Fetcher.getAncestryAsArray(results[0]).slice(0, levelsLeft); - // the ancestry array currently only holds up to 20 values but we can't rely on that so keep recursing - await this.doAncestors(next, levelsLeft, ancestorInfo); - } - - private async doEvents(limit: number, after?: string) { - const query = new EventsQuery( - PaginationBuilder.createBuilder(limit, after), - this.eventsIndexPattern, - this.endpointID - ); - - const { totals, results } = await query.search(this.client, this.id); - if (results.length === 0) { - // return an empty set of results - return createRelatedEvents(this.id); - } - if (!totals[this.id]) { - throw new Error(`Could not find the totals for related events entity_id: ${this.id}`); - } - - return createRelatedEvents( - this.id, - results, - PaginationBuilder.buildCursor(totals[this.id], results) - ); - } - - private async doChildren( - cache: ChildrenNodesHelper, - ids: string[], - limit: number, - levels: number, - after?: string - ) { - if (levels === 0 || ids.length === 0) { - return; - } - - const childrenQuery = new ChildrenQuery( - PaginationBuilder.createBuilder(limit, after), - this.eventsIndexPattern, - this.endpointID - ); - const lifecycleQuery = new LifecycleQuery(this.eventsIndexPattern, this.endpointID); - - const { totals, results } = await childrenQuery.search(this.client, ids); - if (results.length === 0) { - return; - } - - const childIDs = results.map(entityId); - const children = await lifecycleQuery.search(this.client, childIDs); - - cache.addChildren(totals, children); - - await this.doChildren(cache, childIDs, limit, levels - 1); - } - private async doStats(tree: Tree) { const statsQuery = new StatsQuery( [this.eventsIndexPattern, this.alertsIndexPattern], this.endpointID ); const ids = tree.ids(); - const res = await statsQuery.search(this.client, ids); + const res = await statsQuery.searchAndFormat(this.client, ids); const alerts = res.alerts; const events = res.events; ids.forEach((id) => { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts new file mode 100644 index 00000000000000..ab0501e099490b --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/lifecycle_query_handler.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SearchResponse } from 'elasticsearch'; +import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ResolverEvent, ResolverLifecycleNode } from '../../../../../common/endpoint/types'; +import { LifecycleQuery } from '../queries/lifecycle'; +import { QueryInfo } from '../queries/multi_searcher'; +import { SingleQueryHandler } from './fetch'; +import { createLifecycle } from './node'; + +/** + * Retrieve the lifecycle events for a node. + */ +export class LifecycleQueryHandler implements SingleQueryHandler { + private lifecycle: ResolverLifecycleNode | undefined; + private readonly query: LifecycleQuery; + constructor( + private readonly entityID: string, + indexPattern: string, + legacyEndpointID: string | undefined + ) { + this.query = new LifecycleQuery(indexPattern, legacyEndpointID); + } + + private handleResponse = (response: SearchResponse) => { + const results = this.query.formatResponse(response); + if (results.length !== 0) { + this.lifecycle = createLifecycle(this.entityID, results); + } + }; + + /** + * Build the query for retrieving the lifecycle events. This will return undefined once the results have been found. + */ + nextQuery(): QueryInfo | undefined { + if (this.getResults()) { + return; + } + + return { + query: this.query, + ids: this.entityID, + handler: this.handleResponse, + }; + } + + /** + * Get the results from the msearch. + */ + getResults(): ResolverLifecycleNode | undefined { + return this.lifecycle; + } + + /** + * Do a regular search and return the results. + * + * @param client the elasticsearch client. + */ + async search(client: ILegacyScopedClusterClient) { + const results = this.getResults(); + if (results) { + return results; + } + + this.handleResponse(await this.query.search(client, this.entityID)); + return this.getResults() ?? createLifecycle(this.entityID, []); + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts index 57a2ebfcc17929..98180885faf052 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/node.ts @@ -12,6 +12,7 @@ import { ResolverTree, ResolverChildNode, ResolverRelatedAlerts, + ResolverChildren, } from '../../../../../common/endpoint/types'; /** @@ -53,7 +54,6 @@ export function createChild(entityID: string): ResolverChildNode { const lifecycle = createLifecycle(entityID, []); return { ...lifecycle, - nextChild: null, }; } @@ -77,6 +77,19 @@ export function createLifecycle( return { entityID, lifecycle }; } +/** + * Creates a resolver children response. + * + * @param nodes the child nodes to add to the ResolverChildren response + * @param nextChild the cursor for the response + */ +export function createChildren( + nodes: ResolverChildNode[] = [], + nextChild: string | null = null +): ResolverChildren { + return { childNodes: nodes, nextChild }; +} + /** * Creates an empty `Tree` response structure that the tree handler would return * diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts index 74e4e252861e6b..4daa45aec2a740 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.test.ts @@ -18,20 +18,20 @@ describe('Pagination', () => { const root = generator.generateEvent(); const events = Array.from(generator.relatedEventsGenerator(root, 5)); - it('does not build a cursor when all events are present', () => { - expect(PaginationBuilder.buildCursor(0, events)).toBeNull(); + it('does build a cursor when received the same number of events as was requested', () => { + expect(PaginationBuilder.buildCursorRequestLimit(4, events)).not.toBeNull(); }); - it('creates a cursor when not all events are present', () => { - expect(PaginationBuilder.buildCursor(events.length + 1, events)).not.toBeNull(); + it('does not create a cursor when the number of events received is less than the amount requested', () => { + expect(PaginationBuilder.buildCursorRequestLimit(events.length + 1, events)).toBeNull(); }); it('creates a cursor with the right information', () => { - const cursor = PaginationBuilder.buildCursor(events.length + 1, events); + const cursor = PaginationBuilder.buildCursorRequestLimit(events.length, events); expect(cursor).not.toBeNull(); // we are guaranteed that the cursor won't be null from the check above const builder = PaginationBuilder.createBuilder(0, cursor!); - const fields = builder.buildQueryFields(0, '', ''); + const fields = builder.buildQueryFields(''); expect(fields.search_after).toStrictEqual(getSearchAfterInfo(events)); }); }); @@ -39,30 +39,8 @@ describe('Pagination', () => { describe('pagination builder', () => { it('does not include the search after information when no cursor is provided', () => { const builder = PaginationBuilder.createBuilder(100); - const fields = builder.buildQueryFields(1, '', ''); + const fields = builder.buildQueryFields(''); expect(fields).not.toHaveProperty('search_after'); }); - - it('returns no results when the aggregation does not exist in the response', () => { - expect(PaginationBuilder.getTotals()).toStrictEqual({}); - }); - - it('constructs the totals from the aggregation results', () => { - const agg = { - totals: { - buckets: [ - { - key: 'awesome', - doc_count: 5, - }, - { - key: 'soup', - doc_count: 1, - }, - ], - }, - }; - expect(PaginationBuilder.getTotals(agg)).toStrictEqual({ awesome: 5, soup: 1 }); - }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts index 61cb5bdb8f1465..2b107ab1b6db4d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/pagination.ts @@ -8,41 +8,11 @@ import { ResolverEvent } from '../../../../../common/endpoint/types'; import { eventId } from '../../../../../common/endpoint/models/event'; import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/common'; -/** - * Represents a single result bucket of an aggregation - */ -export interface AggBucket { - key: string; - doc_count: number; -} - -interface TotalsAggregation { - totals?: { - buckets?: AggBucket[]; - }; -} - interface PaginationCursor { timestamp: number; eventID: string; } -/** - * The result structure of a query that leverages pagination. This includes totals that can be used to determine if - * additional nodes exist and additional queries need to be made to retrieve the nodes. - */ -export interface PaginatedResults { - /** - * Resulting events returned from the query. - */ - results: ResolverEvent[]; - /** - * Mapping of unique ID to total number of events that exist in ES. The events this references is scoped to the events - * that the query is searching for. - */ - totals: Record; -} - /** * This class handles constructing pagination cursors that resolver can use to return additional events in subsequent * queries. It also constructs an aggregation query to determine the totals for other queries. This class should be used @@ -83,19 +53,28 @@ export class PaginationBuilder { } /** - * Constructs a cursor to use in subsequent queries to retrieve the next set of results. + * Construct a cursor to use in subsequent queries. * - * @param total the total events that exist in ES scoped for a particular query. * @param results the events that were returned by the ES query */ - static buildCursor(total: number, results: ResolverEvent[]): string | null { - if (total > results.length && results.length > 0) { - const lastResult = results[results.length - 1]; - const cursor = { - timestamp: lastResult['@timestamp'], - eventID: eventId(lastResult), - }; - return PaginationBuilder.urlEncodeCursor(cursor); + static buildCursor(results: ResolverEvent[]): string | null { + const lastResult = results[results.length - 1]; + const cursor = { + timestamp: lastResult['@timestamp'], + eventID: eventId(lastResult), + }; + return PaginationBuilder.urlEncodeCursor(cursor); + } + + /** + * Constructs a cursor if the requested limit has not been met. + * + * @param requestLimit the request limit for a query. + * @param results the events that were returned by the ES query + */ + static buildCursorRequestLimit(requestLimit: number, results: ResolverEvent[]): string | null { + if (requestLimit <= results.length && results.length > 0) { + return PaginationBuilder.buildCursor(results); } return null; } @@ -124,45 +103,16 @@ export class PaginationBuilder { /** * Creates an object for adding the pagination fields to a query * - * @param numTerms number of unique IDs that are being search for in this query * @param tiebreaker a unique field to use as the tiebreaker for the search_after - * @param aggregator the field that specifies a unique ID per event (e.g. entity_id) - * @param aggs other aggregations being used with this query * @returns an object containing the pagination information */ - buildQueryFields( - numTerms: number, - tiebreaker: string, - aggregator: string, - aggs: JsonObject = {} - ): JsonObject { + buildQueryFields(tiebreaker: string): JsonObject { const fields: JsonObject = {}; fields.sort = [{ '@timestamp': 'asc' }, { [tiebreaker]: 'asc' }]; - fields.aggs = { ...aggs, totals: { terms: { field: aggregator, size: numTerms } } }; fields.size = this.size; if (this.timestamp && this.eventID) { fields.search_after = [this.timestamp, this.eventID] as Array; } return fields; } - - /** - * Returns the totals found for the specified query - * - * @param aggregations the aggregation field from the ES response - * @returns a mapping of unique ID (e.g. entity_ids) to totals found for those IDs - */ - static getTotals(aggregations?: TotalsAggregation): Record { - if (!aggregations?.totals?.buckets) { - return {}; - } - - return aggregations?.totals?.buckets?.reduce( - (cumulative: Record, bucket: AggBucket) => ({ - ...cumulative, - [bucket.key]: bucket.doc_count, - }), - {} - ); - } } diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts index eeca8ee54e32f3..ace32111005f4c 100644 --- a/x-pack/test/api_integration/apis/endpoint/resolver.ts +++ b/x-pack/test/api_integration/apis/endpoint/resolver.ts @@ -246,6 +246,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC percentWithRelated: 100, numTrees: 1, alwaysGenMaxChildrenPerNode: true, + ancestryArraySize: 2, }; describe('Resolver', () => { @@ -542,13 +543,10 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('returns multiple levels of child process lifecycle events', async () => { const { body }: { body: ResolverChildren } = await supertest - .get( - `/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&generations=1` - ) + .get(`/api/endpoint/resolver/93802/children?legacyEndpointID=${endpointID}&children=10`) .expect(200); + expect(body.childNodes.length).to.eql(10); expect(body.nextChild).to.be(null); - expect(body.childNodes[0].nextChild).to.be(null); - expect(body.childNodes.length).to.eql(8); expect(body.childNodes[0].lifecycle.length).to.eql(1); expect( // for some reason the ts server doesn't think `endgame` exists even though we're using ResolverEvent @@ -615,19 +613,27 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC expect(body.childNodes.length).to.eql(12); // there will be 4 parents, the origin of the tree, and it's 3 children verifyChildren(body.childNodes, tree, 4, 3); + expect(body.nextChild).to.eql(null); }); it('returns a single generation of children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; const { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=1`) + .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=3`) .expect(200); expect(body.childNodes.length).to.eql(3); verifyChildren(body.childNodes, tree, 1, 3); + expect(body.nextChild).to.not.eql(null); }); - it('paginates the children of the origin node', async () => { + it('paginates the children', async () => { + // this gets a node should have 3 children which were created in succession so that the timestamps + // are ordered correctly to be retrieved in a single call + const distantChildEntityID = Array.from(tree.childrenLevels[0].values())[0].id; let { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=1&children=1`) + .get(`/api/endpoint/resolver/${distantChildEntityID}/children?children=1`) .expect(200); expect(body.childNodes.length).to.eql(1); verifyChildren(body.childNodes, tree, 1, 1); @@ -635,49 +641,41 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC ({ body } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}/children?generations=1&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` ) .expect(200)); expect(body.childNodes.length).to.eql(2); verifyChildren(body.childNodes, tree, 1, 2); - expect(body.childNodes[0].nextChild).to.be(null); - expect(body.childNodes[1].nextChild).to.be(null); - }); - - it('paginates the children of different nodes', async () => { - let { body }: { body: ResolverChildren } = await supertest - .get(`/api/endpoint/resolver/${tree.origin.id}/children?generations=2&children=2`) - .expect(200); - // it should return 4 nodes total, 2 for each level - expect(body.childNodes.length).to.eql(4); - verifyChildren(body.childNodes, tree, 2); expect(body.nextChild).to.not.be(null); - expect(body.childNodes[0].nextChild).to.not.be(null); - // the second child will not have any results returned for it so it should not have pagination set (the first) - // request to get it's children should start at the beginning aka not passing any pagination parameter - expect(body.childNodes[1].nextChild).to.be(null); - const firstChild = body.childNodes[0]; - - // get the 3rd child of the origin of the tree ({ body } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}/children?generations=1&children=10&afterChild=${body.nextChild}` + `/api/endpoint/resolver/${distantChildEntityID}/children?children=2&afterChild=${body.nextChild}` ) .expect(200)); - expect(body.childNodes.length).to.be(1); - verifyChildren(body.childNodes, tree, 1, 1); - expect(body.childNodes[0].nextChild).to.be(null); + expect(body.childNodes.length).to.eql(0); + expect(body.nextChild).to.be(null); + }); + + it('gets all children in two queries', async () => { + // should get all the children of the origin + let { body }: { body: ResolverChildren } = await supertest + .get(`/api/endpoint/resolver/${tree.origin.id}/children?children=3`) + .expect(200); + expect(body.childNodes.length).to.eql(3); + verifyChildren(body.childNodes, tree); + expect(body.nextChild).to.not.be(null); + const firstNodes = [...body.childNodes]; - // get the 1 child of the origin of the tree's last child ({ body } = await supertest .get( - `/api/endpoint/resolver/${firstChild.entityID}/children?generations=1&children=10&afterChild=${firstChild.nextChild}` + `/api/endpoint/resolver/${tree.origin.id}/children?children=10&afterChild=${body.nextChild}` ) .expect(200)); - expect(body.childNodes.length).to.be(1); - verifyChildren(body.childNodes, tree, 1, 1); - expect(body.childNodes[0].nextChild).to.be(null); + expect(body.childNodes.length).to.eql(9); + // put all the results together and we should have all the children + verifyChildren([...firstNodes, ...body.childNodes], tree, 4, 3); + expect(body.nextChild).to.be(null); }); }); }); @@ -703,7 +701,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC it('returns a tree', async () => { const { body }: { body: ResolverTree } = await supertest .get( - `/api/endpoint/resolver/${tree.origin.id}?children=100&generations=3&ancestors=5&events=4&alerts=4` + `/api/endpoint/resolver/${tree.origin.id}?children=100&ancestors=5&events=5&alerts=5` ) .expect(200); diff --git a/x-pack/test/api_integration/services/resolver.ts b/x-pack/test/api_integration/services/resolver.ts index 7a100c37aea915..750d2f702fb843 100644 --- a/x-pack/test/api_integration/services/resolver.ts +++ b/x-pack/test/api_integration/services/resolver.ts @@ -18,6 +18,7 @@ export interface Options extends TreeOptions { * Number of trees to generate. */ numTrees?: number; + seed?: string; } /** @@ -38,8 +39,9 @@ export function ResolverGeneratorProvider({ getService }: FtrProviderContext) { eventsIndex: string = 'logs-endpoint.events.process-default', alertsIndex: string = 'logs-endpoint.alerts-default' ): Promise { + const seed = options.seed || 'resolver-seed'; const allTrees: Tree[] = []; - const generator = new EndpointDocGenerator(); + const generator = new EndpointDocGenerator(seed); const numTrees = options.numTrees ?? 1; for (let j = 0; j < numTrees; j++) { const tree = generator.generateTree(options); From 429805d1b878919c0b7be081f425d5dc0ec67c09 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 2 Jul 2020 15:19:21 +0200 Subject: [PATCH 19/31] [APM] Don't fetch dynamic index pattern in setupRequest (#70308) Co-authored-by: Elastic Machine --- .../__tests__/get_buckets.test.ts | 1 - .../convert_ui_filters/get_ui_filters_es.ts | 21 +++------ .../apm/server/lib/helpers/es_client.ts | 44 ++++++++++--------- .../apm/server/lib/helpers/setup_request.ts | 40 ++++++++--------- .../get_timeseries_data/fetcher.test.ts | 1 - .../get_local_filter_query.ts | 5 +-- .../lib/ui_filters/local_ui_filters/index.ts | 3 +- .../apm/server/routes/index_pattern.ts | 15 ++++++- .../plugins/apm/server/routes/ui_filters.ts | 5 +-- 9 files changed, 65 insertions(+), 70 deletions(-) diff --git a/x-pack/plugins/apm/server/lib/errors/distribution/__tests__/get_buckets.test.ts b/x-pack/plugins/apm/server/lib/errors/distribution/__tests__/get_buckets.test.ts index 408cdd387cbd88..5f23a9329a5832 100644 --- a/x-pack/plugins/apm/server/lib/errors/distribution/__tests__/get_buckets.test.ts +++ b/x-pack/plugins/apm/server/lib/errors/distribution/__tests__/get_buckets.test.ts @@ -56,7 +56,6 @@ describe('timeseriesFetcher', () => { apmAgentConfigurationIndex: '.apm-agent-configuration', apmCustomLinkIndex: '.apm-custom-link', }, - dynamicIndexPattern: null as any, }, }); }); diff --git a/x-pack/plugins/apm/server/lib/helpers/convert_ui_filters/get_ui_filters_es.ts b/x-pack/plugins/apm/server/lib/helpers/convert_ui_filters/get_ui_filters_es.ts index c9e9db13cecae0..b34d5535d58cc9 100644 --- a/x-pack/plugins/apm/server/lib/helpers/convert_ui_filters/get_ui_filters_es.ts +++ b/x-pack/plugins/apm/server/lib/helpers/convert_ui_filters/get_ui_filters_es.ts @@ -11,15 +11,9 @@ import { localUIFilters, localUIFilterNames, } from '../../ui_filters/local_ui_filters/config'; -import { - esKuery, - IIndexPattern, -} from '../../../../../../../src/plugins/data/server'; +import { esKuery } from '../../../../../../../src/plugins/data/server'; -export function getUiFiltersES( - indexPattern: IIndexPattern | undefined, - uiFilters: UIFilters -) { +export function getUiFiltersES(uiFilters: UIFilters) { const { kuery, environment, ...localFilterValues } = uiFilters; const mappedFilters = localUIFilterNames .filter((name) => name in localFilterValues) @@ -35,7 +29,7 @@ export function getUiFiltersES( // remove undefined items from list const esFilters = [ - getKueryUiFilterES(indexPattern, uiFilters.kuery), + getKueryUiFilterES(uiFilters.kuery), getEnvironmentUiFilterES(uiFilters.environment), ] .filter((filter) => !!filter) @@ -44,14 +38,11 @@ export function getUiFiltersES( return esFilters; } -function getKueryUiFilterES( - indexPattern: IIndexPattern | undefined, - kuery?: string -) { - if (!kuery || !indexPattern) { +function getKueryUiFilterES(kuery?: string) { + if (!kuery) { return; } const ast = esKuery.fromKueryExpression(kuery); - return esKuery.toElasticsearchQuery(ast, indexPattern) as ESFilter; + return esKuery.toElasticsearchQuery(ast) as ESFilter; } diff --git a/x-pack/plugins/apm/server/lib/helpers/es_client.ts b/x-pack/plugins/apm/server/lib/helpers/es_client.ts index 892f8f0ddd1051..2d730933e24731 100644 --- a/x-pack/plugins/apm/server/lib/helpers/es_client.ts +++ b/x-pack/plugins/apm/server/lib/helpers/es_client.ts @@ -19,11 +19,10 @@ import { ESSearchRequest, ESSearchResponse, } from '../../../typings/elasticsearch'; -import { UI_SETTINGS } from '../../../../../../src/plugins/data/server'; import { OBSERVER_VERSION_MAJOR } from '../../../common/elasticsearch_fieldnames'; import { pickKeys } from '../../../common/utils/pick_keys'; import { APMRequestHandlerContext } from '../../routes/typings'; -import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; +import { ApmIndicesConfig } from '../settings/apm_indices/get_apm_indices'; // `type` was deprecated in 7.0 export type APMIndexDocumentParams = Omit, 'type'>; @@ -85,20 +84,19 @@ function addFilterForLegacyData( } // add additional params for search (aka: read) requests -async function getParamsForSearchRequest( - context: APMRequestHandlerContext, - params: ESSearchRequest, - apmOptions?: APMOptions -) { - const { uiSettings } = context.core; - const [indices, includeFrozen] = await Promise.all([ - getApmIndices({ - savedObjectsClient: context.core.savedObjects.client, - config: context.config, - }), - uiSettings.client.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN), - ]); - +function getParamsForSearchRequest({ + context, + params, + indices, + includeFrozen, + includeLegacyData, +}: { + context: APMRequestHandlerContext; + params: ESSearchRequest; + indices: ApmIndicesConfig; + includeFrozen: boolean; + includeLegacyData?: boolean; +}) { // Get indices for legacy data filter (only those which apply) const apmIndices = Object.values( pickKeys( @@ -112,7 +110,7 @@ async function getParamsForSearchRequest( ) ); return { - ...addFilterForLegacyData(apmIndices, params, apmOptions), // filter out pre-7.0 data + ...addFilterForLegacyData(apmIndices, params, { includeLegacyData }), // filter out pre-7.0 data ignore_throttled: !includeFrozen, // whether to query frozen indices or not }; } @@ -123,6 +121,8 @@ interface APMOptions { interface ClientCreateOptions { clientAsInternalUser?: boolean; + indices: ApmIndicesConfig; + includeFrozen: boolean; } export type ESClient = ReturnType; @@ -134,7 +134,7 @@ function formatObj(obj: Record) { export function getESClient( context: APMRequestHandlerContext, request: KibanaRequest, - { clientAsInternalUser = false }: ClientCreateOptions = {} + { clientAsInternalUser = false, indices, includeFrozen }: ClientCreateOptions ) { const { callAsCurrentUser, @@ -194,11 +194,13 @@ export function getESClient( params: TSearchRequest, apmOptions?: APMOptions ): Promise> => { - const nextParams = await getParamsForSearchRequest( + const nextParams = await getParamsForSearchRequest({ context, params, - apmOptions - ); + indices, + includeFrozen, + ...apmOptions, + }); return callEs('search', nextParams); }, diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts index 2dd8ed01082fd7..14c9378d991928 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts @@ -5,8 +5,8 @@ */ import moment from 'moment'; +import { UI_SETTINGS } from '../../../../../../src/plugins/data/common'; import { KibanaRequest } from '../../../../../../src/core/server'; -import { IIndexPattern } from '../../../../../../src/plugins/data/common'; import { APMConfig } from '../..'; import { getApmIndices, @@ -18,17 +18,13 @@ import { getUiFiltersES } from './convert_ui_filters/get_ui_filters_es'; import { APMRequestHandlerContext } from '../../routes/typings'; import { getESClient } from './es_client'; import { ProcessorEvent } from '../../../common/processor_event'; -import { getDynamicIndexPattern } from '../index_pattern/get_dynamic_index_pattern'; -function decodeUiFilters( - indexPattern: IIndexPattern | undefined, - uiFiltersEncoded?: string -) { - if (!uiFiltersEncoded || !indexPattern) { +function decodeUiFilters(uiFiltersEncoded?: string) { + if (!uiFiltersEncoded) { return []; } const uiFilters = JSON.parse(uiFiltersEncoded); - return getUiFiltersES(indexPattern, uiFilters); + return getUiFiltersES(uiFilters); } // Explicitly type Setup to prevent TS initialization errors // https://github.com/microsoft/TypeScript/issues/34933 @@ -39,7 +35,6 @@ export interface Setup { ml?: ReturnType; config: APMConfig; indices: ApmIndicesConfig; - dynamicIndexPattern?: IIndexPattern; } export interface SetupTimeRange { @@ -75,28 +70,33 @@ export async function setupRequest( const { config } = context; const { query } = context.params; - const indices = await getApmIndices({ - savedObjectsClient: context.core.savedObjects.client, - config, - }); + const [indices, includeFrozen] = await Promise.all([ + getApmIndices({ + savedObjectsClient: context.core.savedObjects.client, + config, + }), + context.core.uiSettings.client.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN), + ]); - const dynamicIndexPattern = await getDynamicIndexPattern({ - context, + const createClientOptions = { indices, - processorEvent: query.processorEvent, - }); + includeFrozen, + }; - const uiFiltersES = decodeUiFilters(dynamicIndexPattern, query.uiFilters); + const uiFiltersES = decodeUiFilters(query.uiFilters); const coreSetupRequest = { indices, - client: getESClient(context, request, { clientAsInternalUser: false }), + client: getESClient(context, request, { + clientAsInternalUser: false, + ...createClientOptions, + }), internalClient: getESClient(context, request, { clientAsInternalUser: true, + ...createClientOptions, }), ml: getMlSetup(context, request), config, - dynamicIndexPattern, }; return { diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.test.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.test.ts index d1f473b485dc32..fb357040f5781d 100644 --- a/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.test.ts +++ b/x-pack/plugins/apm/server/lib/transactions/charts/get_timeseries_data/fetcher.test.ts @@ -44,7 +44,6 @@ describe('timeseriesFetcher', () => { apmAgentConfigurationIndex: 'myIndex', apmCustomLinkIndex: 'myIndex', }, - dynamicIndexPattern: null as any, }, }); }); diff --git a/x-pack/plugins/apm/server/lib/ui_filters/local_ui_filters/get_local_filter_query.ts b/x-pack/plugins/apm/server/lib/ui_filters/local_ui_filters/get_local_filter_query.ts index 5fdd6de06089b4..1cecf14f2eeb8d 100644 --- a/x-pack/plugins/apm/server/lib/ui_filters/local_ui_filters/get_local_filter_query.ts +++ b/x-pack/plugins/apm/server/lib/ui_filters/local_ui_filters/get_local_filter_query.ts @@ -5,7 +5,6 @@ */ import { omit } from 'lodash'; -import { IIndexPattern } from 'src/plugins/data/server'; import { mergeProjection } from '../../../../common/projections/util/merge_projection'; import { Projection } from '../../../../common/projections/typings'; import { UIFilters } from '../../../../typings/ui_filters'; @@ -13,18 +12,16 @@ import { getUiFiltersES } from '../../helpers/convert_ui_filters/get_ui_filters_ import { localUIFilters, LocalUIFilterName } from './config'; export const getLocalFilterQuery = ({ - indexPattern, uiFilters, projection, localUIFilterName, }: { - indexPattern: IIndexPattern | undefined; uiFilters: UIFilters; projection: Projection; localUIFilterName: LocalUIFilterName; }) => { const field = localUIFilters[localUIFilterName]; - const filter = getUiFiltersES(indexPattern, omit(uiFilters, field.name)); + const filter = getUiFiltersES(omit(uiFilters, field.name)); const bucketCountAggregation = projection.body.aggs ? { diff --git a/x-pack/plugins/apm/server/lib/ui_filters/local_ui_filters/index.ts b/x-pack/plugins/apm/server/lib/ui_filters/local_ui_filters/index.ts index 967314644c246e..31bc0563ec13f4 100644 --- a/x-pack/plugins/apm/server/lib/ui_filters/local_ui_filters/index.ts +++ b/x-pack/plugins/apm/server/lib/ui_filters/local_ui_filters/index.ts @@ -26,7 +26,7 @@ export async function getLocalUIFilters({ uiFilters: UIFilters; localFilterNames: LocalUIFilterName[]; }) { - const { client, dynamicIndexPattern } = setup; + const { client } = setup; const projectionWithoutAggs = cloneDeep(projection); @@ -35,7 +35,6 @@ export async function getLocalUIFilters({ return Promise.all( localFilterNames.map(async (name) => { const query = getLocalFilterQuery({ - indexPattern: dynamicIndexPattern, uiFilters, projection, localUIFilterName: name, diff --git a/x-pack/plugins/apm/server/routes/index_pattern.ts b/x-pack/plugins/apm/server/routes/index_pattern.ts index 018a14ac766892..18bc2986d40615 100644 --- a/x-pack/plugins/apm/server/routes/index_pattern.ts +++ b/x-pack/plugins/apm/server/routes/index_pattern.ts @@ -9,6 +9,8 @@ import { createRoute } from './create_route'; import { setupRequest } from '../lib/helpers/setup_request'; import { getInternalSavedObjectsClient } from '../lib/helpers/get_internal_saved_objects_client'; import { getApmIndexPatternTitle } from '../lib/index_pattern/get_apm_index_pattern_title'; +import { getDynamicIndexPattern } from '../lib/index_pattern/get_dynamic_index_pattern'; +import { getApmIndices } from '../lib/settings/apm_indices/get_apm_indices'; export const staticIndexPatternRoute = createRoute((core) => ({ method: 'POST', @@ -34,8 +36,17 @@ export const dynamicIndexPatternRoute = createRoute(() => ({ ]), }), }, - handler: async ({ context, request }) => { - const { dynamicIndexPattern } = await setupRequest(context, request); + handler: async ({ context }) => { + const indices = await getApmIndices({ + config: context.config, + savedObjectsClient: context.core.savedObjects.client, + }); + + const dynamicIndexPattern = await getDynamicIndexPattern({ + context, + indices, + }); + return { dynamicIndexPattern }; }, })); diff --git a/x-pack/plugins/apm/server/routes/ui_filters.ts b/x-pack/plugins/apm/server/routes/ui_filters.ts index 280645d4de8d05..a47d72751dfc47 100644 --- a/x-pack/plugins/apm/server/routes/ui_filters.ts +++ b/x-pack/plugins/apm/server/routes/ui_filters.ts @@ -97,10 +97,7 @@ function createLocalFiltersRoute< query, setup: { ...setup, - uiFiltersES: getUiFiltersES( - setup.dynamicIndexPattern, - omit(parsedUiFilters, filterNames) - ), + uiFiltersES: getUiFiltersES(omit(parsedUiFilters, filterNames)), }, }); From 3f808688e10864e2d12737844ff4387ae0a2aa27 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 2 Jul 2020 15:24:05 +0200 Subject: [PATCH 20/31] redirect to default app if hash can not be forwarded (#70417) --- .../kibana_legacy/public/forward_app/forward_app.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/plugins/kibana_legacy/public/forward_app/forward_app.ts b/src/plugins/kibana_legacy/public/forward_app/forward_app.ts index 89018df1ca7e17..b425091dfbcd9b 100644 --- a/src/plugins/kibana_legacy/public/forward_app/forward_app.ts +++ b/src/plugins/kibana_legacy/public/forward_app/forward_app.ts @@ -20,9 +20,12 @@ import { App, AppMountParameters, CoreSetup } from 'kibana/public'; import { AppNavLinkStatus } from '../../../../core/public'; import { navigateToLegacyKibanaUrl } from './navigate_to_legacy_kibana_url'; -import { ForwardDefinition } from '../plugin'; +import { ForwardDefinition, KibanaLegacyStart } from '../plugin'; -export const createLegacyUrlForwardApp = (core: CoreSetup, forwards: ForwardDefinition[]): App => ({ +export const createLegacyUrlForwardApp = ( + core: CoreSetup<{}, KibanaLegacyStart>, + forwards: ForwardDefinition[] +): App => ({ id: 'kibana', chromeless: true, title: 'Legacy URL migration', @@ -31,7 +34,8 @@ export const createLegacyUrlForwardApp = (core: CoreSetup, forwards: ForwardDefi const hash = params.history.location.hash.substr(1); if (!hash) { - core.fatalErrors.add('Could not forward URL'); + const [, , kibanaLegacyStart] = await core.getStartServices(); + kibanaLegacyStart.navigateToDefaultApp(); } const [ @@ -44,7 +48,8 @@ export const createLegacyUrlForwardApp = (core: CoreSetup, forwards: ForwardDefi const result = await navigateToLegacyKibanaUrl(hash, forwards, basePath, application); if (!result.navigated) { - core.fatalErrors.add('Could not forward URL'); + const [, , kibanaLegacyStart] = await core.getStartServices(); + kibanaLegacyStart.navigateToDefaultApp(); } return () => {}; From 9c76f1918649ead38fcc640495bc923c390236bd Mon Sep 17 00:00:00 2001 From: Thomas Neirynck Date: Thu, 2 Jul 2020 09:24:56 -0400 Subject: [PATCH 21/31] [Maps] Add styling and tooltip support to mapbox mvt vector tile sources (#64488) * tmp commit * rename * more boilerpalte * more boiler * more boilerpalte * typing * fix import * boilerplate * more boiler * enable custom palettes * fix label text and orientation * fix merge errors * remove dupe import * stash commit * tmp commit * debounce settings * return null * slight rearrangement * tooltip guard * minor tweaks * feedback * ts fixes * ts fixes * more ts fixes * ts fixes * jest test * fix typo * spacing * fix typing * add unit test * add more tests * add snapshot test * add snapshot * add field editor snapshot test * fix snapshot * add snapshot * remove unused import * test stub for mvt layer fix optional param more checks * add snapshot test more unit tests more unit tests ts fixes * add data syncing unit test * fix autorefactor * fix merge and replace snapshots * field editor changes * field editor changes * ts fixes * update snapshots * fix things * fix names * fix tooltip * add more error handling * improve copy * styling changes * style option box a little better * ts fixes * fix console error * remove mbProperties from interface * remove unused method * remove cruft * rename for consistency * remove unused param * feedback * feedback * ensure properties are always present * handle possible null values * feedback * typo * update SIEM * feedback * remove cruft * remove unused translations * feedback * improve readability * fix brittle test * fix snapshot after master merge * remove unused method * feedback * revert some feedback * remove micro-optimization * initialize in constructor * simplify wording * add snapshot * naming * add clarifying comment * remove unused import * sanitize tooltips * remove cruft * feedback * fix typo * remove export * Design fixes * clean up supportsAutoDomain * remove patch.txt * cleanup * clean-up * Merge in styling changes * Tweak message format * fix broken import Co-authored-by: Elastic Machine Co-authored-by: miukimiu Co-authored-by: Nathan Reese --- .../validated_range/validated_dual_range.tsx | 4 +- x-pack/plugins/maps/common/constants.ts | 5 + ....d.ts => data_request_descriptor_types.ts} | 0 .../maps/common/descriptor_types/index.ts | 2 +- .../common/descriptor_types/map_descriptor.ts | 4 +- .../{descriptor_types.d.ts => sources.ts} | 31 +- ....ts => style_property_descriptor_types.ts} | 0 .../public/classes/fields/es_agg_field.ts | 4 + .../maps/public/classes/fields/field.ts | 10 + .../maps/public/classes/fields/mvt_field.ts | 59 +++ .../fields/top_term_percentage_field.ts | 4 + .../layers/__tests__/mock_sync_context.ts | 40 ++ .../layers/file_upload_wizard/wizard.tsx | 2 +- .../layers/heatmap_layer/heatmap_layer.js | 2 +- .../maps/public/classes/layers/layer.tsx | 17 +- .../classes/layers/tile_layer/tile_layer.js | 6 +- .../tiled_vector_layer.test.tsx.snap | 8 + .../tiled_vector_layer.test.tsx | 163 ++++++ .../tiled_vector_layer/tiled_vector_layer.tsx | 120 +++-- .../layers/vector_layer/vector_layer.d.ts | 6 + .../layers/vector_layer/vector_layer.js | 23 +- .../ems_file_source/update_source_editor.tsx | 2 +- .../mvt_field_config_editor.test.tsx.snap | 491 ++++++++++++++++++ ...single_layer_source_settings.test.tsx.snap | 211 ++++++++ ...e_layer_vector_source_editor.test.tsx.snap | 30 ++ .../update_source_editor.test.tsx.snap | 57 ++ .../layer_wizard.tsx | 10 +- .../mvt_field_config_editor.test.tsx | 57 ++ .../mvt_field_config_editor.tsx | 210 ++++++++ .../mvt_single_layer_source_settings.test.tsx | 38 ++ .../mvt_single_layer_source_settings.tsx | 191 +++++++ .../mvt_single_layer_vector_source.test.tsx | 91 ++++ .../mvt_single_layer_vector_source.ts | 161 ------ .../mvt_single_layer_vector_source.tsx | 222 ++++++++ ...single_layer_vector_source_editor.test.tsx | 18 + .../mvt_single_layer_vector_source_editor.tsx | 89 ++-- .../mvt_single_layer_vector_source/types.ts | 16 + .../update_source_editor.test.tsx | 35 ++ .../update_source_editor.tsx | 136 +++++ .../maps/public/classes/sources/source.ts | 2 +- .../sources/vector_source/vector_source.d.ts | 10 +- .../components/color/color_map_select.js | 39 +- .../components/color/dynamic_color_form.js | 2 + .../styles/vector/components/field_select.js | 9 +- .../vector/components/style_map_select.js | 26 +- .../components/symbol/dynamic_icon_form.js | 5 +- .../components/symbol/icon_map_select.js | 4 +- .../vector/components/vector_style_editor.js | 5 +- .../dynamic_orientation_property.js | 9 +- .../properties/dynamic_text_property.js | 6 +- .../classes/tooltips/tooltip_property.ts | 2 +- .../connected_components/layer_panel/view.js | 19 +- .../features_tooltip/feature_properties.js | 6 +- .../map/features_tooltip/features_tooltip.js | 1 + .../map/mb/tooltip_control/tooltip_control.js | 23 +- .../tooltip_control/tooltip_control.test.js | 2 +- .../map/mb/tooltip_control/tooltip_popover.js | 16 +- .../map_tool_tip/map_tool_tip.test.tsx | 5 +- .../network/components/embeddables/types.ts | 5 - .../translations/translations/ja-JP.json | 3 - .../translations/translations/zh-CN.json | 3 - .../apps/maps/documents_source/search_hits.js | 6 +- 62 files changed, 2405 insertions(+), 378 deletions(-) rename x-pack/plugins/maps/common/descriptor_types/{data_request_descriptor_types.d.ts => data_request_descriptor_types.ts} (100%) rename x-pack/plugins/maps/common/descriptor_types/{descriptor_types.d.ts => sources.ts} (87%) rename x-pack/plugins/maps/common/descriptor_types/{style_property_descriptor_types.d.ts => style_property_descriptor_types.ts} (100%) create mode 100644 x-pack/plugins/maps/public/classes/fields/mvt_field.ts create mode 100644 x-pack/plugins/maps/public/classes/layers/__tests__/mock_sync_context.ts create mode 100644 x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/__snapshots__/tiled_vector_layer.test.tsx.snap create mode 100644 x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_field_config_editor.test.tsx.snap create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_single_layer_source_settings.test.tsx.snap create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_single_layer_vector_source_editor.test.tsx.snap create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/update_source_editor.test.tsx.snap create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_field_config_editor.test.tsx create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_field_config_editor.tsx create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_source_settings.test.tsx create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_source_settings.tsx create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.test.tsx delete mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.ts create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor.test.tsx create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/types.ts create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/update_source_editor.test.tsx create mode 100644 x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/update_source_editor.tsx diff --git a/src/plugins/kibana_react/public/validated_range/validated_dual_range.tsx b/src/plugins/kibana_react/public/validated_range/validated_dual_range.tsx index 63b9b48ec809eb..45592c8a703af1 100644 --- a/src/plugins/kibana_react/public/validated_range/validated_dual_range.tsx +++ b/src/plugins/kibana_react/public/validated_range/validated_dual_range.tsx @@ -17,7 +17,7 @@ * under the License. */ import { i18n } from '@kbn/i18n'; -import React, { Component } from 'react'; +import React, { Component, ReactNode } from 'react'; import { EuiFormRow, EuiDualRange } from '@elastic/eui'; import { EuiFormRowDisplayKeys } from '@elastic/eui/src/components/form/form_row/form_row'; import { EuiDualRangeProps } from '@elastic/eui/src/components/form/range/dual_range'; @@ -32,7 +32,7 @@ export type ValueMember = EuiDualRangeProps['value'][0]; interface Props extends Omit { value?: Value; allowEmptyRange?: boolean; - label?: string; + label?: string | ReactNode; formRowDisplay?: EuiFormRowDisplayKeys; onChange?: (val: [string, string]) => void; min?: number; diff --git a/x-pack/plugins/maps/common/constants.ts b/x-pack/plugins/maps/common/constants.ts index 25f10c7794fdde..98464427cc3482 100644 --- a/x-pack/plugins/maps/common/constants.ts +++ b/x-pack/plugins/maps/common/constants.ts @@ -223,6 +223,11 @@ export enum SCALING_TYPES { export const RGBA_0000 = 'rgba(0,0,0,0)'; +export enum MVT_FIELD_TYPE { + STRING = 'String', + NUMBER = 'Number', +} + export const SPATIAL_FILTERS_LAYER_ID = 'SPATIAL_FILTERS_LAYER_ID'; export enum INITIAL_LOCATION { diff --git a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.d.ts b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts similarity index 100% rename from x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.d.ts rename to x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts diff --git a/x-pack/plugins/maps/common/descriptor_types/index.ts b/x-pack/plugins/maps/common/descriptor_types/index.ts index af0f4487f471b0..b0ae065856a5de 100644 --- a/x-pack/plugins/maps/common/descriptor_types/index.ts +++ b/x-pack/plugins/maps/common/descriptor_types/index.ts @@ -5,6 +5,6 @@ */ export * from './data_request_descriptor_types'; -export * from './descriptor_types'; +export * from './sources'; export * from './map_descriptor'; export * from './style_property_descriptor_types'; diff --git a/x-pack/plugins/maps/common/descriptor_types/map_descriptor.ts b/x-pack/plugins/maps/common/descriptor_types/map_descriptor.ts index 00380ca12a4865..027cc886cd7f7f 100644 --- a/x-pack/plugins/maps/common/descriptor_types/map_descriptor.ts +++ b/x-pack/plugins/maps/common/descriptor_types/map_descriptor.ts @@ -5,6 +5,7 @@ */ /* eslint-disable @typescript-eslint/consistent-type-definitions */ +import { GeoJsonProperties } from 'geojson'; import { Query } from '../../../../../src/plugins/data/common'; import { DRAW_TYPE, ES_GEO_FIELD_TYPE, ES_SPATIAL_RELATIONS } from '../constants'; @@ -39,8 +40,9 @@ export type Goto = { }; export type TooltipFeature = { - id: number; + id?: number | string; layerId: string; + mbProperties: GeoJsonProperties; }; export type TooltipState = { diff --git a/x-pack/plugins/maps/common/descriptor_types/descriptor_types.d.ts b/x-pack/plugins/maps/common/descriptor_types/sources.ts similarity index 87% rename from x-pack/plugins/maps/common/descriptor_types/descriptor_types.d.ts rename to x-pack/plugins/maps/common/descriptor_types/sources.ts index c7a706ea64f741..86ace0e32cc843 100644 --- a/x-pack/plugins/maps/common/descriptor_types/descriptor_types.d.ts +++ b/x-pack/plugins/maps/common/descriptor_types/sources.ts @@ -7,7 +7,14 @@ import { FeatureCollection } from 'geojson'; import { Query } from 'src/plugins/data/public'; -import { AGG_TYPE, GRID_RESOLUTION, RENDER_AS, SORT_ORDER, SCALING_TYPES } from '../constants'; +import { + AGG_TYPE, + GRID_RESOLUTION, + RENDER_AS, + SORT_ORDER, + SCALING_TYPES, + MVT_FIELD_TYPE, +} from '../constants'; import { StyleDescriptor, VectorStyleDescriptor } from './style_property_descriptor_types'; import { DataRequestDescriptor } from './data_request_descriptor_types'; @@ -96,18 +103,34 @@ export type XYZTMSSourceDescriptor = AbstractSourceDescriptor & urlTemplate: string; }; -export type TiledSingleLayerVectorSourceDescriptor = AbstractSourceDescriptor & { +export type MVTFieldDescriptor = { + name: string; + type: MVT_FIELD_TYPE; +}; + +export type TiledSingleLayerVectorSourceSettings = { urlTemplate: string; layerName: string; // These are the min/max zoom levels of the availability of the a particular layerName in the tileset at urlTemplate. // These are _not_ the visible zoom-range of the data on a map. - // Tiled data can be displayed at higher levels of zoom than that they are stored in the tileset. - // e.g. EMS basemap data from level 14 is at most detailed resolution and can be displayed at higher levels + // These are important so mapbox does not issue invalid requests based on the zoom level. + + // Tiled layer data cannot be displayed at lower levels of zoom than that they are stored in the tileset. + // e.g. building footprints at level 14 cannot be displayed at level 0. minSourceZoom: number; + // Tiled layer data can be displayed at higher levels of zoom than that they are stored in the tileset. + // e.g. EMS basemap data from level 14 is at most detailed resolution and can be displayed at higher levels maxSourceZoom: number; + + fields: MVTFieldDescriptor[]; }; +export type TiledSingleLayerVectorSourceDescriptor = AbstractSourceDescriptor & + TiledSingleLayerVectorSourceSettings & { + tooltipProperties: string[]; + }; + export type GeojsonFileSourceDescriptor = { __featureCollection: FeatureCollection; name: string; diff --git a/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.d.ts b/x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts similarity index 100% rename from x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.d.ts rename to x-pack/plugins/maps/common/descriptor_types/style_property_descriptor_types.ts diff --git a/x-pack/plugins/maps/public/classes/fields/es_agg_field.ts b/x-pack/plugins/maps/public/classes/fields/es_agg_field.ts index 60d437d2321b52..e0f5c79f1d4278 100644 --- a/x-pack/plugins/maps/public/classes/fields/es_agg_field.ts +++ b/x-pack/plugins/maps/public/classes/fields/es_agg_field.ts @@ -128,6 +128,10 @@ export class ESAggField implements IESAggField { async getCategoricalFieldMetaRequest(size: number): Promise { return this._esDocField ? this._esDocField.getCategoricalFieldMetaRequest(size) : null; } + + supportsAutoDomain(): boolean { + return true; + } } export function esAggFieldsFactory( diff --git a/x-pack/plugins/maps/public/classes/fields/field.ts b/x-pack/plugins/maps/public/classes/fields/field.ts index dfd5dc05f7b839..410b38e79ffe4e 100644 --- a/x-pack/plugins/maps/public/classes/fields/field.ts +++ b/x-pack/plugins/maps/public/classes/fields/field.ts @@ -20,6 +20,12 @@ export interface IField { isValid(): boolean; getOrdinalFieldMetaRequest(): Promise; getCategoricalFieldMetaRequest(size: number): Promise; + + // Determines whether Maps-app can automatically determine the domain of the field-values + // if this is not the case (e.g. for .mvt tiled data), + // then styling properties that require the domain to be known cannot use this property. + supportsAutoDomain(): boolean; + supportsFieldMeta(): boolean; } @@ -80,4 +86,8 @@ export class AbstractField implements IField { async getCategoricalFieldMetaRequest(size: number): Promise { return null; } + + supportsAutoDomain(): boolean { + return true; + } } diff --git a/x-pack/plugins/maps/public/classes/fields/mvt_field.ts b/x-pack/plugins/maps/public/classes/fields/mvt_field.ts new file mode 100644 index 00000000000000..eb2bb94b36a690 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/fields/mvt_field.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { AbstractField, IField } from './field'; +import { FIELD_ORIGIN, MVT_FIELD_TYPE } from '../../../common/constants'; +import { ITiledSingleLayerVectorSource, IVectorSource } from '../sources/vector_source'; +import { MVTFieldDescriptor } from '../../../common/descriptor_types'; + +export class MVTField extends AbstractField implements IField { + private readonly _source: ITiledSingleLayerVectorSource; + private readonly _type: MVT_FIELD_TYPE; + constructor({ + fieldName, + type, + source, + origin, + }: { + fieldName: string; + source: ITiledSingleLayerVectorSource; + origin: FIELD_ORIGIN; + type: MVT_FIELD_TYPE; + }) { + super({ fieldName, origin }); + this._source = source; + this._type = type; + } + + getMVTFieldDescriptor(): MVTFieldDescriptor { + return { + type: this._type, + name: this.getName(), + }; + } + + getSource(): IVectorSource { + return this._source; + } + + async getDataType(): Promise { + if (this._type === MVT_FIELD_TYPE.STRING) { + return 'string'; + } else if (this._type === MVT_FIELD_TYPE.NUMBER) { + return 'number'; + } else { + throw new Error(`Unrecognized MVT field-type ${this._type}`); + } + } + + async getLabel(): Promise { + return this.getName(); + } + + supportsAutoDomain() { + return false; + } +} diff --git a/x-pack/plugins/maps/public/classes/fields/top_term_percentage_field.ts b/x-pack/plugins/maps/public/classes/fields/top_term_percentage_field.ts index 6c504daf3e1925..f4625e42ab5deb 100644 --- a/x-pack/plugins/maps/public/classes/fields/top_term_percentage_field.ts +++ b/x-pack/plugins/maps/public/classes/fields/top_term_percentage_field.ts @@ -60,6 +60,10 @@ export class TopTermPercentageField implements IESAggField { return 0; } + supportsAutoDomain(): boolean { + return true; + } + supportsFieldMeta(): boolean { return false; } diff --git a/x-pack/plugins/maps/public/classes/layers/__tests__/mock_sync_context.ts b/x-pack/plugins/maps/public/classes/layers/__tests__/mock_sync_context.ts new file mode 100644 index 00000000000000..8c4eb49d5040d6 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/layers/__tests__/mock_sync_context.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import sinon from 'sinon'; +import { DataRequestContext } from '../../../actions'; +import { DataMeta, MapFilters } from '../../../../common/descriptor_types'; + +export class MockSyncContext implements DataRequestContext { + dataFilters: MapFilters; + isRequestStillActive: (dataId: string, requestToken: symbol) => boolean; + onLoadError: (dataId: string, requestToken: symbol, errorMessage: string) => void; + registerCancelCallback: (requestToken: symbol, callback: () => void) => void; + startLoading: (dataId: string, requestToken: symbol, meta: DataMeta) => void; + stopLoading: (dataId: string, requestToken: symbol, data: object, meta: DataMeta) => void; + updateSourceData: (newData: unknown) => void; + + constructor({ dataFilters }: { dataFilters: Partial }) { + const mapFilters: MapFilters = { + filters: [], + timeFilters: { + from: 'now', + to: '15m', + mode: 'relative', + }, + zoom: 0, + ...dataFilters, + }; + + this.dataFilters = mapFilters; + this.isRequestStillActive = sinon.spy(); + this.onLoadError = sinon.spy(); + this.registerCancelCallback = sinon.spy(); + this.startLoading = sinon.spy(); + this.stopLoading = sinon.spy(); + this.updateSourceData = sinon.spy(); + } +} diff --git a/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx b/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx index 859d6092dc64d7..368dcda6b3a5ff 100644 --- a/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx +++ b/x-pack/plugins/maps/public/classes/layers/file_upload_wizard/wizard.tsx @@ -16,7 +16,7 @@ import { import { getFileUploadComponent } from '../../../kibana_services'; import { GeojsonFileSource } from '../../sources/geojson_file_source'; import { VectorLayer } from '../../layers/vector_layer/vector_layer'; -// @ts-ignore +// @ts-expect-error import { createDefaultLayerDescriptor } from '../../sources/es_search_source'; import { RenderWizardArguments } from '../../layers/layer_wizard_registry'; diff --git a/x-pack/plugins/maps/public/classes/layers/heatmap_layer/heatmap_layer.js b/x-pack/plugins/maps/public/classes/layers/heatmap_layer/heatmap_layer.js index f6b9bd62802901..adcc86b9d1546e 100644 --- a/x-pack/plugins/maps/public/classes/layers/heatmap_layer/heatmap_layer.js +++ b/x-pack/plugins/maps/public/classes/layers/heatmap_layer/heatmap_layer.js @@ -91,7 +91,7 @@ export class HeatmapLayer extends VectorLayer { resolution: this.getSource().getGridResolution(), }); mbMap.setPaintProperty(heatmapLayerId, 'heatmap-opacity', this.getAlpha()); - mbMap.setLayerZoomRange(heatmapLayerId, this._descriptor.minZoom, this._descriptor.maxZoom); + mbMap.setLayerZoomRange(heatmapLayerId, this.getMinZoom(), this.getMaxZoom()); } getLayerTypeIconName() { diff --git a/x-pack/plugins/maps/public/classes/layers/layer.tsx b/x-pack/plugins/maps/public/classes/layers/layer.tsx index 2250d5663378cd..e122d1cda3ed9c 100644 --- a/x-pack/plugins/maps/public/classes/layers/layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/layer.tsx @@ -325,27 +325,28 @@ export class AbstractLayer implements ILayer { return this._source.getMinZoom(); } + _getMbSourceId() { + return this.getId(); + } + _requiresPrevSourceCleanup(mbMap: unknown) { return false; } _removeStaleMbSourcesAndLayers(mbMap: unknown) { if (this._requiresPrevSourceCleanup(mbMap)) { - // @ts-ignore + // @ts-expect-error const mbStyle = mbMap.getStyle(); - // @ts-ignore + // @ts-expect-error mbStyle.layers.forEach((mbLayer) => { - // @ts-ignore if (this.ownsMbLayerId(mbLayer.id)) { - // @ts-ignore + // @ts-expect-error mbMap.removeLayer(mbLayer.id); } }); - // @ts-ignore Object.keys(mbStyle.sources).some((mbSourceId) => { - // @ts-ignore if (this.ownsMbSourceId(mbSourceId)) { - // @ts-ignore + // @ts-expect-error mbMap.removeSource(mbSourceId); } }); @@ -429,7 +430,7 @@ export class AbstractLayer implements ILayer { throw new Error('Should implement AbstractLayer#ownsMbLayerId'); } - ownsMbSourceId(sourceId: string): boolean { + ownsMbSourceId(mbSourceId: string): boolean { throw new Error('Should implement AbstractLayer#ownsMbSourceId'); } diff --git a/x-pack/plugins/maps/public/classes/layers/tile_layer/tile_layer.js b/x-pack/plugins/maps/public/classes/layers/tile_layer/tile_layer.js index 02df8acbfffad3..3e2009c24a2e4d 100644 --- a/x-pack/plugins/maps/public/classes/layers/tile_layer/tile_layer.js +++ b/x-pack/plugins/maps/public/classes/layers/tile_layer/tile_layer.js @@ -74,8 +74,8 @@ export class TileLayer extends AbstractLayer { return; } - const sourceId = this.getId(); - mbMap.addSource(sourceId, { + const mbSourceId = this._getMbSourceId(); + mbMap.addSource(mbSourceId, { type: 'raster', tiles: [tmsSourceData.url], tileSize: 256, @@ -85,7 +85,7 @@ export class TileLayer extends AbstractLayer { mbMap.addLayer({ id: mbLayerId, type: 'raster', - source: sourceId, + source: mbSourceId, minzoom: this._descriptor.minZoom, maxzoom: this._descriptor.maxZoom, }); diff --git a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/__snapshots__/tiled_vector_layer.test.tsx.snap b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/__snapshots__/tiled_vector_layer.test.tsx.snap new file mode 100644 index 00000000000000..f0ae93601ce8a8 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/__snapshots__/tiled_vector_layer.test.tsx.snap @@ -0,0 +1,8 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`icon should use vector icon 1`] = ` +
+`; diff --git a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx new file mode 100644 index 00000000000000..ecd625db344119 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.test.tsx @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { MockSyncContext } from '../__tests__/mock_sync_context'; +import sinon from 'sinon'; + +jest.mock('../../../kibana_services', () => { + return { + getUiSettings() { + return { + get() { + return false; + }, + }; + }, + }; +}); + +import { shallow } from 'enzyme'; + +import { Feature } from 'geojson'; +import { MVTSingleLayerVectorSource } from '../../sources/mvt_single_layer_vector_source'; +import { + DataRequestDescriptor, + TiledSingleLayerVectorSourceDescriptor, + VectorLayerDescriptor, +} from '../../../../common/descriptor_types'; +import { SOURCE_TYPES } from '../../../../common/constants'; +import { TiledVectorLayer } from './tiled_vector_layer'; + +const defaultConfig = { + urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf', + layerName: 'foobar', + minSourceZoom: 4, + maxSourceZoom: 14, +}; + +function createLayer( + layerOptions: Partial = {}, + sourceOptions: Partial = {} +): TiledVectorLayer { + const sourceDescriptor: TiledSingleLayerVectorSourceDescriptor = { + type: SOURCE_TYPES.MVT_SINGLE_LAYER, + ...defaultConfig, + fields: [], + tooltipProperties: [], + ...sourceOptions, + }; + const mvtSource = new MVTSingleLayerVectorSource(sourceDescriptor); + + const defaultLayerOptions = { + ...layerOptions, + sourceDescriptor, + }; + const layerDescriptor = TiledVectorLayer.createDescriptor(defaultLayerOptions); + return new TiledVectorLayer({ layerDescriptor, source: mvtSource }); +} + +describe('visiblity', () => { + it('should get minzoom from source', async () => { + const layer: TiledVectorLayer = createLayer({}, {}); + expect(layer.getMinZoom()).toEqual(4); + }); + it('should get maxzoom from default', async () => { + const layer: TiledVectorLayer = createLayer({}, {}); + expect(layer.getMaxZoom()).toEqual(24); + }); + it('should get maxzoom from layer options', async () => { + const layer: TiledVectorLayer = createLayer({ maxZoom: 10 }, {}); + expect(layer.getMaxZoom()).toEqual(10); + }); +}); + +describe('icon', () => { + it('should use vector icon', async () => { + const layer: TiledVectorLayer = createLayer({}, {}); + + const iconAndTooltipContent = layer.getCustomIconAndTooltipContent(); + const component = shallow(iconAndTooltipContent.icon); + expect(component).toMatchSnapshot(); + }); +}); + +describe('getFeatureById', () => { + it('should return null feature', async () => { + const layer: TiledVectorLayer = createLayer({}, {}); + const feature = layer.getFeatureById('foobar') as Feature; + expect(feature).toEqual(null); + }); +}); + +describe('syncData', () => { + it('Should sync with source-params', async () => { + const layer: TiledVectorLayer = createLayer({}, {}); + + const syncContext = new MockSyncContext({ dataFilters: {} }); + + await layer.syncData(syncContext); + // @ts-expect-error + sinon.assert.calledOnce(syncContext.startLoading); + // @ts-expect-error + sinon.assert.calledOnce(syncContext.stopLoading); + + // @ts-expect-error + const call = syncContext.stopLoading.getCall(0); + expect(call.args[2]).toEqual(defaultConfig); + }); + + it('Should not resync when no changes to source params', async () => { + const layer1: TiledVectorLayer = createLayer({}, {}); + const syncContext1 = new MockSyncContext({ dataFilters: {} }); + + await layer1.syncData(syncContext1); + + const dataRequestDescriptor: DataRequestDescriptor = { + data: { ...defaultConfig }, + dataId: 'source', + }; + const layer2: TiledVectorLayer = createLayer( + { + __dataRequests: [dataRequestDescriptor], + }, + {} + ); + const syncContext2 = new MockSyncContext({ dataFilters: {} }); + await layer2.syncData(syncContext2); + // @ts-expect-error + sinon.assert.notCalled(syncContext2.startLoading); + // @ts-expect-error + sinon.assert.notCalled(syncContext2.stopLoading); + }); + + it('Should resync when changes to source params', async () => { + const layer1: TiledVectorLayer = createLayer({}, {}); + const syncContext1 = new MockSyncContext({ dataFilters: {} }); + + await layer1.syncData(syncContext1); + + const dataRequestDescriptor: DataRequestDescriptor = { + data: defaultConfig, + dataId: 'source', + }; + const layer2: TiledVectorLayer = createLayer( + { + __dataRequests: [dataRequestDescriptor], + }, + { layerName: 'barfoo' } + ); + const syncContext2 = new MockSyncContext({ dataFilters: {} }); + await layer2.syncData(syncContext2); + + // @ts-expect-error + sinon.assert.calledOnce(syncContext2.startLoading); + // @ts-expect-error + sinon.assert.calledOnce(syncContext2.stopLoading); + + // @ts-expect-error + const call = syncContext2.stopLoading.getCall(0); + expect(call.args[2]).toEqual({ ...defaultConfig, layerName: 'barfoo' }); + }); +}); diff --git a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx index a00639aa5fec54..c9ae1c805fa306 100644 --- a/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/tiled_vector_layer/tiled_vector_layer.tsx @@ -6,31 +6,30 @@ import React from 'react'; import { EuiIcon } from '@elastic/eui'; +import { Feature } from 'geojson'; import { VectorStyle } from '../../styles/vector/vector_style'; import { SOURCE_DATA_REQUEST_ID, LAYER_TYPE } from '../../../../common/constants'; import { VectorLayer, VectorLayerArguments } from '../vector_layer/vector_layer'; -import { canSkipSourceUpdate } from '../../util/can_skip_fetch'; import { ITiledSingleLayerVectorSource } from '../../sources/vector_source'; import { DataRequestContext } from '../../../actions'; -import { ISource } from '../../sources/source'; import { VectorLayerDescriptor, VectorSourceRequestMeta, } from '../../../../common/descriptor_types'; -import { MVTSingleLayerVectorSourceConfig } from '../../sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor'; +import { MVTSingleLayerVectorSourceConfig } from '../../sources/mvt_single_layer_vector_source/types'; export class TiledVectorLayer extends VectorLayer { static type = LAYER_TYPE.TILED_VECTOR; static createDescriptor( descriptor: Partial, - mapColors: string[] + mapColors?: string[] ): VectorLayerDescriptor { const layerDescriptor = super.createDescriptor(descriptor, mapColors); layerDescriptor.type = TiledVectorLayer.type; if (!layerDescriptor.style) { - const styleProperties = VectorStyle.createDefaultStyleProperties(mapColors); + const styleProperties = VectorStyle.createDefaultStyleProperties(mapColors ? mapColors : []); layerDescriptor.style = VectorStyle.createDescriptor(styleProperties); } @@ -64,13 +63,16 @@ export class TiledVectorLayer extends VectorLayer { ); const prevDataRequest = this.getSourceDataRequest(); - const canSkip = await canSkipSourceUpdate({ - source: this._source as ISource, - prevDataRequest, - nextMeta: searchFilters, - }); - if (canSkip) { - return null; + if (prevDataRequest) { + const data: MVTSingleLayerVectorSourceConfig = prevDataRequest.getData() as MVTSingleLayerVectorSourceConfig; + const canSkipBecauseNoChanges = + data.layerName === this._source.getLayerName() && + data.minSourceZoom === this._source.getMinZoom() && + data.maxSourceZoom === this._source.getMaxZoom(); + + if (canSkipBecauseNoChanges) { + return null; + } } startLoading(SOURCE_DATA_REQUEST_ID, requestToken, searchFilters); @@ -89,37 +91,41 @@ export class TiledVectorLayer extends VectorLayer { } _syncSourceBindingWithMb(mbMap: unknown) { - // @ts-ignore - const mbSource = mbMap.getSource(this.getId()); - if (!mbSource) { - const sourceDataRequest = this.getSourceDataRequest(); - if (!sourceDataRequest) { - // this is possible if the layer was invisible at startup. - // the actions will not perform any data=syncing as an optimization when a layer is invisible - // when turning the layer back into visible, it's possible the url has not been resovled yet. - return; - } + // @ts-expect-error + const mbSource = mbMap.getSource(this._getMbSourceId()); + if (mbSource) { + return; + } + const sourceDataRequest = this.getSourceDataRequest(); + if (!sourceDataRequest) { + // this is possible if the layer was invisible at startup. + // the actions will not perform any data=syncing as an optimization when a layer is invisible + // when turning the layer back into visible, it's possible the url has not been resovled yet. + return; + } - const sourceMeta: MVTSingleLayerVectorSourceConfig | null = sourceDataRequest.getData() as MVTSingleLayerVectorSourceConfig; - if (!sourceMeta) { - return; - } + const sourceMeta: MVTSingleLayerVectorSourceConfig | null = sourceDataRequest.getData() as MVTSingleLayerVectorSourceConfig; + if (!sourceMeta) { + return; + } - const sourceId = this.getId(); + const mbSourceId = this._getMbSourceId(); + // @ts-expect-error + mbMap.addSource(mbSourceId, { + type: 'vector', + tiles: [sourceMeta.urlTemplate], + minzoom: sourceMeta.minSourceZoom, + maxzoom: sourceMeta.maxSourceZoom, + }); + } - // @ts-ignore - mbMap.addSource(sourceId, { - type: 'vector', - tiles: [sourceMeta.urlTemplate], - minzoom: sourceMeta.minSourceZoom, - maxzoom: sourceMeta.maxSourceZoom, - }); - } + ownsMbSourceId(mbSourceId: string): boolean { + return this._getMbSourceId() === mbSourceId; } _syncStylePropertiesWithMb(mbMap: unknown) { // @ts-ignore - const mbSource = mbMap.getSource(this.getId()); + const mbSource = mbMap.getSource(this._getMbSourceId()); if (!mbSource) { return; } @@ -129,32 +135,52 @@ export class TiledVectorLayer extends VectorLayer { return; } const sourceMeta: MVTSingleLayerVectorSourceConfig = sourceDataRequest.getData() as MVTSingleLayerVectorSourceConfig; + if (sourceMeta.layerName === '') { + return; + } this._setMbPointsProperties(mbMap, sourceMeta.layerName); this._setMbLinePolygonProperties(mbMap, sourceMeta.layerName); } _requiresPrevSourceCleanup(mbMap: unknown): boolean { - // @ts-ignore - const mbTileSource = mbMap.getSource(this.getId()); + // @ts-expect-error + const mbTileSource = mbMap.getSource(this._getMbSourceId()); if (!mbTileSource) { return false; } + const dataRequest = this.getSourceDataRequest(); if (!dataRequest) { return false; } const tiledSourceMeta: MVTSingleLayerVectorSourceConfig | null = dataRequest.getData() as MVTSingleLayerVectorSourceConfig; - if ( - mbTileSource.tiles[0] === tiledSourceMeta.urlTemplate && - mbTileSource.minzoom === tiledSourceMeta.minSourceZoom && - mbTileSource.maxzoom === tiledSourceMeta.maxSourceZoom - ) { - // TileURL and zoom-range captures all the state. If this does not change, no updates are required. + + if (!tiledSourceMeta) { return false; } - return true; + const isSourceDifferent = + mbTileSource.tiles[0] !== tiledSourceMeta.urlTemplate || + mbTileSource.minzoom !== tiledSourceMeta.minSourceZoom || + mbTileSource.maxzoom !== tiledSourceMeta.maxSourceZoom; + + if (isSourceDifferent) { + return true; + } + + const layerIds = this.getMbLayerIds(); + for (let i = 0; i < layerIds.length; i++) { + // @ts-expect-error + const mbLayer = mbMap.getLayer(layerIds[i]); + if (mbLayer && mbLayer.sourceLayer !== tiledSourceMeta.layerName) { + // If the source-pointer of one of the layers is stale, they will all be stale. + // In this case, all the mb-layers need to be removed and re-added. + return true; + } + } + + return false; } syncLayerWithMB(mbMap: unknown) { @@ -171,4 +197,8 @@ export class TiledVectorLayer extends VectorLayer { // higher resolution vector tiles cannot be displayed at lower-res return Math.max(this._source.getMinZoom(), super.getMinZoom()); } + + getFeatureById(id: string | number): Feature | null { + return null; + } } diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.d.ts b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.d.ts index e420087628bc83..77daf9c9af5704 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.d.ts +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.d.ts @@ -5,6 +5,7 @@ */ /* eslint-disable @typescript-eslint/consistent-type-definitions */ +import { Feature, GeoJsonProperties } from 'geojson'; import { AbstractLayer } from '../layer'; import { IVectorSource } from '../../sources/vector_source'; import { @@ -17,6 +18,7 @@ import { IJoin } from '../../joins/join'; import { IVectorStyle } from '../../styles/vector/vector_style'; import { IField } from '../../fields/field'; import { DataRequestContext } from '../../../actions'; +import { ITooltipProperty } from '../../tooltips/tooltip_property'; export type VectorLayerArguments = { source: IVectorSource; @@ -31,6 +33,8 @@ export interface IVectorLayer extends ILayer { getValidJoins(): IJoin[]; getSource(): IVectorSource; getStyle(): IVectorStyle; + getFeatureById(id: string | number): Feature | null; + getPropertiesForTooltip(properties: GeoJsonProperties): Promise; } export class VectorLayer extends AbstractLayer implements IVectorLayer { @@ -75,4 +79,6 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { _setMbLinePolygonProperties(mbMap: unknown, mvtSourceLayer?: string): void; getSource(): IVectorSource; getStyle(): IVectorStyle; + getFeatureById(id: string | number): Feature | null; + getPropertiesForTooltip(properties: GeoJsonProperties): Promise; } diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js index 524ab245c67601..0a4fcfc23060c5 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.js @@ -672,10 +672,10 @@ export class VectorLayer extends AbstractLayer { } this.syncVisibilityWithMb(mbMap, markerLayerId); - mbMap.setLayerZoomRange(markerLayerId, this._descriptor.minZoom, this._descriptor.maxZoom); + mbMap.setLayerZoomRange(markerLayerId, this.getMinZoom(), this.getMaxZoom()); if (markerLayerId !== textLayerId) { this.syncVisibilityWithMb(mbMap, textLayerId); - mbMap.setLayerZoomRange(textLayerId, this._descriptor.minZoom, this._descriptor.maxZoom); + mbMap.setLayerZoomRange(textLayerId, this.getMinZoom(), this.getMaxZoom()); } } @@ -802,14 +802,14 @@ export class VectorLayer extends AbstractLayer { }); this.syncVisibilityWithMb(mbMap, fillLayerId); - mbMap.setLayerZoomRange(fillLayerId, this._descriptor.minZoom, this._descriptor.maxZoom); + mbMap.setLayerZoomRange(fillLayerId, this.getMinZoom(), this.getMaxZoom()); const fillFilterExpr = getFillFilterExpression(hasJoins); if (fillFilterExpr !== mbMap.getFilter(fillLayerId)) { mbMap.setFilter(fillLayerId, fillFilterExpr); } this.syncVisibilityWithMb(mbMap, lineLayerId); - mbMap.setLayerZoomRange(lineLayerId, this._descriptor.minZoom, this._descriptor.maxZoom); + mbMap.setLayerZoomRange(lineLayerId, this.getMinZoom(), this.getMaxZoom()); const lineFilterExpr = getLineFilterExpression(hasJoins); if (lineFilterExpr !== mbMap.getFilter(lineLayerId)) { mbMap.setFilter(lineLayerId, lineFilterExpr); @@ -822,9 +822,9 @@ export class VectorLayer extends AbstractLayer { } _syncSourceBindingWithMb(mbMap) { - const mbSource = mbMap.getSource(this.getId()); + const mbSource = mbMap.getSource(this._getMbSourceId()); if (!mbSource) { - mbMap.addSource(this.getId(), { + mbMap.addSource(this._getMbSourceId(), { type: 'geojson', data: EMPTY_FEATURE_COLLECTION, }); @@ -891,16 +891,17 @@ export class VectorLayer extends AbstractLayer { } async getPropertiesForTooltip(properties) { - let allTooltips = await this.getSource().filterAndFormatPropertiesToHtml(properties); - this._addJoinsToSourceTooltips(allTooltips); + const vectorSource = this.getSource(); + let allProperties = await vectorSource.filterAndFormatPropertiesToHtml(properties); + this._addJoinsToSourceTooltips(allProperties); for (let i = 0; i < this.getJoins().length; i++) { const propsFromJoin = await this.getJoins()[i].filterAndFormatPropertiesForTooltip( properties ); - allTooltips = [...allTooltips, ...propsFromJoin]; + allProperties = [...allProperties, ...propsFromJoin]; } - return allTooltips; + return allProperties; } canShowTooltip() { @@ -912,7 +913,7 @@ export class VectorLayer extends AbstractLayer { getFeatureById(id) { const featureCollection = this._getSourceFeatureCollection(); if (!featureCollection) { - return; + return null; } return featureCollection.features.find((feature) => { diff --git a/x-pack/plugins/maps/public/classes/sources/ems_file_source/update_source_editor.tsx b/x-pack/plugins/maps/public/classes/sources/ems_file_source/update_source_editor.tsx index ac69505a9bed5e..7021859ee9827d 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_file_source/update_source_editor.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_file_source/update_source_editor.tsx @@ -15,7 +15,7 @@ import { OnSourceChangeArgs } from '../../../connected_components/layer_panel/vi interface Props { layerId: string; - onChange: (args: OnSourceChangeArgs) => void; + onChange: (...args: OnSourceChangeArgs[]) => void; source: IEmsFileSource; tooltipFields: IField[]; } diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_field_config_editor.test.tsx.snap b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_field_config_editor.test.tsx.snap new file mode 100644 index 00000000000000..f6d0129e85abf6 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_field_config_editor.test.tsx.snap @@ -0,0 +1,491 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`should render error for dupes 1`] = ` + + + + + + + + + + + + string + + , + "value": "String", + }, + Object { + "inputDisplay": + + + + + number + + , + "value": "Number", + }, + ] + } + valueOfSelected="String" + /> + + + + + + + + + + + + + + + + + string + + , + "value": "String", + }, + Object { + "inputDisplay": + + + + + number + + , + "value": "Number", + }, + ] + } + valueOfSelected="Number" + /> + + + + + + + + + + + Add + + + + +`; + +exports[`should render error for empty name 1`] = ` + + + + + + + + + + + + string + + , + "value": "String", + }, + Object { + "inputDisplay": + + + + + number + + , + "value": "Number", + }, + ] + } + valueOfSelected="String" + /> + + + + + + + + + + + Add + + + + +`; + +exports[`should render field editor 1`] = ` + + + + + + + + + + + + string + + , + "value": "String", + }, + Object { + "inputDisplay": + + + + + number + + , + "value": "Number", + }, + ] + } + valueOfSelected="String" + /> + + + + + + + + + + + + + + + + + string + + , + "value": "String", + }, + Object { + "inputDisplay": + + + + + number + + , + "value": "Number", + }, + ] + } + valueOfSelected="Number" + /> + + + + + + + + + + + Add + + + + +`; diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_single_layer_source_settings.test.tsx.snap b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_single_layer_source_settings.test.tsx.snap new file mode 100644 index 00000000000000..699173bd362fae --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_single_layer_source_settings.test.tsx.snap @@ -0,0 +1,211 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`should not render fields-editor when there is no layername 1`] = ` + + + + + + + Available levels + + + + + } + max={24} + min={0} + onChange={[Function]} + prepend="Zoom" + showInput="inputWithPopover" + showLabels={true} + value={ + Array [ + 4, + 14, + ] + } + /> + +`; + +exports[`should render with fields 1`] = ` + + + + + + + Available levels + + + + + } + max={24} + min={0} + onChange={[Function]} + prepend="Zoom" + showInput="inputWithPopover" + showLabels={true} + value={ + Array [ + 4, + 14, + ] + } + /> + + Fields which are available in + + + foobar + + . + + + These can be used for tooltips and dynamic styling. + + } + delay="regular" + position="top" + > + + Fields + + + + + } + labelType="label" + > + + + +`; + +exports[`should render without fields 1`] = ` + + + + + + + Available levels + + + + + } + max={24} + min={0} + onChange={[Function]} + prepend="Zoom" + showInput="inputWithPopover" + showLabels={true} + value={ + Array [ + 4, + 14, + ] + } + /> + +`; diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_single_layer_vector_source_editor.test.tsx.snap b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_single_layer_vector_source_editor.test.tsx.snap new file mode 100644 index 00000000000000..ccd0e0064d0754 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/mvt_single_layer_vector_source_editor.test.tsx.snap @@ -0,0 +1,30 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`should render source creation editor (fields should _not_ be included) 1`] = ` + + + + + + +`; diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/update_source_editor.test.tsx.snap b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/update_source_editor.test.tsx.snap new file mode 100644 index 00000000000000..bccf2b17e2b5d3 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/__snapshots__/update_source_editor.test.tsx.snap @@ -0,0 +1,57 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`should render update source editor (fields _should_ be included) 1`] = ` + + + +
+ +
+ + + + + + + +
+ +
+ + + + + + +`; diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/layer_wizard.tsx index 067c7f5a47ca35..32fa329be85df5 100644 --- a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/layer_wizard.tsx @@ -6,23 +6,21 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; -import { - MVTSingleLayerVectorSourceEditor, - MVTSingleLayerVectorSourceConfig, -} from './mvt_single_layer_vector_source_editor'; +import { MVTSingleLayerVectorSourceEditor } from './mvt_single_layer_vector_source_editor'; import { MVTSingleLayerVectorSource, sourceTitle } from './mvt_single_layer_vector_source'; import { LayerWizard, RenderWizardArguments } from '../../layers/layer_wizard_registry'; import { TiledVectorLayer } from '../../layers/tiled_vector_layer/tiled_vector_layer'; import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; +import { TiledSingleLayerVectorSourceSettings } from '../../../../common/descriptor_types'; export const mvtVectorSourceWizardConfig: LayerWizard = { categories: [LAYER_WIZARD_CATEGORY.REFERENCE], description: i18n.translate('xpack.maps.source.mvtVectorSourceWizard', { - defaultMessage: 'Vector source wizard', + defaultMessage: 'Data service implementing the Mapbox vector tile specification', }), icon: 'grid', renderWizard: ({ previewLayers, mapColors }: RenderWizardArguments) => { - const onSourceConfigChange = (sourceConfig: MVTSingleLayerVectorSourceConfig) => { + const onSourceConfigChange = (sourceConfig: TiledSingleLayerVectorSourceSettings) => { const sourceDescriptor = MVTSingleLayerVectorSource.createDescriptor(sourceConfig); const layerDescriptor = TiledVectorLayer.createDescriptor({ sourceDescriptor }, mapColors); previewLayers([layerDescriptor]); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_field_config_editor.test.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_field_config_editor.test.tsx new file mode 100644 index 00000000000000..0121dc45cb9ee6 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_field_config_editor.test.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +jest.mock('../../../kibana_services', () => ({})); + +import React from 'react'; +import { shallow } from 'enzyme'; + +import { MVTFieldConfigEditor } from './mvt_field_config_editor'; +import { MVT_FIELD_TYPE } from '../../../../common/constants'; + +test('should render field editor', async () => { + const fields = [ + { + name: 'foo', + type: MVT_FIELD_TYPE.STRING, + }, + { + name: 'bar', + type: MVT_FIELD_TYPE.NUMBER, + }, + ]; + const component = shallow( {}} />); + + expect(component).toMatchSnapshot(); +}); + +test('should render error for empty name', async () => { + const fields = [ + { + name: '', + type: MVT_FIELD_TYPE.STRING, + }, + ]; + const component = shallow( {}} />); + + expect(component).toMatchSnapshot(); +}); + +test('should render error for dupes', async () => { + const fields = [ + { + name: 'foo', + type: MVT_FIELD_TYPE.STRING, + }, + { + name: 'foo', + type: MVT_FIELD_TYPE.NUMBER, + }, + ]; + const component = shallow( {}} />); + + expect(component).toMatchSnapshot(); +}); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_field_config_editor.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_field_config_editor.tsx new file mode 100644 index 00000000000000..b2a93a4ef88ad8 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_field_config_editor.tsx @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +/* eslint-disable @typescript-eslint/consistent-type-definitions */ + +import React, { ChangeEvent, Component, Fragment } from 'react'; +import { + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiButtonEmpty, + EuiSuperSelect, + EuiFieldText, + EuiSpacer, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import _ from 'lodash'; +import { MVTFieldDescriptor } from '../../../../common/descriptor_types'; +import { FieldIcon } from '../../../../../../../src/plugins/kibana_react/public'; +import { MVT_FIELD_TYPE } from '../../../../common/constants'; + +function makeOption({ + value, + icon, + message, +}: { + value: MVT_FIELD_TYPE; + icon: string; + message: string; +}) { + return { + value, + inputDisplay: ( + + + + + {message} + + ), + }; +} + +const FIELD_TYPE_OPTIONS = [ + { + value: MVT_FIELD_TYPE.STRING, + icon: 'string', + message: i18n.translate('xpack.maps.mvtSource.stringFieldLabel', { + defaultMessage: 'string', + }), + }, + { + value: MVT_FIELD_TYPE.NUMBER, + icon: 'number', + message: i18n.translate('xpack.maps.mvtSource.numberFieldLabel', { + defaultMessage: 'number', + }), + }, +].map(makeOption); + +interface Props { + fields: MVTFieldDescriptor[]; + onChange: (fields: MVTFieldDescriptor[]) => void; +} + +interface State { + currentFields: MVTFieldDescriptor[]; +} + +export class MVTFieldConfigEditor extends Component { + state: State = { + currentFields: _.cloneDeep(this.props.fields), + }; + + _notifyChange = _.debounce(() => { + const invalid = this.state.currentFields.some((field: MVTFieldDescriptor) => { + return field.name === ''; + }); + + if (!invalid) { + this.props.onChange(this.state.currentFields); + } + }); + + _fieldChange(newFields: MVTFieldDescriptor[]) { + this.setState( + { + currentFields: newFields, + }, + this._notifyChange + ); + } + + _removeField(index: number) { + const newFields: MVTFieldDescriptor[] = this.state.currentFields.slice(); + newFields.splice(index, 1); + this._fieldChange(newFields); + } + + _addField = () => { + const newFields: MVTFieldDescriptor[] = this.state.currentFields.slice(); + newFields.push({ + type: MVT_FIELD_TYPE.STRING, + name: '', + }); + this._fieldChange(newFields); + }; + + _renderFieldTypeDropDown(mvtFieldConfig: MVTFieldDescriptor, index: number) { + const onChange = (type: MVT_FIELD_TYPE) => { + const newFields = this.state.currentFields.slice(); + newFields[index] = { + type, + name: newFields[index].name, + }; + this._fieldChange(newFields); + }; + + return ( + onChange(value)} + compressed + /> + ); + } + + _renderFieldButtonDelete(index: number) { + return ( + { + this._removeField(index); + }} + title={i18n.translate('xpack.maps.mvtSource.trashButtonTitle', { + defaultMessage: 'Remove field', + })} + aria-label={i18n.translate('xpack.maps.mvtSource.trashButtonAriaLabel', { + defaultMessage: 'Remove field', + })} + /> + ); + } + + _renderFieldNameInput(mvtFieldConfig: MVTFieldDescriptor, index: number) { + const onChange = (e: ChangeEvent) => { + const name = e.target.value; + const newFields = this.state.currentFields.slice(); + newFields[index] = { + name, + type: newFields[index].type, + }; + this._fieldChange(newFields); + }; + + const emptyName = mvtFieldConfig.name === ''; + const hasDupes = + this.state.currentFields.filter((field) => field.name === mvtFieldConfig.name).length > 1; + + return ( + + ); + } + + _renderFieldConfig() { + return this.state.currentFields.map((mvtFieldConfig: MVTFieldDescriptor, index: number) => { + return ( + <> + + {this._renderFieldNameInput(mvtFieldConfig, index)} + {this._renderFieldTypeDropDown(mvtFieldConfig, index)} + {this._renderFieldButtonDelete(index)} + + + + ); + }); + } + + render() { + return ( + + {this._renderFieldConfig()} + + + + + {i18n.translate('xpack.maps.mvtSource.addFieldLabel', { + defaultMessage: 'Add', + })} + + + + + ); + } +} diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_source_settings.test.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_source_settings.test.tsx new file mode 100644 index 00000000000000..b5c75b97e6cb27 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_source_settings.test.tsx @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +jest.mock('../../../kibana_services', () => ({})); + +import React from 'react'; +import { shallow } from 'enzyme'; + +import { MVTSingleLayerSourceSettings } from './mvt_single_layer_source_settings'; + +const defaultSettings = { + handleChange: () => {}, + layerName: 'foobar', + fields: [], + minSourceZoom: 4, + maxSourceZoom: 14, + showFields: true, +}; + +test('should render with fields', async () => { + const component = shallow(); + expect(component).toMatchSnapshot(); +}); + +test('should render without fields', async () => { + const settings = { ...defaultSettings, showFields: false }; + const component = shallow(); + expect(component).toMatchSnapshot(); +}); + +test('should not render fields-editor when there is no layername', async () => { + const settings = { ...defaultSettings, layerName: '' }; + const component = shallow(); + expect(component).toMatchSnapshot(); +}); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_source_settings.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_source_settings.tsx new file mode 100644 index 00000000000000..cd3fd97cf66a63 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_source_settings.tsx @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +/* eslint-disable @typescript-eslint/consistent-type-definitions */ + +import React, { Fragment, Component, ChangeEvent } from 'react'; +import { EuiFieldText, EuiFormRow, EuiToolTip, EuiIcon } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import _ from 'lodash'; +import { MAX_ZOOM, MIN_ZOOM } from '../../../../common/constants'; +import { ValidatedDualRange, Value } from '../../../../../../../src/plugins/kibana_react/public'; +import { MVTFieldConfigEditor } from './mvt_field_config_editor'; +import { MVTFieldDescriptor } from '../../../../common/descriptor_types'; + +export type MVTSettings = { + layerName: string; + fields: MVTFieldDescriptor[]; + minSourceZoom: number; + maxSourceZoom: number; +}; + +interface State { + currentLayerName: string; + currentMinSourceZoom: number; + currentMaxSourceZoom: number; + currentFields: MVTFieldDescriptor[]; +} + +interface Props { + handleChange: (args: MVTSettings) => void; + layerName: string; + fields: MVTFieldDescriptor[]; + minSourceZoom: number; + maxSourceZoom: number; + showFields: boolean; +} + +export class MVTSingleLayerSourceSettings extends Component { + // Tracking in state to allow for debounce. + // Changes to layer-name and/or min/max zoom require heavy operation at map-level (removing and re-adding all sources/layers) + // To preserve snappyness of typing, debounce the dispatches. + state = { + currentLayerName: this.props.layerName, + currentMinSourceZoom: this.props.minSourceZoom, + currentMaxSourceZoom: this.props.maxSourceZoom, + currentFields: _.cloneDeep(this.props.fields), + }; + + _handleChange = _.debounce(() => { + this.props.handleChange({ + layerName: this.state.currentLayerName, + minSourceZoom: this.state.currentMinSourceZoom, + maxSourceZoom: this.state.currentMaxSourceZoom, + fields: this.state.currentFields, + }); + }, 200); + + _handleLayerNameInputChange = (e: ChangeEvent) => { + this.setState({ currentLayerName: e.target.value }, this._handleChange); + }; + + _handleFieldChange = (fields: MVTFieldDescriptor[]) => { + this.setState({ currentFields: fields }, this._handleChange); + }; + + _handleZoomRangeChange = (e: Value) => { + this.setState( + { + currentMinSourceZoom: parseInt(e[0] as string, 10), + currentMaxSourceZoom: parseInt(e[1] as string, 10), + }, + this._handleChange + ); + }; + + render() { + const preMessage = i18n.translate( + 'xpack.maps.source.MVTSingleLayerVectorSourceEditor.fieldsPreHelpMessage', + { + defaultMessage: 'Fields which are available in ', + } + ); + const message = ( + <> + {this.state.currentLayerName}.{' '} + + ); + const postMessage = i18n.translate( + 'xpack.maps.source.MVTSingleLayerVectorSourceEditor.fieldsPostHelpMessage', + { + defaultMessage: 'These can be used for tooltips and dynamic styling.', + } + ); + const fieldEditor = + this.props.showFields && this.state.currentLayerName !== '' ? ( + + {preMessage} + {message} + {postMessage} + + } + > + + {i18n.translate( + 'xpack.maps.source.MVTSingleLayerVectorSourceEditor.fieldsMessage', + { + defaultMessage: 'Fields', + } + )}{' '} + + + + } + > + + + ) : null; + + return ( + + + + + + + + {i18n.translate( + 'xpack.maps.source.MVTSingleLayerVectorSourceEditor.zoomRangeTopMessage', + { + defaultMessage: 'Available levels', + } + )}{' '} + + + + } + formRowDisplay="columnCompressed" + value={[this.state.currentMinSourceZoom, this.state.currentMaxSourceZoom]} + min={MIN_ZOOM} + max={MAX_ZOOM} + onChange={this._handleZoomRangeChange} + allowEmptyRange={false} + showInput="inputWithPopover" + compressed + showLabels + prepend={i18n.translate( + 'xpack.maps.source.MVTSingleLayerVectorSourceEditor.dataZoomRangeMessage', + { + defaultMessage: 'Zoom', + } + )} + /> + {fieldEditor} + + ); + } +} diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.test.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.test.tsx new file mode 100644 index 00000000000000..bc08baad7a8429 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.test.tsx @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { MVTSingleLayerVectorSource } from './mvt_single_layer_vector_source'; +import { MVT_FIELD_TYPE, SOURCE_TYPES } from '../../../../common/constants'; +import { TiledSingleLayerVectorSourceDescriptor } from '../../../../common/descriptor_types'; + +const descriptor: TiledSingleLayerVectorSourceDescriptor = { + type: SOURCE_TYPES.MVT_SINGLE_LAYER, + urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf', + layerName: 'foobar', + minSourceZoom: 4, + maxSourceZoom: 14, + fields: [], + tooltipProperties: [], +}; + +describe('getUrlTemplateWithMeta', () => { + it('should echo configuration', async () => { + const source = new MVTSingleLayerVectorSource(descriptor); + const config = await source.getUrlTemplateWithMeta(); + expect(config.urlTemplate).toEqual(descriptor.urlTemplate); + expect(config.layerName).toEqual(descriptor.layerName); + expect(config.minSourceZoom).toEqual(descriptor.minSourceZoom); + expect(config.maxSourceZoom).toEqual(descriptor.maxSourceZoom); + }); +}); + +describe('canFormatFeatureProperties', () => { + it('false if no tooltips', async () => { + const source = new MVTSingleLayerVectorSource(descriptor); + expect(source.canFormatFeatureProperties()).toEqual(false); + }); + it('true if tooltip', async () => { + const descriptorWithTooltips = { + ...descriptor, + fields: [{ name: 'foobar', type: MVT_FIELD_TYPE.STRING }], + tooltipProperties: ['foobar'], + }; + const source = new MVTSingleLayerVectorSource(descriptorWithTooltips); + expect(source.canFormatFeatureProperties()).toEqual(true); + }); +}); + +describe('filterAndFormatPropertiesToHtml', () => { + const descriptorWithFields = { + ...descriptor, + fields: [ + { + name: 'foo', + type: MVT_FIELD_TYPE.STRING, + }, + { + name: 'food', + type: MVT_FIELD_TYPE.STRING, + }, + { + name: 'fooz', + type: MVT_FIELD_TYPE.NUMBER, + }, + ], + tooltipProperties: ['foo', 'fooz'], + }; + + it('should get tooltipproperties', async () => { + const source = new MVTSingleLayerVectorSource(descriptorWithFields); + const tooltipProperties = await source.filterAndFormatPropertiesToHtml({ + foo: 'bar', + fooz: 123, + }); + expect(tooltipProperties.length).toEqual(2); + expect(tooltipProperties[0].getPropertyName()).toEqual('foo'); + expect(tooltipProperties[0].getHtmlDisplayValue()).toEqual('bar'); + expect(tooltipProperties[1].getPropertyName()).toEqual('fooz'); + expect(tooltipProperties[1].getHtmlDisplayValue()).toEqual('123'); + }); +}); + +describe('getImmutableSourceProperties', () => { + it('should only show immutable props', async () => { + const source = new MVTSingleLayerVectorSource(descriptor); + const properties = await source.getImmutableProperties(); + expect(properties).toEqual([ + { label: 'Data source', value: '.pbf vector tiles' }, + { label: 'Url', value: 'https://example.com/{x}/{y}/{z}.pbf' }, + ]); + }); +}); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.ts b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.ts deleted file mode 100644 index 03b91df22d3cab..00000000000000 --- a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { i18n } from '@kbn/i18n'; -import uuid from 'uuid/v4'; -import { AbstractSource, ImmutableSourceProperty } from '../source'; -import { BoundsFilters, GeoJsonWithMeta, ITiledSingleLayerVectorSource } from '../vector_source'; -import { MAX_ZOOM, MIN_ZOOM, SOURCE_TYPES, VECTOR_SHAPE_TYPE } from '../../../../common/constants'; -import { IField } from '../../fields/field'; -import { registerSource } from '../source_registry'; -import { getDataSourceLabel, getUrlLabel } from '../../../../common/i18n_getters'; -import { - MapExtent, - TiledSingleLayerVectorSourceDescriptor, - VectorSourceSyncMeta, -} from '../../../../common/descriptor_types'; -import { MVTSingleLayerVectorSourceConfig } from './mvt_single_layer_vector_source_editor'; -import { ITooltipProperty } from '../../tooltips/tooltip_property'; - -export const sourceTitle = i18n.translate( - 'xpack.maps.source.MVTSingleLayerVectorSource.sourceTitle', - { - defaultMessage: 'Vector Tile Layer', - } -); - -export class MVTSingleLayerVectorSource extends AbstractSource - implements ITiledSingleLayerVectorSource { - static createDescriptor({ - urlTemplate, - layerName, - minSourceZoom, - maxSourceZoom, - }: MVTSingleLayerVectorSourceConfig) { - return { - type: SOURCE_TYPES.MVT_SINGLE_LAYER, - id: uuid(), - urlTemplate, - layerName, - minSourceZoom: Math.max(MIN_ZOOM, minSourceZoom), - maxSourceZoom: Math.min(MAX_ZOOM, maxSourceZoom), - }; - } - - readonly _descriptor: TiledSingleLayerVectorSourceDescriptor; - - constructor( - sourceDescriptor: TiledSingleLayerVectorSourceDescriptor, - inspectorAdapters?: object - ) { - super(sourceDescriptor, inspectorAdapters); - this._descriptor = sourceDescriptor; - } - - renderSourceSettingsEditor() { - return null; - } - - getFieldNames(): string[] { - return []; - } - - getGeoJsonWithMeta( - layerName: 'string', - searchFilters: unknown[], - registerCancelCallback: (callback: () => void) => void - ): Promise { - // todo: remove this method - // This is a consequence of ITiledSingleLayerVectorSource extending IVectorSource. - throw new Error('Does not implement getGeoJsonWithMeta'); - } - - async getFields(): Promise { - return []; - } - - async getImmutableProperties(): Promise { - return [ - { label: getDataSourceLabel(), value: sourceTitle }, - { label: getUrlLabel(), value: this._descriptor.urlTemplate }, - { - label: i18n.translate('xpack.maps.source.MVTSingleLayerVectorSource.layerNameMessage', { - defaultMessage: 'Layer name', - }), - value: this._descriptor.layerName, - }, - { - label: i18n.translate('xpack.maps.source.MVTSingleLayerVectorSource.minZoomMessage', { - defaultMessage: 'Min zoom', - }), - value: this._descriptor.minSourceZoom.toString(), - }, - { - label: i18n.translate('xpack.maps.source.MVTSingleLayerVectorSource.maxZoomMessage', { - defaultMessage: 'Max zoom', - }), - value: this._descriptor.maxSourceZoom.toString(), - }, - ]; - } - - async getDisplayName(): Promise { - return this._descriptor.layerName; - } - - async getUrlTemplateWithMeta() { - return { - urlTemplate: this._descriptor.urlTemplate, - layerName: this._descriptor.layerName, - minSourceZoom: this._descriptor.minSourceZoom, - maxSourceZoom: this._descriptor.maxSourceZoom, - }; - } - - async getSupportedShapeTypes(): Promise { - return [VECTOR_SHAPE_TYPE.POINT, VECTOR_SHAPE_TYPE.LINE, VECTOR_SHAPE_TYPE.POLYGON]; - } - - canFormatFeatureProperties() { - return false; - } - - getMinZoom() { - return this._descriptor.minSourceZoom; - } - - getMaxZoom() { - return this._descriptor.maxSourceZoom; - } - - getBoundsForFilters( - boundsFilters: BoundsFilters, - registerCancelCallback: (requestToken: symbol, callback: () => void) => void - ): MapExtent | null { - return null; - } - - getFieldByName(fieldName: string): IField | null { - return null; - } - - getSyncMeta(): VectorSourceSyncMeta { - return null; - } - - getApplyGlobalQuery(): boolean { - return false; - } - - async filterAndFormatPropertiesToHtml(properties: unknown): Promise { - return []; - } -} - -registerSource({ - ConstructorFunction: MVTSingleLayerVectorSource, - type: SOURCE_TYPES.MVT_SINGLE_LAYER, -}); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx new file mode 100644 index 00000000000000..ae28828dec5a82 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx @@ -0,0 +1,222 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import uuid from 'uuid/v4'; +import React from 'react'; +import { GeoJsonProperties } from 'geojson'; +import { AbstractSource, ImmutableSourceProperty, SourceEditorArgs } from '../source'; +import { BoundsFilters, GeoJsonWithMeta, ITiledSingleLayerVectorSource } from '../vector_source'; +import { + FIELD_ORIGIN, + MAX_ZOOM, + MIN_ZOOM, + SOURCE_TYPES, + VECTOR_SHAPE_TYPE, +} from '../../../../common/constants'; +import { registerSource } from '../source_registry'; +import { getDataSourceLabel, getUrlLabel } from '../../../../common/i18n_getters'; +import { + MapExtent, + MVTFieldDescriptor, + TiledSingleLayerVectorSourceDescriptor, + VectorSourceSyncMeta, +} from '../../../../common/descriptor_types'; +import { MVTField } from '../../fields/mvt_field'; +import { UpdateSourceEditor } from './update_source_editor'; +import { ITooltipProperty, TooltipProperty } from '../../tooltips/tooltip_property'; + +export const sourceTitle = i18n.translate( + 'xpack.maps.source.MVTSingleLayerVectorSource.sourceTitle', + { + defaultMessage: '.pbf vector tiles', + } +); + +export class MVTSingleLayerVectorSource extends AbstractSource + implements ITiledSingleLayerVectorSource { + static createDescriptor({ + urlTemplate, + layerName, + minSourceZoom, + maxSourceZoom, + fields, + tooltipProperties, + }: Partial) { + return { + type: SOURCE_TYPES.MVT_SINGLE_LAYER, + id: uuid(), + urlTemplate: urlTemplate ? urlTemplate : '', + layerName: layerName ? layerName : '', + minSourceZoom: + typeof minSourceZoom === 'number' ? Math.max(MIN_ZOOM, minSourceZoom) : MIN_ZOOM, + maxSourceZoom: + typeof maxSourceZoom === 'number' ? Math.min(MAX_ZOOM, maxSourceZoom) : MAX_ZOOM, + fields: fields ? fields : [], + tooltipProperties: tooltipProperties ? tooltipProperties : [], + }; + } + + readonly _descriptor: TiledSingleLayerVectorSourceDescriptor; + readonly _tooltipFields: MVTField[]; + + constructor( + sourceDescriptor: TiledSingleLayerVectorSourceDescriptor, + inspectorAdapters?: object + ) { + super(sourceDescriptor, inspectorAdapters); + this._descriptor = MVTSingleLayerVectorSource.createDescriptor(sourceDescriptor); + + this._tooltipFields = this._descriptor.tooltipProperties + .map((fieldName) => { + return this.getFieldByName(fieldName); + }) + .filter((f) => f !== null) as MVTField[]; + } + + async supportsFitToBounds() { + return false; + } + + renderSourceSettingsEditor({ onChange }: SourceEditorArgs) { + return ( + + ); + } + + getFieldNames(): string[] { + return this._descriptor.fields.map((field: MVTFieldDescriptor) => { + return field.name; + }); + } + + getMVTFields(): MVTField[] { + return this._descriptor.fields.map((field: MVTFieldDescriptor) => { + return new MVTField({ + fieldName: field.name, + type: field.type, + source: this, + origin: FIELD_ORIGIN.SOURCE, + }); + }); + } + + getFieldByName(fieldName: string): MVTField | null { + try { + return this.createField({ fieldName }); + } catch (e) { + return null; + } + } + + createField({ fieldName }: { fieldName: string }): MVTField { + const field = this._descriptor.fields.find((f: MVTFieldDescriptor) => { + return f.name === fieldName; + }); + if (!field) { + throw new Error(`Cannot create field for fieldName ${fieldName}`); + } + return new MVTField({ + fieldName: field.name, + type: field.type, + source: this, + origin: FIELD_ORIGIN.SOURCE, + }); + } + + getGeoJsonWithMeta( + layerName: 'string', + searchFilters: unknown[], + registerCancelCallback: (callback: () => void) => void + ): Promise { + // Having this method here is a consequence of ITiledSingleLayerVectorSource extending IVectorSource. + throw new Error('Does not implement getGeoJsonWithMeta'); + } + + async getFields(): Promise { + return this.getMVTFields(); + } + + getLayerName(): string { + return this._descriptor.layerName; + } + + async getImmutableProperties(): Promise { + return [ + { label: getDataSourceLabel(), value: sourceTitle }, + { label: getUrlLabel(), value: this._descriptor.urlTemplate }, + ]; + } + + async getDisplayName(): Promise { + return this.getLayerName(); + } + + async getUrlTemplateWithMeta() { + return { + urlTemplate: this._descriptor.urlTemplate, + layerName: this._descriptor.layerName, + minSourceZoom: this._descriptor.minSourceZoom, + maxSourceZoom: this._descriptor.maxSourceZoom, + }; + } + + async getSupportedShapeTypes(): Promise { + return [VECTOR_SHAPE_TYPE.POINT, VECTOR_SHAPE_TYPE.LINE, VECTOR_SHAPE_TYPE.POLYGON]; + } + + canFormatFeatureProperties() { + return !!this._tooltipFields.length; + } + + getMinZoom() { + return this._descriptor.minSourceZoom; + } + + getMaxZoom() { + return this._descriptor.maxSourceZoom; + } + + getBoundsForFilters( + boundsFilters: BoundsFilters, + registerCancelCallback: (requestToken: symbol, callback: () => void) => void + ): MapExtent | null { + return null; + } + + getSyncMeta(): VectorSourceSyncMeta { + return null; + } + + getApplyGlobalQuery(): boolean { + return false; + } + + async filterAndFormatPropertiesToHtml( + properties: GeoJsonProperties, + featureId?: string | number + ): Promise { + const tooltips = []; + for (const key in properties) { + if (properties.hasOwnProperty(key)) { + for (let i = 0; i < this._tooltipFields.length; i++) { + const mvtField = this._tooltipFields[i]; + if (mvtField.getName() === key) { + const tooltip = new TooltipProperty(key, key, properties[key]); + tooltips.push(tooltip); + break; + } + } + } + } + return tooltips; + } +} + +registerSource({ + ConstructorFunction: MVTSingleLayerVectorSource, + type: SOURCE_TYPES.MVT_SINGLE_LAYER, +}); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor.test.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor.test.tsx new file mode 100644 index 00000000000000..986756f8400145 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor.test.tsx @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +jest.mock('../../../kibana_services', () => ({})); + +import React from 'react'; +import { shallow } from 'enzyme'; + +import { MVTSingleLayerVectorSourceEditor } from './mvt_single_layer_vector_source_editor'; + +test('should render source creation editor (fields should _not_ be included)', async () => { + const component = shallow( {}} />); + + expect(component).toMatchSnapshot(); +}); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor.tsx index 760b8c676cb37e..49487e96a45440 100644 --- a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor.tsx +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source_editor.tsx @@ -10,17 +10,14 @@ import _ from 'lodash'; import { EuiFieldText, EuiFormRow, EuiPanel } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { MAX_ZOOM, MIN_ZOOM } from '../../../../common/constants'; -import { ValidatedDualRange, Value } from '../../../../../../../src/plugins/kibana_react/public'; +import { + MVTFieldDescriptor, + TiledSingleLayerVectorSourceSettings, +} from '../../../../common/descriptor_types'; +import { MVTSingleLayerSourceSettings } from './mvt_single_layer_source_settings'; -export type MVTSingleLayerVectorSourceConfig = { - urlTemplate: string; - layerName: string; - minSourceZoom: number; - maxSourceZoom: number; -}; - -export interface Props { - onSourceConfigChange: (sourceConfig: MVTSingleLayerVectorSourceConfig) => void; +interface Props { + onSourceConfigChange: (sourceConfig: TiledSingleLayerVectorSourceSettings) => void; } interface State { @@ -28,6 +25,7 @@ interface State { layerName: string; minSourceZoom: number; maxSourceZoom: number; + fields?: MVTFieldDescriptor[]; } export class MVTSingleLayerVectorSourceEditor extends Component { @@ -36,6 +34,7 @@ export class MVTSingleLayerVectorSourceEditor extends Component { layerName: '', minSourceZoom: MIN_ZOOM, maxSourceZoom: MAX_ZOOM, + fields: [], }; _sourceConfigChange = _.debounce(() => { @@ -50,6 +49,7 @@ export class MVTSingleLayerVectorSourceEditor extends Component { layerName: this.state.layerName, minSourceZoom: this.state.minSourceZoom, maxSourceZoom: this.state.maxSourceZoom, + fields: this.state.fields, }); } }, 200); @@ -64,23 +64,13 @@ export class MVTSingleLayerVectorSourceEditor extends Component { ); }; - _handleLayerNameInputChange = (e: ChangeEvent) => { - const layerName = e.target.value; - this.setState( - { - layerName, - }, - () => this._sourceConfigChange() - ); - }; - - _handleZoomRangeChange = (e: Value) => { - const minSourceZoom = parseInt(e[0] as string, 10); - const maxSourceZoom = parseInt(e[1] as string, 10); - - if (this.state.minSourceZoom !== minSourceZoom || this.state.maxSourceZoom !== maxSourceZoom) { - this.setState({ minSourceZoom, maxSourceZoom }, () => this._sourceConfigChange()); - } + _handleChange = (state: { + layerName: string; + fields: MVTFieldDescriptor[]; + minSourceZoom: number; + maxSourceZoom: number; + }) => { + this.setState(state, () => this._sourceConfigChange()); }; render() { @@ -90,37 +80,30 @@ export class MVTSingleLayerVectorSourceEditor extends Component { label={i18n.translate('xpack.maps.source.MVTSingleLayerVectorSourceEditor.urlMessage', { defaultMessage: 'Url', })} - > - - - - + - ); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/types.ts b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/types.ts new file mode 100644 index 00000000000000..599eaea73c9a01 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/types.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { MVTFieldDescriptor } from '../../../../common/descriptor_types'; + +export interface MVTSingleLayerVectorSourceConfig { + urlTemplate: string; + layerName: string; + minSourceZoom: number; + maxSourceZoom: number; + fields?: MVTFieldDescriptor[]; + tooltipProperties?: string[]; +} diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/update_source_editor.test.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/update_source_editor.test.tsx new file mode 100644 index 00000000000000..fd19379058e3b5 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/update_source_editor.test.tsx @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +jest.mock('../../../kibana_services', () => ({})); + +import React from 'react'; +import { shallow } from 'enzyme'; + +import { UpdateSourceEditor } from './update_source_editor'; +import { MVTSingleLayerVectorSource } from './mvt_single_layer_vector_source'; +import { TiledSingleLayerVectorSourceDescriptor } from '../../../../common/descriptor_types'; +import { SOURCE_TYPES } from '../../../../common/constants'; + +const descriptor: TiledSingleLayerVectorSourceDescriptor = { + type: SOURCE_TYPES.MVT_SINGLE_LAYER, + urlTemplate: 'https://example.com/{x}/{y}/{z}.pbf', + layerName: 'foobar', + minSourceZoom: 4, + maxSourceZoom: 14, + fields: [], + tooltipProperties: [], +}; + +test('should render update source editor (fields _should_ be included)', async () => { + const source = new MVTSingleLayerVectorSource(descriptor); + + const component = shallow( + {}} /> + ); + + expect(component).toMatchSnapshot(); +}); diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/update_source_editor.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/update_source_editor.tsx new file mode 100644 index 00000000000000..a959912718197e --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/update_source_editor.tsx @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Component, Fragment } from 'react'; +import { EuiTitle, EuiPanel, EuiSpacer } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { TooltipSelector } from '../../../components/tooltip_selector'; +import { MVTField } from '../../fields/mvt_field'; +import { MVTSingleLayerVectorSource } from './mvt_single_layer_vector_source'; +import { MVTSettings, MVTSingleLayerSourceSettings } from './mvt_single_layer_source_settings'; +import { OnSourceChangeArgs } from '../../../connected_components/layer_panel/view'; +import { MVTFieldDescriptor } from '../../../../common/descriptor_types'; + +interface Props { + tooltipFields: MVTField[]; + onChange: (...args: OnSourceChangeArgs[]) => void; + source: MVTSingleLayerVectorSource; +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +interface State {} + +export class UpdateSourceEditor extends Component { + _onTooltipPropertiesSelect = (propertyNames: string[]) => { + this.props.onChange({ propName: 'tooltipProperties', value: propertyNames }); + }; + + _handleChange = (settings: MVTSettings) => { + const changes: OnSourceChangeArgs[] = []; + if (settings.layerName !== this.props.source.getLayerName()) { + changes.push({ propName: 'layerName', value: settings.layerName }); + } + if (settings.minSourceZoom !== this.props.source.getMinZoom()) { + changes.push({ propName: 'minSourceZoom', value: settings.minSourceZoom }); + } + if (settings.maxSourceZoom !== this.props.source.getMaxZoom()) { + changes.push({ propName: 'maxSourceZoom', value: settings.maxSourceZoom }); + } + if (!_.isEqual(settings.fields, this._getFieldDescriptors())) { + changes.push({ propName: 'fields', value: settings.fields }); + + // Remove dangling tooltips. + // This behaves similar to how stale styling properties are removed (e.g. on metric-change in agg sources) + const sanitizedTooltips = []; + for (let i = 0; i < this.props.tooltipFields.length; i++) { + const tooltipName = this.props.tooltipFields[i].getName(); + for (let j = 0; j < settings.fields.length; j++) { + if (settings.fields[j].name === tooltipName) { + sanitizedTooltips.push(tooltipName); + break; + } + } + } + + if (!_.isEqual(sanitizedTooltips, this.props.tooltipFields)) { + changes.push({ propName: 'tooltipProperties', value: sanitizedTooltips }); + } + } + this.props.onChange(...changes); + }; + + _getFieldDescriptors(): MVTFieldDescriptor[] { + return this.props.source.getMVTFields().map((field: MVTField) => { + return field.getMVTFieldDescriptor(); + }); + } + + _renderSourceSettingsCard() { + const fieldDescriptors: MVTFieldDescriptor[] = this._getFieldDescriptors(); + return ( + + + +
+ +
+ + + + + + + + ); + } + + _renderTooltipSelectionCard() { + return ( + + + +
+ +
+ + + + + + + + + + ); + } + + render() { + return ( + + {this._renderSourceSettingsCard()} + {this._renderTooltipSelectionCard()} + + ); + } +} diff --git a/x-pack/plugins/maps/public/classes/sources/source.ts b/x-pack/plugins/maps/public/classes/sources/source.ts index f937eac336532c..c68e22ada8b0c7 100644 --- a/x-pack/plugins/maps/public/classes/sources/source.ts +++ b/x-pack/plugins/maps/public/classes/sources/source.ts @@ -17,7 +17,7 @@ import { MAX_ZOOM, MIN_ZOOM } from '../../../common/constants'; import { OnSourceChangeArgs } from '../../connected_components/layer_panel/view'; export type SourceEditorArgs = { - onChange: (args: OnSourceChangeArgs) => void; + onChange: (...args: OnSourceChangeArgs[]) => void; }; export type ImmutableSourceProperty = { diff --git a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.d.ts b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.d.ts index 99a7478cd83626..42993bf36f6186 100644 --- a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.d.ts +++ b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.d.ts @@ -5,7 +5,7 @@ */ /* eslint-disable @typescript-eslint/consistent-type-definitions */ -import { FeatureCollection } from 'geojson'; +import { FeatureCollection, GeoJsonProperties, Geometry } from 'geojson'; import { Filter, TimeRange } from 'src/plugins/data/public'; import { AbstractSource, ISource } from '../source'; import { IField } from '../../fields/field'; @@ -35,7 +35,7 @@ export type BoundsFilters = { }; export interface IVectorSource extends ISource { - filterAndFormatPropertiesToHtml(properties: unknown): Promise; + filterAndFormatPropertiesToHtml(properties: GeoJsonProperties): Promise; getBoundsForFilters( boundsFilters: BoundsFilters, registerCancelCallback: (requestToken: symbol, callback: () => void) => void @@ -51,10 +51,12 @@ export interface IVectorSource extends ISource { getSyncMeta(): VectorSourceSyncMeta; getFieldNames(): string[]; getApplyGlobalQuery(): boolean; + createField({ fieldName }: { fieldName: string }): IField; + canFormatFeatureProperties(): boolean; } export class AbstractVectorSource extends AbstractSource implements IVectorSource { - filterAndFormatPropertiesToHtml(properties: unknown): Promise; + filterAndFormatPropertiesToHtml(properties: GeoJsonProperties): Promise; getBoundsForFilters( boundsFilters: BoundsFilters, registerCancelCallback: (requestToken: symbol, callback: () => void) => void @@ -72,6 +74,7 @@ export class AbstractVectorSource extends AbstractSource implements IVectorSourc canFormatFeatureProperties(): boolean; getApplyGlobalQuery(): boolean; getFieldNames(): string[]; + createField({ fieldName }: { fieldName: string }): IField; } export interface ITiledSingleLayerVectorSource extends IVectorSource { @@ -83,4 +86,5 @@ export interface ITiledSingleLayerVectorSource extends IVectorSource { }>; getMinZoom(): number; getMaxZoom(): number; + getLayerName(): string; } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/color/color_map_select.js b/x-pack/plugins/maps/public/classes/styles/vector/components/color/color_map_select.js index b7a80562f10cac..fe2f302504a154 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/color/color_map_select.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/color/color_map_select.js @@ -89,7 +89,7 @@ export class ColorMapSelect extends Component { }; _renderColorStopsInput() { - if (!this.props.useCustomColorMap) { + if (!this.props.isCustomOnly && !this.props.useCustomColorMap) { return null; } @@ -102,7 +102,7 @@ export class ColorMapSelect extends Component { swatches={this.props.swatches} /> ); - } else + } else { colorStopEditor = ( ); + } return ( @@ -121,6 +122,10 @@ export class ColorMapSelect extends Component { } _renderColorMapSelections() { + if (this.props.isCustomOnly) { + return null; + } + const colorMapOptionsWithCustom = [ { value: CUSTOM_COLOR_MAP, @@ -146,19 +151,22 @@ export class ColorMapSelect extends Component { ) : null; return ( - - {toggle} - - - - + + + {toggle} + + + + + + ); } @@ -166,7 +174,6 @@ export class ColorMapSelect extends Component { return ( {this._renderColorMapSelections()} - {this._renderColorStopsInput()} ); diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.js b/x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.js index fa13e1cf66664d..90070343a1b48c 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/color/dynamic_color_form.js @@ -90,6 +90,7 @@ export function DynamicColorForm({ if (styleProperty.isOrdinal()) { return ( { + const field = fields.find((field) => { return field.name === selectedFieldName; }); + //Do not spread in all the other unused values (e.g. type, supportsAutoDomain etc...) + if (field) { + selectedOption = { + value: field.value, + label: field.label, + }; + } } return ( diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/style_map_select.js b/x-pack/plugins/maps/public/classes/styles/vector/components/style_map_select.js index e285d91dcd7a4f..e4dc9d1b4d8f6a 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/style_map_select.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/style_map_select.js @@ -46,19 +46,16 @@ export class StyleMapSelect extends Component { }; _renderCustomStopsInput() { - if (!this.props.useCustomMap) { + return !this.props.isCustomOnly && !this.props.useCustomMap + ? null + : this.props.renderCustomStopsInput(this._onCustomMapChange); + } + + _renderMapSelect() { + if (this.props.isCustomOnly) { return null; } - return ( - - - {this.props.renderCustomStopsInput(this._onCustomMapChange)} - - ); - } - - render() { const mapOptionsWithCustom = [ { value: CUSTOM_MAP, @@ -87,6 +84,15 @@ export class StyleMapSelect extends Component { hasDividers={true} compressed /> + + + ); + } + + render() { + return ( + + {this._renderMapSelect()} {this._renderCustomStopsInput()} ); diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/symbol/dynamic_icon_form.js b/x-pack/plugins/maps/public/classes/styles/vector/components/symbol/dynamic_icon_form.js index f9f8a67846470c..e3724d42a783b7 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/symbol/dynamic_icon_form.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/symbol/dynamic_icon_form.js @@ -36,17 +36,20 @@ export function DynamicIconForm({ }; function renderIconMapSelect() { - if (!styleOptions.field || !styleOptions.field.name) { + const field = styleProperty.getField(); + if (!field) { return null; } return ( ); } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/symbol/icon_map_select.js b/x-pack/plugins/maps/public/classes/styles/vector/components/symbol/icon_map_select.js index 08f5dfe4f4ba0f..6cfe656d65a1e3 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/symbol/icon_map_select.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/symbol/icon_map_select.js @@ -8,8 +8,8 @@ import React from 'react'; import { StyleMapSelect } from '../style_map_select'; import { i18n } from '@kbn/i18n'; -import { getIconPaletteOptions } from '../../symbol_utils'; import { IconStops } from './icon_stops'; +import { getIconPaletteOptions } from '../../symbol_utils'; export function IconMapSelect({ customIconStops, @@ -19,6 +19,7 @@ export function IconMapSelect({ styleProperty, symbolOptions, useCustomIconMap, + isCustomOnly, }) { function onMapSelectChange({ customMapStops, selectedMapId, useCustomMap }) { onChange({ @@ -52,6 +53,7 @@ export function IconMapSelect({ useCustomMap={useCustomIconMap} selectedMapId={iconPaletteId} renderCustomStopsInput={renderCustomIconStopsInput} + isCustomOnly={isCustomOnly} /> ); } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js b/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js index 7856a4ddaff395..6528648eff552b 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js @@ -62,6 +62,7 @@ export class VectorStyleEditor extends Component { name: field.getName(), origin: field.getOrigin(), type: await field.getDataType(), + supportsAutoDomain: field.supportsAutoDomain(), }; }; @@ -109,7 +110,9 @@ export class VectorStyleEditor extends Component { } _getOrdinalFields() { - return [...this.state.dateFields, ...this.state.numberFields]; + return [...this.state.dateFields, ...this.state.numberFields].filter((field) => { + return field.supportsAutoDomain; + }); } _handleSelectedFeatureChange = (selectedFeature) => { diff --git a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_orientation_property.js b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_orientation_property.js index ae4d935e2457b7..763eb81ad0f98a 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_orientation_property.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_orientation_property.js @@ -10,11 +10,10 @@ import { VECTOR_STYLES } from '../../../../../common/constants'; export class DynamicOrientationProperty extends DynamicStyleProperty { syncIconRotationWithMb(symbolLayerId, mbMap) { - if (this._options.field && this._options.field.name) { - const targetName = getComputedFieldName( - VECTOR_STYLES.ICON_ORIENTATION, - this._options.field.name - ); + if (this._field && this._field.isValid()) { + const targetName = this._field.supportsAutoDomain() + ? getComputedFieldName(VECTOR_STYLES.ICON_ORIENTATION, this.getFieldName()) + : this._field.getName(); // Using property state instead of feature-state because layout properties do not support feature-state mbMap.setLayoutProperty(symbolLayerId, 'icon-rotate', ['coalesce', ['get', targetName], 0]); } else { diff --git a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_text_property.js b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_text_property.js index de868f3f926506..a7a3130875a955 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_text_property.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/properties/dynamic_text_property.js @@ -10,7 +10,11 @@ import { getComputedFieldName } from '../style_util'; export class DynamicTextProperty extends DynamicStyleProperty { syncTextFieldWithMb(mbLayerId, mbMap) { if (this._field && this._field.isValid()) { - const targetName = getComputedFieldName(this._styleName, this._options.field.name); + // Fields that support auto-domain are normalized with a field-formatter and stored into a computed-field + // Otherwise, the raw value is just carried over and no computed field is created. + const targetName = this._field.supportsAutoDomain() + ? getComputedFieldName(this._styleName, this.getFieldName()) + : this._field.getName(); mbMap.setLayoutProperty(mbLayerId, 'text-field', ['coalesce', ['get', targetName], '']); } else { mbMap.setLayoutProperty(mbLayerId, 'text-field', null); diff --git a/x-pack/plugins/maps/public/classes/tooltips/tooltip_property.ts b/x-pack/plugins/maps/public/classes/tooltips/tooltip_property.ts index 7149fe29f90ecc..7bb79d8d341d35 100644 --- a/x-pack/plugins/maps/public/classes/tooltips/tooltip_property.ts +++ b/x-pack/plugins/maps/public/classes/tooltips/tooltip_property.ts @@ -19,7 +19,7 @@ export interface ITooltipProperty { export interface LoadFeatureProps { layerId: string; - featureId: number; + featureId?: number | string; } export interface FeatureGeometry { diff --git a/x-pack/plugins/maps/public/connected_components/layer_panel/view.js b/x-pack/plugins/maps/public/connected_components/layer_panel/view.js index 14252dcfc067d5..557fe5fd5f7055 100644 --- a/x-pack/plugins/maps/public/connected_components/layer_panel/view.js +++ b/x-pack/plugins/maps/public/connected_components/layer_panel/view.js @@ -43,16 +43,16 @@ export class LayerPanel extends React.Component { componentDidMount() { this._isMounted = true; - this.loadDisplayName(); - this.loadImmutableSourceProperties(); - this.loadLeftJoinFields(); + this._loadDisplayName(); + this._loadImmutableSourceProperties(); + this._loadLeftJoinFields(); } componentWillUnmount() { this._isMounted = false; } - loadDisplayName = async () => { + _loadDisplayName = async () => { if (!this.props.selectedLayer) { return; } @@ -63,7 +63,7 @@ export class LayerPanel extends React.Component { } }; - loadImmutableSourceProperties = async () => { + _loadImmutableSourceProperties = async () => { if (!this.props.selectedLayer) { return; } @@ -74,7 +74,7 @@ export class LayerPanel extends React.Component { } }; - async loadLeftJoinFields() { + async _loadLeftJoinFields() { if (!this.props.selectedLayer || !this.props.selectedLayer.isJoinable()) { return; } @@ -97,8 +97,11 @@ export class LayerPanel extends React.Component { } } - _onSourceChange = ({ propName, value, newLayerType }) => { - this.props.updateSourceProp(this.props.selectedLayer.getId(), propName, value, newLayerType); + _onSourceChange = (...args) => { + for (let i = 0; i < args.length; i++) { + const { propName, value, newLayerType } = args[i]; + this.props.updateSourceProp(this.props.selectedLayer.getId(), propName, value, newLayerType); + } }; _renderFilterSection() { diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_properties.js b/x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_properties.js index 362186a8f5549b..5e2a153b2ccbfb 100644 --- a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_properties.js +++ b/x-pack/plugins/maps/public/connected_components/map/features_tooltip/feature_properties.js @@ -31,14 +31,15 @@ export class FeatureProperties extends React.Component { this._isMounted = false; } - _loadProperties = () => { + _loadProperties = async () => { this._fetchProperties({ nextFeatureId: this.props.featureId, nextLayerId: this.props.layerId, + mbProperties: this.props.mbProperties, }); }; - _fetchProperties = async ({ nextLayerId, nextFeatureId }) => { + _fetchProperties = async ({ nextLayerId, nextFeatureId, mbProperties }) => { if (this.prevLayerId === nextLayerId && this.prevFeatureId === nextFeatureId) { // do not reload same feature properties return; @@ -64,6 +65,7 @@ export class FeatureProperties extends React.Component { properties = await this.props.loadFeatureProperties({ layerId: nextLayerId, featureId: nextFeatureId, + mbProperties: mbProperties, }); } catch (error) { if (this._isMounted) { diff --git a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/features_tooltip.js b/x-pack/plugins/maps/public/connected_components/map/features_tooltip/features_tooltip.js index e5b97947602b0d..d91bc8e803ab9e 100644 --- a/x-pack/plugins/maps/public/connected_components/map/features_tooltip/features_tooltip.js +++ b/x-pack/plugins/maps/public/connected_components/map/features_tooltip/features_tooltip.js @@ -132,6 +132,7 @@ export class FeaturesTooltip extends React.Component { { sinon.assert.notCalled(closeOnClickTooltipStub); sinon.assert.calledWith(openOnClickTooltipStub, { - features: [{ id: 1, layerId: 'tfi3f' }], + features: [{ id: 1, layerId: 'tfi3f', mbProperties: { __kbn__feature_id__: 1 } }], location: [100, 30], }); }); diff --git a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.js b/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.js index 03c2aeb2edd0a8..6c420576804084 100644 --- a/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.js +++ b/x-pack/plugins/maps/public/connected_components/map/mb/tooltip_control/tooltip_popover.js @@ -58,7 +58,7 @@ export class TooltipPopover extends Component { // Mapbox feature geometry is from vector tile and is not the same as the original geometry. _loadFeatureGeometry = ({ layerId, featureId }) => { const tooltipLayer = this._findLayerById(layerId); - if (!tooltipLayer) { + if (!tooltipLayer || typeof featureId === 'undefined') { return null; } @@ -70,22 +70,24 @@ export class TooltipPopover extends Component { return targetFeature.geometry; }; - _loadFeatureProperties = async ({ layerId, featureId }) => { + _loadFeatureProperties = async ({ layerId, featureId, mbProperties }) => { const tooltipLayer = this._findLayerById(layerId); if (!tooltipLayer) { return []; } - const targetFeature = tooltipLayer.getFeatureById(featureId); - if (!targetFeature) { - return []; + let targetFeature; + if (typeof featureId !== 'undefined') { + targetFeature = tooltipLayer.getFeatureById(featureId); } - return await tooltipLayer.getPropertiesForTooltip(targetFeature.properties); + + const properties = targetFeature ? targetFeature.properties : mbProperties; + return await tooltipLayer.getPropertiesForTooltip(properties); }; _loadPreIndexedShape = async ({ layerId, featureId }) => { const tooltipLayer = this._findLayerById(layerId); - if (!tooltipLayer) { + if (!tooltipLayer || typeof featureId === 'undefined') { return null; } diff --git a/x-pack/plugins/security_solution/public/network/components/embeddables/map_tool_tip/map_tool_tip.test.tsx b/x-pack/plugins/security_solution/public/network/components/embeddables/map_tool_tip/map_tool_tip.test.tsx index ff6e8859be0494..98d4d3bd8faba7 100644 --- a/x-pack/plugins/security_solution/public/network/components/embeddables/map_tool_tip/map_tool_tip.test.tsx +++ b/x-pack/plugins/security_solution/public/network/components/embeddables/map_tool_tip/map_tool_tip.test.tsx @@ -7,7 +7,7 @@ import { shallow } from 'enzyme'; import React from 'react'; import { MapToolTipComponent } from './map_tool_tip'; -import { MapFeature } from '../types'; +import { TooltipFeature } from '../../../../../../maps/common/descriptor_types'; describe('MapToolTip', () => { test('placeholder component renders correctly against snapshot', () => { @@ -18,10 +18,11 @@ describe('MapToolTip', () => { test('full component renders correctly against snapshot', () => { const addFilters = jest.fn(); const closeTooltip = jest.fn(); - const features: MapFeature[] = [ + const features: TooltipFeature[] = [ { id: 1, layerId: 'layerId', + mbProperties: {}, }, ]; const getLayerName = jest.fn(); diff --git a/x-pack/plugins/security_solution/public/network/components/embeddables/types.ts b/x-pack/plugins/security_solution/public/network/components/embeddables/types.ts index f91fd677ba7fe8..e3ca3c5b842898 100644 --- a/x-pack/plugins/security_solution/public/network/components/embeddables/types.ts +++ b/x-pack/plugins/security_solution/public/network/components/embeddables/types.ts @@ -36,11 +36,6 @@ export type SetQuery = (params: { refetch: inputsModel.Refetch; }) => void; -export interface MapFeature { - id: number; - layerId: string; -} - export interface FeatureGeometry { coordinates: [number]; type: string; diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 074e93e10fd12b..b5cdad5583e1d5 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9214,9 +9214,6 @@ "xpack.maps.source.kbnTMSDescription": "kibana.yml で構成されたマップタイルです", "xpack.maps.source.kbnTMSTitle": "カスタムタイルマップサービス", "xpack.maps.source.mapSettingsPanel.initialLocationLabel": "マップの初期位置情報", - "xpack.maps.source.MVTSingleLayerVectorSource.layerNameMessage": "レイヤー名", - "xpack.maps.source.MVTSingleLayerVectorSource.maxZoomMessage": "最大ズーム", - "xpack.maps.source.MVTSingleLayerVectorSource.minZoomMessage": "最小ズーム", "xpack.maps.source.MVTSingleLayerVectorSource.sourceTitle": "ベトルタイルレイヤー", "xpack.maps.source.MVTSingleLayerVectorSourceEditor.dataZoomRangeMessage": "ズームレベル", "xpack.maps.source.MVTSingleLayerVectorSourceEditor.layerNameMessage": "レイヤー名", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 9d0bd95526670c..568397912141d2 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9218,9 +9218,6 @@ "xpack.maps.source.kbnTMSDescription": "在 kibana.yml 中配置的地图磁贴", "xpack.maps.source.kbnTMSTitle": "定制磁贴地图服务", "xpack.maps.source.mapSettingsPanel.initialLocationLabel": "初始地图位置", - "xpack.maps.source.MVTSingleLayerVectorSource.layerNameMessage": "图层名称", - "xpack.maps.source.MVTSingleLayerVectorSource.maxZoomMessage": "最大缩放", - "xpack.maps.source.MVTSingleLayerVectorSource.minZoomMessage": "最小缩放", "xpack.maps.source.MVTSingleLayerVectorSource.sourceTitle": "矢量磁贴图层", "xpack.maps.source.MVTSingleLayerVectorSourceEditor.dataZoomRangeMessage": "缩放级别", "xpack.maps.source.MVTSingleLayerVectorSourceEditor.layerNameMessage": "图层名称", diff --git a/x-pack/test/functional/apps/maps/documents_source/search_hits.js b/x-pack/test/functional/apps/maps/documents_source/search_hits.js index 68d3c2536ee0b6..5d75679432c979 100644 --- a/x-pack/test/functional/apps/maps/documents_source/search_hits.js +++ b/x-pack/test/functional/apps/maps/documents_source/search_hits.js @@ -76,7 +76,11 @@ export default function ({ getPageObjects, getService }) { const { lat, lon, zoom } = await PageObjects.maps.getView(); expect(Math.round(lat)).to.equal(41); expect(Math.round(lon)).to.equal(-102); - expect(Math.round(zoom)).to.equal(5); + + // Centering is correct, but screen-size and dpi affect zoom level, + // causing this test to be brittle in different environments + // Expecting zoom-level to be between ]4,5] + expect(Math.ceil(zoom)).to.equal(5); }); }); From 0066c4b5b05879328e247fa5d6576ea3322c8ed2 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Thu, 2 Jul 2020 15:38:24 +0200 Subject: [PATCH 22/31] [S&R] Support data streams (#68078) * Sort endpoint responses into indices and datastreams The server endpoint for policies now returns data streams and filters out backing indices from the indices array it returned previously * Refactor indices switch and field out of the step settings file * Fix indices field form behaviour * WiP on UI. Added the second table per mockup for add and edit. * add support for creating a policy that backs up data streams end to end * wip on restore flow - added data streams to server response * add logic for detecting whether an index is part of a data stream * fix public jest tests * fix server side jest tests * pivot to different solution in UI while we do not have data streams nicely separated * added data stream to snapshot summary details * move the data streams badge file closer to where it used * add data stream badge when restoring snapshots too * update restore copy * fix pattern specification in indices and data streams field * first iteration of complete policy UX * First iteration that is ready for review Given the contraints on working with data streams and indices in policies at the moment the simplest implementation is to just include data streams with indices and have the user select them there for now. The way snapshotting behaviour is currently implemented relies entirely on what is specified inside of "indices", this is also where data streams must be placed. This unfortunately means that capture patterns defined in indices will capture entire data streams too. * delete unused import * fix type issue in tests * added logic for rendering out previous selection as custom pattern * refactor indices fields to make component smaller * added CIT for data streams badge * Data streams > indices * updates to relevant pieces of copy * more copy updates * fix types and remove unused import * removed backing indices from restore view * Added data stream restore warning message * restore CITs * first round of copy feedback * refactor help text to provide clearer feedback, for both restore and policy forms * Restore updates - added spacer between title and data streams callout - added copy to the restore settings tab to indicate that settings also apply to backing indices * further copy refinements * second round of copy feedback * fix i18n * added comment to mock * line spacing fixes and created issue for tracking backing index discovery in snaphots * refactor collapsible list logic and tests * refactor editing managed policy check * refactor copy to be clearer about pluralisation of data streams * refactor file structure in components for data stream badge * added tests for indices and data streams field helper * refactored types and fixed i18n id per guidelines Co-authored-by: Elastic Machine --- .../client_integration/helpers/index.ts | 4 +- .../client_integration/helpers/mocks.tsx | 22 + .../helpers/policy_form.helpers.ts | 3 + .../helpers/restore_snapshot.helpers.ts | 51 ++ .../helpers/setup_environment.tsx | 8 + .../client_integration/policy_add.test.ts | 29 +- .../client_integration/policy_edit.test.ts | 5 +- .../restore_snapshot.test.ts | 49 ++ .../snapshot_restore/common/lib/index.ts | 2 + .../lib/is_data_stream_backing_index.ts | 23 + .../common/lib/snapshot_serialization.test.ts | 1 + .../common/lib/snapshot_serialization.ts | 8 +- .../snapshot_restore/common/lib/utils.ts | 13 + .../snapshot_restore/common/types/index.ts | 1 + .../snapshot_restore/common/types/indices.ts | 10 + .../snapshot_restore/common/types/snapshot.ts | 2 + .../components/collapsible_indices_list.tsx | 76 --- .../collapsible_data_streams_list.tsx | 67 +++ .../collapsible_indices_list.tsx | 66 +++ .../components/collapsible_lists/index.ts | 8 + .../use_collapsible_list.test.ts | 29 ++ .../collapsible_lists/use_collapsible_list.ts | 42 ++ .../components/data_stream_badge.tsx | 18 + .../public/application/components/index.ts | 2 +- .../application/components/lib/helpers.ts | 15 + .../application/components/lib/index.ts | 7 + .../components/policy_form/policy_form.tsx | 9 +- .../components/policy_form/steps/index.ts | 1 + .../policy_form/steps/step_review.tsx | 10 +- .../policy_form/steps/step_settings.tsx | 469 ------------------ .../steps/step_settings/fields/index.ts | 7 + ...ata_streams_and_indices_list_help_text.tsx | 79 +++ .../helpers.test.ts | 69 +++ .../helpers.tsx | 68 +++ .../indices_and_data_streams_field/index.ts | 7 + .../indices_and_data_streams_field.tsx | 348 +++++++++++++ .../policy_form/steps/step_settings/index.ts | 7 + .../steps/step_settings/step_settings.tsx | 206 ++++++++ .../restore_snapshot_form/steps/index.ts | 2 +- ...ata_streams_and_indices_list_help_text.tsx | 79 +++ .../data_streams_global_state_call_out.tsx | 56 +++ .../steps/step_logistics/index.ts | 7 + .../{ => step_logistics}/step_logistics.tsx | 251 ++++++---- .../steps/step_review.tsx | 6 +- .../steps/step_settings.tsx | 20 + .../policy_details/tabs/tab_summary.tsx | 4 +- .../snapshot_details/tabs/tab_summary.tsx | 18 + .../sections/policy_add/policy_add.tsx | 10 +- .../sections/policy_edit/policy_edit.tsx | 7 +- .../services/http/policy_requests.ts | 4 +- .../application/services/http/use_request.ts | 4 +- .../services/validation/validate_policy.ts | 3 +- .../services/validation/validate_restore.ts | 4 +- .../server/routes/api/policy.test.ts | 41 +- .../server/routes/api/policy.ts | 31 +- .../server/routes/api/snapshots.test.ts | 1 + .../plugins/snapshot_restore/server/types.ts | 16 + .../test/fixtures/snapshot.ts | 12 +- .../translations/translations/ja-JP.json | 18 - .../translations/translations/zh-CN.json | 18 - 60 files changed, 1716 insertions(+), 737 deletions(-) create mode 100644 x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/mocks.tsx create mode 100644 x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/restore_snapshot.helpers.ts create mode 100644 x-pack/plugins/snapshot_restore/__jest__/client_integration/restore_snapshot.test.ts create mode 100644 x-pack/plugins/snapshot_restore/common/lib/is_data_stream_backing_index.ts create mode 100644 x-pack/plugins/snapshot_restore/common/lib/utils.ts create mode 100644 x-pack/plugins/snapshot_restore/common/types/indices.ts delete mode 100644 x-pack/plugins/snapshot_restore/public/application/components/collapsible_indices_list.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/collapsible_data_streams_list.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/collapsible_indices_list.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/index.ts create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/use_collapsible_list.test.ts create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/use_collapsible_list.ts create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/data_stream_badge.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/lib/helpers.ts create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/lib/index.ts delete mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/index.ts create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/data_streams_and_indices_list_help_text.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/helpers.test.ts create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/helpers.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/index.ts create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/indices_and_data_streams_field.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/index.ts create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/step_settings.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/data_streams_and_indices_list_help_text.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/data_streams_global_state_call_out.tsx create mode 100644 x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/index.ts rename x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/{ => step_logistics}/step_logistics.tsx (69%) diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/index.ts b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/index.ts index 2f7b75dfba57e5..69d1423f5f8fb6 100644 --- a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/index.ts +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/index.ts @@ -3,12 +3,13 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import './mocks'; import { setup as homeSetup } from './home.helpers'; import { setup as repositoryAddSetup } from './repository_add.helpers'; import { setup as repositoryEditSetup } from './repository_edit.helpers'; import { setup as policyAddSetup } from './policy_add.helpers'; import { setup as policyEditSetup } from './policy_edit.helpers'; +import { setup as restoreSnapshotSetup } from './restore_snapshot.helpers'; export { nextTick, getRandomString, findTestSubject, TestBed } from '../../../../../test_utils'; @@ -20,4 +21,5 @@ export const pageHelpers = { repositoryEdit: { setup: repositoryEditSetup }, policyAdd: { setup: policyAddSetup }, policyEdit: { setup: policyEditSetup }, + restoreSnapshot: { setup: restoreSnapshotSetup }, }; diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/mocks.tsx b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/mocks.tsx new file mode 100644 index 00000000000000..fc02452e373088 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/mocks.tsx @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; + +/* + * Mocking AutoSizer of the react-virtualized because it does not render children in JS DOM. + * This seems related to not being able to properly discover height and width. + */ +jest.mock('react-virtualized', () => { + const original = jest.requireActual('react-virtualized'); + + return { + ...original, + AutoSizer: ({ children }: { children: any }) => ( +
{children({ height: 500, width: 500 })}
+ ), + }; +}); diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/policy_form.helpers.ts b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/policy_form.helpers.ts index 131969b997b532..a3ab829ab642cd 100644 --- a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/policy_form.helpers.ts +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/policy_form.helpers.ts @@ -41,6 +41,8 @@ export type PolicyFormTestSubjects = | 'allIndicesToggle' | 'backButton' | 'deselectIndicesLink' + | 'allDataStreamsToggle' + | 'deselectDataStreamLink' | 'expireAfterValueInput' | 'expireAfterUnitSelect' | 'ignoreUnavailableIndicesToggle' @@ -53,4 +55,5 @@ export type PolicyFormTestSubjects = | 'selectIndicesLink' | 'showAdvancedCronLink' | 'snapshotNameInput' + | 'dataStreamBadge' | 'submitButton'; diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/restore_snapshot.helpers.ts b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/restore_snapshot.helpers.ts new file mode 100644 index 00000000000000..0cfb6fbc979755 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/restore_snapshot.helpers.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* eslint-disable @kbn/eslint/no-restricted-paths */ +import { registerTestBed, TestBed, TestBedConfig } from '../../../../../test_utils'; +import { RestoreSnapshot } from '../../../public/application/sections/restore_snapshot'; +import { WithAppDependencies } from './setup_environment'; + +const testBedConfig: TestBedConfig = { + memoryRouter: { + initialEntries: ['/add_policy'], + componentRoutePath: '/add_policy', + }, + doMountAsync: true, +}; + +const initTestBed = registerTestBed( + WithAppDependencies(RestoreSnapshot), + testBedConfig +); + +const setupActions = (testBed: TestBed) => { + const { find } = testBed; + return { + findDataStreamCallout() { + return find('dataStreamWarningCallOut'); + }, + }; +}; + +type Actions = ReturnType; + +export type RestoreSnapshotTestBed = TestBed & { + actions: Actions; +}; + +export const setup = async (): Promise => { + const testBed = await initTestBed(); + + return { + ...testBed, + actions: setupActions(testBed), + }; +}; + +export type RestoreSnapshotFormTestSubject = + | 'snapshotRestoreStepLogistics' + | 'dataStreamWarningCallOut'; diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/setup_environment.tsx b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/setup_environment.tsx index c4f4876b8a1cdb..e3c0ab0be9bd23 100644 --- a/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/setup_environment.tsx +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/helpers/setup_environment.tsx @@ -64,6 +64,14 @@ export const setupEnvironment = () => { }; }; +/** + * Suppress error messages about Worker not being available in JS DOM. + */ +(window as any).Worker = function Worker() { + this.postMessage = () => {}; + this.terminate = () => {}; +}; + export const WithAppDependencies = (Comp: any) => (props: any) => ( diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_add.test.ts b/x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_add.test.ts index a8e6e976bb16da..17a745fafcc266 100644 --- a/x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_add.test.ts +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_add.test.ts @@ -3,11 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + +// import helpers first, this also sets up the mocks +import { setupEnvironment, pageHelpers, nextTick, getRandomString } from './helpers'; + import { act } from 'react-dom/test-utils'; import * as fixtures from '../../test/fixtures'; -import { setupEnvironment, pageHelpers, nextTick, getRandomString } from './helpers'; import { PolicyFormTestBed } from './helpers/policy_form.helpers'; import { DEFAULT_POLICY_SCHEDULE } from '../../public/application/constants'; @@ -37,7 +40,10 @@ describe('', () => { describe('on component mount', () => { beforeEach(async () => { httpRequestsMockHelpers.setLoadRepositoriesResponse({ repositories: [repository] }); - httpRequestsMockHelpers.setLoadIndicesResponse({ indices: ['my_index'] }); + httpRequestsMockHelpers.setLoadIndicesResponse({ + indices: ['my_index'], + dataStreams: ['my_data_stream', 'my_other_data_stream'], + }); testBed = await setup(); await nextTick(); @@ -96,7 +102,7 @@ describe('', () => { actions.clickNextButton(); }); - test('should require at least one index', async () => { + test('should require at least one index if no data streams are provided', async () => { const { find, form, component } = testBed; await act(async () => { @@ -109,7 +115,22 @@ describe('', () => { // Deselect all indices from list find('deselectIndicesLink').simulate('click'); - expect(form.getErrorsMessages()).toEqual(['You must select at least one index.']); + expect(form.getErrorsMessages()).toEqual([ + 'You must select at least one data stream or index.', + ]); + }); + + test('should correctly indicate data streams with a badge', async () => { + const { find, component, form } = testBed; + + await act(async () => { + // Toggle "All indices" switch + form.toggleEuiSwitch('allIndicesToggle', false); + await nextTick(); + }); + component.update(); + + expect(find('dataStreamBadge').length).toBe(2); }); }); diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_edit.test.ts b/x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_edit.test.ts index 297741755e88bf..7eec80890ca86a 100644 --- a/x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_edit.test.ts +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/policy_edit.test.ts @@ -35,7 +35,10 @@ describe('', () => { describe('on component mount', () => { beforeEach(async () => { httpRequestsMockHelpers.setGetPolicyResponse({ policy: POLICY_EDIT }); - httpRequestsMockHelpers.setLoadIndicesResponse({ indices: ['my_index'] }); + httpRequestsMockHelpers.setLoadIndicesResponse({ + indices: ['my_index'], + dataStreams: ['my_data_stream'], + }); httpRequestsMockHelpers.setLoadRepositoriesResponse({ repositories: [{ name: POLICY_EDIT.repository }], }); diff --git a/x-pack/plugins/snapshot_restore/__jest__/client_integration/restore_snapshot.test.ts b/x-pack/plugins/snapshot_restore/__jest__/client_integration/restore_snapshot.test.ts new file mode 100644 index 00000000000000..17d714c07429f4 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/__jest__/client_integration/restore_snapshot.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { nextTick, pageHelpers, setupEnvironment } from './helpers'; +import { RestoreSnapshotTestBed } from './helpers/restore_snapshot.helpers'; +import * as fixtures from '../../test/fixtures'; + +const { + restoreSnapshot: { setup }, +} = pageHelpers; + +describe('', () => { + const { server, httpRequestsMockHelpers } = setupEnvironment(); + let testBed: RestoreSnapshotTestBed; + + afterAll(() => { + server.restore(); + }); + describe('with data streams', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setGetSnapshotResponse(fixtures.getSnapshot()); + testBed = await setup(); + await nextTick(); + testBed.component.update(); + }); + + it('shows the data streams warning when the snapshot has data streams', () => { + const { exists } = testBed; + expect(exists('dataStreamWarningCallOut')).toBe(true); + }); + }); + + describe('without data streams', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setGetSnapshotResponse(fixtures.getSnapshot({ totalDataStreams: 0 })); + testBed = await setup(); + await nextTick(); + testBed.component.update(); + }); + + it('hides the data streams warning when the snapshot has data streams', () => { + const { exists } = testBed; + expect(exists('dataStreamWarningCallOut')).toBe(false); + }); + }); +}); diff --git a/x-pack/plugins/snapshot_restore/common/lib/index.ts b/x-pack/plugins/snapshot_restore/common/lib/index.ts index 579dae02659392..eaec8054a93abc 100644 --- a/x-pack/plugins/snapshot_restore/common/lib/index.ts +++ b/x-pack/plugins/snapshot_restore/common/lib/index.ts @@ -16,3 +16,5 @@ export { serializeSnapshotRetention, } from './snapshot_serialization'; export { deserializePolicy, serializePolicy } from './policy_serialization'; +export { csvToArray } from './utils'; +export { isDataStreamBackingIndex } from './is_data_stream_backing_index'; diff --git a/x-pack/plugins/snapshot_restore/common/lib/is_data_stream_backing_index.ts b/x-pack/plugins/snapshot_restore/common/lib/is_data_stream_backing_index.ts new file mode 100644 index 00000000000000..3b937670362f7b --- /dev/null +++ b/x-pack/plugins/snapshot_restore/common/lib/is_data_stream_backing_index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/** + * @remark + * WARNING! + * + * This is a very hacky way of determining whether an index is a backing index. + * + * We only do this so that we can show users during a snapshot restore workflow + * that an index is part of a data stream. At the moment there is no way for us + * to get this information from the snapshot itself, even though it contains the + * metadata for the data stream that information is fully opaque to us until after + * we have done the snapshot restore. + * + * Issue for tracking this discussion here: https://github.com/elastic/elasticsearch/issues/58890 + */ +export const isDataStreamBackingIndex = (indexName: string) => { + return indexName.startsWith('.ds'); +}; diff --git a/x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.test.ts b/x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.test.ts index 298fc235fd9cc7..473a3392deb3ee 100644 --- a/x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.test.ts +++ b/x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.test.ts @@ -97,6 +97,7 @@ describe('deserializeSnapshotDetails', () => { version: 'version', // Indices are sorted. indices: ['index1', 'index2', 'index3'], + dataStreams: [], includeGlobalState: false, // Failures are grouped and sorted by index, and the failures themselves are sorted by shard. indexFailures: [ diff --git a/x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.ts b/x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.ts index a636cc1f6326ec..a85b49430eecd5 100644 --- a/x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.ts +++ b/x-pack/plugins/snapshot_restore/common/lib/snapshot_serialization.ts @@ -17,6 +17,8 @@ import { import { deserializeTime, serializeTime } from './time_serialization'; +import { csvToArray } from './utils'; + export function deserializeSnapshotDetails( repository: string, snapshotDetailsEs: SnapshotDetailsEs, @@ -33,6 +35,7 @@ export function deserializeSnapshotDetails( version_id: versionId, version, indices = [], + data_streams: dataStreams = [], include_global_state: includeGlobalState, state, start_time: startTime, @@ -77,6 +80,7 @@ export function deserializeSnapshotDetails( versionId, version, indices: [...indices].sort(), + dataStreams: [...dataStreams].sort(), includeGlobalState, state, startTime, @@ -127,8 +131,10 @@ export function deserializeSnapshotConfig(snapshotConfigEs: SnapshotConfigEs): S export function serializeSnapshotConfig(snapshotConfig: SnapshotConfig): SnapshotConfigEs { const { indices, ignoreUnavailable, includeGlobalState, partial, metadata } = snapshotConfig; + const indicesArray = csvToArray(indices); + const snapshotConfigEs: SnapshotConfigEs = { - indices, + indices: indicesArray, ignore_unavailable: ignoreUnavailable, include_global_state: includeGlobalState, partial, diff --git a/x-pack/plugins/snapshot_restore/common/lib/utils.ts b/x-pack/plugins/snapshot_restore/common/lib/utils.ts new file mode 100644 index 00000000000000..96eb7cb6908d89 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/common/lib/utils.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const csvToArray = (indices?: string | string[]): string[] => { + return indices && Array.isArray(indices) + ? indices + : typeof indices === 'string' + ? indices.split(',') + : []; +}; diff --git a/x-pack/plugins/snapshot_restore/common/types/index.ts b/x-pack/plugins/snapshot_restore/common/types/index.ts index d52584ca737a2a..a12ae904cfee83 100644 --- a/x-pack/plugins/snapshot_restore/common/types/index.ts +++ b/x-pack/plugins/snapshot_restore/common/types/index.ts @@ -8,3 +8,4 @@ export * from './repository'; export * from './snapshot'; export * from './restore'; export * from './policy'; +export * from './indices'; diff --git a/x-pack/plugins/snapshot_restore/common/types/indices.ts b/x-pack/plugins/snapshot_restore/common/types/indices.ts new file mode 100644 index 00000000000000..5e4f2b5fdc1678 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/common/types/indices.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface PolicyIndicesResponse { + indices: string[]; + dataStreams: string[]; +} diff --git a/x-pack/plugins/snapshot_restore/common/types/snapshot.ts b/x-pack/plugins/snapshot_restore/common/types/snapshot.ts index a46f5c7921bfe0..1ff058e1553857 100644 --- a/x-pack/plugins/snapshot_restore/common/types/snapshot.ts +++ b/x-pack/plugins/snapshot_restore/common/types/snapshot.ts @@ -30,6 +30,7 @@ export interface SnapshotDetails { versionId: number; version: string; indices: string[]; + dataStreams: string[]; includeGlobalState: boolean; state: string; /** e.g. '2019-04-05T21:56:40.438Z' */ @@ -52,6 +53,7 @@ export interface SnapshotDetailsEs { version_id: number; version: string; indices: string[]; + data_streams?: string[]; include_global_state: boolean; state: string; /** e.g. '2019-04-05T21:56:40.438Z' */ diff --git a/x-pack/plugins/snapshot_restore/public/application/components/collapsible_indices_list.tsx b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_indices_list.tsx deleted file mode 100644 index 1d8ee726f4cc79..00000000000000 --- a/x-pack/plugins/snapshot_restore/public/application/components/collapsible_indices_list.tsx +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React, { useState } from 'react'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiTitle, EuiLink, EuiIcon, EuiText, EuiSpacer } from '@elastic/eui'; -interface Props { - indices: string[] | string | undefined; -} - -export const CollapsibleIndicesList: React.FunctionComponent = ({ indices }) => { - const [isShowingFullIndicesList, setIsShowingFullIndicesList] = useState(false); - const displayIndices = indices - ? typeof indices === 'string' - ? indices.split(',') - : indices - : undefined; - const hiddenIndicesCount = - displayIndices && displayIndices.length > 10 ? displayIndices.length - 10 : 0; - return ( - <> - {displayIndices ? ( - <> - -
    - {(isShowingFullIndicesList ? displayIndices : [...displayIndices].splice(0, 10)).map( - (index) => ( -
  • - - {index} - -
    • - ) - )} -
- - {hiddenIndicesCount ? ( - <> - - - isShowingFullIndicesList - ? setIsShowingFullIndicesList(false) - : setIsShowingFullIndicesList(true) - } - > - {isShowingFullIndicesList ? ( - - ) : ( - - )}{' '} - - - - ) : null} - - ) : ( - - )} - - ); -}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/collapsible_data_streams_list.tsx b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/collapsible_data_streams_list.tsx new file mode 100644 index 00000000000000..ce1bd7c8d6e45f --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/collapsible_data_streams_list.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiTitle, EuiLink, EuiIcon, EuiText, EuiSpacer } from '@elastic/eui'; + +import { useCollapsibleList } from './use_collapsible_list'; + +interface Props { + dataStreams: string[] | string | undefined; +} + +export const CollapsibleDataStreamsList: React.FunctionComponent = ({ dataStreams }) => { + const { isShowingFullList, setIsShowingFullList, items, hiddenItemsCount } = useCollapsibleList({ + items: dataStreams, + }); + + return items === 'all' ? ( + + ) : ( + <> + +
    + {items.map((dataStream) => ( +
  • + + {dataStream} + +
    • + ))} +
+ + {hiddenItemsCount ? ( + <> + + + isShowingFullList ? setIsShowingFullList(false) : setIsShowingFullList(true) + } + > + {isShowingFullList ? ( + + ) : ( + + )}{' '} + + + + ) : null} + + ); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/collapsible_indices_list.tsx b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/collapsible_indices_list.tsx new file mode 100644 index 00000000000000..ff676a36969418 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/collapsible_indices_list.tsx @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiTitle, EuiLink, EuiIcon, EuiText, EuiSpacer } from '@elastic/eui'; + +import { useCollapsibleList } from './use_collapsible_list'; + +interface Props { + indices: string[] | string | undefined; +} + +export const CollapsibleIndicesList: React.FunctionComponent = ({ indices }) => { + const { hiddenItemsCount, isShowingFullList, items, setIsShowingFullList } = useCollapsibleList({ + items: indices, + }); + return items === 'all' ? ( + + ) : ( + <> + +
    + {items.map((index) => ( +
  • + + {index} + +
    • + ))} +
+ + {hiddenItemsCount ? ( + <> + + + isShowingFullList ? setIsShowingFullList(false) : setIsShowingFullList(true) + } + > + {isShowingFullList ? ( + + ) : ( + + )}{' '} + + + + ) : null} + + ); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/index.ts new file mode 100644 index 00000000000000..d58edc983c5413 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { CollapsibleIndicesList } from './collapsible_indices_list'; +export { CollapsibleDataStreamsList } from './collapsible_data_streams_list'; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/use_collapsible_list.test.ts b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/use_collapsible_list.test.ts new file mode 100644 index 00000000000000..bdeb801117de91 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/use_collapsible_list.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { renderHook } from '@testing-library/react-hooks'; + +import { useCollapsibleList } from './use_collapsible_list'; + +describe('useCollapseList', () => { + it('handles undefined', () => { + const { result } = renderHook(() => useCollapsibleList({ items: undefined })); + expect(result.current.items).toBe('all'); + expect(result.current.hiddenItemsCount).toBe(0); + }); + + it('handles csv', () => { + const { result } = renderHook(() => useCollapsibleList({ items: 'a,b,c' })); + expect(result.current.items).toEqual(['a', 'b', 'c']); + expect(result.current.hiddenItemsCount).toBe(0); + }); + + it('hides items passed a defined maximum (10)', () => { + const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k']; + const { result } = renderHook(() => useCollapsibleList({ items })); + expect(result.current.items).toEqual(items.slice(0, -1)); + expect(result.current.hiddenItemsCount).toBe(1); + }); +}); diff --git a/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/use_collapsible_list.ts b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/use_collapsible_list.ts new file mode 100644 index 00000000000000..275915c5760afd --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/collapsible_lists/use_collapsible_list.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useState } from 'react'; +import { csvToArray } from '../../../../common/lib'; + +type ChildItems = string[] | 'all'; + +interface Arg { + items: string[] | string | undefined; +} + +export interface ReturnValue { + items: ChildItems; + hiddenItemsCount: number; + isShowingFullList: boolean; + setIsShowingFullList: (showAll: boolean) => void; +} + +const maximumItemPreviewCount = 10; + +export const useCollapsibleList = ({ items }: Arg): ReturnValue => { + const [isShowingFullList, setIsShowingFullList] = useState(false); + const itemsArray = csvToArray(items); + const displayItems: ChildItems = + items === undefined + ? 'all' + : itemsArray.slice(0, isShowingFullList ? Infinity : maximumItemPreviewCount); + + const hiddenItemsCount = + itemsArray.length > maximumItemPreviewCount ? itemsArray.length - maximumItemPreviewCount : 0; + + return { + items: displayItems, + hiddenItemsCount, + setIsShowingFullList, + isShowingFullList, + }; +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/data_stream_badge.tsx b/x-pack/plugins/snapshot_restore/public/application/components/data_stream_badge.tsx new file mode 100644 index 00000000000000..e7d3f59bd567a9 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/data_stream_badge.tsx @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { i18n } from '@kbn/i18n'; +import React, { FunctionComponent } from 'react'; +import { EuiBadge } from '@elastic/eui'; + +export const DataStreamBadge: FunctionComponent = () => { + return ( + + {i18n.translate('xpack.snapshotRestore.policyForm.setSettings.dataStreamBadgeContent', { + defaultMessage: 'Data stream', + })} + + ); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/index.ts index f5bb8923898702..91266aae66e278 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/index.ts +++ b/x-pack/plugins/snapshot_restore/public/application/components/index.ts @@ -15,7 +15,7 @@ export { SnapshotDeleteProvider } from './snapshot_delete_provider'; export { RestoreSnapshotForm } from './restore_snapshot_form'; export { PolicyExecuteProvider } from './policy_execute_provider'; export { PolicyDeleteProvider } from './policy_delete_provider'; -export { CollapsibleIndicesList } from './collapsible_indices_list'; +export { CollapsibleIndicesList, CollapsibleDataStreamsList } from './collapsible_lists'; export { RetentionSettingsUpdateModalProvider, UpdateRetentionSettings, diff --git a/x-pack/plugins/snapshot_restore/public/application/components/lib/helpers.ts b/x-pack/plugins/snapshot_restore/public/application/components/lib/helpers.ts new file mode 100644 index 00000000000000..f21576778a0ea2 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/lib/helpers.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const orderDataStreamsAndIndices = ({ + dataStreams, + indices, +}: { + dataStreams: D[]; + indices: D[]; +}) => { + return dataStreams.concat(indices); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/lib/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/lib/index.ts new file mode 100644 index 00000000000000..a40695e9a20e87 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/lib/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { orderDataStreamsAndIndices } from './helpers'; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/policy_form.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/policy_form.tsx index f9cad7cc4e0701..3e1fb9b6500b31 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/policy_form.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/policy_form.tsx @@ -27,6 +27,7 @@ import { PolicyNavigation } from './navigation'; interface Props { policy: SlmPolicyPayload; + dataStreams: string[]; indices: string[]; currentUrl: string; isEditing?: boolean; @@ -39,6 +40,7 @@ interface Props { export const PolicyForm: React.FunctionComponent = ({ policy: originalPolicy, + dataStreams, indices, currentUrl, isEditing, @@ -71,6 +73,8 @@ export const PolicyForm: React.FunctionComponent = ({ }, }); + const isEditingManagedPolicy = Boolean(isEditing && policy.isManagedPolicy); + // Policy validation state const [validation, setValidation] = useState({ isValid: true, @@ -132,6 +136,7 @@ export const PolicyForm: React.FunctionComponent = ({ = ({ {currentStep === lastStep ? ( savePolicy()} isLoading={isSaving} diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/index.ts index 8b251de80a8e1e..a79a6ecb42e459 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/index.ts +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/index.ts @@ -10,6 +10,7 @@ import { PolicyValidation } from '../../../services/validation'; export interface StepProps { policy: SlmPolicyPayload; indices: string[]; + dataStreams: string[]; updatePolicy: (updatedSettings: Partial, validationHelperData?: any) => void; isEditing: boolean; currentUrl: string; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_review.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_review.tsx index b2422be3b78c38..6b253a3fada057 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_review.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_review.tsx @@ -22,7 +22,7 @@ import { import { serializePolicy } from '../../../../../common/lib'; import { useServices } from '../../../app_context'; import { StepProps } from './'; -import { CollapsibleIndicesList } from '../../collapsible_indices_list'; +import { CollapsibleIndicesList } from '../../collapsible_lists'; export const PolicyStepReview: React.FunctionComponent = ({ policy, @@ -148,8 +148,8 @@ export const PolicyStepReview: React.FunctionComponent = ({ @@ -187,8 +187,8 @@ export const PolicyStepReview: React.FunctionComponent = ({ diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings.tsx deleted file mode 100644 index 07a62723123020..00000000000000 --- a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings.tsx +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import React, { Fragment, useState } from 'react'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { - EuiDescribedFormGroup, - EuiTitle, - EuiFormRow, - EuiFlexGroup, - EuiFlexItem, - EuiButtonEmpty, - EuiSpacer, - EuiSwitch, - EuiLink, - EuiSelectable, - EuiPanel, - EuiComboBox, - EuiToolTip, -} from '@elastic/eui'; -import { EuiSelectableOption } from '@elastic/eui'; -import { SlmPolicyPayload, SnapshotConfig } from '../../../../../common/types'; -import { documentationLinksService } from '../../../services/documentation'; -import { useServices } from '../../../app_context'; -import { StepProps } from './'; - -export const PolicyStepSettings: React.FunctionComponent = ({ - policy, - indices, - updatePolicy, - errors, -}) => { - const { i18n } = useServices(); - const { config = {}, isManagedPolicy } = policy; - - const updatePolicyConfig = (updatedFields: Partial): void => { - const newConfig = { ...config, ...updatedFields }; - updatePolicy({ - config: newConfig, - }); - }; - - // States for choosing all indices, or a subset, including caching previously chosen subset list - const [isAllIndices, setIsAllIndices] = useState(!Boolean(config.indices)); - const [indicesSelection, setIndicesSelection] = useState([...indices]); - const [indicesOptions, setIndicesOptions] = useState( - indices.map( - (index): EuiSelectableOption => ({ - label: index, - checked: - isAllIndices || - // If indices is a string, we default to custom input mode, so we mark individual indices - // as selected if user goes back to list mode - typeof config.indices === 'string' || - (Array.isArray(config.indices) && config.indices.includes(index)) - ? 'on' - : undefined, - }) - ) - ); - - // State for using selectable indices list or custom patterns - // Users with more than 100 indices will probably want to use an index pattern to select - // them instead, so we'll default to showing them the index pattern input. - const [selectIndicesMode, setSelectIndicesMode] = useState<'list' | 'custom'>( - typeof config.indices === 'string' || - (Array.isArray(config.indices) && config.indices.length > 100) - ? 'custom' - : 'list' - ); - - // State for custom patterns - const [indexPatterns, setIndexPatterns] = useState( - typeof config.indices === 'string' ? config.indices.split(',') : [] - ); - - const renderIndicesField = () => { - const indicesSwitch = ( - - } - checked={isAllIndices} - disabled={isManagedPolicy} - data-test-subj="allIndicesToggle" - onChange={(e) => { - const isChecked = e.target.checked; - setIsAllIndices(isChecked); - if (isChecked) { - updatePolicyConfig({ indices: undefined }); - } else { - updatePolicyConfig({ - indices: - selectIndicesMode === 'custom' - ? indexPatterns.join(',') - : [...(indicesSelection || [])], - }); - } - }} - /> - ); - - return ( - -

- -

- - } - description={ - - } - fullWidth - > - - - {isManagedPolicy ? ( - - -

- } - > - {indicesSwitch} - - ) : ( - indicesSwitch - )} - {isAllIndices ? null : ( - - - - - - - - { - setSelectIndicesMode('custom'); - updatePolicyConfig({ indices: indexPatterns.join(',') }); - }} - > - - - - - ) : ( - - - - - - { - setSelectIndicesMode('list'); - updatePolicyConfig({ indices: indicesSelection }); - }} - > - - - - - ) - } - helpText={ - selectIndicesMode === 'list' ? ( - 0 ? ( - { - // TODO: Change this to setIndicesOptions() when https://github.com/elastic/eui/issues/2071 is fixed - indicesOptions.forEach((option: EuiSelectableOption) => { - option.checked = undefined; - }); - updatePolicyConfig({ indices: [] }); - setIndicesSelection([]); - }} - > - - - ) : ( - { - // TODO: Change this to setIndicesOptions() when https://github.com/elastic/eui/issues/2071 is fixed - indicesOptions.forEach((option: EuiSelectableOption) => { - option.checked = 'on'; - }); - updatePolicyConfig({ indices: [...indices] }); - setIndicesSelection([...indices]); - }} - > - - - ), - }} - /> - ) : null - } - isInvalid={Boolean(errors.indices)} - error={errors.indices} - > - {selectIndicesMode === 'list' ? ( - { - const newSelectedIndices: string[] = []; - options.forEach(({ label, checked }) => { - if (checked === 'on') { - newSelectedIndices.push(label); - } - }); - setIndicesOptions(options); - updatePolicyConfig({ indices: newSelectedIndices }); - setIndicesSelection(newSelectedIndices); - }} - searchable - height={300} - > - {(list, search) => ( - - {search} - {list} - - )} - - ) : ( - ({ label: index }))} - placeholder={i18n.translate( - 'xpack.snapshotRestore.policyForm.stepSettings.indicesPatternPlaceholder', - { - defaultMessage: 'Enter index patterns, i.e. logstash-*', - } - )} - selectedOptions={indexPatterns.map((pattern) => ({ label: pattern }))} - onCreateOption={(pattern: string) => { - if (!pattern.trim().length) { - return; - } - const newPatterns = [...indexPatterns, pattern]; - setIndexPatterns(newPatterns); - updatePolicyConfig({ - indices: newPatterns.join(','), - }); - }} - onChange={(patterns: Array<{ label: string }>) => { - const newPatterns = patterns.map(({ label }) => label); - setIndexPatterns(newPatterns); - updatePolicyConfig({ - indices: newPatterns.join(','), - }); - }} - /> - )} - - - )} - - - - ); - }; - - const renderIgnoreUnavailableField = () => ( - -

- -

- - } - description={ - - } - fullWidth - > - - - } - checked={Boolean(config.ignoreUnavailable)} - onChange={(e) => { - updatePolicyConfig({ - ignoreUnavailable: e.target.checked, - }); - }} - /> - - - ); - - const renderPartialField = () => ( - -

- -

- - } - description={ - - } - fullWidth - > - - - } - checked={Boolean(config.partial)} - onChange={(e) => { - updatePolicyConfig({ - partial: e.target.checked, - }); - }} - /> - - - ); - - const renderIncludeGlobalStateField = () => ( - -

- -

- - } - description={ - - } - fullWidth - > - - - } - checked={config.includeGlobalState === undefined || config.includeGlobalState} - onChange={(e) => { - updatePolicyConfig({ - includeGlobalState: e.target.checked, - }); - }} - /> - - - ); - return ( -
- {/* Step title and doc link */} - - - -

- -

- - - - - - - - - - - - {renderIndicesField()} - {renderIgnoreUnavailableField()} - {renderPartialField()} - {renderIncludeGlobalStateField()} -
- ); -}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/index.ts new file mode 100644 index 00000000000000..e0d632a58e4e18 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { IndicesAndDataStreamsField } from './indices_and_data_streams_field'; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/data_streams_and_indices_list_help_text.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/data_streams_and_indices_list_help_text.tsx new file mode 100644 index 00000000000000..3570c74fb8fd0d --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/data_streams_and_indices_list_help_text.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiLink } from '@elastic/eui'; + +interface Props { + onSelectionChange: (selection: 'all' | 'none') => void; + selectedIndicesAndDataStreams: string[]; + indices: string[]; + dataStreams: string[]; +} + +export const DataStreamsAndIndicesListHelpText: FunctionComponent = ({ + onSelectionChange, + selectedIndicesAndDataStreams, + indices, + dataStreams, +}) => { + if (selectedIndicesAndDataStreams.length === 0) { + return ( + { + onSelectionChange('all'); + }} + > + + + ), + }} + /> + ); + } + + const indicesCount = selectedIndicesAndDataStreams.reduce( + (acc, v) => (indices.includes(v) ? acc + 1 : acc), + 0 + ); + const dataStreamsCount = selectedIndicesAndDataStreams.reduce( + (acc, v) => (dataStreams.includes(v) ? acc + 1 : acc), + 0 + ); + + return ( + { + onSelectionChange('none'); + }} + > + + + ), + }} + /> + ); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/helpers.test.ts b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/helpers.test.ts new file mode 100644 index 00000000000000..9bf97af6400b52 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/helpers.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { determineListMode } from './helpers'; + +describe('helpers', () => { + describe('determineListMode', () => { + test('list length (> 100)', () => { + expect( + determineListMode({ + indices: Array.from(Array(101).keys()).map(String), + dataStreams: [], + configuredIndices: undefined, + }) + ).toBe('custom'); + + // The length of indices and data streams are cumulative + expect( + determineListMode({ + indices: Array.from(Array(51).keys()).map(String), + dataStreams: Array.from(Array(51).keys()).map(String), + configuredIndices: undefined, + }) + ).toBe('custom'); + + // Other values should result in list mode + expect( + determineListMode({ + indices: [], + dataStreams: [], + configuredIndices: undefined, + }) + ).toBe('list'); + }); + + test('configured indices is a string', () => { + expect( + determineListMode({ + indices: [], + dataStreams: [], + configuredIndices: 'test', + }) + ).toBe('custom'); + }); + + test('configured indices not included in current indices and data streams', () => { + expect( + determineListMode({ + indices: ['a'], + dataStreams: ['b'], + configuredIndices: ['a', 'b', 'c'], + }) + ).toBe('custom'); + }); + + test('configured indices included in current indices and data streams', () => { + expect( + determineListMode({ + indices: ['a'], + dataStreams: ['b'], + configuredIndices: ['a', 'b'], + }) + ).toBe('list'); + }); + }); +}); diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/helpers.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/helpers.tsx new file mode 100644 index 00000000000000..98ad2fe9c54894 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/helpers.tsx @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiSelectableOption } from '@elastic/eui'; +import { orderDataStreamsAndIndices } from '../../../../../lib'; +import { DataStreamBadge } from '../../../../../data_stream_badge'; + +export const mapSelectionToIndicesOptions = ({ + allSelected, + selection, + dataStreams, + indices, +}: { + allSelected: boolean; + selection: string[]; + dataStreams: string[]; + indices: string[]; +}): EuiSelectableOption[] => { + return orderDataStreamsAndIndices({ + dataStreams: dataStreams.map( + (dataStream): EuiSelectableOption => { + return { + label: dataStream, + append: , + checked: allSelected || selection.includes(dataStream) ? 'on' : undefined, + }; + } + ), + indices: indices.map( + (index): EuiSelectableOption => { + return { + label: index, + checked: allSelected || selection.includes(index) ? 'on' : undefined, + }; + } + ), + }); +}; + +/** + * @remark + * Users with more than 100 indices will probably want to use an index pattern to select + * them instead, so we'll default to showing them the index pattern input. Also show the custom + * list if we have no exact matches in the configured array to some existing index. + */ +export const determineListMode = ({ + configuredIndices, + indices, + dataStreams, +}: { + configuredIndices: string | string[] | undefined; + indices: string[]; + dataStreams: string[]; +}): 'custom' | 'list' => { + const indicesAndDataStreams = indices.concat(dataStreams); + return typeof configuredIndices === 'string' || + indicesAndDataStreams.length > 100 || + (Array.isArray(configuredIndices) && + // If not every past configured index maps to an existing index or data stream + // we also show the custom list + !configuredIndices.every((c) => indicesAndDataStreams.some((i) => i === c))) + ? 'custom' + : 'list'; +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/index.ts new file mode 100644 index 00000000000000..e0d632a58e4e18 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { IndicesAndDataStreamsField } from './indices_and_data_streams_field'; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/indices_and_data_streams_field.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/indices_and_data_streams_field.tsx new file mode 100644 index 00000000000000..94854905e66863 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/fields/indices_and_data_streams_field/indices_and_data_streams_field.tsx @@ -0,0 +1,348 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FunctionComponent, useState } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { + EuiComboBox, + EuiDescribedFormGroup, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiLink, + EuiPanel, + EuiSelectable, + EuiSelectableOption, + EuiSpacer, + EuiSwitch, + EuiTitle, + EuiToolTip, +} from '@elastic/eui'; + +import { SlmPolicyPayload } from '../../../../../../../../common/types'; +import { useServices } from '../../../../../../app_context'; +import { PolicyValidation } from '../../../../../../services/validation'; + +import { orderDataStreamsAndIndices } from '../../../../../lib'; +import { DataStreamBadge } from '../../../../../data_stream_badge'; + +import { mapSelectionToIndicesOptions, determineListMode } from './helpers'; + +import { DataStreamsAndIndicesListHelpText } from './data_streams_and_indices_list_help_text'; + +interface Props { + isManagedPolicy: boolean; + policy: SlmPolicyPayload; + indices: string[]; + dataStreams: string[]; + onUpdate: (arg: { indices?: string[] | string }) => void; + errors: PolicyValidation['errors']; +} + +/** + * In future we may be able to split data streams to its own field, but for now + * they share an array "indices" in the snapshot lifecycle policy config. See + * this github issue for progress: https://github.com/elastic/elasticsearch/issues/58474 + */ +export const IndicesAndDataStreamsField: FunctionComponent = ({ + isManagedPolicy, + dataStreams, + indices, + policy, + onUpdate, + errors, +}) => { + const { i18n } = useServices(); + const { config = {} } = policy; + + const indicesAndDataStreams = indices.concat(dataStreams); + + // We assume all indices if the config has no indices entry or if we receive an empty array + const [isAllIndices, setIsAllIndices] = useState( + !config.indices || (Array.isArray(config.indices) && config.indices.length === 0) + ); + + const [indicesAndDataStreamsSelection, setIndicesAndDataStreamsSelection] = useState( + () => + Array.isArray(config.indices) && !isAllIndices + ? indicesAndDataStreams.filter((i) => (config.indices! as string[]).includes(i)) + : [...indicesAndDataStreams] + ); + + // States for choosing all indices, or a subset, including caching previously chosen subset list + const [indicesAndDataStreamsOptions, setIndicesAndDataStreamsOptions] = useState< + EuiSelectableOption[] + >(() => + mapSelectionToIndicesOptions({ + selection: indicesAndDataStreamsSelection, + dataStreams, + indices, + allSelected: isAllIndices || typeof config.indices === 'string', + }) + ); + + // State for using selectable indices list or custom patterns + const [selectIndicesMode, setSelectIndicesMode] = useState<'list' | 'custom'>(() => + determineListMode({ configuredIndices: config.indices, dataStreams, indices }) + ); + + // State for custom patterns + const [indexPatterns, setIndexPatterns] = useState(() => + typeof config.indices === 'string' + ? (config.indices as string).split(',') + : Array.isArray(config.indices) && config.indices + ? config.indices + : [] + ); + + const indicesSwitch = ( + + } + checked={isAllIndices} + disabled={isManagedPolicy} + data-test-subj="allIndicesToggle" + onChange={(e) => { + const isChecked = e.target.checked; + setIsAllIndices(isChecked); + if (isChecked) { + setIndicesAndDataStreamsSelection(indicesAndDataStreams); + setIndicesAndDataStreamsOptions( + mapSelectionToIndicesOptions({ + allSelected: isAllIndices || typeof config.indices === 'string', + dataStreams, + indices, + selection: indicesAndDataStreamsSelection, + }) + ); + onUpdate({ indices: undefined }); + } else { + onUpdate({ + indices: + selectIndicesMode === 'custom' + ? indexPatterns.join(',') + : [...(indicesAndDataStreamsSelection || [])], + }); + } + }} + /> + ); + + return ( + +

+ +

+ + } + description={ + + } + fullWidth + > + + + {isManagedPolicy ? ( + + +

+ } + > + {indicesSwitch} + + ) : ( + indicesSwitch + )} + {isAllIndices ? null : ( + + + + + + + + { + setSelectIndicesMode('custom'); + onUpdate({ indices: indexPatterns.join(',') }); + }} + > + + + + + ) : ( + + + + + + { + setSelectIndicesMode('list'); + onUpdate({ indices: indicesAndDataStreamsSelection }); + }} + > + + + + + ) + } + helpText={ + selectIndicesMode === 'list' ? ( + { + if (selection === 'all') { + // TODO: Change this to setIndicesOptions() when https://github.com/elastic/eui/issues/2071 is fixed + indicesAndDataStreamsOptions.forEach((option: EuiSelectableOption) => { + option.checked = 'on'; + }); + onUpdate({ indices: [...indicesAndDataStreams] }); + setIndicesAndDataStreamsSelection([...indicesAndDataStreams]); + } else { + // TODO: Change this to setIndicesOptions() when https://github.com/elastic/eui/issues/2071 is fixed + indicesAndDataStreamsOptions.forEach((option: EuiSelectableOption) => { + option.checked = undefined; + }); + onUpdate({ indices: [] }); + setIndicesAndDataStreamsSelection([]); + } + }} + selectedIndicesAndDataStreams={indicesAndDataStreamsSelection} + indices={indices} + dataStreams={dataStreams} + /> + ) : null + } + isInvalid={Boolean(errors.indices)} + error={errors.indices} + > + {selectIndicesMode === 'list' ? ( + { + const newSelectedIndices: string[] = []; + options.forEach(({ label, checked }) => { + if (checked === 'on') { + newSelectedIndices.push(label); + } + }); + setIndicesAndDataStreamsOptions(options); + onUpdate({ indices: newSelectedIndices }); + setIndicesAndDataStreamsSelection(newSelectedIndices); + }} + searchable + height={300} + > + {(list, search) => ( + + {search} + {list} + + )} + + ) : ( + ({ + label: index, + value: { isDataStream: false }, + })), + dataStreams: dataStreams.map((dataStream) => ({ + label: dataStream, + value: { isDataStream: true }, + })), + })} + renderOption={({ label, value }) => { + if (value?.isDataStream) { + return ( + + {label} + + + + + ); + } + return label; + }} + placeholder={i18n.translate( + 'xpack.snapshotRestore.policyForm.stepSettings.indicesPatternPlaceholder', + { + defaultMessage: 'Enter index patterns, i.e. logstash-*', + } + )} + selectedOptions={indexPatterns.map((pattern) => ({ label: pattern }))} + onCreateOption={(pattern: string) => { + if (!pattern.trim().length) { + return; + } + const newPatterns = [...indexPatterns, pattern]; + setIndexPatterns(newPatterns); + onUpdate({ + indices: newPatterns.join(','), + }); + }} + onChange={(patterns: Array<{ label: string }>) => { + const newPatterns = patterns.map(({ label }) => label); + setIndexPatterns(newPatterns); + onUpdate({ + indices: newPatterns.join(','), + }); + }} + /> + )} + + + )} + + + + ); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/index.ts new file mode 100644 index 00000000000000..24e9b36e748899 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { PolicyStepSettings } from './step_settings'; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/step_settings.tsx b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/step_settings.tsx new file mode 100644 index 00000000000000..9d43c45d17ea7a --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/policy_form/steps/step_settings/step_settings.tsx @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiDescribedFormGroup, + EuiTitle, + EuiFormRow, + EuiFlexGroup, + EuiFlexItem, + EuiButtonEmpty, + EuiSpacer, + EuiSwitch, +} from '@elastic/eui'; + +import { SlmPolicyPayload } from '../../../../../../common/types'; +import { documentationLinksService } from '../../../../services/documentation'; +import { StepProps } from '../'; + +import { IndicesAndDataStreamsField } from './fields'; + +export const PolicyStepSettings: React.FunctionComponent = ({ + policy, + indices, + dataStreams, + updatePolicy, + errors, +}) => { + const { config = {}, isManagedPolicy } = policy; + + const updatePolicyConfig = (updatedFields: Partial): void => { + const newConfig = { ...config, ...updatedFields }; + updatePolicy({ + config: newConfig, + }); + }; + + const renderIgnoreUnavailableField = () => ( + +

+ +

+ + } + description={ + + } + fullWidth + > + + + } + checked={Boolean(config.ignoreUnavailable)} + onChange={(e) => { + updatePolicyConfig({ + ignoreUnavailable: e.target.checked, + }); + }} + /> + + + ); + + const renderPartialField = () => ( + +

+ +

+ + } + description={ + + } + fullWidth + > + + + } + checked={Boolean(config.partial)} + onChange={(e) => { + updatePolicyConfig({ + partial: e.target.checked, + }); + }} + /> + + + ); + + const renderIncludeGlobalStateField = () => ( + +

+ +

+ + } + description={ + + } + fullWidth + > + + + } + checked={config.includeGlobalState === undefined || config.includeGlobalState} + onChange={(e) => { + updatePolicyConfig({ + includeGlobalState: e.target.checked, + }); + }} + /> + + + ); + return ( +
+ {/* Step title and doc link */} + + + +

+ +

+ + + + + + + + + + + + + + {renderIgnoreUnavailableField()} + {renderPartialField()} + {renderIncludeGlobalStateField()} +
+ ); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/index.ts index 3f3db0ff28eca9..182d4ef8f583a2 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/index.ts +++ b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/index.ts @@ -14,6 +14,6 @@ export interface StepProps { updateCurrentStep: (step: number) => void; } -export { RestoreSnapshotStepLogistics } from './step_logistics'; +export { RestoreSnapshotStepLogistics } from './step_logistics/step_logistics'; export { RestoreSnapshotStepSettings } from './step_settings'; export { RestoreSnapshotStepReview } from './step_review'; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/data_streams_and_indices_list_help_text.tsx b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/data_streams_and_indices_list_help_text.tsx new file mode 100644 index 00000000000000..877dbe89639260 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/data_streams_and_indices_list_help_text.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiLink } from '@elastic/eui'; + +interface Props { + onSelectionChange: (selection: 'all' | 'none') => void; + selectedIndicesAndDataStreams: string[]; + indices: string[]; + dataStreams: string[]; +} + +export const DataStreamsAndIndicesListHelpText: FunctionComponent = ({ + onSelectionChange, + selectedIndicesAndDataStreams, + indices, + dataStreams, +}) => { + if (selectedIndicesAndDataStreams.length === 0) { + return ( + { + onSelectionChange('all'); + }} + > + + + ), + }} + /> + ); + } + + const indicesCount = selectedIndicesAndDataStreams.reduce( + (acc, v) => (indices.includes(v) ? acc + 1 : acc), + 0 + ); + const dataStreamsCount = selectedIndicesAndDataStreams.reduce( + (acc, v) => (dataStreams.includes(v) ? acc + 1 : acc), + 0 + ); + + return ( + { + onSelectionChange('none'); + }} + > + + + ), + }} + /> + ); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/data_streams_global_state_call_out.tsx b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/data_streams_global_state_call_out.tsx new file mode 100644 index 00000000000000..64fce4dcfac431 --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/data_streams_global_state_call_out.tsx @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { FunctionComponent } from 'react'; +import { EuiCallOut, EuiLink } from '@elastic/eui'; + +import { documentationLinksService } from '../../../../services/documentation'; + +const i18nTexts = { + callout: { + title: (count: number) => + i18n.translate('xpack.snapshotRestore.restoreForm.dataStreamsWarningCallOut.title', { + defaultMessage: + 'This snapshot contains {count, plural, one {a data stream} other {data streams}}', + values: { count }, + }), + body: () => ( + + {i18n.translate( + 'xpack.snapshotRestore.restoreForm.dataStreamsWarningCallOut.body.learnMoreLink', + { defaultMessage: 'Learn more' } + )} + + ), + }} + /> + ), + }, +}; + +interface Props { + dataStreamsCount: number; +} + +export const DataStreamsGlobalStateCallOut: FunctionComponent = ({ dataStreamsCount }) => { + return ( + + {i18nTexts.callout.body()} + + ); +}; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/index.ts b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/index.ts new file mode 100644 index 00000000000000..8f4efcf2a91f1a --- /dev/null +++ b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { RestoreSnapshotStepLogistics } from './step_logistics'; diff --git a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics.tsx b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/step_logistics.tsx similarity index 69% rename from x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics.tsx rename to x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/step_logistics.tsx index c80c5a2e4c01d7..d9fd4cca0d614f 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_logistics/step_logistics.tsx @@ -21,10 +21,22 @@ import { EuiComboBox, } from '@elastic/eui'; import { EuiSelectableOption } from '@elastic/eui'; -import { RestoreSettings } from '../../../../../common/types'; -import { documentationLinksService } from '../../../services/documentation'; -import { useServices } from '../../../app_context'; -import { StepProps } from './'; + +import { csvToArray, isDataStreamBackingIndex } from '../../../../../../common/lib'; +import { RestoreSettings } from '../../../../../../common/types'; + +import { documentationLinksService } from '../../../../services/documentation'; + +import { useServices } from '../../../../app_context'; + +import { orderDataStreamsAndIndices } from '../../../lib'; +import { DataStreamBadge } from '../../../data_stream_badge'; + +import { StepProps } from '../index'; + +import { DataStreamsGlobalStateCallOut } from './data_streams_global_state_call_out'; + +import { DataStreamsAndIndicesListHelpText } from './data_streams_and_indices_list_help_text'; export const RestoreSnapshotStepLogistics: React.FunctionComponent = ({ snapshotDetails, @@ -34,10 +46,30 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = }) => { const { i18n } = useServices(); const { - indices: snapshotIndices, + indices: unfilteredSnapshotIndices, + dataStreams: snapshotDataStreams = [], includeGlobalState: snapshotIncludeGlobalState, } = snapshotDetails; + const snapshotIndices = unfilteredSnapshotIndices.filter( + (index) => !isDataStreamBackingIndex(index) + ); + const snapshotIndicesAndDataStreams = snapshotIndices.concat(snapshotDataStreams); + + const comboBoxOptions = orderDataStreamsAndIndices<{ + label: string; + value: { isDataStream: boolean; name: string }; + }>({ + dataStreams: snapshotDataStreams.map((dataStream) => ({ + label: dataStream, + value: { isDataStream: true, name: dataStream }, + })), + indices: snapshotIndices.map((index) => ({ + label: index, + value: { isDataStream: false, name: index }, + })), + }); + const { indices: restoreIndices, renamePattern, @@ -47,28 +79,50 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = } = restoreSettings; // States for choosing all indices, or a subset, including caching previously chosen subset list - const [isAllIndices, setIsAllIndices] = useState(!Boolean(restoreIndices)); - const [indicesOptions, setIndicesOptions] = useState( - snapshotIndices.map( - (index): EuiSelectableOption => ({ - label: index, - checked: - isAllIndices || - // If indices is a string, we default to custom input mode, so we mark individual indices - // as selected if user goes back to list mode - typeof restoreIndices === 'string' || - (Array.isArray(restoreIndices) && restoreIndices.includes(index)) - ? 'on' - : undefined, - }) - ) + const [isAllIndicesAndDataStreams, setIsAllIndicesAndDataStreams] = useState( + !Boolean(restoreIndices) + ); + const [indicesAndDataStreamsOptions, setIndicesAndDataStreamsOptions] = useState< + EuiSelectableOption[] + >(() => + orderDataStreamsAndIndices({ + dataStreams: snapshotDataStreams.map( + (dataStream): EuiSelectableOption => ({ + label: dataStream, + append: , + checked: + isAllIndicesAndDataStreams || + // If indices is a string, we default to custom input mode, so we mark individual indices + // as selected if user goes back to list mode + typeof restoreIndices === 'string' || + (Array.isArray(restoreIndices) && restoreIndices.includes(dataStream)) + ? 'on' + : undefined, + }) + ), + indices: snapshotIndices.map( + (index): EuiSelectableOption => ({ + label: index, + checked: + isAllIndicesAndDataStreams || + // If indices is a string, we default to custom input mode, so we mark individual indices + // as selected if user goes back to list mode + typeof restoreIndices === 'string' || + (Array.isArray(restoreIndices) && restoreIndices.includes(index)) + ? 'on' + : undefined, + }) + ), + }) ); // State for using selectable indices list or custom patterns // Users with more than 100 indices will probably want to use an index pattern to select // them instead, so we'll default to showing them the index pattern input. const [selectIndicesMode, setSelectIndicesMode] = useState<'list' | 'custom'>( - typeof restoreIndices === 'string' || snapshotIndices.length > 100 ? 'custom' : 'list' + typeof restoreIndices === 'string' || snapshotIndicesAndDataStreams.length > 100 + ? 'custom' + : 'list' ); // State for custom patterns @@ -83,13 +137,16 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = // Caching state for togglable settings const [cachedRestoreSettings, setCachedRestoreSettings] = useState({ - indices: [...snapshotIndices], + indices: [...snapshotIndicesAndDataStreams], renamePattern: '', renameReplacement: '', }); return ( -
+
{/* Step title and doc link */} @@ -118,6 +175,14 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = + + {snapshotDataStreams.length ? ( + <> + + + + ) : undefined} + {/* Indices */} @@ -126,16 +191,16 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent =

} description={ } @@ -146,14 +211,14 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = } - checked={isAllIndices} + checked={isAllIndicesAndDataStreams} onChange={(e) => { const isChecked = e.target.checked; - setIsAllIndices(isChecked); + setIsAllIndicesAndDataStreams(isChecked); if (isChecked) { updateRestoreSettings({ indices: undefined }); } else { @@ -166,7 +231,7 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = } }} /> - {isAllIndices ? null : ( + {isAllIndicesAndDataStreams ? null : ( = @@ -210,8 +275,8 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = }} > @@ -220,52 +285,35 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = } helpText={ selectIndicesMode === 'list' ? ( - 0 ? ( - { - // TODO: Change this to setIndicesOptions() when https://github.com/elastic/eui/issues/2071 is fixed - indicesOptions.forEach((option: EuiSelectableOption) => { - option.checked = undefined; - }); - updateRestoreSettings({ indices: [] }); - setCachedRestoreSettings({ - ...cachedRestoreSettings, - indices: [], - }); - }} - > - - - ) : ( - { - // TODO: Change this to setIndicesOptions() when https://github.com/elastic/eui/issues/2071 is fixed - indicesOptions.forEach((option: EuiSelectableOption) => { - option.checked = 'on'; - }); - updateRestoreSettings({ indices: [...snapshotIndices] }); - setCachedRestoreSettings({ - ...cachedRestoreSettings, - indices: [...snapshotIndices], - }); - }} - > - - - ), + { + if (selection === 'all') { + // TODO: Change this to setIndicesOptions() when https://github.com/elastic/eui/issues/2071 is fixed + indicesAndDataStreamsOptions.forEach((option: EuiSelectableOption) => { + option.checked = 'on'; + }); + updateRestoreSettings({ + indices: [...snapshotIndicesAndDataStreams], + }); + setCachedRestoreSettings({ + ...cachedRestoreSettings, + indices: [...snapshotIndicesAndDataStreams], + }); + } else { + // TODO: Change this to setIndicesOptions() when https://github.com/elastic/eui/issues/2071 is fixed + indicesAndDataStreamsOptions.forEach((option: EuiSelectableOption) => { + option.checked = undefined; + }); + updateRestoreSettings({ indices: [] }); + setCachedRestoreSettings({ + ...cachedRestoreSettings, + indices: [], + }); + } }} + selectedIndicesAndDataStreams={csvToArray(restoreIndices)} + indices={snapshotIndices} + dataStreams={snapshotDataStreams} /> ) : null } @@ -275,7 +323,7 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = {selectIndicesMode === 'list' ? ( { const newSelectedIndices: string[] = []; options.forEach(({ label, checked }) => { @@ -283,7 +331,7 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = newSelectedIndices.push(label); } }); - setIndicesOptions(options); + setIndicesAndDataStreamsOptions(options); updateRestoreSettings({ indices: [...newSelectedIndices] }); setCachedRestoreSettings({ ...cachedRestoreSettings, @@ -302,7 +350,24 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = ) : ( ({ label: index }))} + options={comboBoxOptions} + renderOption={({ value }) => { + return value?.isDataStream ? ( + + {value.name} + + + + + ) : ( + value?.name + ); + }} placeholder={i18n.translate( 'xpack.snapshotRestore.restoreForm.stepLogistics.indicesPatternPlaceholder', { @@ -336,22 +401,22 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = - {/* Rename indices */} + {/* Rename data streams and indices */}

} description={ } fullWidth @@ -361,8 +426,8 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = } checked={isRenamingIndices} @@ -405,7 +470,7 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = > { setCachedRestoreSettings({ ...cachedRestoreSettings, @@ -431,7 +496,7 @@ export const RestoreSnapshotStepLogistics: React.FunctionComponent = > { setCachedRestoreSettings({ ...cachedRestoreSettings, diff --git a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_review.tsx b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_review.tsx index 27a3717566d93f..5dacba506fe188 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_review.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_review.tsx @@ -24,7 +24,7 @@ import { import { serializeRestoreSettings } from '../../../../../common/lib'; import { useServices } from '../../../app_context'; import { StepProps } from './'; -import { CollapsibleIndicesList } from '../../collapsible_indices_list'; +import { CollapsibleIndicesList } from '../../collapsible_lists/collapsible_indices_list'; export const RestoreSnapshotStepReview: React.FunctionComponent = ({ restoreSettings, @@ -73,8 +73,8 @@ export const RestoreSnapshotStepReview: React.FunctionComponent = ({ diff --git a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_settings.tsx b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_settings.tsx index 5f3ebf804c5e12..b9a2d7e4b7cd97 100644 --- a/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_settings.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/components/restore_snapshot_form/steps/step_settings.tsx @@ -18,6 +18,7 @@ import { EuiSwitch, EuiTitle, EuiLink, + EuiCallOut, } from '@elastic/eui'; import { RestoreSettings } from '../../../../../common/types'; import { REMOVE_INDEX_SETTINGS_SUGGESTIONS } from '../../../constants'; @@ -28,10 +29,12 @@ import { StepProps } from './'; export const RestoreSnapshotStepSettings: React.FunctionComponent = ({ restoreSettings, updateRestoreSettings, + snapshotDetails, errors, }) => { const { i18n } = useServices(); const { indexSettings, ignoreIndexSettings } = restoreSettings; + const { dataStreams } = snapshotDetails; // State for index setting toggles const [isUsingIndexSettings, setIsUsingIndexSettings] = useState(Boolean(indexSettings)); @@ -96,6 +99,23 @@ export const RestoreSnapshotStepSettings: React.FunctionComponent = ( + {dataStreams?.length ? ( + <> + + + + + + ) : undefined} {/* Modify index settings */} diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_details/tabs/tab_summary.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_details/tabs/tab_summary.tsx index 7bcee4f5f6621d..e69b0fad8014ee 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_details/tabs/tab_summary.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_details/tabs/tab_summary.tsx @@ -236,8 +236,8 @@ export const TabSummary: React.FunctionComponent = ({ policy }) => { diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/snapshot_list/snapshot_details/tabs/tab_summary.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/snapshot_list/snapshot_details/tabs/tab_summary.tsx index 287a77493307da..1a0c26c8544907 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/snapshot_list/snapshot_details/tabs/tab_summary.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/snapshot_list/snapshot_details/tabs/tab_summary.tsx @@ -22,6 +22,7 @@ import { DataPlaceholder, FormattedDateTime, CollapsibleIndicesList, + CollapsibleDataStreamsList, } from '../../../../../components'; import { linkToPolicy } from '../../../../../services/navigation'; import { SnapshotState } from './snapshot_state'; @@ -40,6 +41,7 @@ export const TabSummary: React.FC = ({ snapshotDetails }) => { // TODO: Add a tooltip explaining that: a false value means that the cluster global state // is not stored as part of the snapshot. includeGlobalState, + dataStreams, indices, state, startTimeInMillis, @@ -135,6 +137,22 @@ export const TabSummary: React.FC = ({ snapshotDetails }) => { + + + + + + + + + + + + diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/policy_add/policy_add.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/policy_add/policy_add.tsx index 6d1a432be7f9f0..90cd26c821c5e2 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/policy_add/policy_add.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/policy_add/policy_add.tsx @@ -25,13 +25,8 @@ export const PolicyAdd: React.FunctionComponent = ({ const [isSaving, setIsSaving] = useState(false); const [saveError, setSaveError] = useState(null); - const { - error: errorLoadingIndices, - isLoading: isLoadingIndices, - data: { indices } = { - indices: [], - }, - } = useLoadIndices(); + const { error: errorLoadingIndices, isLoading: isLoadingIndices, data } = useLoadIndices(); + const { indices, dataStreams } = data ?? { indices: [], dataStreams: [] }; // Set breadcrumb and page title useEffect(() => { @@ -123,6 +118,7 @@ export const PolicyAdd: React.FunctionComponent = ({ { }; export const useLoadIndices = () => { - return useRequest({ + return useRequest({ path: `${API_BASE_PATH}policies/indices`, method: 'get', }); diff --git a/x-pack/plugins/snapshot_restore/public/application/services/http/use_request.ts b/x-pack/plugins/snapshot_restore/public/application/services/http/use_request.ts index 27a565ccb74bc9..b4d0493098bbc5 100644 --- a/x-pack/plugins/snapshot_restore/public/application/services/http/use_request.ts +++ b/x-pack/plugins/snapshot_restore/public/application/services/http/use_request.ts @@ -18,6 +18,6 @@ export const sendRequest = (config: SendRequestConfig) => { return _sendRequest(httpService.httpClient, config); }; -export const useRequest = (config: UseRequestConfig) => { - return _useRequest(httpService.httpClient, config); +export const useRequest = (config: UseRequestConfig) => { + return _useRequest(httpService.httpClient, config); }; diff --git a/x-pack/plugins/snapshot_restore/public/application/services/validation/validate_policy.ts b/x-pack/plugins/snapshot_restore/public/application/services/validation/validate_policy.ts index 0720994ca76693..24960b2533230e 100644 --- a/x-pack/plugins/snapshot_restore/public/application/services/validation/validate_policy.ts +++ b/x-pack/plugins/snapshot_restore/public/application/services/validation/validate_policy.ts @@ -48,6 +48,7 @@ export const validatePolicy = ( snapshotName: [], schedule: [], repository: [], + dataStreams: [], indices: [], expireAfterValue: [], minCount: [], @@ -106,7 +107,7 @@ export const validatePolicy = ( if (config && Array.isArray(config.indices) && config.indices.length === 0) { validation.errors.indices.push( i18n.translate('xpack.snapshotRestore.policyValidation.indicesRequiredErrorMessage', { - defaultMessage: 'You must select at least one index.', + defaultMessage: 'You must select at least one data stream or index.', }) ); } diff --git a/x-pack/plugins/snapshot_restore/public/application/services/validation/validate_restore.ts b/x-pack/plugins/snapshot_restore/public/application/services/validation/validate_restore.ts index 5c1a1fbfab12d1..93e278e51f0939 100644 --- a/x-pack/plugins/snapshot_restore/public/application/services/validation/validate_restore.ts +++ b/x-pack/plugins/snapshot_restore/public/application/services/validation/validate_restore.ts @@ -48,7 +48,7 @@ export const validateRestore = (restoreSettings: RestoreSettings): RestoreValida if (Array.isArray(indices) && indices.length === 0) { validation.errors.indices.push( i18n.translate('xpack.snapshotRestore.restoreValidation.indicesRequiredError', { - defaultMessage: 'You must select at least one index.', + defaultMessage: 'You must select at least one data stream or index.', }) ); } @@ -93,7 +93,6 @@ export const validateRestore = (restoreSettings: RestoreSettings): RestoreValida 'xpack.snapshotRestore.restoreValidation.indexSettingsNotModifiableError', { defaultMessage: 'You can’t modify: {settings}', - // @ts-ignore Bug filed: https://github.com/elastic/kibana/issues/39299 values: { settings: unmodifiableSettings.map((setting: string, index: number) => index === 0 ? `${setting} ` : setting @@ -131,7 +130,6 @@ export const validateRestore = (restoreSettings: RestoreSettings): RestoreValida validation.errors.ignoreIndexSettings.push( i18n.translate('xpack.snapshotRestore.restoreValidation.indexSettingsNotRemovableError', { defaultMessage: 'You can’t reset: {settings}', - // @ts-ignore Bug filed: https://github.com/elastic/kibana/issues/39299 values: { settings: unremovableSettings.map((setting: string, index: number) => index === 0 ? `${setting} ` : setting diff --git a/x-pack/plugins/snapshot_restore/server/routes/api/policy.test.ts b/x-pack/plugins/snapshot_restore/server/routes/api/policy.test.ts index eb29b7bad37e66..b96d305fa4a874 100644 --- a/x-pack/plugins/snapshot_restore/server/routes/api/policy.test.ts +++ b/x-pack/plugins/snapshot_restore/server/routes/api/policy.test.ts @@ -6,6 +6,7 @@ import { addBasePath } from '../helpers'; import { registerPolicyRoutes } from './policy'; import { RouterMock, routeDependencies, RequestMock } from '../../test/helpers'; +import { ResolveIndexResponseFromES } from '../../types'; describe('[Snapshot and Restore API Routes] Policy', () => { const mockEsPolicy = { @@ -324,27 +325,45 @@ describe('[Snapshot and Restore API Routes] Policy', () => { }; it('should arrify and sort index names returned from ES', async () => { - const mockEsResponse = [ - { - index: 'fooIndex', - }, - { - index: 'barIndex', - }, - ]; + const mockEsResponse: ResolveIndexResponseFromES = { + indices: [ + { + name: 'fooIndex', + attributes: ['open'], + }, + { + name: 'barIndex', + attributes: ['open'], + data_stream: 'testDataStream', + }, + ], + aliases: [], + data_streams: [ + { + name: 'testDataStream', + backing_indices: ['barIndex'], + timestamp_field: '@timestamp', + }, + ], + }; router.callAsCurrentUserResponses = [mockEsResponse]; const expectedResponse = { - indices: ['barIndex', 'fooIndex'], + indices: ['fooIndex'], + dataStreams: ['testDataStream'], }; await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); }); it('should return empty array if no indices returned from ES', async () => { - const mockEsResponse: any[] = []; + const mockEsResponse: ResolveIndexResponseFromES = { + indices: [], + aliases: [], + data_streams: [], + }; router.callAsCurrentUserResponses = [mockEsResponse]; - const expectedResponse = { indices: [] }; + const expectedResponse = { indices: [], dataStreams: [] }; await expect(router.runRequest(mockRequest)).resolves.toEqual({ body: expectedResponse }); }); diff --git a/x-pack/plugins/snapshot_restore/server/routes/api/policy.ts b/x-pack/plugins/snapshot_restore/server/routes/api/policy.ts index 90667eda23b351..b8e70125295544 100644 --- a/x-pack/plugins/snapshot_restore/server/routes/api/policy.ts +++ b/x-pack/plugins/snapshot_restore/server/routes/api/policy.ts @@ -5,10 +5,10 @@ */ import { schema, TypeOf } from '@kbn/config-schema'; -import { SlmPolicyEs } from '../../../common/types'; +import { SlmPolicyEs, PolicyIndicesResponse } from '../../../common/types'; import { deserializePolicy, serializePolicy } from '../../../common/lib'; import { getManagedPolicyNames } from '../../lib'; -import { RouteDependencies } from '../../types'; +import { RouteDependencies, ResolveIndexResponseFromES } from '../../types'; import { addBasePath } from '../helpers'; import { nameParameterSchema, policySchema } from './validate_schemas'; @@ -232,17 +232,26 @@ export function registerPolicyRoutes({ const { callAsCurrentUser } = ctx.snapshotRestore!.client; try { - const indices: Array<{ - index: string; - }> = await callAsCurrentUser('cat.indices', { - format: 'json', - h: 'index', - }); + const resolvedIndicesResponse: ResolveIndexResponseFromES = await callAsCurrentUser( + 'transport.request', + { + method: 'GET', + path: `_resolve/index/*`, + query: { + expand_wildcards: 'all,hidden', + }, + } + ); + + const body: PolicyIndicesResponse = { + dataStreams: resolvedIndicesResponse.data_streams.map(({ name }) => name).sort(), + indices: resolvedIndicesResponse.indices + .flatMap((index) => (index.data_stream ? [] : index.name)) + .sort(), + }; return res.ok({ - body: { - indices: indices.map(({ index }) => index).sort(), - }, + body, }); } catch (e) { if (isEsError(e)) { diff --git a/x-pack/plugins/snapshot_restore/server/routes/api/snapshots.test.ts b/x-pack/plugins/snapshot_restore/server/routes/api/snapshots.test.ts index f913299fc39925..a7e61d1e7c02a9 100644 --- a/x-pack/plugins/snapshot_restore/server/routes/api/snapshots.test.ts +++ b/x-pack/plugins/snapshot_restore/server/routes/api/snapshots.test.ts @@ -15,6 +15,7 @@ const defaultSnapshot = { versionId: undefined, version: undefined, indices: [], + dataStreams: [], includeGlobalState: undefined, state: undefined, startTime: undefined, diff --git a/x-pack/plugins/snapshot_restore/server/types.ts b/x-pack/plugins/snapshot_restore/server/types.ts index 7794156eb1b888..8cfcaec1a2cd1b 100644 --- a/x-pack/plugins/snapshot_restore/server/types.ts +++ b/x-pack/plugins/snapshot_restore/server/types.ts @@ -31,4 +31,20 @@ export interface RouteDependencies { }; } +/** + * An object representing a resolved index, data stream or alias + */ +interface IndexAndAliasFromEs { + name: string; + // per https://github.com/elastic/elasticsearch/pull/57626 + attributes: Array<'open' | 'closed' | 'hidden' | 'frozen'>; + data_stream?: string; +} + +export interface ResolveIndexResponseFromES { + indices: IndexAndAliasFromEs[]; + aliases: IndexAndAliasFromEs[]; + data_streams: Array<{ name: string; backing_indices: string[]; timestamp_field: string }>; +} + export type CallAsCurrentUser = LegacyScopedClusterClient['callAsCurrentUser']; diff --git a/x-pack/plugins/snapshot_restore/test/fixtures/snapshot.ts b/x-pack/plugins/snapshot_restore/test/fixtures/snapshot.ts index d6a55579b322d4..e59f4689d9e3f6 100644 --- a/x-pack/plugins/snapshot_restore/test/fixtures/snapshot.ts +++ b/x-pack/plugins/snapshot_restore/test/fixtures/snapshot.ts @@ -13,13 +13,23 @@ export const getSnapshot = ({ state = 'SUCCESS', indexFailures = [], totalIndices = getRandomNumber(), -} = {}) => ({ + totalDataStreams = getRandomNumber(), +}: Partial<{ + repository: string; + snapshot: string; + uuid: string; + state: string; + indexFailures: any[]; + totalIndices: number; + totalDataStreams: number; +}> = {}) => ({ repository, snapshot, uuid, versionId: 8000099, version: '8.0.0', indices: new Array(totalIndices).fill('').map(getRandomString), + dataStreams: new Array(totalDataStreams).fill('').map(getRandomString), includeGlobalState: 1, state, startTime: '2019-05-23T06:25:15.896Z', diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index b5cdad5583e1d5..4c1572ddfcad1d 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -13387,7 +13387,6 @@ "xpack.snapshotRestore.policyDetails.includeGlobalStateFalseLabel": "いいえ", "xpack.snapshotRestore.policyDetails.includeGlobalStateLabel": "グローバルステータスを含める", "xpack.snapshotRestore.policyDetails.includeGlobalStateTrueLabel": "はい", - "xpack.snapshotRestore.policyDetails.indicesLabel": "インデックス", "xpack.snapshotRestore.policyDetails.inProgressSnapshotLinkText": "「{snapshotName}」が進行中", "xpack.snapshotRestore.policyDetails.lastFailure.dateLabel": "日付", "xpack.snapshotRestore.policyDetails.lastFailure.detailsAriaLabel": "ポリシー「{name}」の前回のエラーの詳細", @@ -13495,10 +13494,8 @@ "xpack.snapshotRestore.policyForm.stepReview.summaryTab.includeGlobalStateFalseLabel": "いいえ", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.includeGlobalStateLabel": "グローバルステータスを含める", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.includeGlobalStateTrueLabel": "はい", - "xpack.snapshotRestore.policyForm.stepReview.summaryTab.indicesLabel": "インデックス", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.nameLabel": "ポリシー名", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.partialFalseLabel": "いいえ", - "xpack.snapshotRestore.policyForm.stepReview.summaryTab.partialLabel": "部分シャードを許可", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.partialTrueLabel": "はい", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.repositoryLabel": "レポジトリ", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.scheduleLabel": "スケジュール", @@ -13507,7 +13504,6 @@ "xpack.snapshotRestore.policyForm.stepReview.summaryTab.snapshotNameLabel": "スナップショット名", "xpack.snapshotRestore.policyForm.stepReview.summaryTabTitle": "まとめ", "xpack.snapshotRestore.policyForm.stepReviewTitle": "レビューポリシー", - "xpack.snapshotRestore.policyForm.stepSettings.allIndicesLabel": "システムインデックスを含むすべてのインデックス", "xpack.snapshotRestore.policyForm.stepSettings.deselectAllIndicesLink": "すべて選択解除", "xpack.snapshotRestore.policyForm.stepSettings.docsButtonLabel": "スナップショット設定ドキュメント", "xpack.snapshotRestore.policyForm.stepSettings.ignoreUnavailableDescription": "スナップショットの撮影時に利用不可能なインデックスを無視します。これが設定されていない場合、スナップショット全体がエラーになります。", @@ -13515,19 +13511,15 @@ "xpack.snapshotRestore.policyForm.stepSettings.ignoreUnavailableLabel": "利用不可能なインデックスを無視", "xpack.snapshotRestore.policyForm.stepSettings.includeGlobalStateDescription": "スナップショットの一部としてクラスターのグローバルステータスを格納します。", "xpack.snapshotRestore.policyForm.stepSettings.includeGlobalStateDescriptionTitle": "グローバルステータスを含める", - "xpack.snapshotRestore.policyForm.stepSettings.indicesDescription": "バックアップするインデックスです。", "xpack.snapshotRestore.policyForm.stepSettings.indicesPatternLabel": "インデックスパターン", "xpack.snapshotRestore.policyForm.stepSettings.indicesPatternPlaceholder": "logstash-* などのインデックスパターンを入力", - "xpack.snapshotRestore.policyForm.stepSettings.indicesTitle": "インデックス", "xpack.snapshotRestore.policyForm.stepSettings.indicesToggleCustomLink": "インデックスパターンを使用", - "xpack.snapshotRestore.policyForm.stepSettings.indicesToggleListLink": "インデックスを選択", "xpack.snapshotRestore.policyForm.stepSettings.indicesTooltip": "クラウドで管理されたポリシーにはすべてのインデックスが必要です。", "xpack.snapshotRestore.policyForm.stepSettings.partialDescription": "利用不可能なプライマリシャードのインデックスのスナップショットを許可します。これが設定されていない場合、スナップショット全体がエラーになります。", "xpack.snapshotRestore.policyForm.stepSettings.partialDescriptionTitle": "部分インデックスを許可", "xpack.snapshotRestore.policyForm.stepSettings.partialIndicesToggleSwitch": "部分インデックスを許可", "xpack.snapshotRestore.policyForm.stepSettings.policyIncludeGlobalStateLabel": "グローバルステータスを含める", "xpack.snapshotRestore.policyForm.stepSettings.selectAllIndicesLink": "すべて選択", - "xpack.snapshotRestore.policyForm.stepSettings.selectIndicesHelpText": "{count} 件の{count, plural, one {インデックス} other {インデックス}}がバックアップされます。{selectOrDeselectAllLink}", "xpack.snapshotRestore.policyForm.stepSettings.selectIndicesLabel": "インデックスを選択", "xpack.snapshotRestore.policyForm.stepSettingsTitle": "スナップショット設定", "xpack.snapshotRestore.policyList.deniedPrivilegeDescription": "スナップショットライフサイクルポリシーを管理するには、{privilegesCount, plural, one {このクラスター特権} other {これらのクラスター特権}}が必要です: {missingPrivileges}。", @@ -13874,31 +13866,22 @@ "xpack.snapshotRestore.restoreForm.navigation.stepSettingsName": "インデックス設定", "xpack.snapshotRestore.restoreForm.nextButtonLabel": "次へ", "xpack.snapshotRestore.restoreForm.savingButtonLabel": "復元中...", - "xpack.snapshotRestore.restoreForm.stepLogistics.allIndicesLabel": "システムインデックスを含むすべてのインデックス", "xpack.snapshotRestore.restoreForm.stepLogistics.deselectAllIndicesLink": "すべて選択解除", "xpack.snapshotRestore.restoreForm.stepLogistics.docsButtonLabel": "スナップショットと復元ドキュメント", "xpack.snapshotRestore.restoreForm.stepLogistics.includeGlobalStateDescription": "現在クラスターに存在しないテンプレートを復元し、テンプレートを同じ名前で上書きします。永続的な設定も復元します。", "xpack.snapshotRestore.restoreForm.stepLogistics.includeGlobalStateDisabledDescription": "このスナップショットでは使用できません。", "xpack.snapshotRestore.restoreForm.stepLogistics.includeGlobalStateLabel": "グローバル状態の復元", "xpack.snapshotRestore.restoreForm.stepLogistics.includeGlobalStateTitle": "グローバル状態の復元", - "xpack.snapshotRestore.restoreForm.stepLogistics.indicesDescription": "存在しない場合は、新しいインデックスを作成します。閉じていて、スナップショットインデックスと同じ数のシャードがある場合は、既存のインデックスを復元します。", "xpack.snapshotRestore.restoreForm.stepLogistics.indicesPatternLabel": "インデックスパターン", "xpack.snapshotRestore.restoreForm.stepLogistics.indicesPatternPlaceholder": "logstash-* などのインデックスパターンを入力", - "xpack.snapshotRestore.restoreForm.stepLogistics.indicesTitle": "インデックス", "xpack.snapshotRestore.restoreForm.stepLogistics.indicesToggleCustomLink": "インデックスパターンを使用", - "xpack.snapshotRestore.restoreForm.stepLogistics.indicesToggleListLink": "インデックスを選択", "xpack.snapshotRestore.restoreForm.stepLogistics.partialDescription": "すべてのシャードのスナップショットがないインデックスを復元できます。", "xpack.snapshotRestore.restoreForm.stepLogistics.partialLabel": "部分復元", "xpack.snapshotRestore.restoreForm.stepLogistics.partialTitle": "部分復元", - "xpack.snapshotRestore.restoreForm.stepLogistics.renameIndicesDescription": "復元時にインデックス名を変更します。", - "xpack.snapshotRestore.restoreForm.stepLogistics.renameIndicesLabel": "インデックス名の変更", - "xpack.snapshotRestore.restoreForm.stepLogistics.renameIndicesTitle": "インデックス名の変更", "xpack.snapshotRestore.restoreForm.stepLogistics.renamePatternHelpText": "正規表現を使用", "xpack.snapshotRestore.restoreForm.stepLogistics.renamePatternLabel": "取り込みパターン", "xpack.snapshotRestore.restoreForm.stepLogistics.renameReplacementLabel": "置換パターン", "xpack.snapshotRestore.restoreForm.stepLogistics.selectAllIndicesLink": "すべて選択", - "xpack.snapshotRestore.restoreForm.stepLogistics.selectIndicesHelpText": "{count} 件の{count, plural, one {インデックス} other {インデックス}}が復元されます。{selectOrDeselectAllLink}", - "xpack.snapshotRestore.restoreForm.stepLogistics.selectIndicesLabel": "インデックスを選択", "xpack.snapshotRestore.restoreForm.stepLogisticsTitle": "詳細を復元", "xpack.snapshotRestore.restoreForm.stepReview.jsonTab.jsonAriaLabel": "実行する設定を復元", "xpack.snapshotRestore.restoreForm.stepReview.jsonTabTitle": "JSON", @@ -13908,7 +13891,6 @@ "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.includeGlobalStateLabel": "グローバル状態の復元", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.includeGlobalStateTrueValue": "はい", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.indexSettingsLabel": "修正", - "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.indicesLabel": "インデックス", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.noSettingsValue": "インデックス設定の修正はありません", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.partialFalseValue": "いいえ", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.partialLabel": "部分復元", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 568397912141d2..97f10e77dc7177 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -13392,7 +13392,6 @@ "xpack.snapshotRestore.policyDetails.includeGlobalStateFalseLabel": "否", "xpack.snapshotRestore.policyDetails.includeGlobalStateLabel": "包括全局状态", "xpack.snapshotRestore.policyDetails.includeGlobalStateTrueLabel": "是", - "xpack.snapshotRestore.policyDetails.indicesLabel": "索引", "xpack.snapshotRestore.policyDetails.inProgressSnapshotLinkText": "“{snapshotName}”正在进行中", "xpack.snapshotRestore.policyDetails.lastFailure.dateLabel": "日期", "xpack.snapshotRestore.policyDetails.lastFailure.detailsAriaLabel": "策略“{name}”的上次失败详情", @@ -13500,10 +13499,8 @@ "xpack.snapshotRestore.policyForm.stepReview.summaryTab.includeGlobalStateFalseLabel": "否", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.includeGlobalStateLabel": "包括全局状态", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.includeGlobalStateTrueLabel": "是", - "xpack.snapshotRestore.policyForm.stepReview.summaryTab.indicesLabel": "索引", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.nameLabel": "策略名称", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.partialFalseLabel": "否", - "xpack.snapshotRestore.policyForm.stepReview.summaryTab.partialLabel": "允许部分分片", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.partialTrueLabel": "是", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.repositoryLabel": "存储库", "xpack.snapshotRestore.policyForm.stepReview.summaryTab.scheduleLabel": "计划", @@ -13512,7 +13509,6 @@ "xpack.snapshotRestore.policyForm.stepReview.summaryTab.snapshotNameLabel": "快照名称", "xpack.snapshotRestore.policyForm.stepReview.summaryTabTitle": "总结", "xpack.snapshotRestore.policyForm.stepReviewTitle": "复查策略", - "xpack.snapshotRestore.policyForm.stepSettings.allIndicesLabel": "所有索引，包括系统索引", "xpack.snapshotRestore.policyForm.stepSettings.deselectAllIndicesLink": "取消全选", "xpack.snapshotRestore.policyForm.stepSettings.docsButtonLabel": "快照设置文档", "xpack.snapshotRestore.policyForm.stepSettings.ignoreUnavailableDescription": "拍取快照时忽略不可用的索引。否则，整个快照将失败。", @@ -13520,19 +13516,15 @@ "xpack.snapshotRestore.policyForm.stepSettings.ignoreUnavailableLabel": "忽略不可用索引", "xpack.snapshotRestore.policyForm.stepSettings.includeGlobalStateDescription": "将集群的全局状态存储为快照的一部分。", "xpack.snapshotRestore.policyForm.stepSettings.includeGlobalStateDescriptionTitle": "包括全局状态", - "xpack.snapshotRestore.policyForm.stepSettings.indicesDescription": "要备份的索引。", "xpack.snapshotRestore.policyForm.stepSettings.indicesPatternLabel": "索引模式", "xpack.snapshotRestore.policyForm.stepSettings.indicesPatternPlaceholder": "输入索引模式，例如 logstash-*", - "xpack.snapshotRestore.policyForm.stepSettings.indicesTitle": "索引", "xpack.snapshotRestore.policyForm.stepSettings.indicesToggleCustomLink": "使用索引模式", - "xpack.snapshotRestore.policyForm.stepSettings.indicesToggleListLink": "选择索引", "xpack.snapshotRestore.policyForm.stepSettings.indicesTooltip": "云托管的策略需要所有索引。", "xpack.snapshotRestore.policyForm.stepSettings.partialDescription": "允许具有不可用主分片的索引的快照。否则，整个快照将失败。", "xpack.snapshotRestore.policyForm.stepSettings.partialDescriptionTitle": "允许部分索引", "xpack.snapshotRestore.policyForm.stepSettings.partialIndicesToggleSwitch": "允许部分索引", "xpack.snapshotRestore.policyForm.stepSettings.policyIncludeGlobalStateLabel": "包括全局状态", "xpack.snapshotRestore.policyForm.stepSettings.selectAllIndicesLink": "全选", - "xpack.snapshotRestore.policyForm.stepSettings.selectIndicesHelpText": "将备份 {count} 个 {count, plural, one {索引} other {索引}}。{selectOrDeselectAllLink}", "xpack.snapshotRestore.policyForm.stepSettings.selectIndicesLabel": "选择索引", "xpack.snapshotRestore.policyForm.stepSettingsTitle": "快照设置", "xpack.snapshotRestore.policyList.deniedPrivilegeDescription": "要管理快照生命周期策略，必须具有{privilegesCount, plural, one {以下集群权限} other {以下集群权限}}：{missingPrivileges}。", @@ -13879,31 +13871,22 @@ "xpack.snapshotRestore.restoreForm.navigation.stepSettingsName": "索引设置", "xpack.snapshotRestore.restoreForm.nextButtonLabel": "下一步", "xpack.snapshotRestore.restoreForm.savingButtonLabel": "正在还原……", - "xpack.snapshotRestore.restoreForm.stepLogistics.allIndicesLabel": "所有索引，包括系统索引", "xpack.snapshotRestore.restoreForm.stepLogistics.deselectAllIndicesLink": "取消全选", "xpack.snapshotRestore.restoreForm.stepLogistics.docsButtonLabel": "快照和还原文档", "xpack.snapshotRestore.restoreForm.stepLogistics.includeGlobalStateDescription": "还原当前在集群中不存在的模板并覆盖同名模板。同时还原永久性设置。", "xpack.snapshotRestore.restoreForm.stepLogistics.includeGlobalStateDisabledDescription": "不适用于此快照。", "xpack.snapshotRestore.restoreForm.stepLogistics.includeGlobalStateLabel": "还原全局状态", "xpack.snapshotRestore.restoreForm.stepLogistics.includeGlobalStateTitle": "还原全局状态", - "xpack.snapshotRestore.restoreForm.stepLogistics.indicesDescription": "如果不存在，则创建新索引。如果现有索引已关闭且与快照索引有相同数目的分片，则还原现有索引。", "xpack.snapshotRestore.restoreForm.stepLogistics.indicesPatternLabel": "索引模式", "xpack.snapshotRestore.restoreForm.stepLogistics.indicesPatternPlaceholder": "输入索引模式，例如 logstash-*", - "xpack.snapshotRestore.restoreForm.stepLogistics.indicesTitle": "索引", "xpack.snapshotRestore.restoreForm.stepLogistics.indicesToggleCustomLink": "使用索引模式", - "xpack.snapshotRestore.restoreForm.stepLogistics.indicesToggleListLink": "选择索引", "xpack.snapshotRestore.restoreForm.stepLogistics.partialDescription": "允许还原不具有所有分片的快照的索引。", "xpack.snapshotRestore.restoreForm.stepLogistics.partialLabel": "部分还原", "xpack.snapshotRestore.restoreForm.stepLogistics.partialTitle": "部分还原", - "xpack.snapshotRestore.restoreForm.stepLogistics.renameIndicesDescription": "还原时重命名索引。", - "xpack.snapshotRestore.restoreForm.stepLogistics.renameIndicesLabel": "重命名索引", - "xpack.snapshotRestore.restoreForm.stepLogistics.renameIndicesTitle": "重命名索引", "xpack.snapshotRestore.restoreForm.stepLogistics.renamePatternHelpText": "使用正则表达式", "xpack.snapshotRestore.restoreForm.stepLogistics.renamePatternLabel": "捕获模式", "xpack.snapshotRestore.restoreForm.stepLogistics.renameReplacementLabel": "替换模式", "xpack.snapshotRestore.restoreForm.stepLogistics.selectAllIndicesLink": "全选", - "xpack.snapshotRestore.restoreForm.stepLogistics.selectIndicesHelpText": "将还原 {count} 个 {count, plural, one {索引} other {索引}}。{selectOrDeselectAllLink}", - "xpack.snapshotRestore.restoreForm.stepLogistics.selectIndicesLabel": "选择索引", "xpack.snapshotRestore.restoreForm.stepLogisticsTitle": "还原详情", "xpack.snapshotRestore.restoreForm.stepReview.jsonTab.jsonAriaLabel": "还原要执行的设置", "xpack.snapshotRestore.restoreForm.stepReview.jsonTabTitle": "JSON", @@ -13913,7 +13896,6 @@ "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.includeGlobalStateLabel": "还原全局状态", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.includeGlobalStateTrueValue": "鏄", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.indexSettingsLabel": "修改", - "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.indicesLabel": "索引", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.noSettingsValue": "无索引设置修改", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.partialFalseValue": "否", "xpack.snapshotRestore.restoreForm.stepReview.summaryTab.partialLabel": "部分还原", From c6867197ffc2808b8127d25a796c4fd21b51fd5e Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 2 Jul 2020 15:50:43 +0200 Subject: [PATCH 23/31] Allow Saved Object type mappings to set a field's `doc_values` property (#70433) * Allow doc_values to be disabled * Make doc_values optional * doc_values type for CoreFieldMapping * doc_values not doc_value * Update docs Co-authored-by: Elastic Machine --- ...rver.savedobjectscomplexfieldmapping.doc_values.md | 11 +++++++++++ ...gin-core-server.savedobjectscomplexfieldmapping.md | 1 + ...-server.savedobjectscorefieldmapping.doc_values.md | 11 +++++++++++ ...plugin-core-server.savedobjectscorefieldmapping.md | 1 + src/core/server/saved_objects/mappings/types.ts | 2 ++ src/core/server/server.api.md | 4 ++++ 6 files changed, 30 insertions(+) create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md new file mode 100644 index 00000000000000..3f2d81cc97c7c8 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) > [doc\_values](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md) + +## SavedObjectsComplexFieldMapping.doc\_values property + +Signature: + +```typescript +doc_values?: boolean; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.md index a7d13b0015e3fd..cb81686b424ecc 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscomplexfieldmapping.md @@ -18,6 +18,7 @@ export interface SavedObjectsComplexFieldMapping | Property | Type | Description | | --- | --- | --- | +| [doc\_values](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.doc_values.md) | boolean | | | [properties](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.properties.md) | SavedObjectsMappingProperties | | | [type](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.type.md) | string | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md new file mode 100644 index 00000000000000..2a79eafd85a6c2 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsCoreFieldMapping](./kibana-plugin-core-server.savedobjectscorefieldmapping.md) > [doc\_values](./kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md) + +## SavedObjectsCoreFieldMapping.doc\_values property + +Signature: + +```typescript +doc_values?: boolean; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md index 9a31d37b3ff30e..b9e726eac799d7 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectscorefieldmapping.md @@ -16,6 +16,7 @@ export interface SavedObjectsCoreFieldMapping | Property | Type | Description | | --- | --- | --- | +| [doc\_values](./kibana-plugin-core-server.savedobjectscorefieldmapping.doc_values.md) | boolean | | | [enabled](./kibana-plugin-core-server.savedobjectscorefieldmapping.enabled.md) | boolean | | | [fields](./kibana-plugin-core-server.savedobjectscorefieldmapping.fields.md) | {
[subfield: string]: {
type: string;
ignore_above?: number;
};
} | | | [index](./kibana-plugin-core-server.savedobjectscorefieldmapping.index.md) | boolean | | diff --git a/src/core/server/saved_objects/mappings/types.ts b/src/core/server/saved_objects/mappings/types.ts index c037ed733549e6..7521e4a4bee869 100644 --- a/src/core/server/saved_objects/mappings/types.ts +++ b/src/core/server/saved_objects/mappings/types.ts @@ -133,6 +133,7 @@ export interface SavedObjectsCoreFieldMapping { type: string; null_value?: number | boolean | string; index?: boolean; + doc_values?: boolean; enabled?: boolean; fields?: { [subfield: string]: { @@ -153,6 +154,7 @@ export interface SavedObjectsCoreFieldMapping { * @public */ export interface SavedObjectsComplexFieldMapping { + doc_values?: boolean; type?: string; properties: SavedObjectsMappingProperties; } diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 1cabaa57e519ce..cb413be2c19b89 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -1978,6 +1978,8 @@ export interface SavedObjectsClientWrapperOptions { // @public export interface SavedObjectsComplexFieldMapping { + // (undocumented) + doc_values?: boolean; // (undocumented) properties: SavedObjectsMappingProperties; // (undocumented) @@ -1986,6 +1988,8 @@ export interface SavedObjectsComplexFieldMapping { // @public export interface SavedObjectsCoreFieldMapping { + // (undocumented) + doc_values?: boolean; // (undocumented) enabled?: boolean; // (undocumented) From f8ba824ebc7e3d551b791ba7b8c04f31cddcaf10 Mon Sep 17 00:00:00 2001 From: Nick Partridge Date: Thu, 2 Jul 2020 09:02:30 -0500 Subject: [PATCH 24/31] Fix discover, tsvb and Lens chart theming issues (#69695) --- package.json | 2 +- packages/kbn-ui-shared-deps/package.json | 2 +- src/plugins/charts/README.md | 72 +-------------- .../charts/public/services/theme/README.md | 92 +++++++++++++++++++ .../charts/public/services/theme/mock.ts | 14 ++- .../public/services/theme/theme.test.tsx | 64 ++++++++++++- .../charts/public/services/theme/theme.ts | 68 +++++++++++--- .../angular/directives/histogram.tsx | 16 +++- .../components/vis_types/_vis_types.scss | 6 +- .../components/vis_types/timeseries/vis.js | 6 +- .../visualizations/views/timeseries/index.js | 23 +++-- .../views/timeseries/utils/theme.test.ts | 20 ++-- .../views/timeseries/utils/theme.ts | 10 +- x-pack/plugins/lens/kibana.json | 3 +- .../datapanel.test.tsx | 2 + .../indexpattern_datasource/datapanel.tsx | 7 ++ .../field_item.test.tsx | 4 + .../indexpattern_datasource/field_item.tsx | 16 +++- .../fields_accordion.test.tsx | 2 + .../fields_accordion.tsx | 2 + .../public/indexpattern_datasource/index.ts | 5 +- .../indexpattern.test.ts | 2 + .../indexpattern_datasource/indexpattern.tsx | 4 + .../lens/public/pie_visualization/index.ts | 10 +- .../pie_visualization/register_expression.tsx | 8 +- .../render_function.test.tsx | 6 +- .../pie_visualization/render_function.tsx | 12 ++- x-pack/plugins/lens/public/plugin.ts | 34 +++++-- .../__snapshots__/xy_expression.test.tsx.snap | 7 ++ .../lens/public/xy_visualization/index.ts | 9 +- .../xy_visualization/xy_expression.test.tsx | 59 ++++++------ .../public/xy_visualization/xy_expression.tsx | 13 ++- .../threshold/visualization.tsx | 5 +- yarn.lock | 31 ++++++- 34 files changed, 439 insertions(+), 197 deletions(-) create mode 100644 src/plugins/charts/public/services/theme/README.md diff --git a/package.json b/package.json index b520be4df69696..b1dd8686f818bd 100644 --- a/package.json +++ b/package.json @@ -122,7 +122,7 @@ "@babel/plugin-transform-modules-commonjs": "^7.10.1", "@babel/register": "^7.10.1", "@elastic/apm-rum": "^5.2.0", - "@elastic/charts": "19.5.2", + "@elastic/charts": "19.6.3", "@elastic/datemath": "5.0.3", "@elastic/ems-client": "7.9.3", "@elastic/eui": "24.1.0", diff --git a/packages/kbn-ui-shared-deps/package.json b/packages/kbn-ui-shared-deps/package.json index 0e3bb235c3d9f5..ff09d8d4fc5ab6 100644 --- a/packages/kbn-ui-shared-deps/package.json +++ b/packages/kbn-ui-shared-deps/package.json @@ -9,7 +9,7 @@ "kbn:watch": "node scripts/build --dev --watch" }, "dependencies": { - "@elastic/charts": "19.5.2", + "@elastic/charts": "19.6.3", "@elastic/eui": "24.1.0", "@elastic/numeral": "^2.5.0", "@kbn/i18n": "1.0.0", diff --git a/src/plugins/charts/README.md b/src/plugins/charts/README.md index 319da67981aa99..31727b7acb7a1a 100644 --- a/src/plugins/charts/README.md +++ b/src/plugins/charts/README.md @@ -18,7 +18,7 @@ Color mappings in `value`/`text` form ### `getHeatmapColors` -Funciton to retrive heatmap related colors based on `value` and `colorSchemaName` +Function to retrieve heatmap related colors based on `value` and `colorSchemaName` ### `truncatedColorSchemas` @@ -26,72 +26,4 @@ Truncated color mappings in `value`/`text` form ## Theme -the `theme` service offers utilities to interact with theme of kibana. EUI provides a light and dark theme object to work with Elastic-Charts. However, every instance of a Chart would need to pass down this the correctly EUI theme depending on Kibana's light or dark mode. There are several ways you can use the `theme` service to get the correct theme. - -> The current theme (light or dark) of Kibana is typically taken into account for the functions below. - -### `useChartsTheme` - -The simple fetching of the correct EUI theme; a **React hook**. - -```js -import { npStart } from 'ui/new_platform'; -import { Chart, Settings } from '@elastic/charts'; - -export const YourComponent = () => ( - - - -); -``` - -### `chartsTheme$` - -An **observable** of the current charts theme. Use this implementation for more flexible updates to the chart theme without full page refreshes. - -```tsx -import { npStart } from 'ui/new_platform'; -import { EuiChartThemeType } from '@elastic/eui/src/themes/charts/themes'; -import { Subscription } from 'rxjs'; -import { Chart, Settings } from '@elastic/charts'; - -interface YourComponentProps {}; - -interface YourComponentState { - chartsTheme: EuiChartThemeType['theme']; -} - -export class YourComponent extends Component { - private subscription?: Subscription; - public state = { - chartsTheme: npStart.plugins.charts.theme.chartsDefaultTheme, - }; - - componentDidMount() { - this.subscription = npStart.plugins.charts.theme - .chartsTheme$ - .subscribe(chartsTheme => this.setState({ chartsTheme })); - } - - componentWillUnmount() { - if (this.subscription) { - this.subscription.unsubscribe(); - this.subscription = undefined; - } - } - - public render() { - const { chartsTheme } = this.state; - - return ( - - - - ); - } -} -``` - -### `chartsDefaultTheme` - -Returns default charts theme (i.e. light). +See Theme service [docs](public/services/theme/README.md) diff --git a/src/plugins/charts/public/services/theme/README.md b/src/plugins/charts/public/services/theme/README.md new file mode 100644 index 00000000000000..fb4f941f793447 --- /dev/null +++ b/src/plugins/charts/public/services/theme/README.md @@ -0,0 +1,92 @@ +# Theme Service + +The `theme` service offers utilities to interact with the kibana theme. EUI provides a light and dark theme object to supplement the Elastic-Charts `baseTheme`. However, every instance of a Chart would need to pass down the correct EUI theme depending on Kibana's light or dark mode. There are several ways you can use the `theme` service to get the correct shared `theme` and `baseTheme`. + +> The current theme (light or dark) of Kibana is typically taken into account for the functions below. + +## `chartsDefaultBaseTheme` + +Default `baseTheme` from `@elastic/charts` (i.e. light). + +## `chartsDefaultTheme` + +Default `theme` from `@elastic/eui` (i.e. light). + +## `useChartsTheme` and `useChartsBaseTheme` + +A **React hook** for simple fetching of the correct EUI `theme` and `baseTheme`. + +```js +import { npStart } from 'ui/new_platform'; +import { Chart, Settings } from '@elastic/charts'; + +export const YourComponent = () => ( + + + {/* ... */} + +); +``` + +## `chartsTheme$` and `chartsBaseTheme$` + +An **`Observable`** of the current charts `theme` and `baseTheme`. Use this implementation for more flexible updates to the chart theme without full page refreshes. + +```tsx +import { npStart } from 'ui/new_platform'; +import { EuiChartThemeType } from '@elastic/eui/src/themes/charts/themes'; +import { Subscription, combineLatest } from 'rxjs'; +import { Chart, Settings, Theme } from '@elastic/charts'; + +interface YourComponentProps {}; + +interface YourComponentState { + chartsTheme: EuiChartThemeType['theme']; + chartsBaseTheme: Theme; +} + +export class YourComponent extends Component { + private subscriptions: Subscription[] = []; + + public state = { + chartsTheme: npStart.plugins.charts.theme.chartsDefaultTheme, + chartsBaseTheme: npStart.plugins.charts.theme.chartsDefaultBaseTheme, + }; + + componentDidMount() { + this.subscription = combineLatest( + npStart.plugins.charts.theme.chartsTheme$, + npStart.plugins.charts.theme.chartsBaseTheme$ + ).subscribe(([chartsTheme, chartsBaseTheme]) => + this.setState({ chartsTheme, chartsBaseTheme }) + ); + } + + componentWillUnmount() { + if (this.subscription) { + this.subscription.unsubscribe(); + } + } + + public render() { + const { chartsBaseTheme, chartsTheme } = this.state; + + return ( + + + {/* ... */} + + ); + } +} +``` + +## Why have `theme` and `baseTheme`? + +The `theme` prop is a recursive partial `Theme` that overrides properties from the `baseTheme`. This allows changes to the `Theme` TS type in `@elastic/charts` without having to update the `@elastic/eui` themes for every ``. diff --git a/src/plugins/charts/public/services/theme/mock.ts b/src/plugins/charts/public/services/theme/mock.ts index 8aa1a4f2368ac3..7fecb862a3c65f 100644 --- a/src/plugins/charts/public/services/theme/mock.ts +++ b/src/plugins/charts/public/services/theme/mock.ts @@ -21,9 +21,17 @@ import { EUI_CHARTS_THEME_LIGHT } from '@elastic/eui/dist/eui_charts_theme'; import { ThemeService } from './theme'; export const themeServiceMock: ThemeService = { + chartsDefaultTheme: EUI_CHARTS_THEME_LIGHT.theme, chartsTheme$: jest.fn(() => ({ - subsribe: jest.fn(), + subscribe: jest.fn(), })), - chartsDefaultTheme: EUI_CHARTS_THEME_LIGHT.theme, - useChartsTheme: jest.fn(), + chartsBaseTheme$: jest.fn(() => ({ + subscribe: jest.fn(), + })), + darkModeEnabled$: jest.fn(() => ({ + subscribe: jest.fn(), + })), + useDarkMode: jest.fn().mockReturnValue(false), + useChartsTheme: jest.fn().mockReturnValue({}), + useChartsBaseTheme: jest.fn().mockReturnValue({}), } as any; diff --git a/src/plugins/charts/public/services/theme/theme.test.tsx b/src/plugins/charts/public/services/theme/theme.test.tsx index fca503e387ea25..52bc78dfec7dfa 100644 --- a/src/plugins/charts/public/services/theme/theme.test.tsx +++ b/src/plugins/charts/public/services/theme/theme.test.tsx @@ -25,15 +25,35 @@ import { EUI_CHARTS_THEME_DARK, EUI_CHARTS_THEME_LIGHT } from '@elastic/eui/dist import { ThemeService } from './theme'; import { coreMock } from '../../../../../core/public/mocks'; +import { LIGHT_THEME, DARK_THEME } from '@elastic/charts'; const { uiSettings: setupMockUiSettings } = coreMock.createSetup(); describe('ThemeService', () => { - describe('chartsTheme$', () => { + describe('darkModeEnabled$', () => { it('should throw error if service has not been initialized', () => { const themeService = new ThemeService(); - expect(() => themeService.chartsTheme$).toThrowError(); + expect(() => themeService.darkModeEnabled$).toThrowError(); + }); + + it('returns the false when not in dark mode', async () => { + setupMockUiSettings.get$.mockReturnValue(new BehaviorSubject(false)); + const themeService = new ThemeService(); + themeService.init(setupMockUiSettings); + + expect(await themeService.darkModeEnabled$.pipe(take(1)).toPromise()).toBe(false); + }); + + it('returns the true when in dark mode', async () => { + setupMockUiSettings.get$.mockReturnValue(new BehaviorSubject(true)); + const themeService = new ThemeService(); + themeService.init(setupMockUiSettings); + + expect(await themeService.darkModeEnabled$.pipe(take(1)).toPromise()).toBe(true); }); + }); + + describe('chartsTheme$', () => { it('returns the light theme when not in dark mode', async () => { setupMockUiSettings.get$.mockReturnValue(new BehaviorSubject(false)); const themeService = new ThemeService(); @@ -58,6 +78,28 @@ describe('ThemeService', () => { }); }); + describe('chartsBaseTheme$', () => { + it('returns the light theme when not in dark mode', async () => { + setupMockUiSettings.get$.mockReturnValue(new BehaviorSubject(false)); + const themeService = new ThemeService(); + themeService.init(setupMockUiSettings); + + expect(await themeService.chartsBaseTheme$.pipe(take(1)).toPromise()).toEqual(LIGHT_THEME); + }); + + describe('in dark mode', () => { + it(`returns the dark theme`, async () => { + // Fake dark theme turned returning true + setupMockUiSettings.get$.mockReturnValue(new BehaviorSubject(true)); + const themeService = new ThemeService(); + themeService.init(setupMockUiSettings); + const result = await themeService.chartsBaseTheme$.pipe(take(1)).toPromise(); + + expect(result).toEqual(DARK_THEME); + }); + }); + }); + describe('useChartsTheme', () => { it('updates when the uiSettings change', () => { const darkMode$ = new BehaviorSubject(false); @@ -75,4 +117,22 @@ describe('ThemeService', () => { expect(result.current).toBe(EUI_CHARTS_THEME_LIGHT.theme); }); }); + + describe('useBaseChartTheme', () => { + it('updates when the uiSettings change', () => { + const darkMode$ = new BehaviorSubject(false); + setupMockUiSettings.get$.mockReturnValue(darkMode$); + const themeService = new ThemeService(); + themeService.init(setupMockUiSettings); + const { useChartsBaseTheme } = themeService; + + const { result } = renderHook(() => useChartsBaseTheme()); + expect(result.current).toBe(LIGHT_THEME); + + act(() => darkMode$.next(true)); + expect(result.current).toBe(DARK_THEME); + act(() => darkMode$.next(false)); + expect(result.current).toBe(LIGHT_THEME); + }); + }); }); diff --git a/src/plugins/charts/public/services/theme/theme.ts b/src/plugins/charts/public/services/theme/theme.ts index e1e71573caa3a5..2d0c4de8832188 100644 --- a/src/plugins/charts/public/services/theme/theme.ts +++ b/src/plugins/charts/public/services/theme/theme.ts @@ -18,34 +18,56 @@ */ import { useEffect, useState } from 'react'; -import { map } from 'rxjs/operators'; -import { Observable } from 'rxjs'; +import { Observable, BehaviorSubject } from 'rxjs'; import { CoreSetup } from 'kibana/public'; -import { RecursivePartial, Theme } from '@elastic/charts'; +import { DARK_THEME, LIGHT_THEME, PartialTheme, Theme } from '@elastic/charts'; import { EUI_CHARTS_THEME_DARK, EUI_CHARTS_THEME_LIGHT } from '@elastic/eui/dist/eui_charts_theme'; export class ThemeService { - private _chartsTheme$?: Observable>; - /** Returns default charts theme */ public readonly chartsDefaultTheme = EUI_CHARTS_THEME_LIGHT.theme; + public readonly chartsDefaultBaseTheme = LIGHT_THEME; + + private _uiSettingsDarkMode$?: Observable; + private _chartsTheme$ = new BehaviorSubject(this.chartsDefaultTheme); + private _chartsBaseTheme$ = new BehaviorSubject(this.chartsDefaultBaseTheme); /** An observable of the current charts theme */ - public get chartsTheme$(): Observable> { - if (!this._chartsTheme$) { + public chartsTheme$ = this._chartsTheme$.asObservable(); + + /** An observable of the current charts base theme */ + public chartsBaseTheme$ = this._chartsBaseTheme$.asObservable(); + + /** An observable boolean for dark mode of kibana */ + public get darkModeEnabled$(): Observable { + if (!this._uiSettingsDarkMode$) { throw new Error('ThemeService not initialized'); } - return this._chartsTheme$; + return this._uiSettingsDarkMode$; } + /** A React hook for consuming the dark mode value */ + public useDarkMode = (): boolean => { + // eslint-disable-next-line react-hooks/rules-of-hooks + const [value, update] = useState(false); + + // eslint-disable-next-line react-hooks/rules-of-hooks + useEffect(() => { + const s = this.darkModeEnabled$.subscribe(update); + return () => s.unsubscribe(); + }, []); + + return value; + }; + /** A React hook for consuming the charts theme */ - public useChartsTheme = () => { - /* eslint-disable-next-line react-hooks/rules-of-hooks */ + public useChartsTheme = (): PartialTheme => { + // eslint-disable-next-line react-hooks/rules-of-hooks const [value, update] = useState(this.chartsDefaultTheme); - /* eslint-disable-next-line react-hooks/rules-of-hooks */ + // eslint-disable-next-line react-hooks/rules-of-hooks useEffect(() => { const s = this.chartsTheme$.subscribe(update); return () => s.unsubscribe(); @@ -54,12 +76,28 @@ export class ThemeService { return value; }; + /** A React hook for consuming the charts theme */ + public useChartsBaseTheme = (): Theme => { + // eslint-disable-next-line react-hooks/rules-of-hooks + const [value, update] = useState(this.chartsDefaultBaseTheme); + + // eslint-disable-next-line react-hooks/rules-of-hooks + useEffect(() => { + const s = this.chartsBaseTheme$.subscribe(update); + return () => s.unsubscribe(); + }, []); + + return value; + }; + /** initialize service with uiSettings */ public init(uiSettings: CoreSetup['uiSettings']) { - this._chartsTheme$ = uiSettings - .get$('theme:darkMode') - .pipe( - map((darkMode) => (darkMode ? EUI_CHARTS_THEME_DARK.theme : EUI_CHARTS_THEME_LIGHT.theme)) + this._uiSettingsDarkMode$ = uiSettings.get$('theme:darkMode'); + this._uiSettingsDarkMode$.subscribe((darkMode) => { + this._chartsTheme$.next( + darkMode ? EUI_CHARTS_THEME_DARK.theme : EUI_CHARTS_THEME_LIGHT.theme ); + this._chartsBaseTheme$.next(darkMode ? DARK_THEME : LIGHT_THEME); + }); } } diff --git a/src/plugins/discover/public/application/angular/directives/histogram.tsx b/src/plugins/discover/public/application/angular/directives/histogram.tsx index 9afe5e48bc5b82..4c39c8bb255422 100644 --- a/src/plugins/discover/public/application/angular/directives/histogram.tsx +++ b/src/plugins/discover/public/application/angular/directives/histogram.tsx @@ -40,12 +40,13 @@ import { ElementClickListener, XYChartElementEvent, BrushEndListener, + Theme, } from '@elastic/charts'; import { i18n } from '@kbn/i18n'; import { IUiSettingsClient } from 'kibana/public'; import { EuiChartThemeType } from '@elastic/eui/dist/eui_charts_theme'; -import { Subscription } from 'rxjs'; +import { Subscription, combineLatest } from 'rxjs'; import { getServices } from '../../../kibana_services'; import { Chart as IChart } from '../helpers/point_series'; @@ -56,6 +57,7 @@ export interface DiscoverHistogramProps { interface DiscoverHistogramState { chartsTheme: EuiChartThemeType['theme']; + chartsBaseTheme: Theme; } function findIntervalFromDuration( @@ -126,18 +128,21 @@ export class DiscoverHistogram extends Component this.setState({ chartsTheme }) + this.subscription = combineLatest( + getServices().theme.chartsTheme$, + getServices().theme.chartsBaseTheme$ + ).subscribe(([chartsTheme, chartsBaseTheme]) => + this.setState({ chartsTheme, chartsBaseTheme }) ); } componentWillUnmount() { if (this.subscription) { this.subscription.unsubscribe(); - this.subscription = undefined; } } @@ -204,7 +209,7 @@ export class DiscoverHistogram extends Component values.map(({ key, docs }) => ({ @@ -56,7 +56,6 @@ const handleCursorUpdate = (cursor) => { }; export const TimeSeries = ({ - darkMode, backgroundColor, showGrid, legend, @@ -90,15 +89,15 @@ export const TimeSeries = ({ const timeZone = getTimezone(uiSettings); const hasBarChart = series.some(({ bars }) => bars?.show); - // compute the theme based on the bg color - const theme = getTheme(darkMode, backgroundColor); // apply legend style change if bgColor is configured const classes = classNames('tvbVisTimeSeries', getChartClasses(backgroundColor)); // If the color isn't configured by the user, use the color mapping service // to assign a color from the Kibana palette. Colors will be shared across the // session, including dashboards. - const { colors } = getChartsSetup(); + const { colors, theme: themeService } = getChartsSetup(); + const baseTheme = getBaseTheme(themeService.useChartsBaseTheme(), backgroundColor); + colors.mappedColors.mapKeys(series.filter(({ color }) => !color).map(({ label }) => label)); const onBrushEndListener = ({ x }) => { @@ -118,7 +117,7 @@ export const TimeSeries = ({ onBrushEnd={onBrushEndListener} animateData={false} onPointerUpdate={handleCursorUpdate} - theme={ + theme={[ hasBarChart ? {} : { @@ -127,9 +126,14 @@ export const TimeSeries = ({ fill: '#F00', }, }, - } - } - baseTheme={theme} + }, + { + background: { + color: backgroundColor, + }, + }, + ]} + baseTheme={baseTheme} tooltip={{ snap: true, type: tooltipMode === 'show_focused' ? TooltipType.Follow : TooltipType.VerticalCursor, @@ -269,7 +273,6 @@ TimeSeries.defaultProps = { }; TimeSeries.propTypes = { - darkMode: PropTypes.bool, backgroundColor: PropTypes.string, showGrid: PropTypes.bool, legend: PropTypes.bool, diff --git a/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/utils/theme.test.ts b/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/utils/theme.test.ts index 57ca38168ac27f..d7e6560a8dc971 100644 --- a/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/utils/theme.test.ts +++ b/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/utils/theme.test.ts @@ -17,28 +17,30 @@ * under the License. */ -import { getTheme } from './theme'; +import { getBaseTheme } from './theme'; import { LIGHT_THEME, DARK_THEME } from '@elastic/charts'; describe('TSVB theme', () => { it('should return the basic themes if no bg color is specified', () => { // use original dark/light theme - expect(getTheme(false)).toEqual(LIGHT_THEME); - expect(getTheme(true)).toEqual(DARK_THEME); + expect(getBaseTheme(LIGHT_THEME)).toEqual(LIGHT_THEME); + expect(getBaseTheme(DARK_THEME)).toEqual(DARK_THEME); // discard any wrong/missing bg color - expect(getTheme(true, null)).toEqual(DARK_THEME); - expect(getTheme(true, '')).toEqual(DARK_THEME); - expect(getTheme(true, undefined)).toEqual(DARK_THEME); + expect(getBaseTheme(DARK_THEME, null)).toEqual(DARK_THEME); + expect(getBaseTheme(DARK_THEME, '')).toEqual(DARK_THEME); + expect(getBaseTheme(DARK_THEME, undefined)).toEqual(DARK_THEME); }); it('should return a highcontrast color theme for a different background', () => { // red use a near full-black color - expect(getTheme(false, 'red').axes.axisTitleStyle.fill).toEqual('rgb(23,23,23)'); + expect(getBaseTheme(LIGHT_THEME, 'red').axes.axisTitleStyle.fill).toEqual('rgb(23,23,23)'); // violet increased the text color to full white for higer contrast - expect(getTheme(false, '#ba26ff').axes.axisTitleStyle.fill).toEqual('rgb(255,255,255)'); + expect(getBaseTheme(LIGHT_THEME, '#ba26ff').axes.axisTitleStyle.fill).toEqual( + 'rgb(255,255,255)' + ); // light yellow, prefer the LIGHT_THEME fill color because already with a good contrast - expect(getTheme(false, '#fff49f').axes.axisTitleStyle.fill).toEqual('#333'); + expect(getBaseTheme(LIGHT_THEME, '#fff49f').axes.axisTitleStyle.fill).toEqual('#333'); }); }); diff --git a/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/utils/theme.ts b/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/utils/theme.ts index 2694732aa381d2..0e13fd7ef68f96 100644 --- a/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/utils/theme.ts +++ b/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/utils/theme.ts @@ -94,9 +94,15 @@ function isValidColor(color: string | null | undefined): color is string { } } -export function getTheme(darkMode: boolean, bgColor?: string | null): Theme { +/** + * compute base chart theme based on the background color + * + * @param baseTheme + * @param bgColor + */ +export function getBaseTheme(baseTheme: Theme, bgColor?: string | null): Theme { if (!isValidColor(bgColor)) { - return darkMode ? DARK_THEME : LIGHT_THEME; + return baseTheme; } const bgLuminosity = computeRelativeLuminosity(bgColor); diff --git a/x-pack/plugins/lens/kibana.json b/x-pack/plugins/lens/kibana.json index 346a5a24c269f4..7da5eaed5155ef 100644 --- a/x-pack/plugins/lens/kibana.json +++ b/x-pack/plugins/lens/kibana.json @@ -10,7 +10,8 @@ "navigation", "kibanaLegacy", "visualizations", - "dashboard" + "dashboard", + "charts" ], "optionalPlugins": ["embeddable", "usageCollection", "taskManager", "uiActions"], "configPath": ["xpack", "lens"], diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx index f70df855fe0cb3..0d60bd588f710f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx @@ -17,6 +17,7 @@ import { mountWithIntl, shallowWithIntl } from 'test_utils/enzyme_helpers'; import { ChangeIndexPattern } from './change_indexpattern'; import { EuiProgress, EuiLoadingSpinner } from '@elastic/eui'; import { documentField } from './document_field'; +import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; const initialState: IndexPatternPrivateState = { indexPatternRefs: [], @@ -230,6 +231,7 @@ describe('IndexPattern Data Panel', () => { fromDate: 'now-7d', toDate: 'now', }, + charts: chartPluginMock.createSetupContract(), query: { query: '', language: 'lucene' }, filters: [], showNoDataPopover: jest.fn(), diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx index 87fbf81fceba08..0e7cefb58fc28c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx @@ -47,9 +47,11 @@ export type Props = DatasourceDataPanelProps & { state: IndexPatternPrivateState, setState: StateSetter ) => void; + charts: ChartsPluginSetup; }; import { LensFieldIcon } from './lens_field_icon'; import { ChangeIndexPattern } from './change_indexpattern'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; // TODO the typings for EuiContextMenuPanel are incorrect - watchedItemProps is missing. This can be removed when the types are adjusted const FixedEuiContextMenuPanel = (EuiContextMenuPanel as unknown) as React.FunctionComponent< @@ -82,6 +84,7 @@ export function IndexPatternDataPanel({ filters, dateRange, changeIndexPattern, + charts, showNoDataPopover, }: Props) { const { indexPatternRefs, indexPatterns, currentIndexPatternId } = state; @@ -170,6 +173,7 @@ export function IndexPatternDataPanel({ dragDropContext={dragDropContext} core={core} data={data} + charts={charts} onChangeIndexPattern={onChangeIndexPattern} existingFields={state.existingFields} /> @@ -214,6 +218,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ core, data, existingFields, + charts, }: Omit & { data: DataPublicPluginStart; currentIndexPatternId: string; @@ -222,6 +227,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ dragDropContext: DragContextState; onChangeIndexPattern: (newId: string) => void; existingFields: IndexPatternPrivateState['existingFields']; + charts: ChartsPluginSetup; }) { const [localState, setLocalState] = useState({ nameFilter: '', @@ -376,6 +382,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ dateRange, query, filters, + chartsThemeService: charts.theme, }), [core, data, currentIndexPattern, dateRange, query, filters, localState.nameFilter] ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx index e8dfbc250c539a..0a3af97f8ad754 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx @@ -13,6 +13,9 @@ import { mountWithIntl } from 'test_utils/enzyme_helpers'; import { DataPublicPluginStart } from '../../../../../src/plugins/data/public'; import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; import { IndexPattern } from './types'; +import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; + +const chartsThemeService = chartPluginMock.createSetupContract().theme; describe('IndexPattern Field Item', () => { let defaultProps: FieldItemProps; @@ -80,6 +83,7 @@ describe('IndexPattern Field Item', () => { searchable: true, }, exists: true, + chartsThemeService, }; data.fieldFormats = ({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx index 1a1a34d30f8a8e..815725f4331a64 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx @@ -20,7 +20,6 @@ import { EuiText, EuiToolTip, } from '@elastic/eui'; -import { EUI_CHARTS_THEME_DARK, EUI_CHARTS_THEME_LIGHT } from '@elastic/eui/dist/eui_charts_theme'; import { Axis, BarSeries, @@ -41,6 +40,7 @@ import { esQuery, IIndexPattern, } from '../../../../../src/plugins/data/public'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; import { DraggedField } from './indexpattern'; import { DragDrop } from '../drag_drop'; import { DatasourceDataPanelProps, DataType } from '../types'; @@ -60,6 +60,7 @@ export interface FieldItemProps { exists: boolean; query: Query; dateRange: DatasourceDataPanelProps['dateRange']; + chartsThemeService: ChartsPluginSetup['theme']; filters: Filter[]; hideDetails?: boolean; } @@ -254,11 +255,12 @@ function FieldItemPopoverContents(props: State & FieldItemProps) { dateRange, core, sampledValues, + chartsThemeService, data: { fieldFormats }, } = props; - const IS_DARK_THEME = core.uiSettings.get('theme:darkMode'); - const chartTheme = IS_DARK_THEME ? EUI_CHARTS_THEME_DARK.theme : EUI_CHARTS_THEME_LIGHT.theme; + const chartTheme = chartsThemeService.useChartsTheme(); + const chartBaseTheme = chartsThemeService.useChartsBaseTheme(); let histogramDefault = !!props.histogram; const totalValuesCount = @@ -410,6 +412,7 @@ function FieldItemPopoverContents(props: State & FieldItemProps) { - + { let defaultProps: FieldsAccordionProps; @@ -56,6 +57,7 @@ describe('Fields Accordion', () => { }, query: { query: '', language: 'lucene' }, filters: [], + chartsThemeService: chartPluginMock.createSetupContract().theme, }; defaultProps = { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx index b756cf81a90739..7cc049c107b87c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx @@ -19,10 +19,12 @@ import { FieldItem } from './field_item'; import { Query, Filter } from '../../../../../src/plugins/data/public'; import { DatasourceDataPanelProps } from '../types'; import { IndexPattern } from './types'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; export interface FieldItemSharedProps { core: DatasourceDataPanelProps['core']; data: DataPublicPluginStart; + chartsThemeService: ChartsPluginSetup['theme']; indexPattern: IndexPattern; highlight?: string; query: Query; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts index 73fd144b9c7f87..45d0ee45fab4c1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts @@ -9,6 +9,7 @@ import { Storage } from '../../../../../src/plugins/kibana_utils/public'; import { getIndexPatternDatasource } from './indexpattern'; import { renameColumns } from './rename_columns'; import { ExpressionsSetup } from '../../../../../src/plugins/expressions/public'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; import { DataPublicPluginSetup, DataPublicPluginStart, @@ -19,6 +20,7 @@ export interface IndexPatternDatasourceSetupPlugins { expressions: ExpressionsSetup; data: DataPublicPluginSetup; editorFrame: EditorFrameSetup; + charts: ChartsPluginSetup; } export interface IndexPatternDatasourceStartPlugins { @@ -30,7 +32,7 @@ export class IndexPatternDatasource { setup( core: CoreSetup, - { expressions, editorFrame }: IndexPatternDatasourceSetupPlugins + { expressions, editorFrame, charts }: IndexPatternDatasourceSetupPlugins ) { expressions.registerFunction(renameColumns); @@ -40,6 +42,7 @@ export class IndexPatternDatasource { core: coreStart, storage: new Storage(localStorage), data, + charts, }) ) as Promise ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 6a79ce450cd9ad..3bd0685551a4c9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -11,6 +11,7 @@ import { coreMock } from 'src/core/public/mocks'; import { IndexPatternPersistedState, IndexPatternPrivateState } from './types'; import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; import { Ast } from '@kbn/interpreter/common'; +import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; jest.mock('./loader'); jest.mock('../id_generator'); @@ -140,6 +141,7 @@ describe('IndexPattern Data Source', () => { storage: {} as IStorageWrapper, core: coreMock.createStart(), data: dataPluginMock.createStartContract(), + charts: chartPluginMock.createSetupContract(), }); persistedState = { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index a98f63cf9b3606..e9d095bfbcef1d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -46,6 +46,7 @@ import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/p import { DataPublicPluginStart } from '../../../../../src/plugins/data/public'; import { deleteColumn } from './state_helpers'; import { Datasource, StateSetter } from '../index'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; export { OperationType, IndexPatternColumn } from './operations'; @@ -102,10 +103,12 @@ export function getIndexPatternDatasource({ core, storage, data, + charts, }: { core: CoreStart; storage: IStorageWrapper; data: DataPublicPluginStart; + charts: ChartsPluginSetup; }) { const savedObjectsClient = core.savedObjects.client; const uiSettings = core.uiSettings; @@ -212,6 +215,7 @@ export function getIndexPatternDatasource({ }); }} data={data} + charts={charts} {...props} /> , diff --git a/x-pack/plugins/lens/public/pie_visualization/index.ts b/x-pack/plugins/lens/public/pie_visualization/index.ts index dd828c6c35300f..401b6d634c6960 100644 --- a/x-pack/plugins/lens/public/pie_visualization/index.ts +++ b/x-pack/plugins/lens/public/pie_visualization/index.ts @@ -4,18 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EUI_CHARTS_THEME_DARK, EUI_CHARTS_THEME_LIGHT } from '@elastic/eui/dist/eui_charts_theme'; import { CoreSetup } from 'src/core/public'; import { ExpressionsSetup } from 'src/plugins/expressions/public'; import { pieVisualization } from './pie_visualization'; import { pie, getPieRenderer } from './register_expression'; import { EditorFrameSetup, FormatFactory } from '../types'; import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; export interface PieVisualizationPluginSetupPlugins { editorFrame: EditorFrameSetup; expressions: ExpressionsSetup; formatFactory: Promise; + charts: ChartsPluginSetup; } export interface PieVisualizationPluginStartPlugins { @@ -27,17 +28,14 @@ export class PieVisualization { setup( core: CoreSetup, - { expressions, formatFactory, editorFrame }: PieVisualizationPluginSetupPlugins + { expressions, formatFactory, editorFrame, charts }: PieVisualizationPluginSetupPlugins ) { expressions.registerFunction(() => pie); expressions.registerRenderer( getPieRenderer({ formatFactory, - chartTheme: core.uiSettings.get('theme:darkMode') - ? EUI_CHARTS_THEME_DARK.theme - : EUI_CHARTS_THEME_LIGHT.theme, - isDarkMode: core.uiSettings.get('theme:darkMode'), + chartsThemeService: charts.theme, }) ); diff --git a/x-pack/plugins/lens/public/pie_visualization/register_expression.tsx b/x-pack/plugins/lens/public/pie_visualization/register_expression.tsx index bbc6a1dc75c3a6..cea84db8b2794f 100644 --- a/x-pack/plugins/lens/public/pie_visualization/register_expression.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/register_expression.tsx @@ -8,7 +8,6 @@ import React from 'react'; import ReactDOM from 'react-dom'; import { i18n } from '@kbn/i18n'; import { I18nProvider } from '@kbn/i18n/react'; -import { PartialTheme } from '@elastic/charts'; import { IInterpreterRenderHandlers, ExpressionRenderDefinition, @@ -17,6 +16,7 @@ import { import { LensMultiTable, FormatFactory, LensFilterEvent } from '../types'; import { PieExpressionProps, PieExpressionArgs } from './types'; import { PieComponent } from './render_function'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; export interface PieRender { type: 'render'; @@ -93,8 +93,7 @@ export const pie: ExpressionFunctionDefinition< export const getPieRenderer = (dependencies: { formatFactory: Promise; - chartTheme: PartialTheme; - isDarkMode: boolean; + chartsThemeService: ChartsPluginSetup['theme']; }): ExpressionRenderDefinition => ({ name: 'lens_pie_renderer', displayName: i18n.translate('xpack.lens.pie.visualizationName', { @@ -116,10 +115,9 @@ export const getPieRenderer = (dependencies: { , domNode, diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx index 2e29513ba548ba..cfbeb27efb3d0c 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx @@ -11,6 +11,9 @@ import { LensMultiTable } from '../types'; import { PieComponent } from './render_function'; import { PieExpressionArgs } from './types'; import { EmptyPlaceholder } from '../shared_components'; +import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; + +const chartsThemeService = chartPluginMock.createSetupContract().theme; describe('PieVisualization component', () => { let getFormatSpy: jest.Mock; @@ -57,9 +60,8 @@ describe('PieVisualization component', () => { return { data, formatFactory: getFormatSpy, - isDarkMode: false, - chartTheme: {}, onClickValue: jest.fn(), + chartsThemeService, }; } diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx index 36e8d9660ab70c..f349cc4dfd6484 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx @@ -19,7 +19,6 @@ import { PartitionConfig, PartitionLayer, PartitionLayout, - PartialTheme, PartitionFillLabel, RecursivePartial, LayerValue, @@ -32,6 +31,7 @@ import { getSliceValueWithFallback, getFilterContext } from './render_helpers'; import { EmptyPlaceholder } from '../shared_components'; import './visualization.scss'; import { desanitizeFilterContext } from '../utils'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; const EMPTY_SLICE = Symbol('empty_slice'); @@ -40,15 +40,14 @@ const sortedColors = euiPaletteColorBlindBehindText(); export function PieComponent( props: PieExpressionProps & { formatFactory: FormatFactory; - chartTheme: Exclude; - isDarkMode: boolean; + chartsThemeService: ChartsPluginSetup['theme']; onClickValue: (data: LensFilterEvent['data']) => void; } ) { const [firstTable] = Object.values(props.data.tables); const formatters: Record> = {}; - const { chartTheme, isDarkMode, onClickValue } = props; + const { chartsThemeService, onClickValue } = props; const { shape, groups, @@ -60,6 +59,9 @@ export function PieComponent( percentDecimals, hideLabels, } = props.args; + const isDarkMode = chartsThemeService.useDarkMode(); + const chartTheme = chartsThemeService.useChartsTheme(); + const chartBaseTheme = chartsThemeService.useChartsBaseTheme(); if (!hideLabels) { firstTable.columns.forEach((column) => { @@ -245,6 +247,8 @@ export function PieComponent( onClickValue(desanitizeFilterContext(context)); }} + theme={chartTheme} + baseTheme={chartBaseTheme} /> , - { kibanaLegacy, expressions, data, embeddable, visualizations }: LensPluginSetupDependencies + { + kibanaLegacy, + expressions, + data, + embeddable, + visualizations, + charts, + }: LensPluginSetupDependencies ) { const editorFrameSetupInterface = this.editorFrameService.setup(core, { data, embeddable, expressions, }); - const dependencies = { + const dependencies: IndexPatternDatasourceSetupPlugins & + XyVisualizationPluginSetupPlugins & + DatatableVisualizationPluginSetupPlugins & + MetricVisualizationPluginSetupPlugins & + PieVisualizationPluginSetupPlugins = { expressions, data, + charts, editorFrame: editorFrameSetupInterface, formatFactory: core .getStartServices() diff --git a/x-pack/plugins/lens/public/xy_visualization/__snapshots__/xy_expression.test.tsx.snap b/x-pack/plugins/lens/public/xy_visualization/__snapshots__/xy_expression.test.tsx.snap index 48c70e0a4a05b8..8cb30037379da9 100644 --- a/x-pack/plugins/lens/public/xy_visualization/__snapshots__/xy_expression.test.tsx.snap +++ b/x-pack/plugins/lens/public/xy_visualization/__snapshots__/xy_expression.test.tsx.snap @@ -5,6 +5,7 @@ exports[`xy_expression XYChart component it renders area 1`] = ` renderer="canvas" > ; editorFrame: EditorFrameSetup; + charts: ChartsPluginSetup; } function getTimeZone(uiSettings: IUiSettingsClient) { @@ -34,7 +35,7 @@ export class XyVisualization { setup( core: CoreSetup, - { expressions, formatFactory, editorFrame }: XyVisualizationPluginSetupPlugins + { expressions, formatFactory, editorFrame, charts }: XyVisualizationPluginSetupPlugins ) { expressions.registerFunction(() => legendConfig); expressions.registerFunction(() => yAxisConfig); @@ -44,9 +45,7 @@ export class XyVisualization { expressions.registerRenderer( getXyChartRenderer({ formatFactory, - chartTheme: core.uiSettings.get('theme:darkMode') - ? EUI_CHARTS_THEME_DARK.theme - : EUI_CHARTS_THEME_LIGHT.theme, + chartsThemeService: charts.theme, timeZone: getTimeZone(core.uiSettings), histogramBarTarget: core.uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), }) diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_expression.test.tsx b/x-pack/plugins/lens/public/xy_visualization/xy_expression.test.tsx index 34f2a9111253b7..f433a88e3bdbd0 100644 --- a/x-pack/plugins/lens/public/xy_visualization/xy_expression.test.tsx +++ b/x-pack/plugins/lens/public/xy_visualization/xy_expression.test.tsx @@ -24,10 +24,13 @@ import { shallow } from 'enzyme'; import { XYArgs, LegendConfig, legendConfig, layerConfig, LayerArgs } from './types'; import { createMockExecutionContext } from '../../../../../src/plugins/expressions/common/mocks'; import { mountWithIntl } from 'test_utils/enzyme_helpers'; +import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; const onClickValue = jest.fn(); const onSelectRange = jest.fn(); +const chartsThemeService = chartPluginMock.createSetupContract().theme; + const dateHistogramData: LensMultiTable = { type: 'lens_multitable', tables: { @@ -324,7 +327,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -347,7 +350,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'line' }] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -398,7 +401,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -434,7 +437,7 @@ describe('xy_expression', () => { args={multiLayerArgs} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -471,7 +474,7 @@ describe('xy_expression', () => { args={multiLayerArgs} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -509,7 +512,7 @@ describe('xy_expression', () => { args={multiLayerArgs} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -554,7 +557,7 @@ describe('xy_expression', () => { args={multiLayerArgs} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -589,7 +592,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -606,7 +609,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'bar' }] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -626,7 +629,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'area' }] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -646,7 +649,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'bar_horizontal' }] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -671,7 +674,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -721,7 +724,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -758,7 +761,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'bar_stacked' }] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -778,7 +781,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], seriesType: 'area_stacked' }] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -801,7 +804,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -822,7 +825,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="CEST" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -842,7 +845,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [firstLayer] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -869,7 +872,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -890,7 +893,7 @@ describe('xy_expression', () => { }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -1196,7 +1199,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], xScaleType: 'ordinal' }] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -1215,7 +1218,7 @@ describe('xy_expression', () => { args={{ ...args, layers: [{ ...args.layers[0], yScaleType: 'sqrt' }] }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -1234,7 +1237,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -1252,7 +1255,7 @@ describe('xy_expression', () => { data={{ ...data }} args={{ ...args, layers: [{ ...args.layers[0], accessors: ['a'] }] }} formatFactory={getFormatSpy} - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} timeZone="UTC" onClickValue={onClickValue} @@ -1274,7 +1277,7 @@ describe('xy_expression', () => { args={{ ...args }} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -1359,7 +1362,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -1417,7 +1420,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} @@ -1473,7 +1476,7 @@ describe('xy_expression', () => { args={args} formatFactory={getFormatSpy} timeZone="UTC" - chartTheme={{}} + chartsThemeService={chartsThemeService} histogramBarTarget={50} onClickValue={onClickValue} onSelectRange={onSelectRange} diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_expression.tsx b/x-pack/plugins/lens/public/xy_visualization/xy_expression.tsx index 17ed04aa0e9c49..3ff7bd7fda3046 100644 --- a/x-pack/plugins/lens/public/xy_visualization/xy_expression.tsx +++ b/x-pack/plugins/lens/public/xy_visualization/xy_expression.tsx @@ -15,7 +15,6 @@ import { AreaSeries, BarSeries, Position, - PartialTheme, GeometryValue, XYChartSeriesIdentifier, } from '@elastic/charts'; @@ -38,6 +37,7 @@ import { XYArgs, SeriesType, visualizationTypes } from './types'; import { VisualizationContainer } from '../visualization_container'; import { isHorizontalChart } from './state_helpers'; import { parseInterval } from '../../../../../src/plugins/data/common'; +import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; import { EmptyPlaceholder } from '../shared_components'; import { desanitizeFilterContext } from '../utils'; import { getAxesConfiguration } from './axes_configuration'; @@ -59,7 +59,7 @@ export interface XYRender { } type XYChartRenderProps = XYChartProps & { - chartTheme: PartialTheme; + chartsThemeService: ChartsPluginSetup['theme']; formatFactory: FormatFactory; timeZone: string; histogramBarTarget: number; @@ -115,7 +115,7 @@ export const xyChart: ExpressionFunctionDefinition< export const getXyChartRenderer = (dependencies: { formatFactory: Promise; - chartTheme: PartialTheme; + chartsThemeService: ChartsPluginSetup['theme']; histogramBarTarget: number; timeZone: string; }): ExpressionRenderDefinition => ({ @@ -144,7 +144,7 @@ export const getXyChartRenderer = (dependencies: { { return !( @@ -276,6 +278,7 @@ export function XYChart({ legendPosition={legend.position} showLegendExtra={false} theme={chartTheme} + baseTheme={chartBaseTheme} tooltip={{ headerFormatter: (d) => xAxisFormatter.convert(d.value), }} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_alert_types/threshold/visualization.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_alert_types/threshold/visualization.tsx index 244d431930f2eb..a282fa08e8f38c 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_alert_types/threshold/visualization.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/builtin_alert_types/threshold/visualization.tsx @@ -160,7 +160,7 @@ export const ThresholdVisualization: React.FunctionComponent = ({ setLoadingState(LoadingStateType.Idle); } })(); - /* eslint-disable react-hooks/exhaustive-deps */ + // eslint-disable-next-line react-hooks/exhaustive-deps }, [ index, timeField, @@ -175,12 +175,12 @@ export const ThresholdVisualization: React.FunctionComponent = ({ threshold, startVisualizationAt, ]); - /* eslint-enable react-hooks/exhaustive-deps */ if (!charts || !uiSettings || !dataFieldsFormats) { return null; } const chartsTheme = charts.theme.useChartsTheme(); + const chartsBaseTheme = charts.theme.useChartsBaseTheme(); const domain = getDomain(alertInterval, startVisualizationAt); const visualizeOptions = { @@ -261,6 +261,7 @@ export const ThresholdVisualization: React.FunctionComponent = ({ Date: Thu, 2 Jul 2020 16:03:44 +0200 Subject: [PATCH 25/31] [Ingest Manager] Update asset paths to use _ instead of - (#70320) In https://github.com/elastic/package-registry/issues/517 the naming of the file paths inside a package is standardised to only use `_` and not `-`. This adjusts the paths for `ilm-policy`, `component-template`, `index-template` to the correct path. An additional change here is to get rid of assets we don't support yet, like rollup jobs and ml jobs. We will reintroduce these when we support them. --- .../ingest_manager/common/types/models/epm.ts | 19 ++++++++----------- .../ingest_manager/sections/epm/constants.tsx | 8 ++++---- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 5b68cd2beeed43..bf6a8de15182d9 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -32,10 +32,10 @@ export enum KibanaAssetType { } export enum ElasticsearchAssetType { - componentTemplate = 'component-template', - ingestPipeline = 'ingest-pipeline', - indexTemplate = 'index-template', - ilmPolicy = 'ilm-policy', + componentTemplate = 'component_template', + ingestPipeline = 'ingest_pipeline', + indexTemplate = 'index_template', + ilmPolicy = 'ilm_policy', } export enum AgentAssetType { @@ -243,13 +243,10 @@ export type AssetReference = Pick & { * Types of assets which can be installed/removed */ export enum IngestAssetType { - DataFrameTransform = 'data-frame-transform', - IlmPolicy = 'ilm-policy', - IndexTemplate = 'index-template', - ComponentTemplate = 'component-template', - IngestPipeline = 'ingest-pipeline', - MlJob = 'ml-job', - RollupJob = 'rollup-job', + IlmPolicy = 'ilm_policy', + IndexTemplate = 'index_template', + ComponentTemplate = 'component_template', + IngestPipeline = 'ingest_pipeline', } export enum DefaultPackages { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx index 54cb5171f5a3e3..31c6d764464479 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx @@ -17,11 +17,11 @@ export const DisplayedAssets: ServiceNameToAssetTypes = { export const AssetTitleMap: Record = { dashboard: 'Dashboard', - 'ilm-policy': 'ILM Policy', - 'ingest-pipeline': 'Ingest Pipeline', + ilm_policy: 'ILM Policy', + ingest_pipeline: 'Ingest Pipeline', 'index-pattern': 'Index Pattern', - 'index-template': 'Index Template', - 'component-template': 'Component Template', + index_template: 'Index Template', + component_template: 'Component Template', search: 'Saved Search', visualization: 'Visualization', input: 'Agent input', From 854e7a5204502ea1977382c89457c160ee7767b3 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Thu, 2 Jul 2020 07:30:18 -0700 Subject: [PATCH 26/31] [ML] Anomaly Explorer swim lane pagination (#70063) * [ML] use explorer service * [ML] WIP pagination * [ML] add to dashboard without the limit * [ML] WIP * [ML] loading states * [ML] viewBySwimlaneDataLoading on field change * [ML] fix dashboard control * [ML] universal swim lane container, embeddable pagination * [ML] fix css issue * [ML] rename anomalyTimelineService * [ML] rename callback * [ML] rename container component * [ML] empty state, increase pagination margin * [ML] check for loading * [ML] fix i18n * [ML] fix unit test * [ML] improve selected cells * [ML] fix overall selection with changing job selection * [ML] required props for pagination component * [ML] move RESIZE_IGNORED_DIFF_PX * [ML] jest tests * [ML] add test subject * [ML] SWIM_LANE_DEFAULT_PAGE_SIZE * [ML] change empty state styling * [ML] fix agg size for influencer filters * [ML] remove debounce * [ML] SCSS variables, rename swim lane class * [ML] job selector using context * [ML] set padding for embeddable panel * [ML] adjust pagination styles * [ML] replace custom time range subject with timefilter * [ML] change loading indicator to mono * [ML] use swim lane type constant * [ML] change context naming * [ML] update jest snapshot * [ML] fix tests --- x-pack/plugins/ml/public/application/app.tsx | 25 +- .../job_selector/job_selector_flyout.tsx | 8 +- .../date_picker_wrapper.tsx | 6 +- .../contexts/kibana/kibana_context.ts | 4 +- .../application/contexts/ml/ml_context.ts | 3 +- .../application/explorer/_explorer.scss | 20 +- .../explorer/actions/load_explorer_data.ts | 378 ++++++++++-------- .../explorer/add_to_dashboard_control.tsx | 7 +- .../application/explorer/anomaly_timeline.tsx | 179 ++++----- ...explorer_no_influencers_found.test.js.snap | 21 +- .../explorer_no_influencers_found.tsx | 38 +- .../explorer/components/no_overall_data.tsx | 17 + .../public/application/explorer/explorer.js | 154 +++---- .../explorer/explorer_constants.ts | 17 +- .../explorer/explorer_dashboard_service.ts | 25 +- .../explorer/explorer_swimlane.tsx | 10 +- .../application/explorer/explorer_utils.d.ts | 22 +- .../application/explorer/explorer_utils.js | 293 +------------- .../explorer/hooks/use_selected_cells.ts | 87 ++-- .../application/explorer/legacy_utils.ts | 5 - .../clear_influencer_filter_settings.ts | 1 + .../explorer_reducer/job_selection_change.ts | 1 + .../reducers/explorer_reducer/reducer.ts | 43 +- .../set_influencer_filter_settings.ts | 1 + .../reducers/explorer_reducer/state.ts | 12 +- .../explorer/select_limit/index.ts | 7 - .../select_limit/select_limit.test.tsx | 29 -- .../explorer/select_limit/select_limit.tsx | 40 -- .../explorer/swimlane_container.tsx | 150 +++++-- .../explorer/swimlane_pagination.tsx | 108 +++++ .../application/routing/routes/explorer.tsx | 22 +- .../routes/timeseriesexplorer.test.tsx | 64 +-- .../public/application/routing/use_refresh.ts | 38 +- ...service.ts => anomaly_timeline_service.ts} | 85 +++- .../services/dashboard_service.test.ts | 2 +- .../application/services/dashboard_service.ts | 2 +- .../services/ml_api_service/index.ts | 4 +- .../services/ml_api_service/jobs.ts | 57 +-- .../results_service/results_service.d.ts | 16 +- .../results_service/results_service.js | 43 +- .../services/timefilter_refresh_service.tsx | 1 - .../anomaly_swimlane_embeddable.tsx | 30 +- ...omaly_swimlane_embeddable_factory.test.tsx | 5 +- .../anomaly_swimlane_embeddable_factory.ts | 14 +- .../anomaly_swimlane_initializer.tsx | 22 +- .../anomaly_swimlane_setup_flyout.tsx | 99 ++--- ...> embeddable_swim_lane_container.test.tsx} | 66 +-- .../embeddable_swim_lane_container.tsx | 113 ++++++ .../explorer_swimlane_container.tsx | 126 ------ .../swimlane_input_resolver.test.ts | 14 +- .../swimlane_input_resolver.ts | 96 ++++- .../ui_actions/edit_swimlane_panel_action.tsx | 14 +- .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - .../services/ml/anomaly_explorer.ts | 2 +- 55 files changed, 1362 insertions(+), 1288 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/explorer/components/no_overall_data.tsx delete mode 100644 x-pack/plugins/ml/public/application/explorer/select_limit/index.ts delete mode 100644 x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.test.tsx delete mode 100644 x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.tsx create mode 100644 x-pack/plugins/ml/public/application/explorer/swimlane_pagination.tsx rename x-pack/plugins/ml/public/application/services/{explorer_service.ts => anomaly_timeline_service.ts} (82%) rename x-pack/plugins/ml/public/embeddables/anomaly_swimlane/{explorer_swimlane_container.test.tsx => embeddable_swim_lane_container.test.tsx} (73%) create mode 100644 x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.tsx delete mode 100644 x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.tsx diff --git a/x-pack/plugins/ml/public/application/app.tsx b/x-pack/plugins/ml/public/application/app.tsx index 9539d530bab047..9d5125532e5b8f 100644 --- a/x-pack/plugins/ml/public/application/app.tsx +++ b/x-pack/plugins/ml/public/application/app.tsx @@ -7,7 +7,7 @@ import React, { FC } from 'react'; import ReactDOM from 'react-dom'; -import { AppMountParameters, CoreStart } from 'kibana/public'; +import { AppMountParameters, CoreStart, HttpStart } from 'kibana/public'; import { Storage } from '../../../../../src/plugins/kibana_utils/public'; @@ -17,6 +17,8 @@ import { setLicenseCache } from './license'; import { MlSetupDependencies, MlStartDependencies } from '../plugin'; import { MlRouter } from './routing'; +import { mlApiServicesProvider } from './services/ml_api_service'; +import { HttpService } from './services/http_service'; type MlDependencies = MlSetupDependencies & MlStartDependencies; @@ -27,6 +29,23 @@ interface AppProps { const localStorage = new Storage(window.localStorage); +/** + * Provides global services available across the entire ML app. + */ +export function getMlGlobalServices(httpStart: HttpStart) { + const httpService = new HttpService(httpStart); + return { + httpService, + mlApiServices: mlApiServicesProvider(httpService), + }; +} + +export interface MlServicesContext { + mlServices: MlGlobalServices; +} + +export type MlGlobalServices = ReturnType; + const App: FC = ({ coreStart, deps }) => { const pageDeps = { indexPatterns: deps.data.indexPatterns, @@ -47,7 +66,9 @@ const App: FC = ({ coreStart, deps }) => { const I18nContext = coreStart.i18n.Context; return ( - + diff --git a/x-pack/plugins/ml/public/application/components/job_selector/job_selector_flyout.tsx b/x-pack/plugins/ml/public/application/components/job_selector/job_selector_flyout.tsx index 3fb654f35be4de..803281bcd0ce9d 100644 --- a/x-pack/plugins/ml/public/application/components/job_selector/job_selector_flyout.tsx +++ b/x-pack/plugins/ml/public/application/components/job_selector/job_selector_flyout.tsx @@ -27,7 +27,6 @@ import { normalizeTimes, } from './job_select_service_utils'; import { MlJobWithTimeRange } from '../../../../common/types/anomaly_detection_jobs'; -import { ml } from '../../services/ml_api_service'; import { useMlKibana } from '../../contexts/kibana'; import { JobSelectionMaps } from './job_selector'; @@ -66,7 +65,10 @@ export const JobSelectorFlyout: FC = ({ withTimeRangeSelector = true, }) => { const { - services: { notifications }, + services: { + notifications, + mlServices: { mlApiServices }, + }, } = useMlKibana(); const [newSelection, setNewSelection] = useState(selectedIds); @@ -151,7 +153,7 @@ export const JobSelectorFlyout: FC = ({ async function fetchJobs() { try { - const resp = await ml.jobs.jobsWithTimerange(dateFormatTz); + const resp = await mlApiServices.jobs.jobsWithTimerange(dateFormatTz); const normalizedJobs = normalizeTimes(resp.jobs, dateFormatTz, DEFAULT_GANTT_BAR_WIDTH); const { groups: groupsWithTimerange, groupsMap } = getGroupsFromJobs(normalizedJobs); setJobs(normalizedJobs); diff --git a/x-pack/plugins/ml/public/application/components/navigation_menu/date_picker_wrapper/date_picker_wrapper.tsx b/x-pack/plugins/ml/public/application/components/navigation_menu/date_picker_wrapper/date_picker_wrapper.tsx index 27f8c822d68e39..beafae1ecd2f67 100644 --- a/x-pack/plugins/ml/public/application/components/navigation_menu/date_picker_wrapper/date_picker_wrapper.tsx +++ b/x-pack/plugins/ml/public/application/components/navigation_menu/date_picker_wrapper/date_picker_wrapper.tsx @@ -9,10 +9,7 @@ import { Subscription } from 'rxjs'; import { EuiSuperDatePicker, OnRefreshProps } from '@elastic/eui'; import { TimeRange, TimeHistoryContract } from 'src/plugins/data/public'; -import { - mlTimefilterRefresh$, - mlTimefilterTimeChange$, -} from '../../../services/timefilter_refresh_service'; +import { mlTimefilterRefresh$ } from '../../../services/timefilter_refresh_service'; import { useUrlState } from '../../../util/url_state'; import { useMlKibana } from '../../../contexts/kibana'; @@ -108,7 +105,6 @@ export const DatePickerWrapper: FC = () => { timefilter.setTime(newTime); setTime(newTime); setRecentlyUsedRanges(getRecentlyUsedRanges()); - mlTimefilterTimeChange$.next({ lastRefresh: Date.now(), timeRange: { start, end } }); } function updateInterval({ diff --git a/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts b/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts index 2a156b5716ad4d..3bc3b8c2c6dfd2 100644 --- a/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts +++ b/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts @@ -13,6 +13,7 @@ import { import { SecurityPluginSetup } from '../../../../../security/public'; import { LicenseManagementUIPluginSetup } from '../../../../../license_management/public'; import { SharePluginStart } from '../../../../../../../src/plugins/share/public'; +import { MlServicesContext } from '../../app'; interface StartPlugins { data: DataPublicPluginStart; @@ -20,7 +21,8 @@ interface StartPlugins { licenseManagement?: LicenseManagementUIPluginSetup; share: SharePluginStart; } -export type StartServices = CoreStart & StartPlugins & { kibanaVersion: string }; +export type StartServices = CoreStart & + StartPlugins & { kibanaVersion: string } & MlServicesContext; // eslint-disable-next-line react-hooks/rules-of-hooks export const useMlKibana = () => useKibana(); export type MlKibanaReactContextValue = KibanaReactContextValue; diff --git a/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts b/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts index 07d5a153664b75..95ef5e5b2938c9 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts @@ -7,6 +7,7 @@ import React from 'react'; import { IndexPattern, IndexPatternsContract } from '../../../../../../../src/plugins/data/public'; import { SavedSearchSavedObject } from '../../../../common/types/kibana'; +import { MlServicesContext } from '../../app'; export interface MlContextValue { combinedQuery: any; @@ -34,4 +35,4 @@ export type SavedSearchQuery = object; // Multiple custom hooks can be created to access subsets of // the overall context value if necessary too, // see useCurrentIndexPattern() for example. -export const MlContext = React.createContext>({}); +export const MlContext = React.createContext>({}); diff --git a/x-pack/plugins/ml/public/application/explorer/_explorer.scss b/x-pack/plugins/ml/public/application/explorer/_explorer.scss index 7e5f354bbb4024..63c471e66c49ac 100644 --- a/x-pack/plugins/ml/public/application/explorer/_explorer.scss +++ b/x-pack/plugins/ml/public/application/explorer/_explorer.scss @@ -1,3 +1,5 @@ +$borderRadius: $euiBorderRadius / 2; + .ml-swimlane-selector { visibility: hidden; } @@ -104,10 +106,9 @@ // SASSTODO: This entire selector needs to be rewritten. // It looks extremely brittle with very specific sizing units - .ml-explorer-swimlane { + .mlExplorerSwimlane { user-select: none; padding: 0; - margin-bottom: $euiSizeS; line.gridLine { stroke: $euiBorderColor; @@ -218,17 +219,20 @@ div.lane { height: 30px; border-bottom: 0px; - border-radius: 2px; - margin-top: -1px; + border-radius: $borderRadius; white-space: nowrap; + &:not(:first-child) { + margin-top: -1px; + } + div.lane-label { display: inline-block; - font-size: 13px; + font-size: $euiFontSizeXS; height: 30px; text-align: right; vertical-align: middle; - border-radius: 2px; + border-radius: $borderRadius; padding-right: 5px; margin-right: 5px; border: 1px solid transparent; @@ -261,7 +265,7 @@ .sl-cell-inner-dragselect { height: 26px; margin: 1px; - border-radius: 2px; + border-radius: $borderRadius; text-align: center; } @@ -293,7 +297,7 @@ .sl-cell-inner, .sl-cell-inner-dragselect { border: 2px solid $euiColorDarkShade; - border-radius: 2px; + border-radius: $borderRadius; opacity: 1; } } diff --git a/x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts b/x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts index 590a69283a8194..095b42ffac5b78 100644 --- a/x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts +++ b/x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts @@ -11,6 +11,7 @@ import useObservable from 'react-use/lib/useObservable'; import { forkJoin, of, Observable, Subject } from 'rxjs'; import { mergeMap, switchMap, tap } from 'rxjs/operators'; +import { useCallback, useMemo } from 'react'; import { anomalyDataChange } from '../explorer_charts/explorer_charts_container_service'; import { explorerService } from '../explorer_dashboard_service'; import { @@ -22,15 +23,17 @@ import { loadAnomaliesTableData, loadDataForCharts, loadFilteredTopInfluencers, - loadOverallData, loadTopInfluencers, - loadViewBySwimlane, - loadViewByTopFieldValuesForSelectedTime, AppStateSelectedCells, ExplorerJob, TimeRangeBounds, } from '../explorer_utils'; import { ExplorerState } from '../reducers'; +import { useMlKibana, useTimefilter } from '../../contexts/kibana'; +import { AnomalyTimelineService } from '../../services/anomaly_timeline_service'; +import { mlResultsServiceProvider } from '../../services/results_service'; +import { isViewBySwimLaneData } from '../swimlane_container'; +import { ANOMALY_SWIM_LANE_HARD_LIMIT } from '../explorer_constants'; // Memoize the data fetching methods. // wrapWithLastRefreshArg() wraps any given function and preprends a `lastRefresh` argument @@ -39,13 +42,13 @@ import { ExplorerState } from '../reducers'; // about this parameter. The generic type T retains and returns the type information of // the original function. const memoizeIsEqual = (newArgs: any[], lastArgs: any[]) => isEqual(newArgs, lastArgs); -const wrapWithLastRefreshArg = any>(func: T) => { +const wrapWithLastRefreshArg = any>(func: T, context: any = null) => { return function (lastRefresh: number, ...args: Parameters): ReturnType { - return func.apply(null, args); + return func.apply(context, args); }; }; -const memoize = any>(func: T) => { - return memoizeOne(wrapWithLastRefreshArg(func), memoizeIsEqual); +const memoize = any>(func: T, context?: any) => { + return memoizeOne(wrapWithLastRefreshArg(func, context), memoizeIsEqual); }; const memoizedAnomalyDataChange = memoize(anomalyDataChange); @@ -56,9 +59,7 @@ const memoizedLoadDataForCharts = memoize(loadDataForC const memoizedLoadFilteredTopInfluencers = memoize( loadFilteredTopInfluencers ); -const memoizedLoadOverallData = memoize(loadOverallData); const memoizedLoadTopInfluencers = memoize(loadTopInfluencers); -const memoizedLoadViewBySwimlane = memoize(loadViewBySwimlane); const memoizedLoadAnomaliesTableData = memoize(loadAnomaliesTableData); export interface LoadExplorerDataConfig { @@ -73,6 +74,9 @@ export interface LoadExplorerDataConfig { tableInterval: string; tableSeverity: number; viewBySwimlaneFieldName: string; + viewByFromPage: number; + viewByPerPage: number; + swimlaneContainerWidth: number; } export const isLoadExplorerDataConfig = (arg: any): arg is LoadExplorerDataConfig => { @@ -87,183 +91,213 @@ export const isLoadExplorerDataConfig = (arg: any): arg is LoadExplorerDataConfi /** * Fetches the data necessary for the Anomaly Explorer using observables. - * - * @param config LoadExplorerDataConfig - * - * @return Partial */ -function loadExplorerData(config: LoadExplorerDataConfig): Observable> { - if (!isLoadExplorerDataConfig(config)) { - return of({}); - } - - const { - bounds, - lastRefresh, - influencersFilterQuery, - noInfluencersConfigured, - selectedCells, - selectedJobs, - swimlaneBucketInterval, - swimlaneLimit, - tableInterval, - tableSeverity, - viewBySwimlaneFieldName, - } = config; - - const selectionInfluencers = getSelectionInfluencers(selectedCells, viewBySwimlaneFieldName); - const jobIds = getSelectionJobIds(selectedCells, selectedJobs); - const timerange = getSelectionTimeRange( - selectedCells, - swimlaneBucketInterval.asSeconds(), - bounds +const loadExplorerDataProvider = (anomalyTimelineService: AnomalyTimelineService) => { + const memoizedLoadOverallData = memoize( + anomalyTimelineService.loadOverallData, + anomalyTimelineService ); + const memoizedLoadViewBySwimlane = memoize( + anomalyTimelineService.loadViewBySwimlane, + anomalyTimelineService + ); + return (config: LoadExplorerDataConfig): Observable> => { + if (!isLoadExplorerDataConfig(config)) { + return of({}); + } - const dateFormatTz = getDateFormatTz(); - - // First get the data where we have all necessary args at hand using forkJoin: - // annotationsData, anomalyChartRecords, influencers, overallState, tableData, topFieldValues - return forkJoin({ - annotationsData: memoizedLoadAnnotationsTableData( + const { + bounds, lastRefresh, + influencersFilterQuery, + noInfluencersConfigured, selectedCells, selectedJobs, + swimlaneBucketInterval, + swimlaneLimit, + tableInterval, + tableSeverity, + viewBySwimlaneFieldName, + swimlaneContainerWidth, + viewByFromPage, + viewByPerPage, + } = config; + + const selectionInfluencers = getSelectionInfluencers(selectedCells, viewBySwimlaneFieldName); + const jobIds = getSelectionJobIds(selectedCells, selectedJobs); + const timerange = getSelectionTimeRange( + selectedCells, swimlaneBucketInterval.asSeconds(), bounds - ), - anomalyChartRecords: memoizedLoadDataForCharts( - lastRefresh, - jobIds, - timerange.earliestMs, - timerange.latestMs, - selectionInfluencers, - selectedCells, - influencersFilterQuery - ), - influencers: - selectionInfluencers.length === 0 - ? memoizedLoadTopInfluencers( + ); + + const dateFormatTz = getDateFormatTz(); + + // First get the data where we have all necessary args at hand using forkJoin: + // annotationsData, anomalyChartRecords, influencers, overallState, tableData, topFieldValues + return forkJoin({ + annotationsData: memoizedLoadAnnotationsTableData( + lastRefresh, + selectedCells, + selectedJobs, + swimlaneBucketInterval.asSeconds(), + bounds + ), + anomalyChartRecords: memoizedLoadDataForCharts( + lastRefresh, + jobIds, + timerange.earliestMs, + timerange.latestMs, + selectionInfluencers, + selectedCells, + influencersFilterQuery + ), + influencers: + selectionInfluencers.length === 0 + ? memoizedLoadTopInfluencers( + lastRefresh, + jobIds, + timerange.earliestMs, + timerange.latestMs, + [], + noInfluencersConfigured, + influencersFilterQuery + ) + : Promise.resolve({}), + overallState: memoizedLoadOverallData(lastRefresh, selectedJobs, swimlaneContainerWidth), + tableData: memoizedLoadAnomaliesTableData( + lastRefresh, + selectedCells, + selectedJobs, + dateFormatTz, + swimlaneBucketInterval.asSeconds(), + bounds, + viewBySwimlaneFieldName, + tableInterval, + tableSeverity, + influencersFilterQuery + ), + topFieldValues: + selectedCells !== undefined && selectedCells.showTopFieldValues === true + ? anomalyTimelineService.loadViewByTopFieldValuesForSelectedTime( + timerange.earliestMs, + timerange.latestMs, + selectedJobs, + viewBySwimlaneFieldName, + swimlaneLimit, + viewByPerPage, + viewByFromPage, + swimlaneContainerWidth + ) + : Promise.resolve([]), + }).pipe( + // Trigger a side-effect action to reset view-by swimlane, + // show the view-by loading indicator + // and pass on the data we already fetched. + tap(explorerService.setViewBySwimlaneLoading), + // Trigger a side-effect to update the charts. + tap(({ anomalyChartRecords }) => { + if (selectedCells !== undefined && Array.isArray(anomalyChartRecords)) { + memoizedAnomalyDataChange( lastRefresh, - jobIds, + anomalyChartRecords, timerange.earliestMs, timerange.latestMs, + tableSeverity + ); + } else { + memoizedAnomalyDataChange( + lastRefresh, [], - noInfluencersConfigured, - influencersFilterQuery - ) - : Promise.resolve({}), - overallState: memoizedLoadOverallData( - lastRefresh, - selectedJobs, - swimlaneBucketInterval, - bounds - ), - tableData: memoizedLoadAnomaliesTableData( - lastRefresh, - selectedCells, - selectedJobs, - dateFormatTz, - swimlaneBucketInterval.asSeconds(), - bounds, - viewBySwimlaneFieldName, - tableInterval, - tableSeverity, - influencersFilterQuery - ), - topFieldValues: - selectedCells !== undefined && selectedCells.showTopFieldValues === true - ? loadViewByTopFieldValuesForSelectedTime( timerange.earliestMs, timerange.latestMs, - selectedJobs, - viewBySwimlaneFieldName, - swimlaneLimit, - noInfluencersConfigured - ) - : Promise.resolve([]), - }).pipe( - // Trigger a side-effect action to reset view-by swimlane, - // show the view-by loading indicator - // and pass on the data we already fetched. - tap(explorerService.setViewBySwimlaneLoading), - // Trigger a side-effect to update the charts. - tap(({ anomalyChartRecords }) => { - if (selectedCells !== undefined && Array.isArray(anomalyChartRecords)) { - memoizedAnomalyDataChange( - lastRefresh, - anomalyChartRecords, - timerange.earliestMs, - timerange.latestMs, - tableSeverity - ); - } else { - memoizedAnomalyDataChange( - lastRefresh, - [], - timerange.earliestMs, - timerange.latestMs, - tableSeverity - ); - } - }), - // Load view-by swimlane data and filtered top influencers. - // mergeMap is used to have access to the already fetched data and act on it in arg #1. - // In arg #2 of mergeMap we combine the data and pass it on in the action format - // which can be consumed by explorerReducer() later on. - mergeMap( - ({ anomalyChartRecords, influencers, overallState, topFieldValues }) => - forkJoin({ - influencers: - (selectionInfluencers.length > 0 || influencersFilterQuery !== undefined) && - anomalyChartRecords !== undefined && - anomalyChartRecords.length > 0 - ? memoizedLoadFilteredTopInfluencers( - lastRefresh, - jobIds, - timerange.earliestMs, - timerange.latestMs, - anomalyChartRecords, - selectionInfluencers, - noInfluencersConfigured, - influencersFilterQuery - ) - : Promise.resolve(influencers), - viewBySwimlaneState: memoizedLoadViewBySwimlane( - lastRefresh, - topFieldValues, - { - earliest: overallState.overallSwimlaneData.earliest, - latest: overallState.overallSwimlaneData.latest, - }, - selectedJobs, - viewBySwimlaneFieldName, - swimlaneLimit, - influencersFilterQuery, - noInfluencersConfigured - ), - }), - ( - { annotationsData, overallState, tableData }, - { influencers, viewBySwimlaneState } - ): Partial => { - return { - annotationsData, - influencers, - ...overallState, - ...viewBySwimlaneState, - tableData, - }; - } - ) - ); -} - -const loadExplorerData$ = new Subject(); -const explorerData$ = loadExplorerData$.pipe( - switchMap((config: LoadExplorerDataConfig) => loadExplorerData(config)) -); - + tableSeverity + ); + } + }), + // Load view-by swimlane data and filtered top influencers. + // mergeMap is used to have access to the already fetched data and act on it in arg #1. + // In arg #2 of mergeMap we combine the data and pass it on in the action format + // which can be consumed by explorerReducer() later on. + mergeMap( + ({ anomalyChartRecords, influencers, overallState, topFieldValues }) => + forkJoin({ + influencers: + (selectionInfluencers.length > 0 || influencersFilterQuery !== undefined) && + anomalyChartRecords !== undefined && + anomalyChartRecords.length > 0 + ? memoizedLoadFilteredTopInfluencers( + lastRefresh, + jobIds, + timerange.earliestMs, + timerange.latestMs, + anomalyChartRecords, + selectionInfluencers, + noInfluencersConfigured, + influencersFilterQuery + ) + : Promise.resolve(influencers), + viewBySwimlaneState: memoizedLoadViewBySwimlane( + lastRefresh, + topFieldValues, + { + earliest: overallState.earliest, + latest: overallState.latest, + }, + selectedJobs, + viewBySwimlaneFieldName, + ANOMALY_SWIM_LANE_HARD_LIMIT, + viewByPerPage, + viewByFromPage, + swimlaneContainerWidth, + influencersFilterQuery + ), + }), + ( + { annotationsData, overallState, tableData }, + { influencers, viewBySwimlaneState } + ): Partial => { + return { + annotationsData, + influencers, + loading: false, + viewBySwimlaneDataLoading: false, + overallSwimlaneData: overallState, + viewBySwimlaneData: viewBySwimlaneState, + tableData, + swimlaneLimit: isViewBySwimLaneData(viewBySwimlaneState) + ? viewBySwimlaneState.cardinality + : undefined, + }; + } + ) + ); + }; +}; export const useExplorerData = (): [Partial | undefined, (d: any) => void] => { + const timefilter = useTimefilter(); + + const { + services: { + mlServices: { mlApiServices }, + uiSettings, + }, + } = useMlKibana(); + const loadExplorerData = useMemo(() => { + const service = new AnomalyTimelineService( + timefilter, + uiSettings, + mlResultsServiceProvider(mlApiServices) + ); + return loadExplorerDataProvider(service); + }, []); + const loadExplorerData$ = useMemo(() => new Subject(), []); + const explorerData$ = useMemo(() => loadExplorerData$.pipe(switchMap(loadExplorerData)), []); const explorerData = useObservable(explorerData$); - return [explorerData, (c) => loadExplorerData$.next(c)]; + + const update = useCallback((c) => { + loadExplorerData$.next(c); + }, []); + + return [explorerData, update]; }; diff --git a/x-pack/plugins/ml/public/application/explorer/add_to_dashboard_control.tsx b/x-pack/plugins/ml/public/application/explorer/add_to_dashboard_control.tsx index 16e2fb47a209d0..3ad749c9d06314 100644 --- a/x-pack/plugins/ml/public/application/explorer/add_to_dashboard_control.tsx +++ b/x-pack/plugins/ml/public/application/explorer/add_to_dashboard_control.tsx @@ -52,7 +52,6 @@ function getDefaultEmbeddablepaPanelConfig(jobIds: JobId[]) { interface AddToDashboardControlProps { jobIds: JobId[]; viewBy: string; - limit: number; onClose: (callback?: () => Promise) => void; } @@ -63,7 +62,6 @@ export const AddToDashboardControl: FC = ({ onClose, jobIds, viewBy, - limit, }) => { const { notifications: { toasts }, @@ -141,7 +139,6 @@ export const AddToDashboardControl: FC = ({ jobIds, swimlaneType, viewBy, - limit, }, }; } @@ -206,8 +203,8 @@ export const AddToDashboardControl: FC = ({ { id: SWIMLANE_TYPE.VIEW_BY, label: i18n.translate('xpack.ml.explorer.viewByFieldLabel', { - defaultMessage: 'View by {viewByField}, up to {limit} rows', - values: { viewByField: viewBy, limit }, + defaultMessage: 'View by {viewByField}', + values: { viewByField: viewBy }, }), }, ]; diff --git a/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx b/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx index b4d32e2af64b8e..e00e2e1e1e2eb1 100644 --- a/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx +++ b/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx @@ -22,12 +22,11 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { DRAG_SELECT_ACTION, VIEW_BY_JOB_LABEL } from './explorer_constants'; +import { DRAG_SELECT_ACTION, SWIMLANE_TYPE, VIEW_BY_JOB_LABEL } from './explorer_constants'; import { AddToDashboardControl } from './add_to_dashboard_control'; import { useMlKibana } from '../contexts/kibana'; import { TimeBuckets } from '../util/time_buckets'; import { UI_SETTINGS } from '../../../../../../src/plugins/data/common'; -import { SelectLimit } from './select_limit'; import { ALLOW_CELL_RANGE_SELECTION, dragSelect$, @@ -36,9 +35,9 @@ import { import { ExplorerState } from './reducers/explorer_reducer'; import { hasMatchingPoints } from './has_matching_points'; import { ExplorerNoInfluencersFound } from './components/explorer_no_influencers_found/explorer_no_influencers_found'; -import { LoadingIndicator } from '../components/loading_indicator'; import { SwimlaneContainer } from './swimlane_container'; -import { OverallSwimlaneData } from './explorer_utils'; +import { OverallSwimlaneData, ViewBySwimLaneData } from './explorer_utils'; +import { NoOverallData } from './components/no_overall_data'; function mapSwimlaneOptionsToEuiOptions(options: string[]) { return options.map((option) => ({ @@ -132,8 +131,11 @@ export const AnomalyTimeline: FC = React.memo( viewBySwimlaneDataLoading, viewBySwimlaneFieldName, viewBySwimlaneOptions, - swimlaneLimit, selectedJobs, + viewByFromPage, + viewByPerPage, + swimlaneLimit, + loading, } = explorerState; const setSwimlaneSelectActive = useCallback((active: boolean) => { @@ -159,25 +161,18 @@ export const AnomalyTimeline: FC = React.memo( }, []); // Listener for click events in the swimlane to load corresponding anomaly data. - const swimlaneCellClick = useCallback((selectedCellsUpdate: any) => { - // If selectedCells is an empty object we clear any existing selection, - // otherwise we save the new selection in AppState and update the Explorer. - if (Object.keys(selectedCellsUpdate).length === 0) { - setSelectedCells(); - } else { - setSelectedCells(selectedCellsUpdate); - } - }, []); - - const showOverallSwimlane = - overallSwimlaneData !== null && - overallSwimlaneData.laneLabels && - overallSwimlaneData.laneLabels.length > 0; - - const showViewBySwimlane = - viewBySwimlaneData !== null && - viewBySwimlaneData.laneLabels && - viewBySwimlaneData.laneLabels.length > 0; + const swimlaneCellClick = useCallback( + (selectedCellsUpdate: any) => { + // If selectedCells is an empty object we clear any existing selection, + // otherwise we save the new selection in AppState and update the Explorer. + if (Object.keys(selectedCellsUpdate).length === 0) { + setSelectedCells(); + } else { + setSelectedCells(selectedCellsUpdate); + } + }, + [setSelectedCells] + ); const menuItems = useMemo(() => { const items = []; @@ -235,21 +230,6 @@ export const AnomalyTimeline: FC = React.memo( /> - - - - - } - display={'columnCompressed'} - > - - -
{viewByLoadedForTimeFormatted && ( @@ -305,68 +285,84 @@ export const AnomalyTimeline: FC = React.memo(
- {showOverallSwimlane && ( - explorerService.setSwimlaneContainerWidth(width)} - /> - )} + explorerService.setSwimlaneContainerWidth(width)} + isLoading={loading} + noDataWarning={} + />
+ + {viewBySwimlaneOptions.length > 0 && ( <> - {showViewBySwimlane && ( - <> - -
- +
+ explorerService.setSwimlaneContainerWidth(width)} + fromPage={viewByFromPage} + perPage={viewByPerPage} + swimlaneLimit={swimlaneLimit} + onPaginationChange={({ perPage: perPageUpdate, fromPage: fromPageUpdate }) => { + if (perPageUpdate) { + explorerService.setViewByPerPage(perPageUpdate); } - timeBuckets={timeBuckets} - swimlaneCellClick={swimlaneCellClick} - swimlaneData={viewBySwimlaneData as OverallSwimlaneData} - swimlaneType={'viewBy'} - selection={selectedCells} - swimlaneRenderDoneListener={swimlaneRenderDoneListener} - onResize={(width) => explorerService.setSwimlaneContainerWidth(width)} - /> -
- - )} - - {viewBySwimlaneDataLoading && } - - {!showViewBySwimlane && - !viewBySwimlaneDataLoading && - typeof viewBySwimlaneFieldName === 'string' && ( - + ) : ( + + ) + ) : null + } /> - )} +
+ )} @@ -380,7 +376,6 @@ export const AnomalyTimeline: FC = React.memo( }} jobIds={selectedJobs.map(({ id }) => id)} viewBy={viewBySwimlaneFieldName!} - limit={swimlaneLimit} /> )} diff --git a/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/__snapshots__/explorer_no_influencers_found.test.js.snap b/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/__snapshots__/explorer_no_influencers_found.test.js.snap index 3ba4ebb2acdeae..d3190d2ac1dade 100644 --- a/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/__snapshots__/explorer_no_influencers_found.test.js.snap +++ b/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/__snapshots__/explorer_no_influencers_found.test.js.snap @@ -1,20 +1,13 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`ExplorerNoInfluencersFound snapshot 1`] = ` - - - + `; diff --git a/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.tsx b/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.tsx index 639c0f7b785043..24def01108584c 100644 --- a/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.tsx +++ b/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.tsx @@ -7,7 +7,6 @@ import React, { FC } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiEmptyPrompt } from '@elastic/eui'; /* * React component for rendering EuiEmptyPrompt when no influencers were found. @@ -15,26 +14,17 @@ import { EuiEmptyPrompt } from '@elastic/eui'; export const ExplorerNoInfluencersFound: FC<{ viewBySwimlaneFieldName: string; showFilterMessage?: boolean; -}> = ({ viewBySwimlaneFieldName, showFilterMessage = false }) => ( - - {showFilterMessage === false && ( - - )} - {showFilterMessage === true && ( - - )} - - } - /> -); +}> = ({ viewBySwimlaneFieldName, showFilterMessage = false }) => + showFilterMessage === false ? ( + + ) : ( + + ); diff --git a/x-pack/plugins/ml/public/application/explorer/components/no_overall_data.tsx b/x-pack/plugins/ml/public/application/explorer/components/no_overall_data.tsx new file mode 100644 index 00000000000000..e73aac66a0d9fc --- /dev/null +++ b/x-pack/plugins/ml/public/application/explorer/components/no_overall_data.tsx @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +export const NoOverallData: FC = () => { + return ( + + ); +}; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer.js b/x-pack/plugins/ml/public/application/explorer/explorer.js index 71c96840d1b579..df4cea0c079874 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer.js @@ -12,8 +12,6 @@ import PropTypes from 'prop-types'; import React from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { Subject } from 'rxjs'; -import { takeUntil } from 'rxjs/operators'; import { EuiFlexGroup, @@ -27,6 +25,7 @@ import { EuiPageHeaderSection, EuiSpacer, EuiTitle, + EuiLoadingContent, } from '@elastic/eui'; import { AnnotationFlyout } from '../components/annotations/annotation_flyout'; @@ -36,12 +35,10 @@ import { DatePickerWrapper } from '../components/navigation_menu/date_picker_wra import { InfluencersList } from '../components/influencers_list'; import { explorerService } from './explorer_dashboard_service'; import { AnomalyResultsViewSelector } from '../components/anomaly_results_view_selector'; -import { LoadingIndicator } from '../components/loading_indicator/loading_indicator'; import { NavigationMenu } from '../components/navigation_menu'; import { CheckboxShowCharts } from '../components/controls/checkbox_showcharts'; import { JobSelector } from '../components/job_selector'; import { SelectInterval } from '../components/controls/select_interval/select_interval'; -import { limit$ } from './select_limit/select_limit'; import { SelectSeverity } from '../components/controls/select_severity/select_severity'; import { ExplorerQueryBar, @@ -142,19 +139,6 @@ export class Explorer extends React.Component { state = { filterIconTriggeredQuery: undefined, language: DEFAULT_QUERY_LANG }; - _unsubscribeAll = new Subject(); - - componentDidMount() { - limit$.pipe(takeUntil(this._unsubscribeAll)).subscribe(explorerService.setSwimlaneLimit); - } - - componentWillUnmount() { - this._unsubscribeAll.next(); - this._unsubscribeAll.complete(); - } - - viewByChangeHandler = (e) => explorerService.setViewBySwimlaneFieldName(e.target.value); - // Escape regular parens from fieldName as that portion of the query is not wrapped in double quotes // and will cause a syntax error when called with getKqlQueryValues applyFilter = (fieldName, fieldValue, action) => { @@ -240,29 +224,7 @@ export class Explorer extends React.Component { const noJobsFound = selectedJobs === null || selectedJobs.length === 0; const hasResults = overallSwimlaneData.points && overallSwimlaneData.points.length > 0; - if (loading === true) { - return ( - - - - ); - } - - if (noJobsFound) { + if (noJobsFound && !loading) { return ( @@ -270,7 +232,7 @@ export class Explorer extends React.Component { ); } - if (noJobsFound && hasResults === false) { + if (noJobsFound && hasResults === false && !loading) { return ( @@ -320,7 +282,11 @@ export class Explorer extends React.Component { /> - + {loading ? ( + + ) : ( + + )}
)} @@ -352,59 +318,59 @@ export class Explorer extends React.Component { )} - -

- -

- - - - - - - - - - + +

+ +

+ + - - - - {chartsData.seriesToPlot.length > 0 && selectedCells !== undefined && ( - - - - - - )} - - - - -
- {showCharts && } -
- - + + + + + + + + + + + {chartsData.seriesToPlot.length > 0 && selectedCells !== undefined && ( + + + + + + )} + + +
+ {showCharts && } +
+ + + )}
diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts b/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts index d1adf8c7ad744c..21e13cb029d69e 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts +++ b/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts @@ -27,9 +27,10 @@ export const EXPLORER_ACTION = { SET_INFLUENCER_FILTER_SETTINGS: 'setInfluencerFilterSettings', SET_SELECTED_CELLS: 'setSelectedCells', SET_SWIMLANE_CONTAINER_WIDTH: 'setSwimlaneContainerWidth', - SET_SWIMLANE_LIMIT: 'setSwimlaneLimit', SET_VIEW_BY_SWIMLANE_FIELD_NAME: 'setViewBySwimlaneFieldName', SET_VIEW_BY_SWIMLANE_LOADING: 'setViewBySwimlaneLoading', + SET_VIEW_BY_PER_PAGE: 'setViewByPerPage', + SET_VIEW_BY_FROM_PAGE: 'setViewByFromPage', }; export const FILTER_ACTION = { @@ -51,9 +52,23 @@ export const CHART_TYPE = { }; export const MAX_CATEGORY_EXAMPLES = 10; + +/** + * Maximum amount of top influencer to fetch. + */ export const MAX_INFLUENCER_FIELD_VALUES = 10; export const MAX_INFLUENCER_FIELD_NAMES = 50; export const VIEW_BY_JOB_LABEL = i18n.translate('xpack.ml.explorer.jobIdLabel', { defaultMessage: 'job ID', }); +/** + * Hard limitation for the size of terms + * aggregations on influencers values. + */ +export const ANOMALY_SWIM_LANE_HARD_LIMIT = 1000; + +/** + * Default page size fot the anomaly swim lane. + */ +export const SWIM_LANE_DEFAULT_PAGE_SIZE = 10; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_dashboard_service.ts b/x-pack/plugins/ml/public/application/explorer/explorer_dashboard_service.ts index 30ab918983a777..1429bf08583618 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_dashboard_service.ts +++ b/x-pack/plugins/ml/public/application/explorer/explorer_dashboard_service.ts @@ -12,7 +12,7 @@ import { isEqual } from 'lodash'; import { from, isObservable, Observable, Subject } from 'rxjs'; -import { distinctUntilChanged, flatMap, map, scan } from 'rxjs/operators'; +import { distinctUntilChanged, flatMap, map, scan, shareReplay } from 'rxjs/operators'; import { DeepPartial } from '../../../common/types/common'; @@ -49,7 +49,9 @@ const explorerFilteredAction$ = explorerAction$.pipe( // applies action and returns state const explorerState$: Observable = explorerFilteredAction$.pipe( - scan(explorerReducer, getExplorerDefaultState()) + scan(explorerReducer, getExplorerDefaultState()), + // share the last emitted value among new subscribers + shareReplay(1) ); interface ExplorerAppState { @@ -59,6 +61,8 @@ interface ExplorerAppState { selectedTimes?: number[]; showTopFieldValues?: boolean; viewByFieldName?: string; + viewByPerPage?: number; + viewByFromPage?: number; }; mlExplorerFilter: { influencersFilterQuery?: unknown; @@ -88,6 +92,14 @@ const explorerAppState$: Observable = explorerState$.pipe( appState.mlExplorerSwimlane.viewByFieldName = state.viewBySwimlaneFieldName; } + if (state.viewByFromPage !== undefined) { + appState.mlExplorerSwimlane.viewByFromPage = state.viewByFromPage; + } + + if (state.viewByPerPage !== undefined) { + appState.mlExplorerSwimlane.viewByPerPage = state.viewByPerPage; + } + if (state.filterActive) { appState.mlExplorerFilter.influencersFilterQuery = state.influencersFilterQuery; appState.mlExplorerFilter.filterActive = state.filterActive; @@ -153,13 +165,16 @@ export const explorerService = { payload, }); }, - setSwimlaneLimit: (payload: number) => { - explorerAction$.next({ type: EXPLORER_ACTION.SET_SWIMLANE_LIMIT, payload }); - }, setViewBySwimlaneFieldName: (payload: string) => { explorerAction$.next({ type: EXPLORER_ACTION.SET_VIEW_BY_SWIMLANE_FIELD_NAME, payload }); }, setViewBySwimlaneLoading: (payload: any) => { explorerAction$.next({ type: EXPLORER_ACTION.SET_VIEW_BY_SWIMLANE_LOADING, payload }); }, + setViewByFromPage: (payload: number) => { + explorerAction$.next({ type: EXPLORER_ACTION.SET_VIEW_BY_FROM_PAGE, payload }); + }, + setViewByPerPage: (payload: number) => { + explorerAction$.next({ type: EXPLORER_ACTION.SET_VIEW_BY_PER_PAGE, payload }); + }, }; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.tsx b/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.tsx index 4e6dcdcc5129ca..aa386288ac7e08 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.tsx +++ b/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.tsx @@ -29,7 +29,7 @@ import { ChartTooltipService, ChartTooltipValue, } from '../components/chart_tooltip/chart_tooltip_service'; -import { OverallSwimlaneData } from './explorer_utils'; +import { OverallSwimlaneData, ViewBySwimLaneData } from './explorer_utils'; const SCSS = { mlDragselectDragging: 'mlDragselectDragging', @@ -57,7 +57,7 @@ export interface ExplorerSwimlaneProps { maskAll?: boolean; timeBuckets: InstanceType; swimlaneCellClick?: Function; - swimlaneData: OverallSwimlaneData; + swimlaneData: OverallSwimlaneData | ViewBySwimLaneData; swimlaneType: SwimlaneType; selection?: { lanes: any[]; @@ -211,7 +211,7 @@ export class ExplorerSwimlane extends React.Component { const { swimlaneType } = this.props; // This selects both overall and viewby swimlane - const wrapper = d3.selectAll('.ml-explorer-swimlane'); + const wrapper = d3.selectAll('.mlExplorerSwimlane'); wrapper.selectAll('.lane-label').classed('lane-label-masked', true); wrapper @@ -242,7 +242,7 @@ export class ExplorerSwimlane extends React.Component { maskIrrelevantSwimlanes(maskAll: boolean) { if (maskAll === true) { // This selects both overall and viewby swimlane - const allSwimlanes = d3.selectAll('.ml-explorer-swimlane'); + const allSwimlanes = d3.selectAll('.mlExplorerSwimlane'); allSwimlanes.selectAll('.lane-label').classed('lane-label-masked', true); allSwimlanes .selectAll('.sl-cell-inner,.sl-cell-inner-dragselect') @@ -258,7 +258,7 @@ export class ExplorerSwimlane extends React.Component { clearSelection() { // This selects both overall and viewby swimlane - const wrapper = d3.selectAll('.ml-explorer-swimlane'); + const wrapper = d3.selectAll('.mlExplorerSwimlane'); wrapper.selectAll('.lane-label').classed('lane-label-masked', false); wrapper.selectAll('.sl-cell-inner').classed('sl-cell-inner-masked', false); diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_utils.d.ts b/x-pack/plugins/ml/public/application/explorer/explorer_utils.d.ts index 2d49fa737cef60..05fdb52e1ccb28 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_utils.d.ts +++ b/x-pack/plugins/ml/public/application/explorer/explorer_utils.d.ts @@ -8,8 +8,6 @@ import { Moment } from 'moment'; import { CombinedJob } from '../../../common/types/anomaly_detection_jobs'; -import { TimeBucketsInterval } from '../util/time_buckets'; - interface ClearedSelectedAnomaliesState { selectedCells: undefined; viewByLoadedForTimeFormatted: null; @@ -35,6 +33,10 @@ export declare interface OverallSwimlaneData extends SwimlaneData { latest: number; } +export interface ViewBySwimLaneData extends OverallSwimlaneData { + cardinality: number; +} + export declare const getDateFormatTz: () => any; export declare const getDefaultSwimlaneData: () => SwimlaneData; @@ -163,22 +165,6 @@ declare interface LoadOverallDataResponse { overallSwimlaneData: OverallSwimlaneData; } -export declare const loadOverallData: ( - selectedJobs: ExplorerJob[], - interval: TimeBucketsInterval, - bounds: TimeRangeBounds -) => Promise; - -export declare const loadViewBySwimlane: ( - fieldValues: string[], - bounds: SwimlaneBounds, - selectedJobs: ExplorerJob[], - viewBySwimlaneFieldName: string, - swimlaneLimit: number, - influencersFilterQuery: any, - noInfluencersConfigured: boolean -) => Promise; - export declare const loadViewByTopFieldValuesForSelectedTime: ( earliestMs: number, latestMs: number, diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_utils.js b/x-pack/plugins/ml/public/application/explorer/explorer_utils.js index bd6a7ee59c942b..23da9669ee9a59 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_utils.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_utils.js @@ -8,11 +8,9 @@ * utils for Anomaly Explorer. */ -import { chain, each, get, union, uniq } from 'lodash'; +import { chain, get, union, uniq } from 'lodash'; import moment from 'moment-timezone'; -import { i18n } from '@kbn/i18n'; - import { ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE, ANOMALIES_TABLE_DEFAULT_QUERY_SIZE, @@ -27,7 +25,7 @@ import { parseInterval } from '../../../common/util/parse_interval'; import { ml } from '../services/ml_api_service'; import { mlJobService } from '../services/job_service'; import { mlResultsService } from '../services/results_service'; -import { getBoundsRoundedToInterval, getTimeBucketsFromCache } from '../util/time_buckets'; +import { getTimeBucketsFromCache } from '../util/time_buckets'; import { getTimefilter, getUiSettings } from '../util/dependency_cache'; import { @@ -36,7 +34,6 @@ import { SWIMLANE_TYPE, VIEW_BY_JOB_LABEL, } from './explorer_constants'; -import { getSwimlaneContainerWidth } from './legacy_utils'; // create new job objects based on standard job config objects // new job objects just contain job id, bucket span in seconds and a selected flag. @@ -51,6 +48,7 @@ export function getClearedSelectedAnomaliesState() { return { selectedCells: undefined, viewByLoadedForTimeFormatted: null, + swimlaneLimit: undefined, }; } @@ -267,58 +265,6 @@ export function getSwimlaneBucketInterval(selectedJobs, swimlaneContainerWidth) return buckets.getInterval(); } -export function loadViewByTopFieldValuesForSelectedTime( - earliestMs, - latestMs, - selectedJobs, - viewBySwimlaneFieldName, - swimlaneLimit, - noInfluencersConfigured -) { - const selectedJobIds = selectedJobs.map((d) => d.id); - - // Find the top field values for the selected time, and then load the 'view by' - // swimlane over the full time range for those specific field values. - return new Promise((resolve) => { - if (viewBySwimlaneFieldName !== VIEW_BY_JOB_LABEL) { - mlResultsService - .getTopInfluencers(selectedJobIds, earliestMs, latestMs, swimlaneLimit) - .then((resp) => { - if (resp.influencers[viewBySwimlaneFieldName] === undefined) { - resolve([]); - } - - const topFieldValues = []; - const topInfluencers = resp.influencers[viewBySwimlaneFieldName]; - if (Array.isArray(topInfluencers)) { - topInfluencers.forEach((influencerData) => { - if (influencerData.maxAnomalyScore > 0) { - topFieldValues.push(influencerData.influencerFieldValue); - } - }); - } - resolve(topFieldValues); - }); - } else { - mlResultsService - .getScoresByBucket( - selectedJobIds, - earliestMs, - latestMs, - getSwimlaneBucketInterval( - selectedJobs, - getSwimlaneContainerWidth(noInfluencersConfigured) - ).asSeconds() + 's', - swimlaneLimit - ) - .then((resp) => { - const topFieldValues = Object.keys(resp.results); - resolve(topFieldValues); - }); - } - }); -} - // Obtain the list of 'View by' fields per job and viewBySwimlaneFieldName export function getViewBySwimlaneOptions({ currentViewBySwimlaneFieldName, @@ -435,105 +381,6 @@ export function getViewBySwimlaneOptions({ }; } -export function processOverallResults(scoresByTime, searchBounds, interval) { - const overallLabel = i18n.translate('xpack.ml.explorer.overallLabel', { - defaultMessage: 'Overall', - }); - const dataset = { - laneLabels: [overallLabel], - points: [], - interval, - earliest: searchBounds.min.valueOf() / 1000, - latest: searchBounds.max.valueOf() / 1000, - }; - - if (Object.keys(scoresByTime).length > 0) { - // Store the earliest and latest times of the data returned by the ES aggregations, - // These will be used for calculating the earliest and latest times for the swimlane charts. - each(scoresByTime, (score, timeMs) => { - const time = timeMs / 1000; - dataset.points.push({ - laneLabel: overallLabel, - time, - value: score, - }); - - dataset.earliest = Math.min(time, dataset.earliest); - dataset.latest = Math.max(time + dataset.interval, dataset.latest); - }); - } - - return dataset; -} - -export function processViewByResults( - scoresByInfluencerAndTime, - sortedLaneValues, - bounds, - viewBySwimlaneFieldName, - interval -) { - // Processes the scores for the 'view by' swimlane. - // Sorts the lanes according to the supplied array of lane - // values in the order in which they should be displayed, - // or pass an empty array to sort lanes according to max score over all time. - const dataset = { - fieldName: viewBySwimlaneFieldName, - points: [], - interval, - }; - - // Set the earliest and latest to be the same as the overall swimlane. - dataset.earliest = bounds.earliest; - dataset.latest = bounds.latest; - - const laneLabels = []; - const maxScoreByLaneLabel = {}; - - each(scoresByInfluencerAndTime, (influencerData, influencerFieldValue) => { - laneLabels.push(influencerFieldValue); - maxScoreByLaneLabel[influencerFieldValue] = 0; - - each(influencerData, (anomalyScore, timeMs) => { - const time = timeMs / 1000; - dataset.points.push({ - laneLabel: influencerFieldValue, - time, - value: anomalyScore, - }); - maxScoreByLaneLabel[influencerFieldValue] = Math.max( - maxScoreByLaneLabel[influencerFieldValue], - anomalyScore - ); - }); - }); - - const sortValuesLength = sortedLaneValues.length; - if (sortValuesLength === 0) { - // Sort lanes in descending order of max score. - // Note the keys in scoresByInfluencerAndTime received from the ES request - // are not guaranteed to be sorted by score if they can be parsed as numbers - // (e.g. if viewing by HTTP response code). - dataset.laneLabels = laneLabels.sort((a, b) => { - return maxScoreByLaneLabel[b] - maxScoreByLaneLabel[a]; - }); - } else { - // Sort lanes according to supplied order - // e.g. when a cell in the overall swimlane has been selected. - // Find the index of each lane label from the actual data set, - // rather than using sortedLaneValues as-is, just in case they differ. - dataset.laneLabels = laneLabels.sort((a, b) => { - let aIndex = sortedLaneValues.indexOf(a); - let bIndex = sortedLaneValues.indexOf(b); - aIndex = aIndex > -1 ? aIndex : sortValuesLength; - bIndex = bIndex > -1 ? bIndex : sortValuesLength; - return aIndex - bIndex; - }); - } - - return dataset; -} - export function loadAnnotationsTableData(selectedCells, selectedJobs, interval, bounds) { const jobIds = selectedCells !== undefined && selectedCells.viewByFieldName === VIEW_BY_JOB_LABEL @@ -723,138 +570,6 @@ export async function loadDataForCharts( }); } -export function loadOverallData(selectedJobs, interval, bounds) { - return new Promise((resolve) => { - // Loads the overall data components i.e. the overall swimlane and influencers list. - if (selectedJobs === null) { - resolve({ - loading: false, - hasResuts: false, - }); - return; - } - - // Ensure the search bounds align to the bucketing interval used in the swimlane so - // that the first and last buckets are complete. - const searchBounds = getBoundsRoundedToInterval(bounds, interval, false); - const selectedJobIds = selectedJobs.map((d) => d.id); - - // Load the overall bucket scores by time. - // Pass the interval in seconds as the swimlane relies on a fixed number of seconds between buckets - // which wouldn't be the case if e.g. '1M' was used. - // Pass 'true' when obtaining bucket bounds due to the way the overall_buckets endpoint works - // to ensure the search is inclusive of end time. - const overallBucketsBounds = getBoundsRoundedToInterval(bounds, interval, true); - mlResultsService - .getOverallBucketScores( - selectedJobIds, - // Note there is an optimization for when top_n == 1. - // If top_n > 1, we should test what happens when the request takes long - // and refactor the loading calls, if necessary, to avoid delays in loading other components. - 1, - overallBucketsBounds.min.valueOf(), - overallBucketsBounds.max.valueOf(), - interval.asSeconds() + 's' - ) - .then((resp) => { - const overallSwimlaneData = processOverallResults( - resp.results, - searchBounds, - interval.asSeconds() - ); - - resolve({ - loading: false, - overallSwimlaneData, - }); - }); - }); -} - -export function loadViewBySwimlane( - fieldValues, - bounds, - selectedJobs, - viewBySwimlaneFieldName, - swimlaneLimit, - influencersFilterQuery, - noInfluencersConfigured -) { - return new Promise((resolve) => { - const finish = (resp) => { - if (resp !== undefined) { - const viewBySwimlaneData = processViewByResults( - resp.results, - fieldValues, - bounds, - viewBySwimlaneFieldName, - getSwimlaneBucketInterval( - selectedJobs, - getSwimlaneContainerWidth(noInfluencersConfigured) - ).asSeconds() - ); - - resolve({ - viewBySwimlaneData, - viewBySwimlaneDataLoading: false, - }); - } else { - resolve({ viewBySwimlaneDataLoading: false }); - } - }; - - if (selectedJobs === undefined || viewBySwimlaneFieldName === undefined) { - finish(); - return; - } else { - // Ensure the search bounds align to the bucketing interval used in the swimlane so - // that the first and last buckets are complete. - const timefilter = getTimefilter(); - const timefilterBounds = timefilter.getActiveBounds(); - const searchBounds = getBoundsRoundedToInterval( - timefilterBounds, - getSwimlaneBucketInterval(selectedJobs, getSwimlaneContainerWidth(noInfluencersConfigured)), - false - ); - const selectedJobIds = selectedJobs.map((d) => d.id); - - // load scores by influencer/jobId value and time. - // Pass the interval in seconds as the swimlane relies on a fixed number of seconds between buckets - // which wouldn't be the case if e.g. '1M' was used. - const interval = `${getSwimlaneBucketInterval( - selectedJobs, - getSwimlaneContainerWidth(noInfluencersConfigured) - ).asSeconds()}s`; - if (viewBySwimlaneFieldName !== VIEW_BY_JOB_LABEL) { - mlResultsService - .getInfluencerValueMaxScoreByTime( - selectedJobIds, - viewBySwimlaneFieldName, - fieldValues, - searchBounds.min.valueOf(), - searchBounds.max.valueOf(), - interval, - swimlaneLimit, - influencersFilterQuery - ) - .then(finish); - } else { - const jobIds = - fieldValues !== undefined && fieldValues.length > 0 ? fieldValues : selectedJobIds; - mlResultsService - .getScoresByBucket( - jobIds, - searchBounds.min.valueOf(), - searchBounds.max.valueOf(), - interval, - swimlaneLimit - ) - .then(finish); - } - } - }); -} - export async function loadTopInfluencers( selectedJobIds, earliestMs, @@ -871,6 +586,8 @@ export async function loadTopInfluencers( earliestMs, latestMs, MAX_INFLUENCER_FIELD_VALUES, + 10, + 1, influencers, influencersFilterQuery ) diff --git a/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts b/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts index a19750494afdc1..068f43a140c901 100644 --- a/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts +++ b/x-pack/plugins/ml/public/application/explorer/hooks/use_selected_cells.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { useCallback, useMemo } from 'react'; import { useUrlState } from '../../util/url_state'; import { SWIMLANE_TYPE } from '../explorer_constants'; import { AppStateSelectedCells } from '../explorer_utils'; @@ -14,55 +15,55 @@ export const useSelectedCells = (): [ ] => { const [appState, setAppState] = useUrlState('_a'); - let selectedCells: AppStateSelectedCells | undefined; - // keep swimlane selection, restore selectedCells from AppState - if ( - appState && - appState.mlExplorerSwimlane && - appState.mlExplorerSwimlane.selectedType !== undefined - ) { - selectedCells = { - type: appState.mlExplorerSwimlane.selectedType, - lanes: appState.mlExplorerSwimlane.selectedLanes, - times: appState.mlExplorerSwimlane.selectedTimes, - showTopFieldValues: appState.mlExplorerSwimlane.showTopFieldValues, - viewByFieldName: appState.mlExplorerSwimlane.viewByFieldName, - }; - } + const selectedCells = useMemo(() => { + return appState?.mlExplorerSwimlane?.selectedType !== undefined + ? { + type: appState.mlExplorerSwimlane.selectedType, + lanes: appState.mlExplorerSwimlane.selectedLanes, + times: appState.mlExplorerSwimlane.selectedTimes, + showTopFieldValues: appState.mlExplorerSwimlane.showTopFieldValues, + viewByFieldName: appState.mlExplorerSwimlane.viewByFieldName, + } + : undefined; + // TODO fix appState to use memoization + }, [JSON.stringify(appState?.mlExplorerSwimlane)]); - const setSelectedCells = (swimlaneSelectedCells: AppStateSelectedCells) => { - const mlExplorerSwimlane = { ...appState.mlExplorerSwimlane }; + const setSelectedCells = useCallback( + (swimlaneSelectedCells: AppStateSelectedCells) => { + const mlExplorerSwimlane = { ...appState.mlExplorerSwimlane }; - if (swimlaneSelectedCells !== undefined) { - swimlaneSelectedCells.showTopFieldValues = false; + if (swimlaneSelectedCells !== undefined) { + swimlaneSelectedCells.showTopFieldValues = false; - const currentSwimlaneType = selectedCells?.type; - const currentShowTopFieldValues = selectedCells?.showTopFieldValues; - const newSwimlaneType = swimlaneSelectedCells?.type; + const currentSwimlaneType = selectedCells?.type; + const currentShowTopFieldValues = selectedCells?.showTopFieldValues; + const newSwimlaneType = swimlaneSelectedCells?.type; - if ( - (currentSwimlaneType === SWIMLANE_TYPE.OVERALL && - newSwimlaneType === SWIMLANE_TYPE.VIEW_BY) || - newSwimlaneType === SWIMLANE_TYPE.OVERALL || - currentShowTopFieldValues === true - ) { - swimlaneSelectedCells.showTopFieldValues = true; - } + if ( + (currentSwimlaneType === SWIMLANE_TYPE.OVERALL && + newSwimlaneType === SWIMLANE_TYPE.VIEW_BY) || + newSwimlaneType === SWIMLANE_TYPE.OVERALL || + currentShowTopFieldValues === true + ) { + swimlaneSelectedCells.showTopFieldValues = true; + } - mlExplorerSwimlane.selectedType = swimlaneSelectedCells.type; - mlExplorerSwimlane.selectedLanes = swimlaneSelectedCells.lanes; - mlExplorerSwimlane.selectedTimes = swimlaneSelectedCells.times; - mlExplorerSwimlane.showTopFieldValues = swimlaneSelectedCells.showTopFieldValues; - setAppState('mlExplorerSwimlane', mlExplorerSwimlane); - } else { - delete mlExplorerSwimlane.selectedType; - delete mlExplorerSwimlane.selectedLanes; - delete mlExplorerSwimlane.selectedTimes; - delete mlExplorerSwimlane.showTopFieldValues; - setAppState('mlExplorerSwimlane', mlExplorerSwimlane); - } - }; + mlExplorerSwimlane.selectedType = swimlaneSelectedCells.type; + mlExplorerSwimlane.selectedLanes = swimlaneSelectedCells.lanes; + mlExplorerSwimlane.selectedTimes = swimlaneSelectedCells.times; + mlExplorerSwimlane.showTopFieldValues = swimlaneSelectedCells.showTopFieldValues; + setAppState('mlExplorerSwimlane', mlExplorerSwimlane); + } else { + delete mlExplorerSwimlane.selectedType; + delete mlExplorerSwimlane.selectedLanes; + delete mlExplorerSwimlane.selectedTimes; + delete mlExplorerSwimlane.showTopFieldValues; + setAppState('mlExplorerSwimlane', mlExplorerSwimlane); + } + }, + [appState?.mlExplorerSwimlane, selectedCells] + ); return [selectedCells, setSelectedCells]; }; diff --git a/x-pack/plugins/ml/public/application/explorer/legacy_utils.ts b/x-pack/plugins/ml/public/application/explorer/legacy_utils.ts index 3b92ee3fa37f6a..b85b0401c45cab 100644 --- a/x-pack/plugins/ml/public/application/explorer/legacy_utils.ts +++ b/x-pack/plugins/ml/public/application/explorer/legacy_utils.ts @@ -11,8 +11,3 @@ export function getChartContainerWidth() { const chartContainer = document.querySelector('.explorer-charts'); return Math.floor((chartContainer && chartContainer.clientWidth) || 0); } - -export function getSwimlaneContainerWidth() { - const explorerContainer = document.querySelector('.ml-explorer'); - return (explorerContainer && explorerContainer.clientWidth) || 0; -} diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/clear_influencer_filter_settings.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/clear_influencer_filter_settings.ts index 1614da14e355a4..dd1d0516b6173f 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/clear_influencer_filter_settings.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/clear_influencer_filter_settings.ts @@ -19,5 +19,6 @@ export function clearInfluencerFilterSettings(state: ExplorerState): ExplorerSta queryString: '', tableQueryString: '', ...getClearedSelectedAnomaliesState(), + viewByFromPage: 1, }; } diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/job_selection_change.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/job_selection_change.ts index a26c0564c6b16d..49f5794273a04d 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/job_selection_change.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/job_selection_change.ts @@ -17,6 +17,7 @@ export const jobSelectionChange = (state: ExplorerState, payload: ActionPayload) noInfluencersConfigured: getInfluencers(selectedJobs).length === 0, overallSwimlaneData: getDefaultSwimlaneData(), selectedJobs, + viewByFromPage: 1, }; // clear filter if selected jobs have no influencers diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts index c31b26b7adb7b7..c55c06c80ab81a 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts @@ -27,7 +27,7 @@ import { setKqlQueryBarPlaceholder } from './set_kql_query_bar_placeholder'; export const explorerReducer = (state: ExplorerState, nextAction: Action): ExplorerState => { const { type, payload } = nextAction; - let nextState; + let nextState: ExplorerState; switch (type) { case EXPLORER_ACTION.CLEAR_INFLUENCER_FILTER_SETTINGS: @@ -39,6 +39,7 @@ export const explorerReducer = (state: ExplorerState, nextAction: Action): Explo ...state, ...getClearedSelectedAnomaliesState(), loading: false, + viewByFromPage: 1, selectedJobs: [], }; break; @@ -82,22 +83,7 @@ export const explorerReducer = (state: ExplorerState, nextAction: Action): Explo break; case EXPLORER_ACTION.SET_SWIMLANE_CONTAINER_WIDTH: - if (state.noInfluencersConfigured === true) { - // swimlane is full width, minus 30 for the 'no influencers' info icon, - // minus 170 for the lane labels, minus 50 padding - nextState = { ...state, swimlaneContainerWidth: payload - 250 }; - } else { - // swimlane width is 5 sixths of the window, - // minus 170 for the lane labels, minus 50 padding - nextState = { ...state, swimlaneContainerWidth: (payload / 6) * 5 - 220 }; - } - break; - - case EXPLORER_ACTION.SET_SWIMLANE_LIMIT: - nextState = { - ...state, - swimlaneLimit: payload, - }; + nextState = { ...state, swimlaneContainerWidth: payload }; break; case EXPLORER_ACTION.SET_VIEW_BY_SWIMLANE_FIELD_NAME: @@ -117,6 +103,9 @@ export const explorerReducer = (state: ExplorerState, nextAction: Action): Explo ...getClearedSelectedAnomaliesState(), maskAll, viewBySwimlaneFieldName, + viewBySwimlaneData: getDefaultSwimlaneData(), + viewByFromPage: 1, + viewBySwimlaneDataLoading: true, }; break; @@ -125,7 +114,7 @@ export const explorerReducer = (state: ExplorerState, nextAction: Action): Explo nextState = { ...state, annotationsData, - ...overallState, + overallSwimlaneData: overallState, tableData, viewBySwimlaneData: { ...getDefaultSwimlaneData(), @@ -134,6 +123,22 @@ export const explorerReducer = (state: ExplorerState, nextAction: Action): Explo }; break; + case EXPLORER_ACTION.SET_VIEW_BY_FROM_PAGE: + nextState = { + ...state, + viewByFromPage: payload, + }; + break; + + case EXPLORER_ACTION.SET_VIEW_BY_PER_PAGE: + nextState = { + ...state, + // reset current page on the page size change + viewByFromPage: 1, + viewByPerPage: payload, + }; + break; + default: nextState = state; } @@ -155,7 +160,7 @@ export const explorerReducer = (state: ExplorerState, nextAction: Action): Explo filteredFields: nextState.filteredFields, isAndOperator: nextState.isAndOperator, selectedJobs: nextState.selectedJobs, - selectedCells: nextState.selectedCells, + selectedCells: nextState.selectedCells!, }); const { bounds, selectedCells } = nextState; diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/set_influencer_filter_settings.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/set_influencer_filter_settings.ts index 819f6ca1cac922..be87de7da8c883 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/set_influencer_filter_settings.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/set_influencer_filter_settings.ts @@ -57,5 +57,6 @@ export function setInfluencerFilterSettings( filteredFields.includes(selectedViewByFieldName) === false, viewBySwimlaneFieldName: selectedViewByFieldName, viewBySwimlaneOptions: filteredViewBySwimlaneOptions, + viewByFromPage: 1, }; } diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts index 4e1a2af9b13a60..892b46467345b3 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts @@ -19,7 +19,9 @@ import { TimeRangeBounds, OverallSwimlaneData, SwimlaneData, + ViewBySwimLaneData, } from '../../explorer_utils'; +import { SWIM_LANE_DEFAULT_PAGE_SIZE } from '../../explorer_constants'; export interface ExplorerState { annotationsData: any[]; @@ -42,14 +44,16 @@ export interface ExplorerState { selectedJobs: ExplorerJob[] | null; swimlaneBucketInterval: any; swimlaneContainerWidth: number; - swimlaneLimit: number; tableData: AnomaliesTableData; tableQueryString: string; viewByLoadedForTimeFormatted: string | null; - viewBySwimlaneData: SwimlaneData | OverallSwimlaneData; + viewBySwimlaneData: SwimlaneData | ViewBySwimLaneData; viewBySwimlaneDataLoading: boolean; viewBySwimlaneFieldName?: string; + viewByPerPage: number; + viewByFromPage: number; viewBySwimlaneOptions: string[]; + swimlaneLimit?: number; } function getDefaultIndexPattern() { @@ -78,7 +82,6 @@ export function getExplorerDefaultState(): ExplorerState { selectedJobs: null, swimlaneBucketInterval: undefined, swimlaneContainerWidth: 0, - swimlaneLimit: 10, tableData: { anomalies: [], examplesByJobId: [''], @@ -92,5 +95,8 @@ export function getExplorerDefaultState(): ExplorerState { viewBySwimlaneDataLoading: false, viewBySwimlaneFieldName: undefined, viewBySwimlaneOptions: [], + viewByPerPage: SWIM_LANE_DEFAULT_PAGE_SIZE, + viewByFromPage: 1, + swimlaneLimit: undefined, }; } diff --git a/x-pack/plugins/ml/public/application/explorer/select_limit/index.ts b/x-pack/plugins/ml/public/application/explorer/select_limit/index.ts deleted file mode 100644 index 5b7040e5c3606a..00000000000000 --- a/x-pack/plugins/ml/public/application/explorer/select_limit/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export { useSwimlaneLimit, SelectLimit } from './select_limit'; diff --git a/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.test.tsx b/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.test.tsx deleted file mode 100644 index cf65419e4bd801..00000000000000 --- a/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.test.tsx +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React from 'react'; -import { act } from 'react-dom/test-utils'; -import { shallow } from 'enzyme'; -import { SelectLimit } from './select_limit'; - -describe('SelectLimit', () => { - test('creates correct initial selected value', () => { - const wrapper = shallow(); - expect(wrapper.props().value).toEqual(10); - }); - - test('state for currently selected value is updated correctly on click', () => { - const wrapper = shallow(); - expect(wrapper.props().value).toEqual(10); - - act(() => { - wrapper.simulate('change', { target: { value: 25 } }); - }); - wrapper.update(); - - expect(wrapper.props().value).toEqual(10); - }); -}); diff --git a/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.tsx b/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.tsx deleted file mode 100644 index 7a2df1a0f05350..00000000000000 --- a/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.tsx +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -/* - * React component for rendering a select element with limit options. - */ -import React from 'react'; -import useObservable from 'react-use/lib/useObservable'; -import { BehaviorSubject } from 'rxjs'; - -import { EuiSelect } from '@elastic/eui'; - -const limitOptions = [5, 10, 25, 50]; - -const euiOptions = limitOptions.map((limit) => ({ - value: limit, - text: `${limit}`, -})); - -export const defaultLimit = limitOptions[1]; -export const limit$ = new BehaviorSubject(defaultLimit); - -export const useSwimlaneLimit = (): [number, (newLimit: number) => void] => { - const limit = useObservable(limit$, defaultLimit); - - return [limit!, (newLimit: number) => limit$.next(newLimit)]; -}; - -export const SelectLimit = () => { - const [limit, setLimit] = useSwimlaneLimit(); - - function onChange(e: React.ChangeEvent) { - setLimit(parseInt(e.target.value, 10)); - } - - return ; -}; diff --git a/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx index 57d1fd81000b7e..e34e1d26c9cab2 100644 --- a/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx +++ b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx @@ -5,7 +5,14 @@ */ import React, { FC, useCallback, useState } from 'react'; -import { EuiResizeObserver, EuiText } from '@elastic/eui'; +import { + EuiText, + EuiLoadingChart, + EuiResizeObserver, + EuiFlexGroup, + EuiFlexItem, + EuiEmptyPrompt, +} from '@elastic/eui'; import { throttle } from 'lodash'; import { @@ -14,48 +21,139 @@ import { } from '../../application/explorer/explorer_swimlane'; import { MlTooltipComponent } from '../../application/components/chart_tooltip'; +import { SwimLanePagination } from './swimlane_pagination'; +import { SWIMLANE_TYPE } from './explorer_constants'; +import { ViewBySwimLaneData } from './explorer_utils'; +/** + * Ignore insignificant resize, e.g. browser scrollbar appearance. + */ +const RESIZE_IGNORED_DIFF_PX = 20; const RESIZE_THROTTLE_TIME_MS = 500; +export function isViewBySwimLaneData(arg: any): arg is ViewBySwimLaneData { + return arg && arg.hasOwnProperty('cardinality'); +} + +/** + * Anomaly swim lane container responsible for handling resizing, pagination and injecting + * tooltip service. + * + * @param children + * @param onResize + * @param perPage + * @param fromPage + * @param swimlaneLimit + * @param onPaginationChange + * @param props + * @constructor + */ export const SwimlaneContainer: FC< Omit & { onResize: (width: number) => void; + fromPage?: number; + perPage?: number; + swimlaneLimit?: number; + onPaginationChange?: (arg: { perPage?: number; fromPage?: number }) => void; + isLoading: boolean; + noDataWarning: string | JSX.Element | null; } -> = ({ children, onResize, ...props }) => { +> = ({ + children, + onResize, + perPage, + fromPage, + swimlaneLimit, + onPaginationChange, + isLoading, + noDataWarning, + ...props +}) => { const [chartWidth, setChartWidth] = useState(0); const resizeHandler = useCallback( throttle((e: { width: number; height: number }) => { const labelWidth = 200; - setChartWidth(e.width - labelWidth); - onResize(e.width); + const resultNewWidth = e.width - labelWidth; + if (Math.abs(resultNewWidth - chartWidth) > RESIZE_IGNORED_DIFF_PX) { + setChartWidth(resultNewWidth); + onResize(resultNewWidth); + } }, RESIZE_THROTTLE_TIME_MS), - [] + [chartWidth] ); + const showSwimlane = + props.swimlaneData && + props.swimlaneData.laneLabels && + props.swimlaneData.laneLabels.length > 0 && + props.swimlaneData.points.length > 0; + + const isPaginationVisible = + (showSwimlane || isLoading) && + swimlaneLimit !== undefined && + onPaginationChange && + props.swimlaneType === SWIMLANE_TYPE.VIEW_BY && + fromPage && + perPage; + return ( - - {(resizeRef) => ( -
{ - resizeRef(el); - }} - > -
- - - {(tooltipService) => ( - + + {(resizeRef) => ( + { + resizeRef(el); + }} + data-test-subj="mlSwimLaneContainer" + > + + + {showSwimlane && !isLoading && ( + + {(tooltipService) => ( + + )} + + )} + {isLoading && ( + + + + )} + {!isLoading && !showSwimlane && ( + {noDataWarning}} /> )} - - -
-
- )} - + + + {isPaginationVisible && ( + + + + )} + + )} + + ); }; diff --git a/x-pack/plugins/ml/public/application/explorer/swimlane_pagination.tsx b/x-pack/plugins/ml/public/application/explorer/swimlane_pagination.tsx new file mode 100644 index 00000000000000..0607f7fd35fad5 --- /dev/null +++ b/x-pack/plugins/ml/public/application/explorer/swimlane_pagination.tsx @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useCallback, useState } from 'react'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiPopover, + EuiContextMenuPanel, + EuiPagination, + EuiContextMenuItem, + EuiButtonEmpty, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; + +interface SwimLanePaginationProps { + fromPage: number; + perPage: number; + cardinality: number; + onPaginationChange: (arg: { perPage?: number; fromPage?: number }) => void; +} + +export const SwimLanePagination: FC = ({ + cardinality, + fromPage, + perPage, + onPaginationChange, +}) => { + const componentFromPage = fromPage - 1; + + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + + const onButtonClick = () => setIsPopoverOpen(() => !isPopoverOpen); + const closePopover = () => setIsPopoverOpen(false); + + const goToPage = useCallback((pageNumber: number) => { + onPaginationChange({ fromPage: pageNumber + 1 }); + }, []); + + const setPerPage = useCallback((perPageUpdate: number) => { + onPaginationChange({ perPage: perPageUpdate }); + }, []); + + const pageCount = Math.ceil(cardinality / perPage); + + const items = [5, 10, 20, 50, 100].map((v) => { + return ( + { + closePopover(); + setPerPage(v); + }} + > + + + ); + }); + + return ( + + + + + + } + isOpen={isPopoverOpen} + closePopover={closePopover} + panelPaddingSize="none" + > + + + + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx index 2e355c6073abd9..52b4408d1ac5bb 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/explorer.tsx @@ -22,7 +22,6 @@ import { ml } from '../../services/ml_api_service'; import { useExplorerData } from '../../explorer/actions'; import { explorerService } from '../../explorer/explorer_dashboard_service'; import { getDateFormatTz } from '../../explorer/explorer_utils'; -import { useSwimlaneLimit } from '../../explorer/select_limit'; import { useJobSelection } from '../../components/job_selector/use_job_selection'; import { useShowCharts } from '../../components/controls/checkbox_showcharts'; import { useTableInterval } from '../../components/controls/select_interval'; @@ -30,6 +29,7 @@ import { useTableSeverity } from '../../components/controls/select_severity'; import { useUrlState } from '../../util/url_state'; import { ANOMALY_DETECTION_BREADCRUMB, ML_BREADCRUMB } from '../breadcrumbs'; import { useTimefilter } from '../../contexts/kibana'; +import { isViewBySwimLaneData } from '../../explorer/swimlane_container'; const breadcrumbs = [ ML_BREADCRUMB, @@ -151,10 +151,6 @@ const ExplorerUrlStateManager: FC = ({ jobsWithTim const [showCharts] = useShowCharts(); const [tableInterval] = useTableInterval(); const [tableSeverity] = useTableSeverity(); - const [swimlaneLimit] = useSwimlaneLimit(); - useEffect(() => { - explorerService.setSwimlaneLimit(swimlaneLimit); - }, [swimlaneLimit]); const [selectedCells, setSelectedCells] = useSelectedCells(); useEffect(() => { @@ -170,14 +166,26 @@ const ExplorerUrlStateManager: FC = ({ jobsWithTim selectedCells, selectedJobs: explorerState.selectedJobs, swimlaneBucketInterval: explorerState.swimlaneBucketInterval, - swimlaneLimit: explorerState.swimlaneLimit, tableInterval: tableInterval.val, tableSeverity: tableSeverity.val, viewBySwimlaneFieldName: explorerState.viewBySwimlaneFieldName, + swimlaneContainerWidth: explorerState.swimlaneContainerWidth, + viewByPerPage: explorerState.viewByPerPage, + viewByFromPage: explorerState.viewByFromPage, }) || undefined; + useEffect(() => { - loadExplorerData(loadExplorerDataConfig); + if (explorerState && explorerState.swimlaneContainerWidth > 0) { + loadExplorerData({ + ...loadExplorerDataConfig, + swimlaneLimit: + explorerState?.viewBySwimlaneData && + isViewBySwimLaneData(explorerState?.viewBySwimlaneData) + ? explorerState?.viewBySwimlaneData.cardinality + : undefined, + }); + } }, [JSON.stringify(loadExplorerDataConfig)]); if (explorerState === undefined || refresh === undefined || showCharts === undefined) { diff --git a/x-pack/plugins/ml/public/application/routing/routes/timeseriesexplorer.test.tsx b/x-pack/plugins/ml/public/application/routing/routes/timeseriesexplorer.test.tsx index ac4882b0055ae0..11ec074bac1db4 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/timeseriesexplorer.test.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/timeseriesexplorer.test.tsx @@ -12,41 +12,47 @@ import { I18nProvider } from '@kbn/i18n/react'; import { TimeSeriesExplorerUrlStateManager } from './timeseriesexplorer'; -jest.mock('../../contexts/kibana/kibana_context', () => ({ - useMlKibana: () => { - return { - services: { - uiSettings: { get: jest.fn() }, - data: { - query: { - timefilter: { +jest.mock('../../contexts/kibana/kibana_context', () => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { of } = require('rxjs'); + return { + useMlKibana: () => { + return { + services: { + uiSettings: { get: jest.fn() }, + data: { + query: { timefilter: { - disableTimeRangeSelector: jest.fn(), - disableAutoRefreshSelector: jest.fn(), - enableTimeRangeSelector: jest.fn(), - enableAutoRefreshSelector: jest.fn(), - getRefreshInterval: jest.fn(), - setRefreshInterval: jest.fn(), - getTime: jest.fn(), - isAutoRefreshSelectorEnabled: jest.fn(), - isTimeRangeSelectorEnabled: jest.fn(), - getRefreshIntervalUpdate$: jest.fn(), - getTimeUpdate$: jest.fn(), - getEnabledUpdated$: jest.fn(), + timefilter: { + disableTimeRangeSelector: jest.fn(), + disableAutoRefreshSelector: jest.fn(), + enableTimeRangeSelector: jest.fn(), + enableAutoRefreshSelector: jest.fn(), + getRefreshInterval: jest.fn(), + setRefreshInterval: jest.fn(), + getTime: jest.fn(), + isAutoRefreshSelectorEnabled: jest.fn(), + isTimeRangeSelectorEnabled: jest.fn(), + getRefreshIntervalUpdate$: jest.fn(), + getTimeUpdate$: jest.fn(() => { + return of(); + }), + getEnabledUpdated$: jest.fn(), + }, + history: { get: jest.fn() }, }, - history: { get: jest.fn() }, }, }, - }, - notifications: { - toasts: { - addDanger: () => {}, + notifications: { + toasts: { + addDanger: () => {}, + }, }, }, - }, - }; - }, -})); + }; + }, + }; +}); jest.mock('../../util/dependency_cache', () => ({ getToastNotifications: () => ({ addSuccess: jest.fn(), addDanger: jest.fn() }), diff --git a/x-pack/plugins/ml/public/application/routing/use_refresh.ts b/x-pack/plugins/ml/public/application/routing/use_refresh.ts index f0b93c876526ba..c247fd9765e966 100644 --- a/x-pack/plugins/ml/public/application/routing/use_refresh.ts +++ b/x-pack/plugins/ml/public/application/routing/use_refresh.ts @@ -5,26 +5,40 @@ */ import { useObservable } from 'react-use'; -import { merge, Observable } from 'rxjs'; -import { map } from 'rxjs/operators'; +import { merge } from 'rxjs'; +import { map, skip } from 'rxjs/operators'; +import { useMemo } from 'react'; import { annotationsRefresh$ } from '../services/annotations_service'; -import { - mlTimefilterRefresh$, - mlTimefilterTimeChange$, -} from '../services/timefilter_refresh_service'; +import { mlTimefilterRefresh$ } from '../services/timefilter_refresh_service'; +import { useTimefilter } from '../contexts/kibana'; export interface Refresh { lastRefresh: number; timeRange?: { start: string; end: string }; } -const refresh$: Observable = merge( - mlTimefilterRefresh$, - mlTimefilterTimeChange$, - annotationsRefresh$.pipe(map((d) => ({ lastRefresh: d }))) -); - +/** + * Hook that provides the latest refresh timestamp + * and the most recent applied time range. + */ export const useRefresh = () => { + const timefilter = useTimefilter(); + + const refresh$ = useMemo(() => { + return merge( + mlTimefilterRefresh$, + timefilter.getTimeUpdate$().pipe( + // skip initially emitted value + skip(1), + map((_) => { + const { from, to } = timefilter.getTime(); + return { lastRefresh: Date.now(), timeRange: { start: from, end: to } }; + }) + ), + annotationsRefresh$.pipe(map((d) => ({ lastRefresh: d }))) + ); + }, []); + return useObservable(refresh$); }; diff --git a/x-pack/plugins/ml/public/application/services/explorer_service.ts b/x-pack/plugins/ml/public/application/services/anomaly_timeline_service.ts similarity index 82% rename from x-pack/plugins/ml/public/application/services/explorer_service.ts rename to x-pack/plugins/ml/public/application/services/anomaly_timeline_service.ts index 0944328db00523..f2e362f754f2b9 100644 --- a/x-pack/plugins/ml/public/application/services/explorer_service.ts +++ b/x-pack/plugins/ml/public/application/services/anomaly_timeline_service.ts @@ -12,14 +12,19 @@ import { UI_SETTINGS, } from '../../../../../../src/plugins/data/public'; import { getBoundsRoundedToInterval, TimeBuckets, TimeRangeBounds } from '../util/time_buckets'; -import { ExplorerJob, OverallSwimlaneData, SwimlaneData } from '../explorer/explorer_utils'; +import { + ExplorerJob, + OverallSwimlaneData, + SwimlaneData, + ViewBySwimLaneData, +} from '../explorer/explorer_utils'; import { VIEW_BY_JOB_LABEL } from '../explorer/explorer_constants'; import { MlResultsService } from './results_service'; /** - * Anomaly Explorer Service + * Service for retrieving anomaly swim lanes data. */ -export class ExplorerService { +export class AnomalyTimelineService { private timeBuckets: TimeBuckets; private _customTimeRange: TimeRange | undefined; @@ -130,12 +135,27 @@ export class ExplorerService { return overallSwimlaneData; } + /** + * Fetches view by swim lane data. + * + * @param fieldValues + * @param bounds + * @param selectedJobs + * @param viewBySwimlaneFieldName + * @param swimlaneLimit + * @param perPage + * @param fromPage + * @param swimlaneContainerWidth + * @param influencersFilterQuery + */ public async loadViewBySwimlane( fieldValues: string[], bounds: { earliest: number; latest: number }, selectedJobs: ExplorerJob[], viewBySwimlaneFieldName: string, swimlaneLimit: number, + perPage: number, + fromPage: number, swimlaneContainerWidth: number, influencersFilterQuery?: any ): Promise { @@ -172,7 +192,8 @@ export class ExplorerService { searchBounds.min.valueOf(), searchBounds.max.valueOf(), interval, - swimlaneLimit + perPage, + fromPage ); } else { response = await this.mlResultsService.getInfluencerValueMaxScoreByTime( @@ -183,6 +204,8 @@ export class ExplorerService { searchBounds.max.valueOf(), interval, swimlaneLimit, + perPage, + fromPage, influencersFilterQuery ); } @@ -193,6 +216,7 @@ export class ExplorerService { const viewBySwimlaneData = this.processViewByResults( response.results, + response.cardinality, fieldValues, bounds, viewBySwimlaneFieldName, @@ -204,6 +228,55 @@ export class ExplorerService { return viewBySwimlaneData; } + public async loadViewByTopFieldValuesForSelectedTime( + earliestMs: number, + latestMs: number, + selectedJobs: ExplorerJob[], + viewBySwimlaneFieldName: string, + swimlaneLimit: number, + perPage: number, + fromPage: number, + swimlaneContainerWidth: number + ) { + const selectedJobIds = selectedJobs.map((d) => d.id); + + // Find the top field values for the selected time, and then load the 'view by' + // swimlane over the full time range for those specific field values. + if (viewBySwimlaneFieldName !== VIEW_BY_JOB_LABEL) { + const resp = await this.mlResultsService.getTopInfluencers( + selectedJobIds, + earliestMs, + latestMs, + swimlaneLimit, + perPage, + fromPage + ); + if (resp.influencers[viewBySwimlaneFieldName] === undefined) { + return []; + } + + const topFieldValues: any[] = []; + const topInfluencers = resp.influencers[viewBySwimlaneFieldName]; + if (Array.isArray(topInfluencers)) { + topInfluencers.forEach((influencerData) => { + if (influencerData.maxAnomalyScore > 0) { + topFieldValues.push(influencerData.influencerFieldValue); + } + }); + } + return topFieldValues; + } else { + const resp = await this.mlResultsService.getScoresByBucket( + selectedJobIds, + earliestMs, + latestMs, + this.getSwimlaneBucketInterval(selectedJobs, swimlaneContainerWidth).asSeconds() + 's', + swimlaneLimit + ); + return Object.keys(resp.results); + } + } + private getTimeBounds(): TimeRangeBounds { return this._customTimeRange !== undefined ? this.timeFilter.calculateBounds(this._customTimeRange) @@ -245,6 +318,7 @@ export class ExplorerService { private processViewByResults( scoresByInfluencerAndTime: Record, + cardinality: number, sortedLaneValues: string[], bounds: any, viewBySwimlaneFieldName: string, @@ -254,7 +328,7 @@ export class ExplorerService { // Sorts the lanes according to the supplied array of lane // values in the order in which they should be displayed, // or pass an empty array to sort lanes according to max score over all time. - const dataset: OverallSwimlaneData = { + const dataset: ViewBySwimLaneData = { fieldName: viewBySwimlaneFieldName, points: [], laneLabels: [], @@ -262,6 +336,7 @@ export class ExplorerService { // Set the earliest and latest to be the same as the overall swim lane. earliest: bounds.earliest, latest: bounds.latest, + cardinality, }; const maxScoreByLaneLabel: Record = {}; diff --git a/x-pack/plugins/ml/public/application/services/dashboard_service.test.ts b/x-pack/plugins/ml/public/application/services/dashboard_service.test.ts index a618534d7ae005..00adb2d3258339 100644 --- a/x-pack/plugins/ml/public/application/services/dashboard_service.test.ts +++ b/x-pack/plugins/ml/public/application/services/dashboard_service.test.ts @@ -37,7 +37,7 @@ describe('DashboardService', () => { // assert expect(mockSavedObjectClient.find).toHaveBeenCalledWith({ type: 'dashboard', - perPage: 10, + perPage: 1000, search: `test*`, searchFields: ['title^3', 'description'], }); diff --git a/x-pack/plugins/ml/public/application/services/dashboard_service.ts b/x-pack/plugins/ml/public/application/services/dashboard_service.ts index 7f2bb71d18eb98..d6ccfc2f203e9b 100644 --- a/x-pack/plugins/ml/public/application/services/dashboard_service.ts +++ b/x-pack/plugins/ml/public/application/services/dashboard_service.ts @@ -34,7 +34,7 @@ export function dashboardServiceProvider( async fetchDashboards(query?: string) { return await savedObjectClient.find({ type: 'dashboard', - perPage: 10, + perPage: 1000, search: query ? `${query}*` : '', searchFields: ['title^3', 'description'], }); diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index af6944d7ae2d24..d1b6f95f32bed5 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -12,7 +12,7 @@ import { annotations } from './annotations'; import { dataFrameAnalytics } from './data_frame_analytics'; import { filters } from './filters'; import { resultsApiProvider } from './results'; -import { jobs } from './jobs'; +import { jobsApiProvider } from './jobs'; import { fileDatavisualizer } from './datavisualizer'; import { MlServerDefaults, MlServerLimits } from '../../../../common/types/ml_server_info'; @@ -726,7 +726,7 @@ export function mlApiServicesProvider(httpService: HttpService) { dataFrameAnalytics, filters, results: resultsApiProvider(httpService), - jobs, + jobs: jobsApiProvider(httpService), fileDatavisualizer, }; } diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts index 6aa62da3f0768c..d356fc0ef339b9 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { http } from '../http_service'; +import { HttpService } from '../http_service'; import { basePath } from './index'; import { Dictionary } from '../../../../common/types/common'; @@ -24,10 +24,10 @@ import { import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '../../../../common/constants/categorization_job'; import { Category } from '../../../../common/types/categories'; -export const jobs = { +export const jobsApiProvider = (httpService: HttpService) => ({ jobsSummary(jobIds: string[]) { const body = JSON.stringify({ jobIds }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/jobs_summary`, method: 'POST', body, @@ -36,7 +36,10 @@ export const jobs = { jobsWithTimerange(dateFormatTz: string) { const body = JSON.stringify({ dateFormatTz }); - return http<{ jobs: MlJobWithTimeRange[]; jobsMap: Dictionary }>({ + return httpService.http<{ + jobs: MlJobWithTimeRange[]; + jobsMap: Dictionary; + }>({ path: `${basePath()}/jobs/jobs_with_time_range`, method: 'POST', body, @@ -45,7 +48,7 @@ export const jobs = { jobs(jobIds: string[]) { const body = JSON.stringify({ jobIds }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/jobs`, method: 'POST', body, @@ -53,7 +56,7 @@ export const jobs = { }, groups() { - return http({ + return httpService.http({ path: `${basePath()}/jobs/groups`, method: 'GET', }); @@ -61,7 +64,7 @@ export const jobs = { updateGroups(updatedJobs: string[]) { const body = JSON.stringify({ jobs: updatedJobs }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/update_groups`, method: 'POST', body, @@ -75,7 +78,7 @@ export const jobs = { end, }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/force_start_datafeeds`, method: 'POST', body, @@ -84,7 +87,7 @@ export const jobs = { stopDatafeeds(datafeedIds: string[]) { const body = JSON.stringify({ datafeedIds }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/stop_datafeeds`, method: 'POST', body, @@ -93,7 +96,7 @@ export const jobs = { deleteJobs(jobIds: string[]) { const body = JSON.stringify({ jobIds }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/delete_jobs`, method: 'POST', body, @@ -102,7 +105,7 @@ export const jobs = { closeJobs(jobIds: string[]) { const body = JSON.stringify({ jobIds }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/close_jobs`, method: 'POST', body, @@ -111,7 +114,7 @@ export const jobs = { forceStopAndCloseJob(jobId: string) { const body = JSON.stringify({ jobId }); - return http<{ success: boolean }>({ + return httpService.http<{ success: boolean }>({ path: `${basePath()}/jobs/force_stop_and_close_job`, method: 'POST', body, @@ -121,7 +124,7 @@ export const jobs = { jobAuditMessages(jobId: string, from?: number) { const jobIdString = jobId !== undefined ? `/${jobId}` : ''; const query = from !== undefined ? { from } : {}; - return http({ + return httpService.http({ path: `${basePath()}/job_audit_messages/messages${jobIdString}`, method: 'GET', query, @@ -129,7 +132,7 @@ export const jobs = { }, deletingJobTasks() { - return http({ + return httpService.http({ path: `${basePath()}/jobs/deleting_jobs_tasks`, method: 'GET', }); @@ -137,7 +140,7 @@ export const jobs = { jobsExist(jobIds: string[]) { const body = JSON.stringify({ jobIds }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/jobs_exist`, method: 'POST', body, @@ -146,7 +149,7 @@ export const jobs = { newJobCaps(indexPatternTitle: string, isRollup: boolean = false) { const query = isRollup === true ? { rollup: true } : {}; - return http({ + return httpService.http({ path: `${basePath()}/jobs/new_job_caps/${indexPatternTitle}`, method: 'GET', query, @@ -175,7 +178,7 @@ export const jobs = { splitFieldName, splitFieldValue, }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/new_job_line_chart`, method: 'POST', body, @@ -202,7 +205,7 @@ export const jobs = { aggFieldNamePairs, splitFieldName, }); - return http({ + return httpService.http({ path: `${basePath()}/jobs/new_job_population_chart`, method: 'POST', body, @@ -210,7 +213,7 @@ export const jobs = { }, getAllJobAndGroupIds() { - return http({ + return httpService.http({ path: `${basePath()}/jobs/all_jobs_and_group_ids`, method: 'GET', }); @@ -222,7 +225,7 @@ export const jobs = { start, end, }); - return http<{ progress: number; isRunning: boolean; isJobClosed: boolean }>({ + return httpService.http<{ progress: number; isRunning: boolean; isJobClosed: boolean }>({ path: `${basePath()}/jobs/look_back_progress`, method: 'POST', body, @@ -249,7 +252,7 @@ export const jobs = { end, analyzer, }); - return http<{ + return httpService.http<{ examples: CategoryFieldExample[]; sampleSize: number; overallValidStatus: CATEGORY_EXAMPLES_VALIDATION_STATUS; @@ -263,7 +266,10 @@ export const jobs = { topCategories(jobId: string, count: number) { const body = JSON.stringify({ jobId, count }); - return http<{ total: number; categories: Array<{ count?: number; category: Category }> }>({ + return httpService.http<{ + total: number; + categories: Array<{ count?: number; category: Category }>; + }>({ path: `${basePath()}/jobs/top_categories`, method: 'POST', body, @@ -278,10 +284,13 @@ export const jobs = { calendarEvents?: Array<{ start: number; end: number; description: string }> ) { const body = JSON.stringify({ jobId, snapshotId, replay, end, calendarEvents }); - return http<{ total: number; categories: Array<{ count?: number; category: Category }> }>({ + return httpService.http<{ + total: number; + categories: Array<{ count?: number; category: Category }>; + }>({ path: `${basePath()}/jobs/revert_model_snapshot`, method: 'POST', body, }); }, -}; +}); diff --git a/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts b/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts index 1b2c01ab73fcef..b26528b76037b2 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts @@ -14,9 +14,19 @@ export function resultsServiceProvider( earliestMs: number, latestMs: number, interval: string | number, - maxResults: number + perPage?: number, + fromPage?: number + ): Promise; + getTopInfluencers( + selectedJobIds: string[], + earliestMs: number, + latestMs: number, + maxFieldValues: number, + perPage?: number, + fromPage?: number, + influencers?: any[], + influencersFilterQuery?: any ): Promise; - getTopInfluencers(): Promise; getTopInfluencerValues(): Promise; getOverallBucketScores( jobIds: any, @@ -33,6 +43,8 @@ export function resultsServiceProvider( latestMs: number, interval: string, maxResults: number, + perPage: number, + fromPage: number, influencersFilterQuery: any ): Promise; getRecordInfluencers(): Promise; diff --git a/x-pack/plugins/ml/public/application/services/results_service/results_service.js b/x-pack/plugins/ml/public/application/services/results_service/results_service.js index 9e3fed189b6f48..55ddb1de3529e7 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/results_service.js +++ b/x-pack/plugins/ml/public/application/services/results_service/results_service.js @@ -9,6 +9,10 @@ import _ from 'lodash'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; import { escapeForElasticsearchQuery } from '../../util/string_utils'; import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; +import { + ANOMALY_SWIM_LANE_HARD_LIMIT, + SWIM_LANE_DEFAULT_PAGE_SIZE, +} from '../../explorer/explorer_constants'; /** * Service for carrying out Elasticsearch queries to obtain data for the Ml Results dashboards. @@ -24,7 +28,7 @@ export function resultsServiceProvider(mlApiServices) { // Pass an empty array or ['*'] to search over all job IDs. // Returned response contains a results property, with a key for job // which has results for the specified time range. - getScoresByBucket(jobIds, earliestMs, latestMs, interval, maxResults) { + getScoresByBucket(jobIds, earliestMs, latestMs, interval, perPage = 10, fromPage = 1) { return new Promise((resolve, reject) => { const obj = { success: true, @@ -88,7 +92,7 @@ export function resultsServiceProvider(mlApiServices) { jobId: { terms: { field: 'job_id', - size: maxResults !== undefined ? maxResults : 5, + size: jobIds?.length ?? 1, order: { anomalyScore: 'desc', }, @@ -99,6 +103,12 @@ export function resultsServiceProvider(mlApiServices) { field: 'anomaly_score', }, }, + bucketTruncate: { + bucket_sort: { + from: (fromPage - 1) * perPage, + size: perPage === 0 ? 1 : perPage, + }, + }, byTime: { date_histogram: { field: 'timestamp', @@ -158,7 +168,9 @@ export function resultsServiceProvider(mlApiServices) { jobIds, earliestMs, latestMs, - maxFieldValues = 10, + maxFieldValues = ANOMALY_SWIM_LANE_HARD_LIMIT, + perPage = 10, + fromPage = 1, influencers = [], influencersFilterQuery ) { @@ -272,6 +284,12 @@ export function resultsServiceProvider(mlApiServices) { }, }, aggs: { + bucketTruncate: { + bucket_sort: { + from: (fromPage - 1) * perPage, + size: perPage, + }, + }, maxAnomalyScore: { max: { field: 'influencer_score', @@ -472,7 +490,9 @@ export function resultsServiceProvider(mlApiServices) { earliestMs, latestMs, interval, - maxResults, + maxResults = ANOMALY_SWIM_LANE_HARD_LIMIT, + perPage = SWIM_LANE_DEFAULT_PAGE_SIZE, + fromPage = 1, influencersFilterQuery ) { return new Promise((resolve, reject) => { @@ -565,10 +585,15 @@ export function resultsServiceProvider(mlApiServices) { }, }, aggs: { + influencerValuesCardinality: { + cardinality: { + field: 'influencer_field_value', + }, + }, influencerFieldValues: { terms: { field: 'influencer_field_value', - size: maxResults !== undefined ? maxResults : 10, + size: !!maxResults ? maxResults : ANOMALY_SWIM_LANE_HARD_LIMIT, order: { maxAnomalyScore: 'desc', }, @@ -579,6 +604,12 @@ export function resultsServiceProvider(mlApiServices) { field: 'influencer_score', }, }, + bucketTruncate: { + bucket_sort: { + from: (fromPage - 1) * perPage, + size: perPage, + }, + }, byTime: { date_histogram: { field: 'timestamp', @@ -618,6 +649,8 @@ export function resultsServiceProvider(mlApiServices) { obj.results[fieldValue] = fieldValues; }); + obj.cardinality = resp.aggregations?.influencerValuesCardinality?.value ?? 0; + resolve(obj); }) .catch((resp) => { diff --git a/x-pack/plugins/ml/public/application/services/timefilter_refresh_service.tsx b/x-pack/plugins/ml/public/application/services/timefilter_refresh_service.tsx index 86c07a3577f7b6..4f5d0723d65a4f 100644 --- a/x-pack/plugins/ml/public/application/services/timefilter_refresh_service.tsx +++ b/x-pack/plugins/ml/public/application/services/timefilter_refresh_service.tsx @@ -9,4 +9,3 @@ import { Subject } from 'rxjs'; import { Refresh } from '../routing/use_refresh'; export const mlTimefilterRefresh$ = new Subject>(); -export const mlTimefilterTimeChange$ = new Subject>(); diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable.tsx index 3b4562628051e0..83070a5d94ba09 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable.tsx @@ -16,10 +16,10 @@ import { IContainer, } from '../../../../../../src/plugins/embeddable/public'; import { MlStartDependencies } from '../../plugin'; -import { ExplorerSwimlaneContainer } from './explorer_swimlane_container'; +import { EmbeddableSwimLaneContainer } from './embeddable_swim_lane_container'; import { AnomalyDetectorService } from '../../application/services/anomaly_detector_service'; import { JobId } from '../../../common/types/anomaly_detection_jobs'; -import { ExplorerService } from '../../application/services/explorer_service'; +import { AnomalyTimelineService } from '../../application/services/anomaly_timeline_service'; import { Filter, Query, @@ -40,7 +40,7 @@ export interface AnomalySwimlaneEmbeddableCustomInput { jobIds: JobId[]; swimlaneType: SwimlaneType; viewBy?: string; - limit?: number; + perPage?: number; // Embeddable inputs which are not included in the default interface filters: Filter[]; @@ -58,12 +58,12 @@ export interface AnomalySwimlaneEmbeddableCustomOutput { jobIds: JobId[]; swimlaneType: SwimlaneType; viewBy?: string; - limit?: number; + perPage?: number; } export interface AnomalySwimlaneServices { anomalyDetectorService: AnomalyDetectorService; - explorerService: ExplorerService; + anomalyTimelineService: AnomalyTimelineService; } export type AnomalySwimlaneEmbeddableServices = [ @@ -101,14 +101,20 @@ export class AnomalySwimlaneEmbeddable extends Embeddable< super.render(node); this.node = node; + const I18nContext = this.services[0].i18n.Context; + ReactDOM.render( - this.updateOutput(output)} - />, + + { + this.updateInput(input); + }} + /> + , node ); } diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.test.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.test.tsx index 6b2ab89de8a5d0..243369982ac1f8 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.test.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.test.tsx @@ -46,6 +46,9 @@ describe('AnomalySwimlaneEmbeddableFactory', () => { }); expect(Object.keys(createServices[0])).toEqual(Object.keys(coreStart)); expect(createServices[1]).toMatchObject(pluginsStart); - expect(Object.keys(createServices[2])).toEqual(['anomalyDetectorService', 'explorerService']); + expect(Object.keys(createServices[2])).toEqual([ + 'anomalyDetectorService', + 'anomalyTimelineService', + ]); }); }); diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts index 37c2cfb3e029b8..0d587b428d89b6 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts @@ -22,7 +22,7 @@ import { import { MlStartDependencies } from '../../plugin'; import { HttpService } from '../../application/services/http_service'; import { AnomalyDetectorService } from '../../application/services/anomaly_detector_service'; -import { ExplorerService } from '../../application/services/explorer_service'; +import { AnomalyTimelineService } from '../../application/services/anomaly_timeline_service'; import { mlResultsServiceProvider } from '../../application/services/results_service'; import { resolveAnomalySwimlaneUserInput } from './anomaly_swimlane_setup_flyout'; import { mlApiServicesProvider } from '../../application/services/ml_api_service'; @@ -44,14 +44,10 @@ export class AnomalySwimlaneEmbeddableFactory } public async getExplicitInput(): Promise> { - const [{ overlays, uiSettings }, , { anomalyDetectorService }] = await this.getServices(); + const [coreStart] = await this.getServices(); try { - return await resolveAnomalySwimlaneUserInput({ - anomalyDetectorService, - overlays, - uiSettings, - }); + return await resolveAnomalySwimlaneUserInput(coreStart); } catch (e) { return Promise.reject(); } @@ -62,13 +58,13 @@ export class AnomalySwimlaneEmbeddableFactory const httpService = new HttpService(coreStart.http); const anomalyDetectorService = new AnomalyDetectorService(httpService); - const explorerService = new ExplorerService( + const anomalyTimelineService = new AnomalyTimelineService( pluginsStart.data.query.timefilter.timefilter, coreStart.uiSettings, mlResultsServiceProvider(mlApiServicesProvider(httpService)) ); - return [coreStart, pluginsStart, { anomalyDetectorService, explorerService }]; + return [coreStart, pluginsStart, { anomalyDetectorService, anomalyTimelineService }]; } public async create( diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_initializer.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_initializer.tsx index 4977ece54bb57f..be9a332e51dbcc 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_initializer.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_initializer.tsx @@ -27,7 +27,7 @@ export interface AnomalySwimlaneInitializerProps { defaultTitle: string; influencers: string[]; initialInput?: Partial< - Pick + Pick >; onCreate: (swimlaneProps: { panelTitle: string; @@ -38,11 +38,6 @@ export interface AnomalySwimlaneInitializerProps { onCancel: () => void; } -const limitOptions = [5, 10, 25, 50].map((limit) => ({ - value: limit, - text: `${limit}`, -})); - export const AnomalySwimlaneInitializer: FC = ({ defaultTitle, influencers, @@ -55,7 +50,6 @@ export const AnomalySwimlaneInitializer: FC = ( initialInput?.swimlaneType ?? SWIMLANE_TYPE.OVERALL ); const [viewBySwimlaneFieldName, setViewBySwimlaneFieldName] = useState(initialInput?.viewBy); - const [limit, setLimit] = useState(initialInput?.limit ?? 5); const swimlaneTypeOptions = [ { @@ -154,19 +148,6 @@ export const AnomalySwimlaneInitializer: FC = ( onChange={(e) => setViewBySwimlaneFieldName(e.target.value)} /> - - } - > - setLimit(Number(e.target.value))} - /> - )} @@ -186,7 +167,6 @@ export const AnomalySwimlaneInitializer: FC = ( panelTitle, swimlaneType, viewBy: viewBySwimlaneFieldName, - limit, })} fill > diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx index 54f50d2d3da326..1ffdadb60aaa33 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx @@ -5,10 +5,13 @@ */ import React from 'react'; -import { IUiSettingsClient, OverlayStart } from 'kibana/public'; +import { CoreStart } from 'kibana/public'; import moment from 'moment'; import { VIEW_BY_JOB_LABEL } from '../../application/explorer/explorer_constants'; -import { toMountPoint } from '../../../../../../src/plugins/kibana_react/public'; +import { + KibanaContextProvider, + toMountPoint, +} from '../../../../../../src/plugins/kibana_react/public'; import { AnomalySwimlaneInitializer } from './anomaly_swimlane_initializer'; import { JobSelectorFlyout } from '../../application/components/job_selector/job_selector_flyout'; import { AnomalyDetectorService } from '../../application/services/anomaly_detector_service'; @@ -17,19 +20,17 @@ import { AnomalySwimlaneEmbeddableInput, getDefaultPanelTitle, } from './anomaly_swimlane_embeddable'; +import { getMlGlobalServices } from '../../application/app'; +import { HttpService } from '../../application/services/http_service'; export async function resolveAnomalySwimlaneUserInput( - { - overlays, - anomalyDetectorService, - uiSettings, - }: { - anomalyDetectorService: AnomalyDetectorService; - overlays: OverlayStart; - uiSettings: IUiSettingsClient; - }, + coreStart: CoreStart, input?: AnomalySwimlaneEmbeddableInput ): Promise> { + const { http, uiSettings, overlays } = coreStart; + + const anomalyDetectorService = new AnomalyDetectorService(new HttpService(http)); + return new Promise(async (resolve, reject) => { const maps = { groupsMap: getInitialGroupsMap([]), @@ -41,48 +42,50 @@ export async function resolveAnomalySwimlaneUserInput( const selectedIds = input?.jobIds; - const flyoutSession = overlays.openFlyout( + const flyoutSession = coreStart.overlays.openFlyout( toMountPoint( - { - flyoutSession.close(); - reject(); - }} - onSelectionConfirmed={async ({ jobIds, groups }) => { - const title = input?.title ?? getDefaultPanelTitle(jobIds); + + { + flyoutSession.close(); + reject(); + }} + onSelectionConfirmed={async ({ jobIds, groups }) => { + const title = input?.title ?? getDefaultPanelTitle(jobIds); - const jobs = await anomalyDetectorService.getJobs$(jobIds).toPromise(); + const jobs = await anomalyDetectorService.getJobs$(jobIds).toPromise(); - const influencers = anomalyDetectorService.extractInfluencers(jobs); - influencers.push(VIEW_BY_JOB_LABEL); + const influencers = anomalyDetectorService.extractInfluencers(jobs); + influencers.push(VIEW_BY_JOB_LABEL); - await flyoutSession.close(); + await flyoutSession.close(); - const modalSession = overlays.openModal( - toMountPoint( - { - modalSession.close(); - resolve({ jobIds, title: panelTitle, swimlaneType, viewBy, limit }); - }} - onCancel={() => { - modalSession.close(); - reject(); - }} - /> - ) - ); - }} - maps={maps} - /> + const modalSession = overlays.openModal( + toMountPoint( + { + modalSession.close(); + resolve({ jobIds, title: panelTitle, swimlaneType, viewBy }); + }} + onCancel={() => { + modalSession.close(); + reject(); + }} + /> + ) + ); + }} + maps={maps} + /> + ), { 'data-test-subj': 'mlAnomalySwimlaneEmbeddable', diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.test.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.test.tsx similarity index 73% rename from x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.test.tsx rename to x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.test.tsx index 63ae89b5acdd1e..846a3f543c2d4d 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.test.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.test.tsx @@ -6,7 +6,7 @@ import React from 'react'; import { render } from '@testing-library/react'; -import { ExplorerSwimlaneContainer } from './explorer_swimlane_container'; +import { EmbeddableSwimLaneContainer } from './embeddable_swim_lane_container'; import { BehaviorSubject, Observable } from 'rxjs'; import { I18nProvider } from '@kbn/i18n/react'; import { @@ -17,6 +17,7 @@ import { CoreStart } from 'kibana/public'; import { MlStartDependencies } from '../../plugin'; import { useSwimlaneInputResolver } from './swimlane_input_resolver'; import { SWIMLANE_TYPE } from '../../application/explorer/explorer_constants'; +import { SwimlaneContainer } from '../../application/explorer/swimlane_container'; jest.mock('./swimlane_input_resolver', () => ({ useSwimlaneInputResolver: jest.fn(() => { @@ -24,12 +25,11 @@ jest.mock('./swimlane_input_resolver', () => ({ }), })); -jest.mock('../../application/explorer/explorer_swimlane', () => ({ - ExplorerSwimlane: jest.fn(), -})); - -jest.mock('../../application/components/chart_tooltip', () => ({ - MlTooltipComponent: jest.fn(), +jest.mock('../../application/explorer/swimlane_container', () => ({ + SwimlaneContainer: jest.fn(() => { + return null; + }), + isViewBySwimLaneData: jest.fn(), })); const defaultOptions = { wrapper: I18nProvider }; @@ -38,6 +38,7 @@ describe('ExplorerSwimlaneContainer', () => { let embeddableInput: BehaviorSubject>; let refresh: BehaviorSubject; let services: [CoreStart, MlStartDependencies, AnomalySwimlaneServices]; + const onInputChange = jest.fn(); beforeEach(() => { embeddableInput = new BehaviorSubject({ @@ -61,25 +62,39 @@ describe('ExplorerSwimlaneContainer', () => { }; (useSwimlaneInputResolver as jest.Mock).mockReturnValueOnce([ - mockOverallData, SWIMLANE_TYPE.OVERALL, - undefined, + mockOverallData, + 10, + jest.fn(), + {}, + false, + null, ]); - const { findByTestId } = render( - } services={services} refresh={refresh} + onInputChange={onInputChange} />, defaultOptions ); - expect( - await findByTestId('mlMaxAnomalyScoreEmbeddable_test-swimlane-embeddable') - ).toBeDefined(); + + const calledWith = ((SwimlaneContainer as unknown) as jest.Mock).mock + .calls[0][0]; + + expect(calledWith).toMatchObject({ + perPage: 10, + swimlaneType: SWIMLANE_TYPE.OVERALL, + swimlaneData: mockOverallData, + isLoading: false, + swimlaneLimit: undefined, + fromPage: 1, + }); }); test('should render an error in case it could not fetch the ML swimlane data', async () => { @@ -87,38 +102,25 @@ describe('ExplorerSwimlaneContainer', () => { undefined, undefined, undefined, + undefined, + undefined, + false, { message: 'Something went wrong' }, ]); const { findByText } = render( - } services={services} refresh={refresh} + onInputChange={onInputChange} />, defaultOptions ); const errorMessage = await findByText('Something went wrong'); expect(errorMessage).toBeDefined(); }); - - test('should render a loading indicator during the data fetching', async () => { - const { findByTestId } = render( - - } - services={services} - refresh={refresh} - />, - defaultOptions - ); - expect( - await findByTestId('loading_mlMaxAnomalyScoreEmbeddable_test-swimlane-embeddable') - ).toBeDefined(); - }); }); diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.tsx new file mode 100644 index 00000000000000..5d91bdb41df6af --- /dev/null +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/embeddable_swim_lane_container.tsx @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useState } from 'react'; +import { EuiCallOut } from '@elastic/eui'; +import { Observable } from 'rxjs'; + +import { CoreStart } from 'kibana/public'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { MlStartDependencies } from '../../plugin'; +import { + AnomalySwimlaneEmbeddableInput, + AnomalySwimlaneEmbeddableOutput, + AnomalySwimlaneServices, +} from './anomaly_swimlane_embeddable'; +import { useSwimlaneInputResolver } from './swimlane_input_resolver'; +import { SwimlaneType } from '../../application/explorer/explorer_constants'; +import { + isViewBySwimLaneData, + SwimlaneContainer, +} from '../../application/explorer/swimlane_container'; + +export interface ExplorerSwimlaneContainerProps { + id: string; + embeddableInput: Observable; + services: [CoreStart, MlStartDependencies, AnomalySwimlaneServices]; + refresh: Observable; + onInputChange: (output: Partial) => void; +} + +export const EmbeddableSwimLaneContainer: FC = ({ + id, + embeddableInput, + services, + refresh, + onInputChange, +}) => { + const [chartWidth, setChartWidth] = useState(0); + const [fromPage, setFromPage] = useState(1); + + const [ + swimlaneType, + swimlaneData, + perPage, + setPerPage, + timeBuckets, + isLoading, + error, + ] = useSwimlaneInputResolver( + embeddableInput, + onInputChange, + refresh, + services, + chartWidth, + fromPage + ); + + if (error) { + return ( + + } + color="danger" + iconType="alert" + style={{ width: '100%' }} + > +

{error.message}

+ + ); + } + + return ( +
+ { + setChartWidth(width); + }} + onPaginationChange={(update) => { + if (update.fromPage) { + setFromPage(update.fromPage); + } + if (update.perPage) { + setFromPage(1); + setPerPage(update.perPage); + } + }} + isLoading={isLoading} + noDataWarning={ + + } + /> +
+ ); +}; diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.tsx deleted file mode 100644 index db2b9d55cfabbf..00000000000000 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.tsx +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React, { FC, useCallback, useState } from 'react'; -import { - EuiCallOut, - EuiFlexGroup, - EuiFlexItem, - EuiLoadingChart, - EuiResizeObserver, - EuiSpacer, - EuiText, -} from '@elastic/eui'; -import { Observable } from 'rxjs'; - -import { throttle } from 'lodash'; -import { CoreStart } from 'kibana/public'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { ExplorerSwimlane } from '../../application/explorer/explorer_swimlane'; -import { MlStartDependencies } from '../../plugin'; -import { - AnomalySwimlaneEmbeddableInput, - AnomalySwimlaneEmbeddableOutput, - AnomalySwimlaneServices, -} from './anomaly_swimlane_embeddable'; -import { MlTooltipComponent } from '../../application/components/chart_tooltip'; -import { useSwimlaneInputResolver } from './swimlane_input_resolver'; -import { SwimlaneType } from '../../application/explorer/explorer_constants'; - -const RESIZE_THROTTLE_TIME_MS = 500; - -export interface ExplorerSwimlaneContainerProps { - id: string; - embeddableInput: Observable; - services: [CoreStart, MlStartDependencies, AnomalySwimlaneServices]; - refresh: Observable; - onOutputChange?: (output: Partial) => void; -} - -export const ExplorerSwimlaneContainer: FC = ({ - id, - embeddableInput, - services, - refresh, -}) => { - const [chartWidth, setChartWidth] = useState(0); - - const [swimlaneType, swimlaneData, timeBuckets, error] = useSwimlaneInputResolver( - embeddableInput, - refresh, - services, - chartWidth - ); - - const onResize = useCallback( - throttle((e: { width: number; height: number }) => { - const labelWidth = 200; - setChartWidth(e.width - labelWidth); - }, RESIZE_THROTTLE_TIME_MS), - [] - ); - - if (error) { - return ( - - } - color="danger" - iconType="alert" - style={{ width: '100%' }} - > -

{error.message}

- - ); - } - - return ( - - {(resizeRef) => ( -
{ - resizeRef(el); - }} - > -
- - - {chartWidth > 0 && swimlaneData && swimlaneType ? ( - - - {(tooltipService) => ( - - )} - - - ) : ( - - - - - - )} -
-
- )} - - ); -}; diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.test.ts b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.test.ts index 890c2bde6305db..a34955adebf62c 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.test.ts +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.test.ts @@ -19,6 +19,7 @@ describe('useSwimlaneInputResolver', () => { let embeddableInput: BehaviorSubject>; let refresh: Subject; let services: [CoreStart, MlStartDependencies, AnomalySwimlaneServices]; + let onInputChange: jest.Mock; beforeEach(() => { jest.useFakeTimers(); @@ -41,7 +42,7 @@ describe('useSwimlaneInputResolver', () => { } as CoreStart, (null as unknown) as MlStartDependencies, ({ - explorerService: { + anomalyTimelineService: { setTimeRange: jest.fn(), loadOverallData: jest.fn(() => Promise.resolve({ @@ -69,6 +70,7 @@ describe('useSwimlaneInputResolver', () => { }, } as unknown) as AnomalySwimlaneServices, ]; + onInputChange = jest.fn(); }); afterEach(() => { jest.useRealTimers(); @@ -79,9 +81,11 @@ describe('useSwimlaneInputResolver', () => { const { result, waitForNextUpdate } = renderHook(() => useSwimlaneInputResolver( embeddableInput as Observable, + onInputChange, refresh, services, - 1000 + 1000, + 1 ) ); @@ -94,7 +98,7 @@ describe('useSwimlaneInputResolver', () => { }); expect(services[2].anomalyDetectorService.getJobs$).toHaveBeenCalledTimes(1); - expect(services[2].explorerService.loadOverallData).toHaveBeenCalledTimes(1); + expect(services[2].anomalyTimelineService.loadOverallData).toHaveBeenCalledTimes(1); await act(async () => { embeddableInput.next({ @@ -109,7 +113,7 @@ describe('useSwimlaneInputResolver', () => { }); expect(services[2].anomalyDetectorService.getJobs$).toHaveBeenCalledTimes(2); - expect(services[2].explorerService.loadOverallData).toHaveBeenCalledTimes(2); + expect(services[2].anomalyTimelineService.loadOverallData).toHaveBeenCalledTimes(2); await act(async () => { embeddableInput.next({ @@ -124,7 +128,7 @@ describe('useSwimlaneInputResolver', () => { }); expect(services[2].anomalyDetectorService.getJobs$).toHaveBeenCalledTimes(2); - expect(services[2].explorerService.loadOverallData).toHaveBeenCalledTimes(3); + expect(services[2].anomalyTimelineService.loadOverallData).toHaveBeenCalledTimes(3); }); }); diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.ts b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.ts index 3829bbce5e5c96..9ed6f88150f68d 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.ts +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.ts @@ -16,23 +16,31 @@ import { skipWhile, startWith, switchMap, + tap, } from 'rxjs/operators'; import { CoreStart } from 'kibana/public'; import { TimeBuckets } from '../../application/util/time_buckets'; import { AnomalySwimlaneEmbeddableInput, + AnomalySwimlaneEmbeddableOutput, AnomalySwimlaneServices, } from './anomaly_swimlane_embeddable'; import { MlStartDependencies } from '../../plugin'; -import { SWIMLANE_TYPE, SwimlaneType } from '../../application/explorer/explorer_constants'; +import { + ANOMALY_SWIM_LANE_HARD_LIMIT, + SWIM_LANE_DEFAULT_PAGE_SIZE, + SWIMLANE_TYPE, + SwimlaneType, +} from '../../application/explorer/explorer_constants'; import { Filter } from '../../../../../../src/plugins/data/common/es_query/filters'; import { Query } from '../../../../../../src/plugins/data/common/query'; import { esKuery, UI_SETTINGS } from '../../../../../../src/plugins/data/public'; import { ExplorerJob, OverallSwimlaneData } from '../../application/explorer/explorer_utils'; import { parseInterval } from '../../../common/util/parse_interval'; import { AnomalyDetectorService } from '../../application/services/anomaly_detector_service'; +import { isViewBySwimLaneData } from '../../application/explorer/swimlane_container'; +import { ViewMode } from '../../../../../../src/plugins/embeddable/public'; -const RESIZE_IGNORED_DIFF_PX = 20; const FETCH_RESULTS_DEBOUNCE_MS = 500; function getJobsObservable( @@ -48,17 +56,31 @@ function getJobsObservable( export function useSwimlaneInputResolver( embeddableInput: Observable, + onInputChange: (output: Partial) => void, refresh: Observable, services: [CoreStart, MlStartDependencies, AnomalySwimlaneServices], - chartWidth: number -): [string | undefined, OverallSwimlaneData | undefined, TimeBuckets, Error | null | undefined] { - const [{ uiSettings }, , { explorerService, anomalyDetectorService }] = services; + chartWidth: number, + fromPage: number +): [ + string | undefined, + OverallSwimlaneData | undefined, + number, + (perPage: number) => void, + TimeBuckets, + boolean, + Error | null | undefined +] { + const [{ uiSettings }, , { anomalyTimelineService, anomalyDetectorService }] = services; const [swimlaneData, setSwimlaneData] = useState(); const [swimlaneType, setSwimlaneType] = useState(); const [error, setError] = useState(); + const [perPage, setPerPage] = useState(); + const [isLoading, setIsLoading] = useState(false); const chartWidth$ = useMemo(() => new Subject(), []); + const fromPage$ = useMemo(() => new Subject(), []); + const perPage$ = useMemo(() => new Subject(), []); const timeBuckets = useMemo(() => { return new TimeBuckets({ @@ -73,28 +95,32 @@ export function useSwimlaneInputResolver( const subscription = combineLatest([ getJobsObservable(embeddableInput, anomalyDetectorService), embeddableInput, - chartWidth$.pipe( - skipWhile((v) => !v), - distinctUntilChanged((prev, curr) => { - // emit only if the width has been changed significantly - return Math.abs(curr - prev) < RESIZE_IGNORED_DIFF_PX; - }) + chartWidth$.pipe(skipWhile((v) => !v)), + fromPage$, + perPage$.pipe( + startWith(undefined), + // no need to emit when the initial value has been set + distinctUntilChanged( + (prev, curr) => prev === undefined && curr === SWIM_LANE_DEFAULT_PAGE_SIZE + ) ), refresh.pipe(startWith(null)), ]) .pipe( + tap(setIsLoading.bind(null, true)), debounceTime(FETCH_RESULTS_DEBOUNCE_MS), - switchMap(([jobs, input, swimlaneContainerWidth]) => { + switchMap(([jobs, input, swimlaneContainerWidth, fromPageInput, perPageFromState]) => { const { viewBy, swimlaneType: swimlaneTypeInput, - limit, + perPage: perPageInput, timeRange, filters, query, + viewMode, } = input; - explorerService.setTimeRange(timeRange); + anomalyTimelineService.setTimeRange(timeRange); if (!swimlaneType) { setSwimlaneType(swimlaneTypeInput); @@ -118,18 +144,34 @@ export function useSwimlaneInputResolver( return of(undefined); } - return from(explorerService.loadOverallData(explorerJobs, swimlaneContainerWidth)).pipe( + return from( + anomalyTimelineService.loadOverallData(explorerJobs, swimlaneContainerWidth) + ).pipe( switchMap((overallSwimlaneData) => { const { earliest, latest } = overallSwimlaneData; if (overallSwimlaneData && swimlaneTypeInput === SWIMLANE_TYPE.VIEW_BY) { + if (perPageFromState === undefined) { + // set initial pagination from the input or default one + setPerPage(perPageInput ?? SWIM_LANE_DEFAULT_PAGE_SIZE); + } + + if (viewMode === ViewMode.EDIT && perPageFromState !== perPageInput) { + // store per page value when the dashboard is in the edit mode + onInputChange({ perPage: perPageFromState }); + } + return from( - explorerService.loadViewBySwimlane( + anomalyTimelineService.loadViewBySwimlane( [], { earliest, latest }, explorerJobs, viewBy!, - limit!, + isViewBySwimLaneData(swimlaneData) + ? swimlaneData.cardinality + : ANOMALY_SWIM_LANE_HARD_LIMIT, + perPageFromState ?? perPageInput ?? SWIM_LANE_DEFAULT_PAGE_SIZE, + fromPageInput, swimlaneContainerWidth, appliedFilters ) @@ -156,6 +198,7 @@ export function useSwimlaneInputResolver( if (data !== undefined) { setError(null); setSwimlaneData(data); + setIsLoading(false); } }); @@ -164,11 +207,28 @@ export function useSwimlaneInputResolver( }; }, []); + useEffect(() => { + fromPage$.next(fromPage); + }, [fromPage]); + + useEffect(() => { + if (perPage === undefined) return; + perPage$.next(perPage); + }, [perPage]); + useEffect(() => { chartWidth$.next(chartWidth); }, [chartWidth]); - return [swimlaneType, swimlaneData, timeBuckets, error]; + return [ + swimlaneType, + swimlaneData, + perPage ?? SWIM_LANE_DEFAULT_PAGE_SIZE, + setPerPage, + timeBuckets, + isLoading, + error, + ]; } export function processFilters(filters: Filter[], query: Query) { diff --git a/x-pack/plugins/ml/public/ui_actions/edit_swimlane_panel_action.tsx b/x-pack/plugins/ml/public/ui_actions/edit_swimlane_panel_action.tsx index 312b9f31124b13..0db41c1ed104e0 100644 --- a/x-pack/plugins/ml/public/ui_actions/edit_swimlane_panel_action.tsx +++ b/x-pack/plugins/ml/public/ui_actions/edit_swimlane_panel_action.tsx @@ -14,8 +14,6 @@ import { AnomalySwimlaneEmbeddableOutput, } from '../embeddables/anomaly_swimlane/anomaly_swimlane_embeddable'; import { resolveAnomalySwimlaneUserInput } from '../embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout'; -import { HttpService } from '../application/services/http_service'; -import { AnomalyDetectorService } from '../application/services/anomaly_detector_service'; export const EDIT_SWIMLANE_PANEL_ACTION = 'editSwimlanePanelAction'; @@ -39,18 +37,10 @@ export function createEditSwimlanePanelAction(getStartServices: CoreSetup['getSt throw new Error('Not possible to execute an action without the embeddable context'); } - const [{ overlays, uiSettings, http }] = await getStartServices(); - const anomalyDetectorService = new AnomalyDetectorService(new HttpService(http)); + const [coreStart] = await getStartServices(); try { - const result = await resolveAnomalySwimlaneUserInput( - { - anomalyDetectorService, - overlays, - uiSettings, - }, - embeddable.getInput() - ); + const result = await resolveAnomalySwimlaneUserInput(coreStart, embeddable.getInput()); embeddable.updateInput(result); } catch (e) { return Promise.reject(); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 4c1572ddfcad1d..5c5d270d324ffc 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9790,8 +9790,6 @@ "xpack.ml.explorer.jobIdLabel": "ジョブ ID", "xpack.ml.explorer.jobScoreAcrossAllInfluencersLabel": "(すべての影響因子のジョブスコア)", "xpack.ml.explorer.kueryBar.filterPlaceholder": "影響因子フィールドでフィルタリング… ({queryExample})", - "xpack.ml.explorer.limitLabel": "制限", - "xpack.ml.explorer.loadingLabel": "読み込み中", "xpack.ml.explorer.noConfiguredInfluencersTooltip": "選択されたジョブに影響因子が構成されていないため、トップインフルエンスリストは非表示になっています。", "xpack.ml.explorer.noInfluencersFoundTitle": "{viewBySwimlaneFieldName}影響因子が見つかりません", "xpack.ml.explorer.noInfluencersFoundTitleFilterMessage": "指定されたフィルターの{viewBySwimlaneFieldName} 影響因子が見つかりません", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 97f10e77dc7177..c71215d2bfb740 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9794,8 +9794,6 @@ "xpack.ml.explorer.jobIdLabel": "作业 ID", "xpack.ml.explorer.jobScoreAcrossAllInfluencersLabel": "（所有影响因素的作业分数）", "xpack.ml.explorer.kueryBar.filterPlaceholder": "按影响因素字段筛选……（{queryExample}）", - "xpack.ml.explorer.limitLabel": "限制", - "xpack.ml.explorer.loadingLabel": "正在加载", "xpack.ml.explorer.noConfiguredInfluencersTooltip": "“顶级影响因素”列表被隐藏，因为没有为所选作业配置影响因素。", "xpack.ml.explorer.noInfluencersFoundTitle": "未找到任何 {viewBySwimlaneFieldName} 影响因素", "xpack.ml.explorer.noInfluencersFoundTitleFilterMessage": "对于指定筛选找不到任何 {viewBySwimlaneFieldName} 影响因素", diff --git a/x-pack/test/functional/services/ml/anomaly_explorer.ts b/x-pack/test/functional/services/ml/anomaly_explorer.ts index 7c479a4234673e..80df235bf6ff8d 100644 --- a/x-pack/test/functional/services/ml/anomaly_explorer.ts +++ b/x-pack/test/functional/services/ml/anomaly_explorer.ts @@ -76,7 +76,7 @@ export function MachineLearningAnomalyExplorerProvider({ getService }: FtrProvid async addAndEditSwimlaneInDashboard(dashboardTitle: string) { await this.filterWithSearchString(dashboardTitle); await testSubjects.isDisplayed('mlDashboardSelectionTable > checkboxSelectAll'); - await testSubjects.click('mlDashboardSelectionTable > checkboxSelectAll'); + await testSubjects.clickWhenNotDisabled('mlDashboardSelectionTable > checkboxSelectAll'); expect(await testSubjects.isChecked('mlDashboardSelectionTable > checkboxSelectAll')).to.be( true ); From 3c56371153cad2c99d94511299b2aba7dd4225ea Mon Sep 17 00:00:00 2001 From: Andrea Villaverde Date: Thu, 2 Jul 2020 17:01:31 +0200 Subject: [PATCH 27/31] Update known-plugins.asciidoc (#69370) Co-authored-by: Elastic Machine --- docs/plugins/known-plugins.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/plugins/known-plugins.asciidoc b/docs/plugins/known-plugins.asciidoc index cd07596ad37ef5..8fc2b7381de835 100644 --- a/docs/plugins/known-plugins.asciidoc +++ b/docs/plugins/known-plugins.asciidoc @@ -59,6 +59,7 @@ This list of plugins is not guaranteed to work on your version of Kibana. Instea * https://github.com/sbeyn/kibana-plugin-traffic-sg[Traffic] (sbeyn) * https://github.com/PhaedrusTheGreek/transform_vis[Transform Visualization] (PhaedrusTheGreek) * https://github.com/nyurik/kibana-vega-vis[Vega-based visualizations] (nyurik) - Support for user-defined graphs, external data sources, maps, images, and user-defined interactivity. +* https://github.com/Camichan/kbn_aframe[VR Graph Visualizations] (Camichan) [float] === Other From 59ece7992b29597b83f2acc78268c51fc1675de5 Mon Sep 17 00:00:00 2001 From: CJ Cenizal Date: Thu, 2 Jul 2020 08:24:42 -0700 Subject: [PATCH 28/31] Make Index Management functional and API integration tests robust against side effects introduced by Ingest Manager. (#70533) --- .../apis/management/index_management/templates.js | 8 ++++---- .../test/functional/apps/index_management/home_page.ts | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/test/api_integration/apis/management/index_management/templates.js b/x-pack/test/api_integration/apis/management/index_management/templates.js index 003fb21b09ccc5..fcee8ed6a183fc 100644 --- a/x-pack/test/api_integration/apis/management/index_management/templates.js +++ b/x-pack/test/api_integration/apis/management/index_management/templates.js @@ -24,8 +24,7 @@ export default function ({ getService }) { updateTemplate, } = registerHelpers({ supertest }); - // blocking es snapshot promotion: https://github.com/elastic/kibana/issues/70532 - describe.skip('index templates', () => { + describe('index templates', () => { after(() => Promise.all([cleanUpEsResources()])); describe('get all', () => { @@ -41,8 +40,9 @@ export default function ({ getService }) { it('should list all the index templates with the expected parameters', async () => { const { body: allTemplates } = await getAllTemplates().expect(200); - // Composable templates - expect(allTemplates.templates).to.eql([]); + // Composable index templates may have been created by other apps, e.g. Ingest Manager, + // so we don't make any assertion about these contents. + expect(allTemplates.templates).to.be.an('array'); // Legacy templates const legacyTemplate = allTemplates.legacyTemplates.find( diff --git a/x-pack/test/functional/apps/index_management/home_page.ts b/x-pack/test/functional/apps/index_management/home_page.ts index b5b0197aad4b32..eab90e1fc19cff 100644 --- a/x-pack/test/functional/apps/index_management/home_page.ts +++ b/x-pack/test/functional/apps/index_management/home_page.ts @@ -13,8 +13,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const log = getService('log'); const browser = getService('browser'); - // blocking es snapshot promotion: https://github.com/elastic/kibana/issues/70532 - describe.skip('Home page', function () { + describe('Home page', function () { before(async () => { await pageObjects.common.navigateToApp('indexManagement'); }); @@ -82,9 +81,10 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const url = await browser.getCurrentUrl(); expect(url).to.contain(`/component_templates`); - // There should be no component templates by default, so we verify the empty prompt displays - const componentTemplateEmptyPrompt = await testSubjects.exists('emptyList'); - expect(componentTemplateEmptyPrompt).to.be(true); + // Verify content. Component templates may have been created by other apps, e.g. Ingest Manager, + // so we don't make any assertion about the presence or absence of component templates. + const componentTemplateList = await testSubjects.exists('componentTemplateList'); + expect(componentTemplateList).to.be(true); }); }); }); From 8a09f247e30987eff89824fc007953b815670fa3 Mon Sep 17 00:00:00 2001 From: Michael Hirsch Date: Thu, 2 Jul 2020 11:35:40 -0400 Subject: [PATCH 29/31] [ML] Updates APM Module to Work with Service Maps (#70361) * updates apm integration job to work with service maps * rename apm job in setup_module test * modifies detector description Co-authored-by: Elastic Machine --- .../modules/apm_transaction/manifest.json | 16 ++++----- ...afeed_high_mean_transaction_duration.json} | 2 +- .../ml/high_mean_response_time.json | 30 ---------------- .../ml/high_mean_transaction_duration.json | 35 +++++++++++++++++++ .../apis/ml/modules/setup_module.ts | 2 +- 5 files changed, 45 insertions(+), 40 deletions(-) rename x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/{datafeed_high_mean_response_time.json => datafeed_high_mean_transaction_duration.json} (75%) delete mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_response_time.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json index 5e185e80a60386..f8feaef3be5f85 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/manifest.json @@ -1,29 +1,29 @@ { "id": "apm_transaction", "title": "APM", - "description": "Detect anomalies in high mean of transaction duration (ECS).", + "description": "Detect anomalies in transactions from your APM services.", "type": "Transaction data", "logoFile": "logo.json", - "defaultIndexPattern": "apm-*", + "defaultIndexPattern": "apm-*-transaction", "query": { "bool": { "filter": [ { "term": { "processor.event": "transaction" } }, - { "term": { "transaction.type": "request" } } + { "exists": { "field": "transaction.duration" } } ] } }, "jobs": [ { - "id": "high_mean_response_time", - "file": "high_mean_response_time.json" + "id": "high_mean_transaction_duration", + "file": "high_mean_transaction_duration.json" } ], "datafeeds": [ { - "id": "datafeed-high_mean_response_time", - "file": "datafeed_high_mean_response_time.json", - "job_id": "high_mean_response_time" + "id": "datafeed-high_mean_transaction_duration", + "file": "datafeed_high_mean_transaction_duration.json", + "job_id": "high_mean_transaction_duration" } ] } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_response_time.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json similarity index 75% rename from x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_response_time.json rename to x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json index dc37d05d181110..d312577902f517 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_response_time.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/datafeed_high_mean_transaction_duration.json @@ -7,7 +7,7 @@ "bool": { "filter": [ { "term": { "processor.event": "transaction" } }, - { "term": { "transaction.type": "request" } } + { "exists": { "field": "transaction.duration.us" } } ] } } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_response_time.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_response_time.json deleted file mode 100644 index f6c230a6792fb3..00000000000000 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_response_time.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "job_type": "anomaly_detector", - "groups": [ - "apm" - ], - "description": "Detect anomalies in high mean of transaction duration", - "analysis_config": { - "bucket_span": "15m", - "detectors": [ - { - "detector_description": "high_mean(\"transaction.duration.us\")", - "function": "high_mean", - "field_name": "transaction.duration.us" - } - ], - "influencers": [] - }, - "analysis_limits": { - "model_memory_limit": "10mb" - }, - "data_description": { - "time_field": "@timestamp" - }, - "model_plot_config": { - "enabled": true - }, - "custom_settings": { - "created_by": "ml-module-apm-transaction" - } -} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json new file mode 100644 index 00000000000000..77284cb275cd8d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apm_transaction/ml/high_mean_transaction_duration.json @@ -0,0 +1,35 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "Detect transaction duration anomalies across transaction types for your APM services.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "high duration by transaction type for an APM service", + "function": "high_mean", + "field_name": "transaction.duration.us", + "by_field_name": "transaction.type", + "partition_field_name": "service.name" + } + ], + "influencers": [ + "transaction.type", + "service.name" + ] + }, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "model_plot_config": { + "enabled": true + }, + "custom_settings": { + "created_by": "ml-module-apm-transaction" + } +} diff --git a/x-pack/test/api_integration/apis/ml/modules/setup_module.ts b/x-pack/test/api_integration/apis/ml/modules/setup_module.ts index 1c98cd3a4e379b..10c0f00234abc5 100644 --- a/x-pack/test/api_integration/apis/ml/modules/setup_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/setup_module.ts @@ -218,7 +218,7 @@ export default ({ getService }: FtrProviderContext) => { responseCode: 200, jobs: [ { - jobId: 'pf5_high_mean_response_time', + jobId: 'pf5_high_mean_transaction_duration', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, modelMemoryLimit: '11mb', From 55922cb9a083007f7f14775d18873478ac09d1ae Mon Sep 17 00:00:00 2001 From: patrykkopycinski Date: Thu, 2 Jul 2020 17:37:29 +0200 Subject: [PATCH 30/31] [Security Solution] Reposition EuiPopovers on scroll (#69433) * [Security Solution] Reposition EuiPopovers on scroll * update snapshots Co-authored-by: Elastic Machine --- src/plugins/data/public/ui/filter_bar/filter_bar.tsx | 1 + src/plugins/data/public/ui/filter_bar/filter_options.tsx | 1 + .../data/public/ui/query_string_input/language_switcher.tsx | 1 + .../saved_query_management/saved_query_management_component.tsx | 1 + .../rule_actions_overflow/__snapshots__/index.test.tsx.snap | 1 + .../alerts/components/rules/rule_actions_overflow/index.tsx | 1 + .../rules/all/rules_table_filters/tags_filter_popover.tsx | 1 + .../public/cases/components/filter_popover/index.tsx | 1 + .../public/cases/components/property_actions/index.tsx | 1 + .../components/exceptions/viewer/exceptions_pagination.tsx | 1 + .../ml/score/__snapshots__/anomaly_score.test.tsx.snap | 1 + .../public/common/components/ml/score/anomaly_score.tsx | 1 + .../filters/__snapshots__/groups_filter_popover.test.tsx.snap | 1 + .../ml_popover/jobs_table/filters/groups_filter_popover.tsx | 1 + .../public/common/components/ml_popover/ml_popover.tsx | 2 ++ .../public/common/components/paginated_table/index.tsx | 1 + .../public/common/components/tables/helpers.tsx | 1 + .../public/common/components/utility_bar/utility_bar_action.tsx | 1 + .../public/management/pages/policy/view/policy_list.tsx | 1 + .../plugins/security_solution/public/resolver/view/submenu.tsx | 1 + .../timelines/components/field_renderers/field_renderers.tsx | 1 + .../components/timeline/body/events/event_column_view.tsx | 1 + .../components/timeline/insert_timeline_popover/index.tsx | 1 + .../components/timeline/properties/properties_right.tsx | 1 + 24 files changed, 25 insertions(+) diff --git a/src/plugins/data/public/ui/filter_bar/filter_bar.tsx b/src/plugins/data/public/ui/filter_bar/filter_bar.tsx index 43dba150bf8d44..fdd952e2207d93 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_bar.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_bar.tsx @@ -109,6 +109,7 @@ function FilterBarUI(props: Props) { panelPaddingSize="none" ownFocus={true} initialFocus=".filterEditor__hiddenItem" + repositionOnScroll >
diff --git a/src/plugins/data/public/ui/filter_bar/filter_options.tsx b/src/plugins/data/public/ui/filter_bar/filter_options.tsx index 3fb7f198d5466a..b97e0e33f2400b 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_options.tsx +++ b/src/plugins/data/public/ui/filter_bar/filter_options.tsx @@ -167,6 +167,7 @@ class FilterOptionsUI extends Component { anchorPosition="rightUp" panelPaddingSize="none" withTitle + repositionOnScroll > setIsPopoverOpen(false)} withTitle + repositionOnScroll >
diff --git a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/all/rules_table_filters/tags_filter_popover.tsx b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/all/rules_table_filters/tags_filter_popover.tsx index b453125223c307..fd75c229d479d2 100644 --- a/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/all/rules_table_filters/tags_filter_popover.tsx +++ b/x-pack/plugins/security_solution/public/alerts/pages/detection_engine/rules/all/rules_table_filters/tags_filter_popover.tsx @@ -61,6 +61,7 @@ export const TagsFilterPopoverComponent = ({ isOpen={isTagPopoverOpen} closePopover={() => setIsTagPopoverOpen(!isTagPopoverOpen)} panelPaddingSize="none" + repositionOnScroll > {tags.map((tag, index) => ( diff --git a/x-pack/plugins/security_solution/public/cases/components/filter_popover/index.tsx b/x-pack/plugins/security_solution/public/cases/components/filter_popover/index.tsx index 7b66bcffc89a12..4c16a8c0f3243d 100644 --- a/x-pack/plugins/security_solution/public/cases/components/filter_popover/index.tsx +++ b/x-pack/plugins/security_solution/public/cases/components/filter_popover/index.tsx @@ -84,6 +84,7 @@ export const FilterPopoverComponent = ({ isOpen={isPopoverOpen} closePopover={setIsPopoverOpenCb} panelPaddingSize="none" + repositionOnScroll > {options.map((option, index) => ( diff --git a/x-pack/plugins/security_solution/public/cases/components/property_actions/index.tsx b/x-pack/plugins/security_solution/public/cases/components/property_actions/index.tsx index 6b8e00921abcb7..29f1a2c5a14955 100644 --- a/x-pack/plugins/security_solution/public/cases/components/property_actions/index.tsx +++ b/x-pack/plugins/security_solution/public/cases/components/property_actions/index.tsx @@ -71,6 +71,7 @@ export const PropertyActions = React.memo(({ propertyActio id="settingsPopover" isOpen={showActions} closePopover={onClosePopover} + repositionOnScroll > diff --git a/x-pack/plugins/security_solution/public/common/components/ml/score/__snapshots__/anomaly_score.test.tsx.snap b/x-pack/plugins/security_solution/public/common/components/ml/score/__snapshots__/anomaly_score.test.tsx.snap index a9ec474a7b6840..6694cec53987b2 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml/score/__snapshots__/anomaly_score.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/common/components/ml/score/__snapshots__/anomaly_score.test.tsx.snap @@ -95,6 +95,7 @@ exports[`anomaly_scores renders correctly against snapshot 1`] = ` onClick={[Function]} ownFocus={false} panelPaddingSize="m" + repositionOnScroll={true} > setIsOpen(!isOpen)} closePopover={() => setIsOpen(!isOpen)} button={} + repositionOnScroll > setIsGroupPopoverOpen(!isGroupPopoverOpen)} panelPaddingSize="none" + repositionOnScroll > {uniqueGroups.map((group, index) => ( { } isOpen={isPopoverOpen} closePopover={() => setIsPopoverOpen(!isPopoverOpen)} + repositionOnScroll > @@ -147,6 +148,7 @@ export const MlPopover = React.memo(() => { } isOpen={isPopoverOpen} closePopover={() => setIsPopoverOpen(!isPopoverOpen)} + repositionOnScroll > {i18n.ML_JOB_SETTINGS} diff --git a/x-pack/plugins/security_solution/public/common/components/paginated_table/index.tsx b/x-pack/plugins/security_solution/public/common/components/paginated_table/index.tsx index 3b3130af77cfd7..9f95284d989a94 100644 --- a/x-pack/plugins/security_solution/public/common/components/paginated_table/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/paginated_table/index.tsx @@ -273,6 +273,7 @@ const PaginatedTableComponent: FC = ({ isOpen={isPopoverOpen} closePopover={closePopover} panelPaddingSize="none" + repositionOnScroll > diff --git a/x-pack/plugins/security_solution/public/common/components/tables/helpers.tsx b/x-pack/plugins/security_solution/public/common/components/tables/helpers.tsx index b8ea32969c015b..55e5758775504f 100644 --- a/x-pack/plugins/security_solution/public/common/components/tables/helpers.tsx +++ b/x-pack/plugins/security_solution/public/common/components/tables/helpers.tsx @@ -195,6 +195,7 @@ export const PopoverComponent = ({ closePopover={() => setIsOpen(!isOpen)} id={`${idPrefix}-popover`} isOpen={isOpen} + repositionOnScroll > {children} diff --git a/x-pack/plugins/security_solution/public/common/components/utility_bar/utility_bar_action.tsx b/x-pack/plugins/security_solution/public/common/components/utility_bar/utility_bar_action.tsx index 250ed75f134c13..f072b27274ed70 100644 --- a/x-pack/plugins/security_solution/public/common/components/utility_bar/utility_bar_action.tsx +++ b/x-pack/plugins/security_solution/public/common/components/utility_bar/utility_bar_action.tsx @@ -33,6 +33,7 @@ const Popover = React.memo( } closePopover={() => setPopoverState(false)} isOpen={popoverState} + repositionOnScroll > {popoverContent?.(closePopover)} diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_list.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_list.tsx index 08c6ec89ff051f..447a70ef998a94 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_list.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_list.tsx @@ -92,6 +92,7 @@ export const TableRowActions = React.memo<{ items: EuiContextMenuPanelProps['ite } isOpen={isOpen} closePopover={handleCloseMenu} + repositionOnScroll > diff --git a/x-pack/plugins/security_solution/public/resolver/view/submenu.tsx b/x-pack/plugins/security_solution/public/resolver/view/submenu.tsx index d3bb6123ce04da..ce126bf695559d 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/submenu.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/submenu.tsx @@ -215,6 +215,7 @@ const NodeSubMenuComponents = React.memo( button={submenuPopoverButton} isOpen={menuIsOpen} closePopover={closePopover} + repositionOnScroll > {menuIsOpen && typeof optionsWithActions === 'object' && ( diff --git a/x-pack/plugins/security_solution/public/timelines/components/field_renderers/field_renderers.tsx b/x-pack/plugins/security_solution/public/timelines/components/field_renderers/field_renderers.tsx index 7296e0ee4b9719..80fe7cb33779a2 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/field_renderers/field_renderers.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/field_renderers/field_renderers.tsx @@ -274,6 +274,7 @@ export const DefaultFieldRendererOverflow = React.memo setIsOpen(!isOpen)} + repositionOnScroll > ( closePopover={closePopover} panelPaddingSize="none" anchorPosition="downLeft" + repositionOnScroll > diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/insert_timeline_popover/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/insert_timeline_popover/index.tsx index 83417cdb51b699..0adf7673082695 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/insert_timeline_popover/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/insert_timeline_popover/index.tsx @@ -100,6 +100,7 @@ export const InsertTimelinePopoverComponent: React.FC = ({ button={insertTimelineButton} isOpen={isPopoverOpen} closePopover={handleClosePopover} + repositionOnScroll > = ({ id="timelineSettingsPopover" isOpen={showActions} closePopover={onClosePopover} + repositionOnScroll > {capabilitiesCanUserCRUD && ( From 0b6674edf5bf201064fdd9787e93ca479fbd0e8c Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Thu, 2 Jul 2020 08:37:37 -0700 Subject: [PATCH 31/31] Adds FOSSA CLI configuration file (#70137) FOSSA analysis by default checks for dependencies in the following order: 1. Parse output from `npm ls --json --production` - Runs if npm exists on the system and provides an accurate list of all dependencies needed to build the production project. 2. Parse `package.json` - Runs if `package.json` can be successfully parsed into a dependency graph. 3. Run yarn list --json - This command verifies through yarn what the actual dependencies which are installed on the system are. This strategy runs with `NODE_ENV=production` by default to find production dependencies. 4. Parse `yarn.lock` - Detects dependencies based on the yarn lockfile. 5. Parse `npm-shrinkwrap.json` - Detects dependencies based on the lockfile. 6. Parse `package-lock.json` - Detects dependencies based on the lockfile. Since our dependencies specified in `package.json` use compatible version matching (`^`), the reported version would often not be what the `yarn.lock` is currently specified to use. Because of this, we are defining a single module with a strategy on `yarn.lock`. Our `yarn.lock` file includes all dependencies. Signed-off-by: Tyler Smalley --- .fossa.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 .fossa.yml diff --git a/.fossa.yml b/.fossa.yml new file mode 100755 index 00000000000000..17d86d1f855218 --- /dev/null +++ b/.fossa.yml @@ -0,0 +1,15 @@ +# Generated by FOSSA CLI (https://github.com/fossas/fossa-cli) +# Visit https://fossa.com to learn more + +version: 2 +cli: + server: https://app.fossa.com + fetcher: custom + project: kibana +analyze: + modules: + - name: kibana + type: nodejs + strategy: yarn.lock + target: . + path: .