From 7fa30ba33e5981d726269891be514b387b5f52cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Tue, 2 Feb 2021 15:42:27 +0100 Subject: [PATCH 01/12] [Logs UI] Load entries via async searches (#86899) This PR replaces the usage of plain HTTP routes to load the log stream entries with async search strategy calls. --- x-pack/plugins/infra/common/graphql/types.ts | 337 --------- .../log_sources/log_source_configuration.ts | 1 + .../infra/common/log_entry/log_entry.ts | 74 +- .../common/log_entry/log_entry_cursor.ts | 16 +- .../log_entries/log_entries.ts | 74 ++ .../log_entries/log_entry.ts | 12 +- x-pack/plugins/infra/common/typed_json.ts | 2 + .../log_stream/log_stream.stories.mdx | 77 +- .../components/log_stream/log_stream.tsx | 18 +- .../log_entry_actions_menu.test.tsx | 16 +- .../log_entry_fields_table.tsx | 8 +- .../logging/log_text_stream/item.ts | 1 - .../logging/log_text_stream/log_entry_row.tsx | 1 - .../infra/public/containers/logs/log_entry.ts | 18 +- .../log_entry_highlights.gql_query.ts | 42 -- .../log_highlights/log_entry_highlights.tsx | 7 +- .../containers/logs/log_stream/index.ts | 272 ++++---- .../log_stream/use_fetch_log_entries_after.ts | 157 +++++ .../use_fetch_log_entries_around.ts | 184 +++++ .../use_fetch_log_entries_before.ts | 158 +++++ .../infra/public/graphql/introspection.json | 660 ------------------ x-pack/plugins/infra/public/graphql/types.ts | 337 --------- .../category_example_message.tsx | 1 + .../infra/public/test_utils/entries.ts | 1 + .../utils/data_search/data_search.stories.mdx | 36 +- .../flatten_data_search_response.ts | 29 + .../infra/public/utils/data_search/index.ts | 3 + .../normalize_data_search_responses.ts | 77 ++ .../infra/public/utils/data_search/types.ts | 19 +- .../use_data_search_request.test.tsx | 18 +- .../data_search/use_data_search_request.ts | 104 +-- .../use_data_search_response_state.ts | 35 + ...test_partial_data_search_response.test.tsx | 77 +- ...use_latest_partial_data_search_response.ts | 125 +--- .../infra/public/utils/log_entry/log_entry.ts | 6 - .../utils/log_entry/log_entry_highlight.ts | 8 - .../infra/public/utils/use_observable.ts | 20 +- x-pack/plugins/infra/server/graphql/types.ts | 406 ----------- .../log_entries/kibana_log_entries_adapter.ts | 1 + .../log_entries_domain/log_entries_domain.ts | 9 +- .../log_entry_categories_analysis.ts | 2 +- .../log_entries_search_strategy.test.ts | 318 +++++++++ .../log_entries_search_strategy.ts | 245 +++++++ .../log_entries/log_entries_service.ts | 6 + .../log_entry_search_strategy.test.ts | 2 +- .../log_entries/log_entry_search_strategy.ts | 2 +- .../builtin_rules/filebeat_apache2.test.ts | 0 .../builtin_rules/filebeat_apache2.ts | 0 .../builtin_rules/filebeat_auditd.test.ts | 0 .../message}/builtin_rules/filebeat_auditd.ts | 0 .../builtin_rules/filebeat_haproxy.test.ts | 0 .../builtin_rules/filebeat_haproxy.ts | 0 .../builtin_rules/filebeat_icinga.test.ts | 0 .../message}/builtin_rules/filebeat_icinga.ts | 0 .../builtin_rules/filebeat_iis.test.ts | 0 .../message}/builtin_rules/filebeat_iis.ts | 0 .../builtin_rules/filebeat_kafka.test.ts | 0 .../builtin_rules/filebeat_logstash.test.ts | 0 .../builtin_rules/filebeat_logstash.ts | 0 .../builtin_rules/filebeat_mongodb.test.ts | 0 .../builtin_rules/filebeat_mongodb.ts | 0 .../builtin_rules/filebeat_mysql.test.ts | 0 .../message}/builtin_rules/filebeat_mysql.ts | 0 .../builtin_rules/filebeat_nginx.test.ts | 0 .../message}/builtin_rules/filebeat_nginx.ts | 0 .../builtin_rules/filebeat_osquery.test.ts | 0 .../builtin_rules/filebeat_osquery.ts | 0 .../message}/builtin_rules/filebeat_redis.ts | 0 .../message}/builtin_rules/filebeat_system.ts | 0 .../builtin_rules/filebeat_traefik.test.ts | 0 .../builtin_rules/filebeat_traefik.ts | 0 .../message}/builtin_rules/generic.test.ts | 0 .../message}/builtin_rules/generic.ts | 0 .../builtin_rules/generic_webserver.ts | 0 .../message}/builtin_rules/helpers.ts | 0 .../message}/builtin_rules/index.ts | 0 .../services/log_entries/message/index.ts | 8 + .../log_entries/message}/message.ts | 0 .../log_entries/message}/rule_types.ts | 0 .../services/log_entries/queries/common.ts | 32 + .../log_entries/queries/log_entries.ts | 141 ++++ .../apis/metrics_ui/log_entries.ts | 8 +- 82 files changed, 1931 insertions(+), 2280 deletions(-) create mode 100644 x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts delete mode 100644 x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.gql_query.ts create mode 100644 x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts create mode 100644 x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts create mode 100644 x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts create mode 100644 x-pack/plugins/infra/public/utils/data_search/flatten_data_search_response.ts create mode 100644 x-pack/plugins/infra/public/utils/data_search/normalize_data_search_responses.ts create mode 100644 x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts create mode 100644 x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts create mode 100644 x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_apache2.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_apache2.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_auditd.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_auditd.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_haproxy.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_haproxy.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_icinga.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_icinga.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_iis.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_iis.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_kafka.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_logstash.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_logstash.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_mongodb.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_mongodb.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_mysql.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_mysql.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_nginx.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_nginx.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_osquery.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_osquery.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_redis.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_system.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_traefik.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/filebeat_traefik.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/generic.test.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/generic.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/generic_webserver.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/helpers.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/builtin_rules/index.ts (100%) create mode 100644 x-pack/plugins/infra/server/services/log_entries/message/index.ts rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/message.ts (100%) rename x-pack/plugins/infra/server/{lib/domains/log_entries_domain => services/log_entries/message}/rule_types.ts (100%) create mode 100644 x-pack/plugins/infra/server/services/log_entries/queries/common.ts create mode 100644 x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts diff --git a/x-pack/plugins/infra/common/graphql/types.ts b/x-pack/plugins/infra/common/graphql/types.ts index 4a18c3d5ff3340..ee536feb1ce653 100644 --- a/x-pack/plugins/infra/common/graphql/types.ts +++ b/x-pack/plugins/infra/common/graphql/types.ts @@ -28,12 +28,6 @@ export interface InfraSource { configuration: InfraSourceConfiguration; /** The status of the source */ status: InfraSourceStatus; - /** A consecutive span of log entries surrounding a point in time */ - logEntriesAround: InfraLogEntryInterval; - /** A consecutive span of log entries within an interval */ - logEntriesBetween: InfraLogEntryInterval; - /** Sequences of log entries matching sets of highlighting queries within an interval */ - logEntryHighlights: InfraLogEntryInterval[]; /** A snapshot of nodes */ snapshot?: InfraSnapshotResponse | null; @@ -129,80 +123,6 @@ export interface InfraIndexField { /** Whether the field should be displayed based on event.module and a ECS allowed list */ displayable: boolean; } -/** A consecutive sequence of log entries */ -export interface InfraLogEntryInterval { - /** The key corresponding to the start of the interval covered by the entries */ - start?: InfraTimeKey | null; - /** The key corresponding to the end of the interval covered by the entries */ - end?: InfraTimeKey | null; - /** Whether there are more log entries available before the start */ - hasMoreBefore: boolean; - /** Whether there are more log entries available after the end */ - hasMoreAfter: boolean; - /** The query the log entries were filtered by */ - filterQuery?: string | null; - /** The query the log entries were highlighted with */ - highlightQuery?: string | null; - /** A list of the log entries */ - entries: InfraLogEntry[]; -} -/** A representation of the log entry's position in the event stream */ -export interface InfraTimeKey { - /** The timestamp of the event that the log entry corresponds to */ - time: number; - /** The tiebreaker that disambiguates events with the same timestamp */ - tiebreaker: number; -} -/** A log entry */ -export interface InfraLogEntry { - /** A unique representation of the log entry's position in the event stream */ - key: InfraTimeKey; - /** The log entry's id */ - gid: string; - /** The source id */ - source: string; - /** The columns used for rendering the log entry */ - columns: InfraLogEntryColumn[]; -} -/** A special built-in column that contains the log entry's timestamp */ -export interface InfraLogEntryTimestampColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The timestamp */ - timestamp: number; -} -/** A special built-in column that contains the log entry's constructed message */ -export interface InfraLogEntryMessageColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** A list of the formatted log entry segments */ - message: InfraLogMessageSegment[]; -} -/** A segment of the log entry message that was derived from a field */ -export interface InfraLogMessageFieldSegment { - /** The field the segment was derived from */ - field: string; - /** The segment's message */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} -/** A segment of the log entry message that was derived from a string literal */ -export interface InfraLogMessageConstantSegment { - /** The segment's message */ - constant: string; -} -/** A column that contains the value of a field of the log entry */ -export interface InfraLogEntryFieldColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The field name of the column */ - field: string; - /** The value of the field in the log entry */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} export interface InfraSnapshotResponse { /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ @@ -276,21 +196,6 @@ export interface DeleteSourceResult { // InputTypes // ==================================================== -export interface InfraTimeKeyInput { - time: number; - - tiebreaker: number; -} -/** A highlighting definition */ -export interface InfraLogEntryHighlightInput { - /** The query to highlight by */ - query: string; - /** The number of highlighted documents to include beyond the beginning of the interval */ - countBefore: number; - /** The number of highlighted documents to include beyond the end of the interval */ - countAfter: number; -} - export interface InfraTimerangeInput { /** The interval string to use for last bucket. The format is '{value}{unit}'. For example '5m' would return the metrics for the last 5 minutes of the timespan. */ interval: string; @@ -381,34 +286,6 @@ export interface SourceQueryArgs { /** The id of the source */ id: string; } -export interface LogEntriesAroundInfraSourceArgs { - /** The sort key that corresponds to the point in time */ - key: InfraTimeKeyInput; - /** The maximum number of preceding to return */ - countBefore?: number | null; - /** The maximum number of following to return */ - countAfter?: number | null; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntriesBetweenInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntryHighlightsInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - /** The highlighting to apply to the log entries */ - highlights: InfraLogEntryHighlightInput[]; -} export interface SnapshotInfraSourceArgs { timerange: InfraTimerangeInput; @@ -565,15 +442,6 @@ export type InfraSourceLogColumn = | InfraSourceMessageLogColumn | InfraSourceFieldLogColumn; -/** A column of a log entry */ -export type InfraLogEntryColumn = - | InfraLogEntryTimestampColumn - | InfraLogEntryMessageColumn - | InfraLogEntryFieldColumn; - -/** A segment of the log entry message */ -export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessageConstantSegment; - // ==================================================== // END: Typescript template // ==================================================== @@ -582,46 +450,6 @@ export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessa // Documents // ==================================================== -export namespace LogEntryHighlightsQuery { - export type Variables = { - sourceId?: string | null; - startKey: InfraTimeKeyInput; - endKey: InfraTimeKeyInput; - filterQuery?: string | null; - highlights: InfraLogEntryHighlightInput[]; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'InfraSource'; - - id: string; - - logEntryHighlights: LogEntryHighlights[]; - }; - - export type LogEntryHighlights = { - __typename?: 'InfraLogEntryInterval'; - - start?: Start | null; - - end?: End | null; - - entries: Entries[]; - }; - - export type Start = InfraTimeKeyFields.Fragment; - - export type End = InfraTimeKeyFields.Fragment; - - export type Entries = InfraLogEntryHighlightFields.Fragment; -} - export namespace MetricsQuery { export type Variables = { sourceId: string; @@ -820,50 +648,6 @@ export namespace WaffleNodesQuery { }; } -export namespace LogEntries { - export type Variables = { - sourceId?: string | null; - timeKey: InfraTimeKeyInput; - countBefore?: number | null; - countAfter?: number | null; - filterQuery?: string | null; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'InfraSource'; - - id: string; - - logEntriesAround: LogEntriesAround; - }; - - export type LogEntriesAround = { - __typename?: 'InfraLogEntryInterval'; - - start?: Start | null; - - end?: End | null; - - hasMoreBefore: boolean; - - hasMoreAfter: boolean; - - entries: Entries[]; - }; - - export type Start = InfraTimeKeyFields.Fragment; - - export type End = InfraTimeKeyFields.Fragment; - - export type Entries = InfraLogEntryFields.Fragment; -} - export namespace SourceConfigurationFields { export type Fragment = { __typename?: 'InfraSourceConfiguration'; @@ -994,124 +778,3 @@ export namespace InfraSourceFields { origin: string; }; } - -export namespace InfraLogEntryFields { - export type Fragment = { - __typename?: 'InfraLogEntry'; - - gid: string; - - key: Key; - - columns: Columns[]; - }; - - export type Key = { - __typename?: 'InfraTimeKey'; - - time: number; - - tiebreaker: number; - }; - - export type Columns = - | InfraLogEntryTimestampColumnInlineFragment - | InfraLogEntryMessageColumnInlineFragment - | InfraLogEntryFieldColumnInlineFragment; - - export type InfraLogEntryTimestampColumnInlineFragment = { - __typename?: 'InfraLogEntryTimestampColumn'; - - columnId: string; - - timestamp: number; - }; - - export type InfraLogEntryMessageColumnInlineFragment = { - __typename?: 'InfraLogEntryMessageColumn'; - - columnId: string; - - message: Message[]; - }; - - export type Message = - | InfraLogMessageFieldSegmentInlineFragment - | InfraLogMessageConstantSegmentInlineFragment; - - export type InfraLogMessageFieldSegmentInlineFragment = { - __typename?: 'InfraLogMessageFieldSegment'; - - field: string; - - value: string; - }; - - export type InfraLogMessageConstantSegmentInlineFragment = { - __typename?: 'InfraLogMessageConstantSegment'; - - constant: string; - }; - - export type InfraLogEntryFieldColumnInlineFragment = { - __typename?: 'InfraLogEntryFieldColumn'; - - columnId: string; - - field: string; - - value: string; - }; -} - -export namespace InfraLogEntryHighlightFields { - export type Fragment = { - __typename?: 'InfraLogEntry'; - - gid: string; - - key: Key; - - columns: Columns[]; - }; - - export type Key = { - __typename?: 'InfraTimeKey'; - - time: number; - - tiebreaker: number; - }; - - export type Columns = - | InfraLogEntryMessageColumnInlineFragment - | InfraLogEntryFieldColumnInlineFragment; - - export type InfraLogEntryMessageColumnInlineFragment = { - __typename?: 'InfraLogEntryMessageColumn'; - - columnId: string; - - message: Message[]; - }; - - export type Message = InfraLogMessageFieldSegmentInlineFragment; - - export type InfraLogMessageFieldSegmentInlineFragment = { - __typename?: 'InfraLogMessageFieldSegment'; - - field: string; - - highlights: string[]; - }; - - export type InfraLogEntryFieldColumnInlineFragment = { - __typename?: 'InfraLogEntryFieldColumn'; - - columnId: string; - - field: string; - - highlights: string[]; - }; -} diff --git a/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts b/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts index 7581e29692356f..df7d80d33f1e60 100644 --- a/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts +++ b/x-pack/plugins/infra/common/http_api/log_sources/log_source_configuration.ts @@ -53,6 +53,7 @@ export const logSourceColumnConfigurationRT = rt.union([ logSourceMessageColumnConfigurationRT, logSourceFieldColumnConfigurationRT, ]); +export type LogSourceColumnConfiguration = rt.TypeOf; export const logSourceConfigurationPropertiesRT = rt.strict({ name: rt.string, diff --git a/x-pack/plugins/infra/common/log_entry/log_entry.ts b/x-pack/plugins/infra/common/log_entry/log_entry.ts index eec1fb59f3091c..bf3f9ceb0b0887 100644 --- a/x-pack/plugins/infra/common/log_entry/log_entry.ts +++ b/x-pack/plugins/infra/common/log_entry/log_entry.ts @@ -6,87 +6,79 @@ import * as rt from 'io-ts'; import { TimeKey } from '../time'; -import { logEntryCursorRT } from './log_entry_cursor'; import { jsonArrayRT } from '../typed_json'; - -export interface LogEntryOrigin { - id: string; - index: string; - type: string; -} +import { logEntryCursorRT } from './log_entry_cursor'; export type LogEntryTime = TimeKey; -export interface LogEntryFieldsMapping { - message: string; - tiebreaker: string; - time: string; -} - -export function isEqual(time1: LogEntryTime, time2: LogEntryTime) { - return time1.time === time2.time && time1.tiebreaker === time2.tiebreaker; -} - -export function isLess(time1: LogEntryTime, time2: LogEntryTime) { - return ( - time1.time < time2.time || (time1.time === time2.time && time1.tiebreaker < time2.tiebreaker) - ); -} - -export function isLessOrEqual(time1: LogEntryTime, time2: LogEntryTime) { - return ( - time1.time < time2.time || (time1.time === time2.time && time1.tiebreaker <= time2.tiebreaker) - ); -} - -export function isBetween(min: LogEntryTime, max: LogEntryTime, operand: LogEntryTime) { - return isLessOrEqual(min, operand) && isLessOrEqual(operand, max); -} +/** + * message parts + */ export const logMessageConstantPartRT = rt.type({ constant: rt.string, }); +export type LogMessageConstantPart = rt.TypeOf; + export const logMessageFieldPartRT = rt.type({ field: rt.string, value: jsonArrayRT, highlights: rt.array(rt.string), }); +export type LogMessageFieldPart = rt.TypeOf; export const logMessagePartRT = rt.union([logMessageConstantPartRT, logMessageFieldPartRT]); +export type LogMessagePart = rt.TypeOf; + +/** + * columns + */ export const logTimestampColumnRT = rt.type({ columnId: rt.string, timestamp: rt.number }); +export type LogTimestampColumn = rt.TypeOf; + export const logFieldColumnRT = rt.type({ columnId: rt.string, field: rt.string, value: jsonArrayRT, highlights: rt.array(rt.string), }); +export type LogFieldColumn = rt.TypeOf; + export const logMessageColumnRT = rt.type({ columnId: rt.string, message: rt.array(logMessagePartRT), }); +export type LogMessageColumn = rt.TypeOf; export const logColumnRT = rt.union([logTimestampColumnRT, logFieldColumnRT, logMessageColumnRT]); +export type LogColumn = rt.TypeOf; +/** + * fields + */ export const logEntryContextRT = rt.union([ rt.type({}), rt.type({ 'container.id': rt.string }), rt.type({ 'host.name': rt.string, 'log.file.path': rt.string }), ]); +export type LogEntryContext = rt.TypeOf; + +export const logEntryFieldRT = rt.type({ + field: rt.string, + value: jsonArrayRT, +}); +export type LogEntryField = rt.TypeOf; + +/** + * entry + */ export const logEntryRT = rt.type({ id: rt.string, + index: rt.string, cursor: logEntryCursorRT, columns: rt.array(logColumnRT), context: logEntryContextRT, }); - -export type LogMessageConstantPart = rt.TypeOf; -export type LogMessageFieldPart = rt.TypeOf; -export type LogMessagePart = rt.TypeOf; -export type LogEntryContext = rt.TypeOf; export type LogEntry = rt.TypeOf; -export type LogTimestampColumn = rt.TypeOf; -export type LogFieldColumn = rt.TypeOf; -export type LogMessageColumn = rt.TypeOf; -export type LogColumn = rt.TypeOf; diff --git a/x-pack/plugins/infra/common/log_entry/log_entry_cursor.ts b/x-pack/plugins/infra/common/log_entry/log_entry_cursor.ts index 280403dd5438d4..b11a48822e758f 100644 --- a/x-pack/plugins/infra/common/log_entry/log_entry_cursor.ts +++ b/x-pack/plugins/infra/common/log_entry/log_entry_cursor.ts @@ -11,9 +11,23 @@ export const logEntryCursorRT = rt.type({ time: rt.number, tiebreaker: rt.number, }); - export type LogEntryCursor = rt.TypeOf; +export const logEntryBeforeCursorRT = rt.type({ + before: rt.union([logEntryCursorRT, rt.literal('last')]), +}); +export type LogEntryBeforeCursor = rt.TypeOf; + +export const logEntryAfterCursorRT = rt.type({ + after: rt.union([logEntryCursorRT, rt.literal('first')]), +}); +export type LogEntryAfterCursor = rt.TypeOf; + +export const logEntryAroundCursorRT = rt.type({ + center: logEntryCursorRT, +}); +export type LogEntryAroundCursor = rt.TypeOf; + export const getLogEntryCursorFromHit = (hit: { sort: [number, number] }) => decodeOrThrow(logEntryCursorRT)({ time: hit.sort[0], diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts new file mode 100644 index 00000000000000..b2a879c3b72fd3 --- /dev/null +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entries.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; +import { DslQuery } from '../../../../../../src/plugins/data/common'; +import { logSourceColumnConfigurationRT } from '../../http_api/log_sources'; +import { + logEntryAfterCursorRT, + logEntryBeforeCursorRT, + logEntryCursorRT, + logEntryRT, +} from '../../log_entry'; +import { JsonObject, jsonObjectRT } from '../../typed_json'; +import { searchStrategyErrorRT } from '../common/errors'; + +export const LOG_ENTRIES_SEARCH_STRATEGY = 'infra-log-entries'; + +const logEntriesBaseSearchRequestParamsRT = rt.intersection([ + rt.type({ + sourceId: rt.string, + startTimestamp: rt.number, + endTimestamp: rt.number, + size: rt.number, + }), + rt.partial({ + query: jsonObjectRT, + columns: rt.array(logSourceColumnConfigurationRT), + highlightPhrase: rt.string, + }), +]); + +export const logEntriesBeforeSearchRequestParamsRT = rt.intersection([ + logEntriesBaseSearchRequestParamsRT, + logEntryBeforeCursorRT, +]); + +export const logEntriesAfterSearchRequestParamsRT = rt.intersection([ + logEntriesBaseSearchRequestParamsRT, + logEntryAfterCursorRT, +]); + +export const logEntriesSearchRequestParamsRT = rt.union([ + logEntriesBaseSearchRequestParamsRT, + logEntriesBeforeSearchRequestParamsRT, + logEntriesAfterSearchRequestParamsRT, +]); + +export type LogEntriesSearchRequestParams = rt.TypeOf; + +export type LogEntriesSearchRequestQuery = JsonObject | DslQuery; + +export const logEntriesSearchResponsePayloadRT = rt.intersection([ + rt.type({ + data: rt.intersection([ + rt.type({ + entries: rt.array(logEntryRT), + topCursor: rt.union([logEntryCursorRT, rt.null]), + bottomCursor: rt.union([logEntryCursorRT, rt.null]), + }), + rt.partial({ + hasMoreBefore: rt.boolean, + hasMoreAfter: rt.boolean, + }), + ]), + }), + rt.partial({ + errors: rt.array(searchStrategyErrorRT), + }), +]); + +export type LogEntriesSearchResponsePayload = rt.TypeOf; diff --git a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entry.ts b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entry.ts index af6bd203f980ea..986f6baf044881 100644 --- a/x-pack/plugins/infra/common/search_strategies/log_entries/log_entry.ts +++ b/x-pack/plugins/infra/common/search_strategies/log_entries/log_entry.ts @@ -5,8 +5,7 @@ */ import * as rt from 'io-ts'; -import { logEntryCursorRT } from '../../log_entry'; -import { jsonArrayRT } from '../../typed_json'; +import { logEntryCursorRT, logEntryFieldRT } from '../../log_entry'; import { searchStrategyErrorRT } from '../common/errors'; export const LOG_ENTRY_SEARCH_STRATEGY = 'infra-log-entry'; @@ -18,18 +17,11 @@ export const logEntrySearchRequestParamsRT = rt.type({ export type LogEntrySearchRequestParams = rt.TypeOf; -const logEntryFieldRT = rt.type({ - field: rt.string, - value: jsonArrayRT, -}); - -export type LogEntryField = rt.TypeOf; - export const logEntryRT = rt.type({ id: rt.string, index: rt.string, fields: rt.array(logEntryFieldRT), - key: logEntryCursorRT, + cursor: logEntryCursorRT, }); export type LogEntry = rt.TypeOf; diff --git a/x-pack/plugins/infra/common/typed_json.ts b/x-pack/plugins/infra/common/typed_json.ts index f3e7608910e095..5aec8d3eaf2ccf 100644 --- a/x-pack/plugins/infra/common/typed_json.ts +++ b/x-pack/plugins/infra/common/typed_json.ts @@ -7,6 +7,8 @@ import * as rt from 'io-ts'; import { JsonArray, JsonObject, JsonValue } from '../../../../src/plugins/kibana_utils/common'; +export { JsonArray, JsonObject, JsonValue }; + export const jsonScalarRT = rt.union([rt.null, rt.boolean, rt.number, rt.string]); export const jsonValueRT: rt.Type = rt.recursion('JsonValue', () => diff --git a/x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx b/x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx index bda52d9323eb60..901a4b6a8383e3 100644 --- a/x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx +++ b/x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx @@ -1,10 +1,12 @@ import { Meta, Story, Canvas, ArgsTable } from '@storybook/addon-docs/blocks'; -import { Subject } from 'rxjs'; +import { defer, of, Subject } from 'rxjs'; +import { delay } from 'rxjs/operators'; import { I18nProvider } from '@kbn/i18n/react'; import { EuiThemeProvider } from '../../../../../../src/plugins/kibana_react/common'; import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public'; +import { LOG_ENTRIES_SEARCH_STRATEGY } from '../../../common/search_strategies/log_entries/log_entries'; import { DEFAULT_SOURCE_CONFIGURATION } from '../../test_utils/source_configuration'; import { generateFakeEntries, ENTRIES_EMPTY } from '../../test_utils/entries'; @@ -15,30 +17,61 @@ import { LogStream } from './'; export const startTimestamp = 1595145600000; export const endTimestamp = startTimestamp + 15 * 60 * 1000; +export const dataMock = { + search: { + search: ({ params }, options) => { + return defer(() => { + switch (options.strategy) { + case LOG_ENTRIES_SEARCH_STRATEGY: + if (params.after?.time === params.endTimestamp || params.before?.time === params.startTimestamp) { + return of({ + id: 'EMPTY_FAKE_RESPONSE', + total: 1, + loaded: 1, + isRunning: false, + isPartial: false, + rawResponse: ENTRIES_EMPTY, + }); + } else { + const entries = generateFakeEntries( + 200, + params.startTimestamp, + params.endTimestamp, + params.columns || DEFAULT_SOURCE_CONFIGURATION.data.configuration.logColumns + ); + return of({ + id: 'FAKE_RESPONSE', + total: 1, + loaded: 1, + isRunning: false, + isPartial: false, + rawResponse: { + data: { + entries, + topCursor: entries[0].cursor, + bottomCursor: entries[entries.length - 1].cursor, + hasMoreBefore: false, + }, + errors: [], + } + }); + } + default: + return of({ + id: 'FAKE_RESPONSE', + rawResponse: {}, + }); + } + }).pipe(delay(2000)); + }, + }, +}; + + export const fetch = function (url, params) { switch (url) { case '/api/infra/log_source_configurations/default': return DEFAULT_SOURCE_CONFIGURATION; - case '/api/log_entries/entries': - const body = JSON.parse(params.body); - if (body.after?.time === body.endTimestamp || body.before?.time === body.startTimestamp) { - return ENTRIES_EMPTY; - } else { - const entries = generateFakeEntries( - 200, - body.startTimestamp, - body.endTimestamp, - body.columns || DEFAULT_SOURCE_CONFIGURATION.data.configuration.logColumns - ); - return { - data: { - entries, - topCursor: entries[0].cursor, - bottomCursor: entries[entries.length - 1].cursor, - hasMoreBefore: false, - }, - }; - } default: return {}; } @@ -67,7 +100,7 @@ export const Template = (args) => ; (story) => ( - + {story()} diff --git a/x-pack/plugins/infra/public/components/log_stream/log_stream.tsx b/x-pack/plugins/infra/public/components/log_stream/log_stream.tsx index b7410fda6f6fd4..ab9bc0099f1969 100644 --- a/x-pack/plugins/infra/public/components/log_stream/log_stream.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/log_stream.tsx @@ -101,14 +101,14 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re // Internal state const { - loadingState, - pageLoadingState, entries, - hasMoreBefore, - hasMoreAfter, fetchEntries, - fetchPreviousEntries, fetchNextEntries, + fetchPreviousEntries, + hasMoreAfter, + hasMoreBefore, + isLoadingMore, + isReloading, } = useLogStream({ sourceId, startTimestamp, @@ -118,12 +118,6 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re columns: customColumns, }); - // Derived state - const isReloading = - isLoadingSourceConfiguration || loadingState === 'uninitialized' || loadingState === 'loading'; - - const isLoadingMore = pageLoadingState === 'loading'; - const columnConfigurations = useMemo(() => { return sourceConfiguration ? customColumns ?? sourceConfiguration.configuration.logColumns : []; }, [sourceConfiguration, customColumns]); @@ -177,7 +171,7 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re items={streamItems} scale="medium" wrap={true} - isReloading={isReloading} + isReloading={isLoadingSourceConfiguration || isReloading} isLoadingMore={isLoadingMore} hasMoreBeforeStart={hasMoreBefore} hasMoreAfterEnd={hasMoreAfter} diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx index f578292d6d6fcb..447e6afbbf1fdb 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx @@ -32,7 +32,7 @@ describe('LogEntryActionsMenu component', () => { fields: [{ field: 'host.ip', value: ['HOST_IP'] }], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -62,7 +62,7 @@ describe('LogEntryActionsMenu component', () => { fields: [{ field: 'container.id', value: ['CONTAINER_ID'] }], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -92,7 +92,7 @@ describe('LogEntryActionsMenu component', () => { fields: [{ field: 'kubernetes.pod.uid', value: ['POD_UID'] }], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -126,7 +126,7 @@ describe('LogEntryActionsMenu component', () => { ], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -158,7 +158,7 @@ describe('LogEntryActionsMenu component', () => { fields: [], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -192,7 +192,7 @@ describe('LogEntryActionsMenu component', () => { fields: [{ field: 'trace.id', value: ['1234567'] }], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -226,7 +226,7 @@ describe('LogEntryActionsMenu component', () => { ], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, @@ -256,7 +256,7 @@ describe('LogEntryActionsMenu component', () => { fields: [], id: 'ITEM_ID', index: 'INDEX', - key: { + cursor: { time: 0, tiebreaker: 0, }, diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx index 44e9902e0413f1..b3c80a3a4924a2 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx @@ -7,10 +7,8 @@ import { EuiBasicTableColumn, EuiInMemoryTable } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React, { useMemo } from 'react'; -import { - LogEntry, - LogEntryField, -} from '../../../../common/search_strategies/log_entries/log_entry'; +import { LogEntryField } from '../../../../common/log_entry'; +import { LogEntry } from '../../../../common/search_strategies/log_entries/log_entry'; import { TimeKey } from '../../../../common/time'; import { FieldValue } from '../log_text_stream/field_value'; @@ -22,7 +20,7 @@ export const LogEntryFieldsTable: React.FC<{ () => onSetFieldFilter ? (field: LogEntryField) => () => { - onSetFieldFilter?.(`${field.field}:"${field.value}"`, logEntry.id, logEntry.key); + onSetFieldFilter?.(`${field.field}:"${field.value}"`, logEntry.id, logEntry.cursor); } : undefined, [logEntry, onSetFieldFilter] diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/item.ts b/x-pack/plugins/infra/public/components/logging/log_text_stream/item.ts index b0ff36574bedef..13d5b7b889465a 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/item.ts +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/item.ts @@ -5,7 +5,6 @@ */ import { bisector } from 'd3-array'; - import { compareToTimeKey, TimeKey } from '../../../../common/time'; import { LogEntry } from '../../../../common/log_entry'; diff --git a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_row.tsx b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_row.tsx index 1a472df2b5c906..036818317011c2 100644 --- a/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_row.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_text_stream/log_entry_row.tsx @@ -7,7 +7,6 @@ import React, { memo, useState, useCallback, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { isEmpty } from 'lodash'; - import { euiStyled } from '../../../../../../../src/plugins/kibana_react/common'; import { useUiTracker } from '../../../../../observability/public'; import { isTimestampColumn } from '../../../utils/log_entry'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_entry.ts b/x-pack/plugins/infra/public/containers/logs/log_entry.ts index af8618b8be5657..60000e0b8baba1 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_entry.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_entry.ts @@ -11,7 +11,11 @@ import { logEntrySearchResponsePayloadRT, LOG_ENTRY_SEARCH_STRATEGY, } from '../../../common/search_strategies/log_entries/log_entry'; -import { useDataSearch, useLatestPartialDataSearchResponse } from '../../utils/data_search'; +import { + normalizeDataSearchResponses, + useDataSearch, + useLatestPartialDataSearchResponse, +} from '../../utils/data_search'; export const useLogEntry = ({ sourceId, @@ -31,6 +35,7 @@ export const useLogEntry = ({ } : null; }, [sourceId, logEntryId]), + parseResponses: parseLogEntrySearchResponses, }); const { @@ -41,11 +46,7 @@ export const useLogEntry = ({ latestResponseErrors, loaded, total, - } = useLatestPartialDataSearchResponse( - logEntrySearchRequests$, - null, - decodeLogEntrySearchResponse - ); + } = useLatestPartialDataSearchResponse(logEntrySearchRequests$); return { cancelRequest, @@ -59,4 +60,7 @@ export const useLogEntry = ({ }; }; -const decodeLogEntrySearchResponse = decodeOrThrow(logEntrySearchResponsePayloadRT); +const parseLogEntrySearchResponses = normalizeDataSearchResponses( + null, + decodeOrThrow(logEntrySearchResponsePayloadRT) +); diff --git a/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.gql_query.ts b/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.gql_query.ts deleted file mode 100644 index 9d9fab58754275..00000000000000 --- a/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.gql_query.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import gql from 'graphql-tag'; - -import { sharedFragments } from '../../../../common/graphql/shared'; - -export const logEntryHighlightsQuery = gql` - query LogEntryHighlightsQuery( - $sourceId: ID = "default" - $startKey: InfraTimeKeyInput! - $endKey: InfraTimeKeyInput! - $filterQuery: String - $highlights: [InfraLogEntryHighlightInput!]! - ) { - source(id: $sourceId) { - id - logEntryHighlights( - startKey: $startKey - endKey: $endKey - filterQuery: $filterQuery - highlights: $highlights - ) { - start { - ...InfraTimeKeyFields - } - end { - ...InfraTimeKeyFields - } - entries { - ...InfraLogEntryHighlightFields - } - } - } - } - - ${sharedFragments.InfraTimeKey} - ${sharedFragments.InfraLogEntryHighlightFields} -`; diff --git a/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.tsx b/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.tsx index fb72874df54099..caac28a0756a16 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_highlights/log_entry_highlights.tsx @@ -5,13 +5,12 @@ */ import { useEffect, useMemo, useState } from 'react'; - -import { TimeKey } from '../../../../common/time'; -import { useTrackedPromise } from '../../../utils/use_tracked_promise'; -import { fetchLogEntriesHighlights } from './api/fetch_log_entries_highlights'; import { LogEntriesHighlightsResponse } from '../../../../common/http_api'; import { LogEntry } from '../../../../common/log_entry'; +import { TimeKey } from '../../../../common/time'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; +import { useTrackedPromise } from '../../../utils/use_tracked_promise'; +import { fetchLogEntriesHighlights } from './api/fetch_log_entries_highlights'; export const useLogEntryHighlights = ( sourceId: string, diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts index 1d9a7a1b1d7779..8343525c828628 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/index.ts @@ -4,15 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useMemo, useEffect } from 'react'; -import useSetState from 'react-use/lib/useSetState'; +import { useCallback, useEffect, useMemo } from 'react'; import usePrevious from 'react-use/lib/usePrevious'; +import useSetState from 'react-use/lib/useSetState'; import { esKuery, esQuery, Query } from '../../../../../../../src/plugins/data/public'; -import { fetchLogEntries } from '../log_entries/api/fetch_log_entries'; -import { useTrackedPromise } from '../../../utils/use_tracked_promise'; -import { LogEntryCursor, LogEntry } from '../../../../common/log_entry'; -import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; +import { LogEntry, LogEntryCursor } from '../../../../common/log_entry'; +import { useSubscription } from '../../../utils/use_observable'; import { LogSourceConfigurationProperties } from '../log_source'; +import { useFetchLogEntriesAfter } from './use_fetch_log_entries_after'; +import { useFetchLogEntriesAround } from './use_fetch_log_entries_around'; +import { useFetchLogEntriesBefore } from './use_fetch_log_entries_before'; interface LogStreamProps { sourceId: string; @@ -31,16 +32,6 @@ interface LogStreamState { hasMoreAfter: boolean; } -type LoadingState = 'uninitialized' | 'loading' | 'success' | 'error'; - -interface LogStreamReturn extends LogStreamState { - fetchEntries: () => void; - fetchPreviousEntries: () => void; - fetchNextEntries: () => void; - loadingState: LoadingState; - pageLoadingState: LoadingState; -} - const INITIAL_STATE: LogStreamState = { entries: [], topCursor: null, @@ -50,11 +41,7 @@ const INITIAL_STATE: LogStreamState = { hasMoreAfter: true, }; -const EMPTY_DATA = { - entries: [], - topCursor: null, - bottomCursor: null, -}; +const LOG_ENTRIES_CHUNK_SIZE = 200; export function useLogStream({ sourceId, @@ -63,8 +50,7 @@ export function useLogStream({ query, center, columns, -}: LogStreamProps): LogStreamReturn { - const { services } = useKibanaContextForPlugin(); +}: LogStreamProps) { const [state, setState] = useSetState(INITIAL_STATE); // Ensure the pagination keeps working when the timerange gets extended @@ -85,175 +71,151 @@ export function useLogStream({ const parsedQuery = useMemo(() => { if (!query) { - return null; - } - - let q; - - if (typeof query === 'string') { - q = esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query)); + return undefined; + } else if (typeof query === 'string') { + return esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query)); } else if (query.language === 'kuery') { - q = esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query.query as string)); + return esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(query.query as string)); } else if (query.language === 'lucene') { - q = esQuery.luceneStringToDsl(query.query as string); + return esQuery.luceneStringToDsl(query.query as string); + } else { + return undefined; } - - return JSON.stringify(q); }, [query]); - // Callbacks - const [entriesPromise, fetchEntries] = useTrackedPromise( - { - cancelPreviousOn: 'creation', - createPromise: () => { - setState(INITIAL_STATE); - const fetchPosition = center ? { center } : { before: 'last' }; + const commonFetchArguments = useMemo( + () => ({ + sourceId, + startTimestamp, + endTimestamp, + query: parsedQuery, + columnOverrides: columns, + }), + [columns, endTimestamp, parsedQuery, sourceId, startTimestamp] + ); - return fetchLogEntries( - { - sourceId, - startTimestamp, - endTimestamp, - query: parsedQuery, - columns, - ...fetchPosition, - }, - services.http.fetch - ); - }, - onResolve: ({ data }) => { + const { + fetchLogEntriesAround, + isRequestRunning: isLogEntriesAroundRequestRunning, + logEntriesAroundSearchResponses$, + } = useFetchLogEntriesAround(commonFetchArguments); + + useSubscription(logEntriesAroundSearchResponses$, { + next: ({ before, after, combined }) => { + if ((before.response.data != null || after?.response.data != null) && !combined.isPartial) { setState((prevState) => ({ - ...data, - hasMoreBefore: data.hasMoreBefore ?? prevState.hasMoreBefore, - hasMoreAfter: data.hasMoreAfter ?? prevState.hasMoreAfter, + ...prevState, + entries: combined.entries, + hasMoreAfter: combined.hasMoreAfter ?? prevState.hasMoreAfter, + hasMoreBefore: combined.hasMoreAfter ?? prevState.hasMoreAfter, + bottomCursor: combined.bottomCursor, + topCursor: combined.topCursor, })); - }, + } }, - [sourceId, startTimestamp, endTimestamp, query] - ); + }); - const [previousEntriesPromise, fetchPreviousEntries] = useTrackedPromise( - { - cancelPreviousOn: 'creation', - createPromise: () => { - if (state.topCursor === null) { - throw new Error( - 'useLogState: Cannot fetch previous entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' - ); - } + const { + fetchLogEntriesBefore, + isRequestRunning: isLogEntriesBeforeRequestRunning, + logEntriesBeforeSearchResponse$, + } = useFetchLogEntriesBefore(commonFetchArguments); - if (!state.hasMoreBefore) { - return Promise.resolve({ data: EMPTY_DATA }); - } - - return fetchLogEntries( - { - sourceId, - startTimestamp, - endTimestamp, - query: parsedQuery, - before: state.topCursor, - }, - services.http.fetch - ); - }, - onResolve: ({ data }) => { - if (!data.entries.length) { - return; - } + useSubscription(logEntriesBeforeSearchResponse$, { + next: ({ response: { data, isPartial } }) => { + if (data != null && !isPartial) { setState((prevState) => ({ + ...prevState, entries: [...data.entries, ...prevState.entries], hasMoreBefore: data.hasMoreBefore ?? prevState.hasMoreBefore, topCursor: data.topCursor ?? prevState.topCursor, + bottomCursor: prevState.bottomCursor ?? data.bottomCursor, })); - }, + } }, - [sourceId, startTimestamp, endTimestamp, query, state.topCursor] - ); + }); - const [nextEntriesPromise, fetchNextEntries] = useTrackedPromise( - { - cancelPreviousOn: 'creation', - createPromise: () => { - if (state.bottomCursor === null) { - throw new Error( - 'useLogState: Cannot fetch next entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' - ); - } + const fetchPreviousEntries = useCallback(() => { + if (state.topCursor === null) { + throw new Error( + 'useLogState: Cannot fetch previous entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' + ); + } + + if (!state.hasMoreBefore) { + return; + } - if (!state.hasMoreAfter) { - return Promise.resolve({ data: EMPTY_DATA }); - } + fetchLogEntriesBefore(state.topCursor, LOG_ENTRIES_CHUNK_SIZE); + }, [fetchLogEntriesBefore, state.topCursor, state.hasMoreBefore]); - return fetchLogEntries( - { - sourceId, - startTimestamp, - endTimestamp, - query: parsedQuery, - after: state.bottomCursor, - }, - services.http.fetch - ); - }, - onResolve: ({ data }) => { - if (!data.entries.length) { - return; - } + const { + fetchLogEntriesAfter, + isRequestRunning: isLogEntriesAfterRequestRunning, + logEntriesAfterSearchResponse$, + } = useFetchLogEntriesAfter(commonFetchArguments); + + useSubscription(logEntriesAfterSearchResponse$, { + next: ({ response: { data, isPartial } }) => { + if (data != null && !isPartial) { setState((prevState) => ({ + ...prevState, entries: [...prevState.entries, ...data.entries], hasMoreAfter: data.hasMoreAfter ?? prevState.hasMoreAfter, + topCursor: prevState.topCursor ?? data.topCursor, bottomCursor: data.bottomCursor ?? prevState.bottomCursor, })); - }, + } }, - [sourceId, startTimestamp, endTimestamp, query, state.bottomCursor] - ); - - const loadingState = useMemo( - () => convertPromiseStateToLoadingState(entriesPromise.state), - [entriesPromise.state] - ); + }); - const pageLoadingState = useMemo(() => { - const states = [previousEntriesPromise.state, nextEntriesPromise.state]; - - if (states.includes('pending')) { - return 'loading'; + const fetchNextEntries = useCallback(() => { + if (state.bottomCursor === null) { + throw new Error( + 'useLogState: Cannot fetch next entries. No cursor is set.\nEnsure you have called `fetchEntries` at least once.' + ); } - if (states.includes('rejected')) { - return 'error'; + if (!state.hasMoreAfter) { + return; } - if (states.includes('resolved')) { - return 'success'; + fetchLogEntriesAfter(state.bottomCursor, LOG_ENTRIES_CHUNK_SIZE); + }, [fetchLogEntriesAfter, state.bottomCursor, state.hasMoreAfter]); + + const fetchEntries = useCallback(() => { + setState(INITIAL_STATE); + + if (center) { + fetchLogEntriesAround(center, LOG_ENTRIES_CHUNK_SIZE); + } else { + fetchLogEntriesBefore('last', LOG_ENTRIES_CHUNK_SIZE); } + }, [center, fetchLogEntriesAround, fetchLogEntriesBefore, setState]); + + const isReloading = useMemo( + () => + isLogEntriesAroundRequestRunning || + (state.bottomCursor == null && state.topCursor == null && isLogEntriesBeforeRequestRunning), + [ + isLogEntriesAroundRequestRunning, + isLogEntriesBeforeRequestRunning, + state.bottomCursor, + state.topCursor, + ] + ); - return 'uninitialized'; - }, [previousEntriesPromise.state, nextEntriesPromise.state]); + const isLoadingMore = useMemo( + () => isLogEntriesBeforeRequestRunning || isLogEntriesAfterRequestRunning, + [isLogEntriesAfterRequestRunning, isLogEntriesBeforeRequestRunning] + ); return { ...state, fetchEntries, - fetchPreviousEntries, fetchNextEntries, - loadingState, - pageLoadingState, + fetchPreviousEntries, + isLoadingMore, + isReloading, }; } - -function convertPromiseStateToLoadingState( - state: 'uninitialized' | 'pending' | 'resolved' | 'rejected' -): LoadingState { - switch (state) { - case 'uninitialized': - return 'uninitialized'; - case 'pending': - return 'loading'; - case 'resolved': - return 'success'; - case 'rejected': - return 'error'; - } -} diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts new file mode 100644 index 00000000000000..c7076ec51db6a3 --- /dev/null +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_after.ts @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { Observable } from 'rxjs'; +import { exhaustMap } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../../src/plugins/data/public'; +import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; +import { LogEntryAfterCursor } from '../../../../common/log_entry'; +import { decodeOrThrow } from '../../../../common/runtime_types'; +import { + logEntriesSearchRequestParamsRT, + LogEntriesSearchRequestQuery, + LogEntriesSearchResponsePayload, + logEntriesSearchResponsePayloadRT, + LOG_ENTRIES_SEARCH_STRATEGY, +} from '../../../../common/search_strategies/log_entries/log_entries'; +import { + flattenDataSearchResponseDescriptor, + normalizeDataSearchResponses, + ParsedDataSearchRequestDescriptor, + useDataSearch, + useDataSearchResponseState, +} from '../../../utils/data_search'; +import { useOperator } from '../../../utils/use_observable'; + +export const useLogEntriesAfterRequest = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: LogEntriesSearchRequestQuery; + sourceId: string; + startTimestamp: number; +}) => { + const { search: fetchLogEntriesAfter, requests$: logEntriesAfterSearchRequests$ } = useDataSearch( + { + getRequest: useCallback( + (cursor: LogEntryAfterCursor['after'], size: number) => { + return !!sourceId + ? { + request: { + params: logEntriesSearchRequestParamsRT.encode({ + after: cursor, + columns: columnOverrides, + endTimestamp, + highlightPhrase, + query, + size, + sourceId, + startTimestamp, + }), + }, + options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, + } + : null; + }, + [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] + ), + parseResponses: parseLogEntriesAfterSearchResponses, + } + ); + + return { + fetchLogEntriesAfter, + logEntriesAfterSearchRequests$, + }; +}; + +export const useLogEntriesAfterResponse = ( + logEntriesAfterSearchRequests$: Observable< + ParsedDataSearchRequestDescriptor + > +) => { + const logEntriesAfterSearchResponse$ = useOperator( + logEntriesAfterSearchRequests$, + flattenLogEntriesAfterSearchResponse + ); + + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + total, + } = useDataSearchResponseState(logEntriesAfterSearchResponse$); + + return { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesAfterSearchRequests$, + logEntriesAfterSearchResponse$, + total, + }; +}; + +export const useFetchLogEntriesAfter = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: LogEntriesSearchRequestQuery; + sourceId: string; + startTimestamp: number; +}) => { + const { fetchLogEntriesAfter, logEntriesAfterSearchRequests$ } = useLogEntriesAfterRequest({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, + }); + + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesAfterSearchResponse$, + total, + } = useLogEntriesAfterResponse(logEntriesAfterSearchRequests$); + + return { + cancelRequest, + fetchLogEntriesAfter, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesAfterSearchResponse$, + total, + }; +}; + +export const parseLogEntriesAfterSearchResponses = normalizeDataSearchResponses( + null, + decodeOrThrow(logEntriesSearchResponsePayloadRT) +); + +const flattenLogEntriesAfterSearchResponse = exhaustMap(flattenDataSearchResponseDescriptor); diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts new file mode 100644 index 00000000000000..01f6336e0d5c8b --- /dev/null +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { combineLatest, Observable, Subject } from 'rxjs'; +import { last, map, startWith, switchMap } from 'rxjs/operators'; +import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; +import { LogEntryCursor } from '../../../../common/log_entry'; +import { LogEntriesSearchRequestQuery } from '../../../../common/search_strategies/log_entries/log_entries'; +import { flattenDataSearchResponseDescriptor } from '../../../utils/data_search'; +import { useObservable, useObservableState } from '../../../utils/use_observable'; +import { useLogEntriesAfterRequest } from './use_fetch_log_entries_after'; +import { useLogEntriesBeforeRequest } from './use_fetch_log_entries_before'; + +export const useFetchLogEntriesAround = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: LogEntriesSearchRequestQuery; + sourceId: string; + startTimestamp: number; +}) => { + const { fetchLogEntriesBefore } = useLogEntriesBeforeRequest({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, + }); + + const { fetchLogEntriesAfter } = useLogEntriesAfterRequest({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, + }); + + type LogEntriesBeforeRequest = NonNullable>; + type LogEntriesAfterRequest = NonNullable>; + + const logEntriesAroundSearchRequests$ = useObservable( + () => new Subject<[LogEntriesBeforeRequest, Observable]>(), + [] + ); + + const fetchLogEntriesAround = useCallback( + (cursor: LogEntryCursor, size: number) => { + const logEntriesBeforeSearchRequest = fetchLogEntriesBefore(cursor, Math.floor(size / 2)); + + if (logEntriesBeforeSearchRequest == null) { + return; + } + + const logEntriesAfterSearchRequest$ = flattenDataSearchResponseDescriptor( + logEntriesBeforeSearchRequest + ).pipe( + last(), // in the future we could start earlier if we receive partial results already + map((lastBeforeSearchResponse) => { + const cursorAfter = lastBeforeSearchResponse.response.data?.bottomCursor ?? { + time: cursor.time - 1, + tiebreaker: 0, + }; + + const logEntriesAfterSearchRequest = fetchLogEntriesAfter( + cursorAfter, + Math.ceil(size / 2) + ); + + if (logEntriesAfterSearchRequest == null) { + throw new Error('Failed to create request: no request args given'); + } + + return logEntriesAfterSearchRequest; + }) + ); + + logEntriesAroundSearchRequests$.next([ + logEntriesBeforeSearchRequest, + logEntriesAfterSearchRequest$, + ]); + }, + [fetchLogEntriesAfter, fetchLogEntriesBefore, logEntriesAroundSearchRequests$] + ); + + const logEntriesAroundSearchResponses$ = useObservable( + (inputs$) => + inputs$.pipe( + switchMap(([currentSearchRequests$]) => + currentSearchRequests$.pipe( + switchMap(([beforeRequest, afterRequest$]) => { + const beforeResponse$ = flattenDataSearchResponseDescriptor(beforeRequest); + const afterResponse$ = afterRequest$.pipe( + switchMap(flattenDataSearchResponseDescriptor), + startWith(undefined) // emit "before" response even if "after" hasn't started yet + ); + return combineLatest([beforeResponse$, afterResponse$]); + }), + map(([beforeResponse, afterResponse]) => { + const loadedBefore = beforeResponse.response.loaded; + const loadedAfter = afterResponse?.response.loaded; + const totalBefore = beforeResponse.response.total; + const totalAfter = afterResponse?.response.total; + + return { + before: beforeResponse, + after: afterResponse, + combined: { + isRunning: + (beforeResponse.response.isRunning || afterResponse?.response.isRunning) ?? + false, + isPartial: + (beforeResponse.response.isPartial || afterResponse?.response.isPartial) ?? + false, + loaded: + loadedBefore != null || loadedAfter != null + ? (loadedBefore ?? 0) + (loadedAfter ?? 0) + : undefined, + total: + totalBefore != null || totalAfter != null + ? (totalBefore ?? 0) + (totalAfter ?? 0) + : undefined, + entries: [ + ...(beforeResponse.response.data?.entries ?? []), + ...(afterResponse?.response.data?.entries ?? []), + ], + errors: [ + ...(beforeResponse.response.errors ?? []), + ...(afterResponse?.response.errors ?? []), + ], + hasMoreBefore: beforeResponse.response.data?.hasMoreBefore, + hasMoreAfter: afterResponse?.response.data?.hasMoreAfter, + topCursor: beforeResponse.response.data?.topCursor, + bottomCursor: afterResponse?.response.data?.bottomCursor, + }, + }; + }) + ) + ) + ), + [logEntriesAroundSearchRequests$] + ); + + const { + latestValue: { + before: latestBeforeResponse, + after: latestAfterResponse, + combined: latestCombinedResponse, + }, + } = useObservableState(logEntriesAroundSearchResponses$, initialCombinedResponse); + + const cancelRequest = useCallback(() => { + latestBeforeResponse?.abortController.abort(); + latestAfterResponse?.abortController.abort(); + }, [latestBeforeResponse, latestAfterResponse]); + + return { + cancelRequest, + fetchLogEntriesAround, + isRequestRunning: latestCombinedResponse?.isRunning ?? false, + isResponsePartial: latestCombinedResponse?.isPartial ?? false, + loaded: latestCombinedResponse?.loaded, + logEntriesAroundSearchResponses$, + total: latestCombinedResponse?.total, + }; +}; + +const initialCombinedResponse = { + before: undefined, + after: undefined, + combined: undefined, +} as const; diff --git a/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts new file mode 100644 index 00000000000000..5553be11b9fef3 --- /dev/null +++ b/x-pack/plugins/infra/public/containers/logs/log_stream/use_fetch_log_entries_before.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { Observable } from 'rxjs'; +import { exhaustMap } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../../src/plugins/data/public'; +import { LogSourceColumnConfiguration } from '../../../../common/http_api/log_sources'; +import { LogEntryBeforeCursor } from '../../../../common/log_entry'; +import { decodeOrThrow } from '../../../../common/runtime_types'; +import { + logEntriesSearchRequestParamsRT, + LogEntriesSearchRequestQuery, + LogEntriesSearchResponsePayload, + logEntriesSearchResponsePayloadRT, + LOG_ENTRIES_SEARCH_STRATEGY, +} from '../../../../common/search_strategies/log_entries/log_entries'; +import { + flattenDataSearchResponseDescriptor, + normalizeDataSearchResponses, + ParsedDataSearchRequestDescriptor, + useDataSearch, + useDataSearchResponseState, +} from '../../../utils/data_search'; +import { useOperator } from '../../../utils/use_observable'; + +export const useLogEntriesBeforeRequest = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: LogEntriesSearchRequestQuery; + sourceId: string; + startTimestamp: number; +}) => { + const { + search: fetchLogEntriesBefore, + requests$: logEntriesBeforeSearchRequests$, + } = useDataSearch({ + getRequest: useCallback( + (cursor: LogEntryBeforeCursor['before'], size: number) => { + return !!sourceId + ? { + request: { + params: logEntriesSearchRequestParamsRT.encode({ + before: cursor, + columns: columnOverrides, + endTimestamp, + highlightPhrase, + query, + size, + sourceId, + startTimestamp, + }), + }, + options: { strategy: LOG_ENTRIES_SEARCH_STRATEGY }, + } + : null; + }, + [columnOverrides, endTimestamp, highlightPhrase, query, sourceId, startTimestamp] + ), + parseResponses: parseLogEntriesBeforeSearchResponses, + }); + + return { + fetchLogEntriesBefore, + logEntriesBeforeSearchRequests$, + }; +}; + +export const useLogEntriesBeforeResponse = ( + logEntriesBeforeSearchRequests$: Observable< + ParsedDataSearchRequestDescriptor + > +) => { + const logEntriesBeforeSearchResponse$ = useOperator( + logEntriesBeforeSearchRequests$, + flattenLogEntriesBeforeSearchResponse + ); + + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + total, + } = useDataSearchResponseState(logEntriesBeforeSearchResponse$); + + return { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesBeforeSearchRequests$, + logEntriesBeforeSearchResponse$, + total, + }; +}; + +export const useFetchLogEntriesBefore = ({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, +}: { + columnOverrides?: LogSourceColumnConfiguration[]; + endTimestamp: number; + highlightPhrase?: string; + query?: LogEntriesSearchRequestQuery; + sourceId: string; + startTimestamp: number; +}) => { + const { fetchLogEntriesBefore, logEntriesBeforeSearchRequests$ } = useLogEntriesBeforeRequest({ + columnOverrides, + endTimestamp, + highlightPhrase, + query, + sourceId, + startTimestamp, + }); + + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesBeforeSearchResponse$, + total, + } = useLogEntriesBeforeResponse(logEntriesBeforeSearchRequests$); + + return { + cancelRequest, + fetchLogEntriesBefore, + isRequestRunning, + isResponsePartial, + loaded, + logEntriesBeforeSearchResponse$, + total, + }; +}; + +export const parseLogEntriesBeforeSearchResponses = normalizeDataSearchResponses( + null, + decodeOrThrow(logEntriesSearchResponsePayloadRT) +); + +const flattenLogEntriesBeforeSearchResponse = exhaustMap(flattenDataSearchResponseDescriptor); diff --git a/x-pack/plugins/infra/public/graphql/introspection.json b/x-pack/plugins/infra/public/graphql/introspection.json index 5d351f3259ac5f..efdca72c1383aa 100644 --- a/x-pack/plugins/infra/public/graphql/introspection.json +++ b/x-pack/plugins/infra/public/graphql/introspection.json @@ -137,155 +137,6 @@ "isDeprecated": false, "deprecationReason": null }, - { - "name": "logEntriesAround", - "description": "A consecutive span of log entries surrounding a point in time", - "args": [ - { - "name": "key", - "description": "The sort key that corresponds to the point in time", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "countBefore", - "description": "The maximum number of preceding to return", - "type": { "kind": "SCALAR", "name": "Int", "ofType": null }, - "defaultValue": "0" - }, - { - "name": "countAfter", - "description": "The maximum number of following to return", - "type": { "kind": "SCALAR", "name": "Int", "ofType": null }, - "defaultValue": "0" - }, - { - "name": "filterQuery", - "description": "The query to filter the log entries by", - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraLogEntryInterval", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "logEntriesBetween", - "description": "A consecutive span of log entries within an interval", - "args": [ - { - "name": "startKey", - "description": "The sort key that corresponds to the start of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "endKey", - "description": "The sort key that corresponds to the end of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "filterQuery", - "description": "The query to filter the log entries by", - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraLogEntryInterval", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "logEntryHighlights", - "description": "Sequences of log entries matching sets of highlighting queries within an interval", - "args": [ - { - "name": "startKey", - "description": "The sort key that corresponds to the start of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "endKey", - "description": "The sort key that corresponds to the end of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "INPUT_OBJECT", "name": "InfraTimeKeyInput", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "filterQuery", - "description": "The query to filter the log entries by", - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "defaultValue": null - }, - { - "name": "highlights", - "description": "The highlighting to apply to the log entries", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "InfraLogEntryHighlightInput", - "ofType": null - } - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraLogEntryInterval", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, { "name": "snapshot", "description": "A snapshot of nodes", @@ -993,37 +844,6 @@ "enumValues": null, "possibleTypes": null }, - { - "kind": "INPUT_OBJECT", - "name": "InfraTimeKeyInput", - "description": "", - "fields": null, - "inputFields": [ - { - "name": "time", - "description": "", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "tiebreaker", - "description": "", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, { "kind": "SCALAR", "name": "Int", @@ -1034,486 +854,6 @@ "enumValues": null, "possibleTypes": null }, - { - "kind": "OBJECT", - "name": "InfraLogEntryInterval", - "description": "A consecutive sequence of log entries", - "fields": [ - { - "name": "start", - "description": "The key corresponding to the start of the interval covered by the entries", - "args": [], - "type": { "kind": "OBJECT", "name": "InfraTimeKey", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "end", - "description": "The key corresponding to the end of the interval covered by the entries", - "args": [], - "type": { "kind": "OBJECT", "name": "InfraTimeKey", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hasMoreBefore", - "description": "Whether there are more log entries available before the start", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Boolean", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hasMoreAfter", - "description": "Whether there are more log entries available after the end", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Boolean", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "filterQuery", - "description": "The query the log entries were filtered by", - "args": [], - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "highlightQuery", - "description": "The query the log entries were highlighted with", - "args": [], - "type": { "kind": "SCALAR", "name": "String", "ofType": null }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "entries", - "description": "A list of the log entries", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraLogEntry", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraTimeKey", - "description": "A representation of the log entry's position in the event stream", - "fields": [ - { - "name": "time", - "description": "The timestamp of the event that the log entry corresponds to", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "tiebreaker", - "description": "The tiebreaker that disambiguates events with the same timestamp", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraLogEntry", - "description": "A log entry", - "fields": [ - { - "name": "key", - "description": "A unique representation of the log entry's position in the event stream", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "OBJECT", "name": "InfraTimeKey", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "gid", - "description": "The log entry's id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "source", - "description": "The source id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "columns", - "description": "The columns used for rendering the log entry", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "UNION", "name": "InfraLogEntryColumn", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "UNION", - "name": "InfraLogEntryColumn", - "description": "A column of a log entry", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { "kind": "OBJECT", "name": "InfraLogEntryTimestampColumn", "ofType": null }, - { "kind": "OBJECT", "name": "InfraLogEntryMessageColumn", "ofType": null }, - { "kind": "OBJECT", "name": "InfraLogEntryFieldColumn", "ofType": null } - ] - }, - { - "kind": "OBJECT", - "name": "InfraLogEntryTimestampColumn", - "description": "A special built-in column that contains the log entry's timestamp", - "fields": [ - { - "name": "columnId", - "description": "The id of the corresponding column configuration", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "ID", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "timestamp", - "description": "The timestamp", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Float", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraLogEntryMessageColumn", - "description": "A special built-in column that contains the log entry's constructed message", - "fields": [ - { - "name": "columnId", - "description": "The id of the corresponding column configuration", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "ID", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "message", - "description": "A list of the formatted log entry segments", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "UNION", "name": "InfraLogMessageSegment", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "UNION", - "name": "InfraLogMessageSegment", - "description": "A segment of the log entry message", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { "kind": "OBJECT", "name": "InfraLogMessageFieldSegment", "ofType": null }, - { "kind": "OBJECT", "name": "InfraLogMessageConstantSegment", "ofType": null } - ] - }, - { - "kind": "OBJECT", - "name": "InfraLogMessageFieldSegment", - "description": "A segment of the log entry message that was derived from a field", - "fields": [ - { - "name": "field", - "description": "The field the segment was derived from", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "value", - "description": "The segment's message", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "highlights", - "description": "A list of highlighted substrings of the value", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraLogMessageConstantSegment", - "description": "A segment of the log entry message that was derived from a string literal", - "fields": [ - { - "name": "constant", - "description": "The segment's message", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InfraLogEntryFieldColumn", - "description": "A column that contains the value of a field of the log entry", - "fields": [ - { - "name": "columnId", - "description": "The id of the corresponding column configuration", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "ID", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "field", - "description": "The field name of the column", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "value", - "description": "The value of the field in the log entry", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "highlights", - "description": "A list of highlighted substrings of the value", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "InfraLogEntryHighlightInput", - "description": "A highlighting definition", - "fields": null, - "inputFields": [ - { - "name": "query", - "description": "The query to highlight by", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "String", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "countBefore", - "description": "The number of highlighted documents to include beyond the beginning of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Int", "ofType": null } - }, - "defaultValue": null - }, - { - "name": "countAfter", - "description": "The number of highlighted documents to include beyond the end of the interval", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { "kind": "SCALAR", "name": "Int", "ofType": null } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, { "kind": "INPUT_OBJECT", "name": "InfraTimerangeInput", diff --git a/x-pack/plugins/infra/public/graphql/types.ts b/x-pack/plugins/infra/public/graphql/types.ts index f0f74c34a19e6c..eb025ee4efd79b 100644 --- a/x-pack/plugins/infra/public/graphql/types.ts +++ b/x-pack/plugins/infra/public/graphql/types.ts @@ -30,12 +30,6 @@ export interface InfraSource { configuration: InfraSourceConfiguration; /** The status of the source */ status: InfraSourceStatus; - /** A consecutive span of log entries surrounding a point in time */ - logEntriesAround: InfraLogEntryInterval; - /** A consecutive span of log entries within an interval */ - logEntriesBetween: InfraLogEntryInterval; - /** Sequences of log entries matching sets of highlighting queries within an interval */ - logEntryHighlights: InfraLogEntryInterval[]; /** A snapshot of nodes */ snapshot?: InfraSnapshotResponse | null; @@ -135,80 +129,6 @@ export interface InfraIndexField { /** Whether the field should be displayed based on event.module and a ECS allowed list */ displayable: boolean; } -/** A consecutive sequence of log entries */ -export interface InfraLogEntryInterval { - /** The key corresponding to the start of the interval covered by the entries */ - start?: InfraTimeKey | null; - /** The key corresponding to the end of the interval covered by the entries */ - end?: InfraTimeKey | null; - /** Whether there are more log entries available before the start */ - hasMoreBefore: boolean; - /** Whether there are more log entries available after the end */ - hasMoreAfter: boolean; - /** The query the log entries were filtered by */ - filterQuery?: string | null; - /** The query the log entries were highlighted with */ - highlightQuery?: string | null; - /** A list of the log entries */ - entries: InfraLogEntry[]; -} -/** A representation of the log entry's position in the event stream */ -export interface InfraTimeKey { - /** The timestamp of the event that the log entry corresponds to */ - time: number; - /** The tiebreaker that disambiguates events with the same timestamp */ - tiebreaker: number; -} -/** A log entry */ -export interface InfraLogEntry { - /** A unique representation of the log entry's position in the event stream */ - key: InfraTimeKey; - /** The log entry's id */ - gid: string; - /** The source id */ - source: string; - /** The columns used for rendering the log entry */ - columns: InfraLogEntryColumn[]; -} -/** A special built-in column that contains the log entry's timestamp */ -export interface InfraLogEntryTimestampColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The timestamp */ - timestamp: number; -} -/** A special built-in column that contains the log entry's constructed message */ -export interface InfraLogEntryMessageColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** A list of the formatted log entry segments */ - message: InfraLogMessageSegment[]; -} -/** A segment of the log entry message that was derived from a field */ -export interface InfraLogMessageFieldSegment { - /** The field the segment was derived from */ - field: string; - /** The segment's message */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} -/** A segment of the log entry message that was derived from a string literal */ -export interface InfraLogMessageConstantSegment { - /** The segment's message */ - constant: string; -} -/** A column that contains the value of a field of the log entry */ -export interface InfraLogEntryFieldColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The field name of the column */ - field: string; - /** The value of the field in the log entry */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} export interface InfraSnapshotResponse { /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ @@ -282,21 +202,6 @@ export interface DeleteSourceResult { // InputTypes // ==================================================== -export interface InfraTimeKeyInput { - time: number; - - tiebreaker: number; -} -/** A highlighting definition */ -export interface InfraLogEntryHighlightInput { - /** The query to highlight by */ - query: string; - /** The number of highlighted documents to include beyond the beginning of the interval */ - countBefore: number; - /** The number of highlighted documents to include beyond the end of the interval */ - countAfter: number; -} - export interface InfraTimerangeInput { /** The interval string to use for last bucket. The format is '{value}{unit}'. For example '5m' would return the metrics for the last 5 minutes of the timespan. */ interval: string; @@ -387,34 +292,6 @@ export interface SourceQueryArgs { /** The id of the source */ id: string; } -export interface LogEntriesAroundInfraSourceArgs { - /** The sort key that corresponds to the point in time */ - key: InfraTimeKeyInput; - /** The maximum number of preceding to return */ - countBefore?: number | null; - /** The maximum number of following to return */ - countAfter?: number | null; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntriesBetweenInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntryHighlightsInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - /** The highlighting to apply to the log entries */ - highlights: InfraLogEntryHighlightInput[]; -} export interface SnapshotInfraSourceArgs { timerange: InfraTimerangeInput; @@ -571,15 +448,6 @@ export type InfraSourceLogColumn = | InfraSourceMessageLogColumn | InfraSourceFieldLogColumn; -/** A column of a log entry */ -export type InfraLogEntryColumn = - | InfraLogEntryTimestampColumn - | InfraLogEntryMessageColumn - | InfraLogEntryFieldColumn; - -/** A segment of the log entry message */ -export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessageConstantSegment; - // ==================================================== // END: Typescript template // ==================================================== @@ -588,46 +456,6 @@ export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessa // Documents // ==================================================== -export namespace LogEntryHighlightsQuery { - export type Variables = { - sourceId?: string | null; - startKey: InfraTimeKeyInput; - endKey: InfraTimeKeyInput; - filterQuery?: string | null; - highlights: InfraLogEntryHighlightInput[]; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'InfraSource'; - - id: string; - - logEntryHighlights: LogEntryHighlights[]; - }; - - export type LogEntryHighlights = { - __typename?: 'InfraLogEntryInterval'; - - start?: Start | null; - - end?: End | null; - - entries: Entries[]; - }; - - export type Start = InfraTimeKeyFields.Fragment; - - export type End = InfraTimeKeyFields.Fragment; - - export type Entries = InfraLogEntryHighlightFields.Fragment; -} - export namespace MetricsQuery { export type Variables = { sourceId: string; @@ -826,50 +654,6 @@ export namespace WaffleNodesQuery { }; } -export namespace LogEntries { - export type Variables = { - sourceId?: string | null; - timeKey: InfraTimeKeyInput; - countBefore?: number | null; - countAfter?: number | null; - filterQuery?: string | null; - }; - - export type Query = { - __typename?: 'Query'; - - source: Source; - }; - - export type Source = { - __typename?: 'InfraSource'; - - id: string; - - logEntriesAround: LogEntriesAround; - }; - - export type LogEntriesAround = { - __typename?: 'InfraLogEntryInterval'; - - start?: Start | null; - - end?: End | null; - - hasMoreBefore: boolean; - - hasMoreAfter: boolean; - - entries: Entries[]; - }; - - export type Start = InfraTimeKeyFields.Fragment; - - export type End = InfraTimeKeyFields.Fragment; - - export type Entries = InfraLogEntryFields.Fragment; -} - export namespace SourceConfigurationFields { export type Fragment = { __typename?: 'InfraSourceConfiguration'; @@ -1000,124 +784,3 @@ export namespace InfraSourceFields { origin: string; }; } - -export namespace InfraLogEntryFields { - export type Fragment = { - __typename?: 'InfraLogEntry'; - - gid: string; - - key: Key; - - columns: Columns[]; - }; - - export type Key = { - __typename?: 'InfraTimeKey'; - - time: number; - - tiebreaker: number; - }; - - export type Columns = - | InfraLogEntryTimestampColumnInlineFragment - | InfraLogEntryMessageColumnInlineFragment - | InfraLogEntryFieldColumnInlineFragment; - - export type InfraLogEntryTimestampColumnInlineFragment = { - __typename?: 'InfraLogEntryTimestampColumn'; - - columnId: string; - - timestamp: number; - }; - - export type InfraLogEntryMessageColumnInlineFragment = { - __typename?: 'InfraLogEntryMessageColumn'; - - columnId: string; - - message: Message[]; - }; - - export type Message = - | InfraLogMessageFieldSegmentInlineFragment - | InfraLogMessageConstantSegmentInlineFragment; - - export type InfraLogMessageFieldSegmentInlineFragment = { - __typename?: 'InfraLogMessageFieldSegment'; - - field: string; - - value: string; - }; - - export type InfraLogMessageConstantSegmentInlineFragment = { - __typename?: 'InfraLogMessageConstantSegment'; - - constant: string; - }; - - export type InfraLogEntryFieldColumnInlineFragment = { - __typename?: 'InfraLogEntryFieldColumn'; - - columnId: string; - - field: string; - - value: string; - }; -} - -export namespace InfraLogEntryHighlightFields { - export type Fragment = { - __typename?: 'InfraLogEntry'; - - gid: string; - - key: Key; - - columns: Columns[]; - }; - - export type Key = { - __typename?: 'InfraTimeKey'; - - time: number; - - tiebreaker: number; - }; - - export type Columns = - | InfraLogEntryMessageColumnInlineFragment - | InfraLogEntryFieldColumnInlineFragment; - - export type InfraLogEntryMessageColumnInlineFragment = { - __typename?: 'InfraLogEntryMessageColumn'; - - columnId: string; - - message: Message[]; - }; - - export type Message = InfraLogMessageFieldSegmentInlineFragment; - - export type InfraLogMessageFieldSegmentInlineFragment = { - __typename?: 'InfraLogMessageFieldSegment'; - - field: string; - - highlights: string[]; - }; - - export type InfraLogEntryFieldColumnInlineFragment = { - __typename?: 'InfraLogEntryFieldColumn'; - - columnId: string; - - field: string; - - highlights: string[]; - }; -} diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx index e24fdd06bc6d9b..83659ace3ce548 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx @@ -127,6 +127,7 @@ export const CategoryExampleMessage: React.FunctionComponent<{ onClick: () => { const logEntry: LogEntry = { id, + index: '', // TODO: use real index when loading via async search context, cursor: { time: timestamp, tiebreaker }, columns: [], diff --git a/x-pack/plugins/infra/public/test_utils/entries.ts b/x-pack/plugins/infra/public/test_utils/entries.ts index 96737fb1753650..1633b9d8dc0760 100644 --- a/x-pack/plugins/infra/public/test_utils/entries.ts +++ b/x-pack/plugins/infra/public/test_utils/entries.ts @@ -28,6 +28,7 @@ export function generateFakeEntries( const timestamp = i === count - 1 ? endTimestamp : startTimestamp + timestampStep * i; entries.push({ id: `entry-${i}`, + index: 'logs-fake', context: {}, cursor: { time: timestamp, tiebreaker: i }, columns: columns.map((column) => { diff --git a/x-pack/plugins/infra/public/utils/data_search/data_search.stories.mdx b/x-pack/plugins/infra/public/utils/data_search/data_search.stories.mdx index a698b806b4cd70..a8854692caa36c 100644 --- a/x-pack/plugins/infra/public/utils/data_search/data_search.stories.mdx +++ b/x-pack/plugins/infra/public/utils/data_search/data_search.stories.mdx @@ -43,15 +43,35 @@ be issued by calling the returned `search()` function. For each new request the hook emits an object describing the request and its state in the `requests$` `Observable`. +Since the specific response shape depends on the data strategy used, the hook +takes a projection function, that is responsible for decoding the response in +an appropriate way. Because most response projections follow a similar pattern +there's a helper `normalizeDataSearchResponses(initialResponse, +parseRawResponse)`, which generates an RxJS operator, that... + +- emits an initial response containing the given `initialResponse` value +- applies `parseRawResponse` to the `rawResponse` property of each emitted response +- transforms transport layer errors as well as parsing errors into + `SearchStrategyError`s + ```typescript +const parseMyCustomSearchResponse = normalizeDataSearchResponses( + 'initial value', + decodeOrThrow(myCustomSearchResponsePayloadRT) +); + const { search, requests$ } = useDataSearch({ getRequest: useCallback((searchTerm: string) => ({ request: { params: { searchTerm - } - } - }), []); + }, + options: { + strategy: 'my-custom-search-strategy', + }, + }, + }), []), + parseResponses: parseMyCustomSearchResponse, }); ``` @@ -68,10 +88,6 @@ observables are unsubscribed from for proper cancellation if a new request has been created. This uses RxJS's `switchMap()` operator under the hood. The hook also makes sure that all observables are unsubscribed from on unmount. -Since the specific response shape depends on the data strategy used, the hook -takes a projection function, that is responsible for decoding the response in -an appropriate way. - A request can fail due to various reasons that include servers-side errors, Elasticsearch shard failures and network failures. The intention is to map all of them to a common `SearchStrategyError` interface. While the @@ -94,11 +110,7 @@ const { latestResponseErrors, loaded, total, -} = useLatestPartialDataSearchResponse( - requests$, - 'initialValue', - useMemo(() => decodeOrThrow(mySearchStrategyResponsePayloadRT), []), -); +} = useLatestPartialDataSearchResponse(requests$); ``` ## Representing the request state to the user diff --git a/x-pack/plugins/infra/public/utils/data_search/flatten_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/flatten_data_search_response.ts new file mode 100644 index 00000000000000..98df6d441bd806 --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/flatten_data_search_response.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { map } from 'rxjs/operators'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { ParsedDataSearchRequestDescriptor } from './types'; + +export const flattenDataSearchResponseDescriptor = < + Request extends IKibanaSearchRequest, + Response +>({ + abortController, + options, + request, + response$, +}: ParsedDataSearchRequestDescriptor) => + response$.pipe( + map((response) => { + return { + abortController, + options, + request, + response, + }; + }) + ); diff --git a/x-pack/plugins/infra/public/utils/data_search/index.ts b/x-pack/plugins/infra/public/utils/data_search/index.ts index c08ab0727fd904..10beba4aa4fdcd 100644 --- a/x-pack/plugins/infra/public/utils/data_search/index.ts +++ b/x-pack/plugins/infra/public/utils/data_search/index.ts @@ -4,6 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ +export * from './flatten_data_search_response'; +export * from './normalize_data_search_responses'; export * from './types'; export * from './use_data_search_request'; +export * from './use_data_search_response_state'; export * from './use_latest_partial_data_search_response'; diff --git a/x-pack/plugins/infra/public/utils/data_search/normalize_data_search_responses.ts b/x-pack/plugins/infra/public/utils/data_search/normalize_data_search_responses.ts new file mode 100644 index 00000000000000..5046cc128a835a --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/normalize_data_search_responses.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Observable, of } from 'rxjs'; +import { catchError, map, startWith } from 'rxjs/operators'; +import { IKibanaSearchResponse } from '../../../../../../src/plugins/data/public'; +import { AbortError } from '../../../../../../src/plugins/kibana_utils/public'; +import { SearchStrategyError } from '../../../common/search_strategies/common/errors'; +import { ParsedKibanaSearchResponse } from './types'; + +export type RawResponseParser = ( + rawResponse: RawResponse +) => { data: Response; errors?: SearchStrategyError[] }; + +/** + * An operator factory that normalizes each {@link IKibanaSearchResponse} by + * parsing it into a {@link ParsedKibanaSearchResponse} and adding initial + * responses and error handling. + * + * @param initialResponse - The initial value to emit when a new request is + * handled. + * @param projectResponse - The projection function to apply to each response + * payload. It should validate that the response payload is of the type {@link + * RawResponse} and decode it to a {@link Response}. + * + * @return An operator that adds parsing and error handling transformations to + * each response payload using the arguments given above. + */ +export const normalizeDataSearchResponses = ( + initialResponse: InitialResponse, + parseRawResponse: RawResponseParser +) => ( + response$: Observable> +): Observable> => + response$.pipe( + map((response) => { + const { data, errors = [] } = parseRawResponse(response.rawResponse); + return { + data, + errors, + isPartial: response.isPartial ?? false, + isRunning: response.isRunning ?? false, + loaded: response.loaded, + total: response.total, + }; + }), + startWith({ + data: initialResponse, + errors: [], + isPartial: true, + isRunning: true, + loaded: 0, + total: undefined, + }), + catchError((error) => + of({ + data: initialResponse, + errors: [ + error instanceof AbortError + ? { + type: 'aborted' as const, + } + : { + type: 'generic' as const, + message: `${error.message ?? error}`, + }, + ], + isPartial: true, + isRunning: false, + loaded: 0, + total: undefined, + }) + ) + ); diff --git a/x-pack/plugins/infra/public/utils/data_search/types.ts b/x-pack/plugins/infra/public/utils/data_search/types.ts index ba0a4c639dae4a..4fcb5898ea5bd8 100644 --- a/x-pack/plugins/infra/public/utils/data_search/types.ts +++ b/x-pack/plugins/infra/public/utils/data_search/types.ts @@ -19,7 +19,17 @@ export interface DataSearchRequestDescriptor { +export interface ParsedDataSearchRequestDescriptor< + Request extends IKibanaSearchRequest, + ResponseData +> { + request: Request; + options: ISearchOptions; + response$: Observable>; + abortController: AbortController; +} + +export interface ParsedKibanaSearchResponse { total?: number; loaded?: number; isRunning: boolean; @@ -28,9 +38,12 @@ export interface NormalizedKibanaSearchResponse { errors: SearchStrategyError[]; } -export interface DataSearchResponseDescriptor { +export interface ParsedDataSearchResponseDescriptor< + Request extends IKibanaSearchRequest, + Response +> { request: Request; options: ISearchOptions; - response: NormalizedKibanaSearchResponse; + response: ParsedKibanaSearchResponse; abortController: AbortController; } diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.test.tsx b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.test.tsx index 87c091f12ad90a..780476abb7b1bf 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.test.tsx +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.test.tsx @@ -17,6 +17,7 @@ import { import { dataPluginMock } from '../../../../../../src/plugins/data/public/mocks'; import { createKibanaReactContext } from '../../../../../../src/plugins/kibana_react/public'; import { PluginKibanaContextValue } from '../../hooks/use_kibana'; +import { normalizeDataSearchResponses } from './normalize_data_search_responses'; import { useDataSearch } from './use_data_search_request'; describe('useDataSearch hook', () => { @@ -34,6 +35,7 @@ describe('useDataSearch hook', () => { () => useDataSearch({ getRequest, + parseResponses: noopParseResponse, }), { wrapper: ({ children }) => {children}, @@ -48,7 +50,7 @@ describe('useDataSearch hook', () => { expect(dataMock.search.search).not.toHaveBeenCalled(); }); - it('creates search requests with the given params and options', async () => { + it('creates search requests with the given params and options and parses the responses', async () => { const dataMock = createDataPluginMock(); const searchResponseMock$ = of({ rawResponse: { @@ -78,6 +80,7 @@ describe('useDataSearch hook', () => { () => useDataSearch({ getRequest, + parseResponses: noopParseResponse, }), { wrapper: ({ children }) => {children}, @@ -112,10 +115,11 @@ describe('useDataSearch hook', () => { }); expect(firstRequest).toHaveProperty('options.strategy', 'test-search-strategy'); expect(firstRequest).toHaveProperty('response$', expect.any(Observable)); - await expect(firstRequest.response$.toPromise()).resolves.toEqual({ - rawResponse: { - firstKey: 'firstValue', + await expect(firstRequest.response$.toPromise()).resolves.toMatchObject({ + data: { + firstKey: 'firstValue', // because this specific response parser just copies the raw response }, + errors: [], }); }); @@ -145,6 +149,7 @@ describe('useDataSearch hook', () => { () => useDataSearch({ getRequest, + parseResponses: noopParseResponse, }), { wrapper: ({ children }) => {children}, @@ -186,3 +191,8 @@ const createDataPluginMock = () => { }; return dataMock; }; + +const noopParseResponse = normalizeDataSearchResponses( + null, + (response: Response) => ({ data: response }) +); diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts index a23f06adc0353c..0f1686a93be829 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_request.ts @@ -5,8 +5,8 @@ */ import { useCallback } from 'react'; -import { Subject } from 'rxjs'; -import { map, share, switchMap, tap } from 'rxjs/operators'; +import { OperatorFunction, Subject } from 'rxjs'; +import { share, tap } from 'rxjs/operators'; import { IKibanaSearchRequest, IKibanaSearchResponse, @@ -14,6 +14,7 @@ import { } from '../../../../../../src/plugins/data/public'; import { useKibanaContextForPlugin } from '../../hooks/use_kibana'; import { tapUnsubscribe, useObservable } from '../use_observable'; +import { ParsedDataSearchRequestDescriptor, ParsedKibanaSearchResponse } from './types'; export type DataSearchRequestFactory = ( ...args: Args @@ -25,69 +26,74 @@ export type DataSearchRequestFactory = OperatorFunction< + IKibanaSearchResponse, + ParsedKibanaSearchResponse +>; + export const useDataSearch = < RequestFactoryArgs extends any[], - Request extends IKibanaSearchRequest, - RawResponse + RequestParams, + Request extends IKibanaSearchRequest, + RawResponse, + Response >({ getRequest, + parseResponses, }: { getRequest: DataSearchRequestFactory; + parseResponses: ParseResponsesOperator; }) => { const { services } = useKibanaContextForPlugin(); - const request$ = useObservable( - () => new Subject<{ request: Request; options: ISearchOptions }>(), - [] - ); const requests$ = useObservable( - (inputs$) => - inputs$.pipe( - switchMap(([currentRequest$]) => currentRequest$), - map(({ request, options }) => { - const abortController = new AbortController(); - let isAbortable = true; - - return { - abortController, - request, - options, - response$: services.data.search - .search>(request, { - abortSignal: abortController.signal, - ...options, - }) - .pipe( - // avoid aborting failed or completed requests - tap({ - error: () => { - isAbortable = false; - }, - complete: () => { - isAbortable = false; - }, - }), - tapUnsubscribe(() => { - if (isAbortable) { - abortController.abort(); - } - }), - share() - ), - }; - }) - ), - [request$] + () => new Subject>(), + [] ); const search = useCallback( (...args: RequestFactoryArgs) => { - const request = getRequest(...args); + const requestArgs = getRequest(...args); - if (request) { - request$.next(request); + if (requestArgs == null) { + return; } + + const abortController = new AbortController(); + let isAbortable = true; + + const newRequestDescriptor = { + ...requestArgs, + abortController, + response$: services.data.search + .search>(requestArgs.request, { + abortSignal: abortController.signal, + ...requestArgs.options, + }) + .pipe( + // avoid aborting failed or completed requests + tap({ + error: () => { + isAbortable = false; + }, + complete: () => { + isAbortable = false; + }, + }), + tapUnsubscribe(() => { + if (isAbortable) { + abortController.abort(); + } + }), + parseResponses, + share() + ), + }; + + requests$.next(newRequestDescriptor); + + return newRequestDescriptor; }, - [getRequest, request$] + [getRequest, services.data.search, parseResponses, requests$] ); return { diff --git a/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts b/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts new file mode 100644 index 00000000000000..3b37b80f26cdc2 --- /dev/null +++ b/x-pack/plugins/infra/public/utils/data_search/use_data_search_response_state.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useCallback } from 'react'; +import { Observable } from 'rxjs'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { useObservableState } from '../use_observable'; +import { ParsedDataSearchResponseDescriptor } from './types'; + +export const useDataSearchResponseState = < + Request extends IKibanaSearchRequest, + Response, + InitialResponse +>( + response$: Observable> +) => { + const { latestValue } = useObservableState(response$, undefined); + + const cancelRequest = useCallback(() => { + latestValue?.abortController.abort(); + }, [latestValue]); + + return { + cancelRequest, + isRequestRunning: latestValue?.response.isRunning ?? false, + isResponsePartial: latestValue?.response.isPartial ?? false, + latestResponseData: latestValue?.response.data, + latestResponseErrors: latestValue?.response.errors, + loaded: latestValue?.response.loaded, + total: latestValue?.response.total, + }; +}; diff --git a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.test.tsx b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.test.tsx index 4c336aa1107a22..864d92f43bc17e 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.test.tsx +++ b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.test.tsx @@ -5,12 +5,9 @@ */ import { act, renderHook } from '@testing-library/react-hooks'; -import { Observable, of, Subject } from 'rxjs'; -import { - IKibanaSearchRequest, - IKibanaSearchResponse, -} from '../../../../../../src/plugins/data/public'; -import { DataSearchRequestDescriptor } from './types'; +import { BehaviorSubject, Observable, of, Subject } from 'rxjs'; +import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; +import { ParsedDataSearchRequestDescriptor, ParsedKibanaSearchResponse } from './types'; import { useLatestPartialDataSearchResponse } from './use_latest_partial_data_search_response'; describe('useLatestPartialDataSearchResponse hook', () => { @@ -19,25 +16,31 @@ describe('useLatestPartialDataSearchResponse hook', () => { abortController: new AbortController(), options: {}, request: { params: 'firstRequestParam' }, - response$: new Subject>(), + response$: new BehaviorSubject>({ + data: 'initial', + isRunning: true, + isPartial: true, + errors: [], + }), }; const secondRequest = { abortController: new AbortController(), options: {}, request: { params: 'secondRequestParam' }, - response$: new Subject>(), + response$: new BehaviorSubject>({ + data: 'initial', + isRunning: true, + isPartial: true, + errors: [], + }), }; const requests$ = new Subject< - DataSearchRequestDescriptor, string> + ParsedDataSearchRequestDescriptor, string> >(); - const { result } = renderHook(() => - useLatestPartialDataSearchResponse(requests$, 'initial', (response) => ({ - data: `projection of ${response}`, - })) - ); + const { result } = renderHook(() => useLatestPartialDataSearchResponse(requests$)); expect(result).toHaveProperty('current.isRequestRunning', false); expect(result).toHaveProperty('current.latestResponseData', undefined); @@ -52,37 +55,43 @@ describe('useLatestPartialDataSearchResponse hook', () => { // first response of the first request arrives act(() => { - firstRequest.response$.next({ rawResponse: 'request-1-response-1', isRunning: true }); + firstRequest.response$.next({ + data: 'request-1-response-1', + isRunning: true, + isPartial: true, + errors: [], + }); }); expect(result).toHaveProperty('current.isRequestRunning', true); - expect(result).toHaveProperty( - 'current.latestResponseData', - 'projection of request-1-response-1' - ); + expect(result).toHaveProperty('current.latestResponseData', 'request-1-response-1'); // second request is started before the second response of the first request arrives act(() => { requests$.next(secondRequest); - secondRequest.response$.next({ rawResponse: 'request-2-response-1', isRunning: true }); + secondRequest.response$.next({ + data: 'request-2-response-1', + isRunning: true, + isPartial: true, + errors: [], + }); }); expect(result).toHaveProperty('current.isRequestRunning', true); - expect(result).toHaveProperty( - 'current.latestResponseData', - 'projection of request-2-response-1' - ); + expect(result).toHaveProperty('current.latestResponseData', 'request-2-response-1'); // second response of the second request arrives act(() => { - secondRequest.response$.next({ rawResponse: 'request-2-response-2', isRunning: false }); + secondRequest.response$.next({ + data: 'request-2-response-2', + isRunning: false, + isPartial: false, + errors: [], + }); }); expect(result).toHaveProperty('current.isRequestRunning', false); - expect(result).toHaveProperty( - 'current.latestResponseData', - 'projection of request-2-response-2' - ); + expect(result).toHaveProperty('current.latestResponseData', 'request-2-response-2'); }); it("unsubscribes from the latest request's response observable on unmount", () => { @@ -92,20 +101,16 @@ describe('useLatestPartialDataSearchResponse hook', () => { abortController: new AbortController(), options: {}, request: { params: 'firstRequestParam' }, - response$: new Observable>(() => { + response$: new Observable>(() => { return onUnsubscribe; }), }; - const requests$ = of, string>>( + const requests$ = of, string>>( firstRequest ); - const { unmount } = renderHook(() => - useLatestPartialDataSearchResponse(requests$, 'initial', (response) => ({ - data: `projection of ${response}`, - })) - ); + const { unmount } = renderHook(() => useLatestPartialDataSearchResponse(requests$)); expect(onUnsubscribe).not.toHaveBeenCalled(); diff --git a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts index 71fd96283d0ef9..9366df8adbaf71 100644 --- a/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts +++ b/x-pack/plugins/infra/public/utils/data_search/use_latest_partial_data_search_response.ts @@ -4,111 +4,40 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useCallback } from 'react'; -import { Observable, of } from 'rxjs'; -import { catchError, map, startWith, switchMap } from 'rxjs/operators'; +import { Observable } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; import { IKibanaSearchRequest } from '../../../../../../src/plugins/data/public'; -import { AbortError } from '../../../../../../src/plugins/kibana_utils/public'; -import { SearchStrategyError } from '../../../common/search_strategies/common/errors'; -import { useLatest, useObservable, useObservableState } from '../use_observable'; -import { DataSearchRequestDescriptor, DataSearchResponseDescriptor } from './types'; +import { useOperator } from '../use_observable'; +import { flattenDataSearchResponseDescriptor } from './flatten_data_search_response'; +import { ParsedDataSearchRequestDescriptor, ParsedDataSearchResponseDescriptor } from './types'; +import { useDataSearchResponseState } from './use_data_search_response_state'; -export const useLatestPartialDataSearchResponse = < - Request extends IKibanaSearchRequest, - RawResponse, - Response, - InitialResponse ->( - requests$: Observable>, - initialResponse: InitialResponse, - projectResponse: (rawResponse: RawResponse) => { data: Response; errors?: SearchStrategyError[] } +export const useLatestPartialDataSearchResponse = ( + requests$: Observable> ) => { - const latestInitialResponse = useLatest(initialResponse); - const latestProjectResponse = useLatest(projectResponse); - const latestResponse$: Observable< - DataSearchResponseDescriptor - > = useObservable( - (inputs$) => - inputs$.pipe( - switchMap(([currentRequests$]) => - currentRequests$.pipe( - switchMap(({ abortController, options, request, response$ }) => - response$.pipe( - map((response) => { - const { data, errors = [] } = latestProjectResponse.current(response.rawResponse); - return { - abortController, - options, - request, - response: { - data, - errors, - isPartial: response.isPartial ?? false, - isRunning: response.isRunning ?? false, - loaded: response.loaded, - total: response.total, - }, - }; - }), - startWith({ - abortController, - options, - request, - response: { - data: latestInitialResponse.current, - errors: [], - isPartial: true, - isRunning: true, - loaded: 0, - total: undefined, - }, - }), - catchError((error) => - of({ - abortController, - options, - request, - response: { - data: latestInitialResponse.current, - errors: [ - error instanceof AbortError - ? { - type: 'aborted' as const, - } - : { - type: 'generic' as const, - message: `${error.message ?? error}`, - }, - ], - isPartial: true, - isRunning: false, - loaded: 0, - total: undefined, - }, - }) - ) - ) - ) - ) - ) - ), - [requests$] as const - ); - - const { latestValue } = useObservableState(latestResponse$, undefined); + ParsedDataSearchResponseDescriptor + > = useOperator(requests$, flattenLatestDataSearchResponse); - const cancelRequest = useCallback(() => { - latestValue?.abortController.abort(); - }, [latestValue]); + const { + cancelRequest, + isRequestRunning, + isResponsePartial, + latestResponseData, + latestResponseErrors, + loaded, + total, + } = useDataSearchResponseState(latestResponse$); return { cancelRequest, - isRequestRunning: latestValue?.response.isRunning ?? false, - isResponsePartial: latestValue?.response.isPartial ?? false, - latestResponseData: latestValue?.response.data, - latestResponseErrors: latestValue?.response.errors, - loaded: latestValue?.response.loaded, - total: latestValue?.response.total, + isRequestRunning, + isResponsePartial, + latestResponseData, + latestResponseErrors, + loaded, + total, }; }; + +const flattenLatestDataSearchResponse = switchMap(flattenDataSearchResponseDescriptor); diff --git a/x-pack/plugins/infra/public/utils/log_entry/log_entry.ts b/x-pack/plugins/infra/public/utils/log_entry/log_entry.ts index c69104ad6177e7..60034aea6be639 100644 --- a/x-pack/plugins/infra/public/utils/log_entry/log_entry.ts +++ b/x-pack/plugins/infra/public/utils/log_entry/log_entry.ts @@ -5,9 +5,7 @@ */ import { bisector } from 'd3-array'; - import { compareToTimeKey, getIndexAtTimeKey, TimeKey, UniqueTimeKey } from '../../../common/time'; -import { InfraLogEntryFields } from '../../graphql/types'; import { LogEntry, LogColumn, @@ -19,10 +17,6 @@ import { LogMessageConstantPart, } from '../../../common/log_entry'; -export type LogEntryMessageSegment = InfraLogEntryFields.Message; -export type LogEntryConstantMessageSegment = InfraLogEntryFields.InfraLogMessageConstantSegmentInlineFragment; -export type LogEntryFieldMessageSegment = InfraLogEntryFields.InfraLogMessageFieldSegmentInlineFragment; - export const getLogEntryKey = (entry: { cursor: TimeKey }) => entry.cursor; export const getUniqueLogEntryKey = (entry: { id: string; cursor: TimeKey }): UniqueTimeKey => ({ diff --git a/x-pack/plugins/infra/public/utils/log_entry/log_entry_highlight.ts b/x-pack/plugins/infra/public/utils/log_entry/log_entry_highlight.ts index 208316c693d4db..e14d938c426f90 100644 --- a/x-pack/plugins/infra/public/utils/log_entry/log_entry_highlight.ts +++ b/x-pack/plugins/infra/public/utils/log_entry/log_entry_highlight.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import { InfraLogEntryHighlightFields } from '../../graphql/types'; import { LogEntry, LogColumn, @@ -14,13 +13,6 @@ import { LogMessageFieldPart, } from '../../../common/log_entry'; -export type LogEntryHighlightColumn = InfraLogEntryHighlightFields.Columns; -export type LogEntryHighlightMessageColumn = InfraLogEntryHighlightFields.InfraLogEntryMessageColumnInlineFragment; -export type LogEntryHighlightFieldColumn = InfraLogEntryHighlightFields.InfraLogEntryFieldColumnInlineFragment; - -export type LogEntryHighlightMessageSegment = InfraLogEntryHighlightFields.Message | {}; -export type LogEntryHighlightFieldMessageSegment = InfraLogEntryHighlightFields.InfraLogMessageFieldSegmentInlineFragment; - export interface LogEntryHighlightsMap { [entryId: string]: LogEntry[]; } diff --git a/x-pack/plugins/infra/public/utils/use_observable.ts b/x-pack/plugins/infra/public/utils/use_observable.ts index 342aa5aa797b13..508684f8d72686 100644 --- a/x-pack/plugins/infra/public/utils/use_observable.ts +++ b/x-pack/plugins/infra/public/utils/use_observable.ts @@ -5,7 +5,8 @@ */ import { useEffect, useRef, useState } from 'react'; -import { BehaviorSubject, Observable, PartialObserver, Subscription } from 'rxjs'; +import { BehaviorSubject, Observable, OperatorFunction, PartialObserver, Subscription } from 'rxjs'; +import { switchMap } from 'rxjs/operators'; export const useLatest = (value: Value) => { const valueRef = useRef(value); @@ -62,7 +63,9 @@ export const useSubscription = ( const fixedUnsubscribe = latestUnsubscribe.current; const subscription = input$.subscribe({ - next: (value) => latestNext.current?.(value), + next: (value) => { + return latestNext.current?.(value); + }, error: (value) => latestError.current?.(value), complete: () => latestComplete.current?.(), }); @@ -78,6 +81,19 @@ export const useSubscription = ( return latestSubscription.current; }; +export const useOperator = ( + input$: Observable, + operator: OperatorFunction +) => { + const latestOperator = useLatest(operator); + + return useObservable( + (inputs$) => + inputs$.pipe(switchMap(([currentInput$]) => latestOperator.current(currentInput$))), + [input$] as const + ); +}; + export const tapUnsubscribe = (onUnsubscribe: () => void) => (source$: Observable) => { return new Observable((subscriber) => { const subscription = source$.subscribe({ diff --git a/x-pack/plugins/infra/server/graphql/types.ts b/x-pack/plugins/infra/server/graphql/types.ts index 02dcd76e8b34c8..712438ce2bfe02 100644 --- a/x-pack/plugins/infra/server/graphql/types.ts +++ b/x-pack/plugins/infra/server/graphql/types.ts @@ -56,12 +56,6 @@ export interface InfraSource { configuration: InfraSourceConfiguration; /** The status of the source */ status: InfraSourceStatus; - /** A consecutive span of log entries surrounding a point in time */ - logEntriesAround: InfraLogEntryInterval; - /** A consecutive span of log entries within an interval */ - logEntriesBetween: InfraLogEntryInterval; - /** Sequences of log entries matching sets of highlighting queries within an interval */ - logEntryHighlights: InfraLogEntryInterval[]; /** A snapshot of nodes */ snapshot?: InfraSnapshotResponse | null; @@ -157,80 +151,6 @@ export interface InfraIndexField { /** Whether the field should be displayed based on event.module and a ECS allowed list */ displayable: boolean; } -/** A consecutive sequence of log entries */ -export interface InfraLogEntryInterval { - /** The key corresponding to the start of the interval covered by the entries */ - start?: InfraTimeKey | null; - /** The key corresponding to the end of the interval covered by the entries */ - end?: InfraTimeKey | null; - /** Whether there are more log entries available before the start */ - hasMoreBefore: boolean; - /** Whether there are more log entries available after the end */ - hasMoreAfter: boolean; - /** The query the log entries were filtered by */ - filterQuery?: string | null; - /** The query the log entries were highlighted with */ - highlightQuery?: string | null; - /** A list of the log entries */ - entries: InfraLogEntry[]; -} -/** A representation of the log entry's position in the event stream */ -export interface InfraTimeKey { - /** The timestamp of the event that the log entry corresponds to */ - time: number; - /** The tiebreaker that disambiguates events with the same timestamp */ - tiebreaker: number; -} -/** A log entry */ -export interface InfraLogEntry { - /** A unique representation of the log entry's position in the event stream */ - key: InfraTimeKey; - /** The log entry's id */ - gid: string; - /** The source id */ - source: string; - /** The columns used for rendering the log entry */ - columns: InfraLogEntryColumn[]; -} -/** A special built-in column that contains the log entry's timestamp */ -export interface InfraLogEntryTimestampColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The timestamp */ - timestamp: number; -} -/** A special built-in column that contains the log entry's constructed message */ -export interface InfraLogEntryMessageColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** A list of the formatted log entry segments */ - message: InfraLogMessageSegment[]; -} -/** A segment of the log entry message that was derived from a field */ -export interface InfraLogMessageFieldSegment { - /** The field the segment was derived from */ - field: string; - /** The segment's message */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} -/** A segment of the log entry message that was derived from a string literal */ -export interface InfraLogMessageConstantSegment { - /** The segment's message */ - constant: string; -} -/** A column that contains the value of a field of the log entry */ -export interface InfraLogEntryFieldColumn { - /** The id of the corresponding column configuration */ - columnId: string; - /** The field name of the column */ - field: string; - /** The value of the field in the log entry */ - value: string; - /** A list of highlighted substrings of the value */ - highlights: string[]; -} export interface InfraSnapshotResponse { /** Nodes of type host, container or pod grouped by 0, 1 or 2 terms */ @@ -304,21 +224,6 @@ export interface DeleteSourceResult { // InputTypes // ==================================================== -export interface InfraTimeKeyInput { - time: number; - - tiebreaker: number; -} -/** A highlighting definition */ -export interface InfraLogEntryHighlightInput { - /** The query to highlight by */ - query: string; - /** The number of highlighted documents to include beyond the beginning of the interval */ - countBefore: number; - /** The number of highlighted documents to include beyond the end of the interval */ - countAfter: number; -} - export interface InfraTimerangeInput { /** The interval string to use for last bucket. The format is '{value}{unit}'. For example '5m' would return the metrics for the last 5 minutes of the timespan. */ interval: string; @@ -409,34 +314,6 @@ export interface SourceQueryArgs { /** The id of the source */ id: string; } -export interface LogEntriesAroundInfraSourceArgs { - /** The sort key that corresponds to the point in time */ - key: InfraTimeKeyInput; - /** The maximum number of preceding to return */ - countBefore?: number | null; - /** The maximum number of following to return */ - countAfter?: number | null; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntriesBetweenInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; -} -export interface LogEntryHighlightsInfraSourceArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - /** The highlighting to apply to the log entries */ - highlights: InfraLogEntryHighlightInput[]; -} export interface SnapshotInfraSourceArgs { timerange: InfraTimerangeInput; @@ -593,15 +470,6 @@ export type InfraSourceLogColumn = | InfraSourceMessageLogColumn | InfraSourceFieldLogColumn; -/** A column of a log entry */ -export type InfraLogEntryColumn = - | InfraLogEntryTimestampColumn - | InfraLogEntryMessageColumn - | InfraLogEntryFieldColumn; - -/** A segment of the log entry message */ -export type InfraLogMessageSegment = InfraLogMessageFieldSegment | InfraLogMessageConstantSegment; - // ==================================================== // END: Typescript template // ==================================================== @@ -650,12 +518,6 @@ export namespace InfraSourceResolvers { configuration?: ConfigurationResolver; /** The status of the source */ status?: StatusResolver; - /** A consecutive span of log entries surrounding a point in time */ - logEntriesAround?: LogEntriesAroundResolver; - /** A consecutive span of log entries within an interval */ - logEntriesBetween?: LogEntriesBetweenResolver; - /** Sequences of log entries matching sets of highlighting queries within an interval */ - logEntryHighlights?: LogEntryHighlightsResolver; /** A snapshot of nodes */ snapshot?: SnapshotResolver; @@ -693,51 +555,6 @@ export namespace InfraSourceResolvers { Parent = InfraSource, Context = InfraContext > = Resolver; - export type LogEntriesAroundResolver< - R = InfraLogEntryInterval, - Parent = InfraSource, - Context = InfraContext - > = Resolver; - export interface LogEntriesAroundArgs { - /** The sort key that corresponds to the point in time */ - key: InfraTimeKeyInput; - /** The maximum number of preceding to return */ - countBefore?: number | null; - /** The maximum number of following to return */ - countAfter?: number | null; - /** The query to filter the log entries by */ - filterQuery?: string | null; - } - - export type LogEntriesBetweenResolver< - R = InfraLogEntryInterval, - Parent = InfraSource, - Context = InfraContext - > = Resolver; - export interface LogEntriesBetweenArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - } - - export type LogEntryHighlightsResolver< - R = InfraLogEntryInterval[], - Parent = InfraSource, - Context = InfraContext - > = Resolver; - export interface LogEntryHighlightsArgs { - /** The sort key that corresponds to the start of the interval */ - startKey: InfraTimeKeyInput; - /** The sort key that corresponds to the end of the interval */ - endKey: InfraTimeKeyInput; - /** The query to filter the log entries by */ - filterQuery?: string | null; - /** The highlighting to apply to the log entries */ - highlights: InfraLogEntryHighlightInput[]; - } export type SnapshotResolver< R = InfraSnapshotResponse | null, @@ -1059,229 +876,6 @@ export namespace InfraIndexFieldResolvers { Context = InfraContext > = Resolver; } -/** A consecutive sequence of log entries */ -export namespace InfraLogEntryIntervalResolvers { - export interface Resolvers { - /** The key corresponding to the start of the interval covered by the entries */ - start?: StartResolver; - /** The key corresponding to the end of the interval covered by the entries */ - end?: EndResolver; - /** Whether there are more log entries available before the start */ - hasMoreBefore?: HasMoreBeforeResolver; - /** Whether there are more log entries available after the end */ - hasMoreAfter?: HasMoreAfterResolver; - /** The query the log entries were filtered by */ - filterQuery?: FilterQueryResolver; - /** The query the log entries were highlighted with */ - highlightQuery?: HighlightQueryResolver; - /** A list of the log entries */ - entries?: EntriesResolver; - } - - export type StartResolver< - R = InfraTimeKey | null, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type EndResolver< - R = InfraTimeKey | null, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type HasMoreBeforeResolver< - R = boolean, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type HasMoreAfterResolver< - R = boolean, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type FilterQueryResolver< - R = string | null, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type HighlightQueryResolver< - R = string | null, - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; - export type EntriesResolver< - R = InfraLogEntry[], - Parent = InfraLogEntryInterval, - Context = InfraContext - > = Resolver; -} -/** A representation of the log entry's position in the event stream */ -export namespace InfraTimeKeyResolvers { - export interface Resolvers { - /** The timestamp of the event that the log entry corresponds to */ - time?: TimeResolver; - /** The tiebreaker that disambiguates events with the same timestamp */ - tiebreaker?: TiebreakerResolver; - } - - export type TimeResolver = Resolver< - R, - Parent, - Context - >; - export type TiebreakerResolver< - R = number, - Parent = InfraTimeKey, - Context = InfraContext - > = Resolver; -} -/** A log entry */ -export namespace InfraLogEntryResolvers { - export interface Resolvers { - /** A unique representation of the log entry's position in the event stream */ - key?: KeyResolver; - /** The log entry's id */ - gid?: GidResolver; - /** The source id */ - source?: SourceResolver; - /** The columns used for rendering the log entry */ - columns?: ColumnsResolver; - } - - export type KeyResolver< - R = InfraTimeKey, - Parent = InfraLogEntry, - Context = InfraContext - > = Resolver; - export type GidResolver = Resolver< - R, - Parent, - Context - >; - export type SourceResolver = Resolver< - R, - Parent, - Context - >; - export type ColumnsResolver< - R = InfraLogEntryColumn[], - Parent = InfraLogEntry, - Context = InfraContext - > = Resolver; -} -/** A special built-in column that contains the log entry's timestamp */ -export namespace InfraLogEntryTimestampColumnResolvers { - export interface Resolvers { - /** The id of the corresponding column configuration */ - columnId?: ColumnIdResolver; - /** The timestamp */ - timestamp?: TimestampResolver; - } - - export type ColumnIdResolver< - R = string, - Parent = InfraLogEntryTimestampColumn, - Context = InfraContext - > = Resolver; - export type TimestampResolver< - R = number, - Parent = InfraLogEntryTimestampColumn, - Context = InfraContext - > = Resolver; -} -/** A special built-in column that contains the log entry's constructed message */ -export namespace InfraLogEntryMessageColumnResolvers { - export interface Resolvers { - /** The id of the corresponding column configuration */ - columnId?: ColumnIdResolver; - /** A list of the formatted log entry segments */ - message?: MessageResolver; - } - - export type ColumnIdResolver< - R = string, - Parent = InfraLogEntryMessageColumn, - Context = InfraContext - > = Resolver; - export type MessageResolver< - R = InfraLogMessageSegment[], - Parent = InfraLogEntryMessageColumn, - Context = InfraContext - > = Resolver; -} -/** A segment of the log entry message that was derived from a field */ -export namespace InfraLogMessageFieldSegmentResolvers { - export interface Resolvers { - /** The field the segment was derived from */ - field?: FieldResolver; - /** The segment's message */ - value?: ValueResolver; - /** A list of highlighted substrings of the value */ - highlights?: HighlightsResolver; - } - - export type FieldResolver< - R = string, - Parent = InfraLogMessageFieldSegment, - Context = InfraContext - > = Resolver; - export type ValueResolver< - R = string, - Parent = InfraLogMessageFieldSegment, - Context = InfraContext - > = Resolver; - export type HighlightsResolver< - R = string[], - Parent = InfraLogMessageFieldSegment, - Context = InfraContext - > = Resolver; -} -/** A segment of the log entry message that was derived from a string literal */ -export namespace InfraLogMessageConstantSegmentResolvers { - export interface Resolvers { - /** The segment's message */ - constant?: ConstantResolver; - } - - export type ConstantResolver< - R = string, - Parent = InfraLogMessageConstantSegment, - Context = InfraContext - > = Resolver; -} -/** A column that contains the value of a field of the log entry */ -export namespace InfraLogEntryFieldColumnResolvers { - export interface Resolvers { - /** The id of the corresponding column configuration */ - columnId?: ColumnIdResolver; - /** The field name of the column */ - field?: FieldResolver; - /** The value of the field in the log entry */ - value?: ValueResolver; - /** A list of highlighted substrings of the value */ - highlights?: HighlightsResolver; - } - - export type ColumnIdResolver< - R = string, - Parent = InfraLogEntryFieldColumn, - Context = InfraContext - > = Resolver; - export type FieldResolver< - R = string, - Parent = InfraLogEntryFieldColumn, - Context = InfraContext - > = Resolver; - export type ValueResolver< - R = string, - Parent = InfraLogEntryFieldColumn, - Context = InfraContext - > = Resolver; - export type HighlightsResolver< - R = string[], - Parent = InfraLogEntryFieldColumn, - Context = InfraContext - > = Resolver; -} export namespace InfraSnapshotResponseResolvers { export interface Resolvers { diff --git a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index ffbc750af14f82..6702a43cb2316e 100644 --- a/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -215,6 +215,7 @@ function mapHitsToLogEntryDocuments(hits: SortedSearchHit[], fields: string[]): return { id: hit._id, + index: hit._index, cursor: { time: hit.sort[0], tiebreaker: hit.sort[1] }, fields: logFields, highlights: hit.highlight || {}, diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts index 4c5debe58ed260..e318075045522f 100644 --- a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -12,19 +12,19 @@ import { LogEntriesSummaryHighlightsBucket, LogEntriesRequest, } from '../../../../common/http_api'; -import { LogEntry, LogColumn } from '../../../../common/log_entry'; +import { LogColumn, LogEntryCursor, LogEntry } from '../../../../common/log_entry'; import { InfraSourceConfiguration, InfraSources, SavedSourceConfigurationFieldColumnRuntimeType, } from '../../sources'; -import { getBuiltinRules } from './builtin_rules'; +import { getBuiltinRules } from '../../../services/log_entries/message/builtin_rules'; import { CompiledLogMessageFormattingRule, Fields, Highlights, compileFormattingRules, -} from './message'; +} from '../../../services/log_entries/message/message'; import { KibanaFramework } from '../../adapters/framework/kibana_framework_adapter'; import { decodeOrThrow } from '../../../../common/runtime_types'; import { @@ -33,7 +33,6 @@ import { CompositeDatasetKey, createLogEntryDatasetsQuery, } from './queries/log_entry_datasets'; -import { LogEntryCursor } from '../../../../common/log_entry'; export interface LogEntriesParams { startTimestamp: number; @@ -156,6 +155,7 @@ export class InfraLogEntriesDomain { const entries = documents.map((doc) => { return { id: doc.id, + index: doc.index, cursor: doc.cursor, columns: columnDefinitions.map( (column): LogColumn => { @@ -317,6 +317,7 @@ export type LogEntryQuery = JsonObject; export interface LogEntryDocument { id: string; + index: string; fields: Fields; highlights: Highlights; cursor: LogEntryCursor; diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts index 071a8a94e009be..5d4846598d2045 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts @@ -5,7 +5,6 @@ */ import type { ILegacyScopedClusterClient } from 'src/core/server'; -import { LogEntryContext } from '../../../common/log_entry'; import { compareDatasetsByMaximumAnomalyScore, getJobId, @@ -13,6 +12,7 @@ import { logEntryCategoriesJobTypes, CategoriesSort, } from '../../../common/log_analysis'; +import { LogEntryContext } from '../../../common/log_entry'; import { startTracingSpan } from '../../../common/performance_tracing'; import { decodeOrThrow } from '../../../common/runtime_types'; import type { MlAnomalyDetectors, MlSystem } from '../../types'; diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts new file mode 100644 index 00000000000000..f07ee0508fa6c4 --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.test.ts @@ -0,0 +1,318 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { of, throwError } from 'rxjs'; +import { + elasticsearchServiceMock, + savedObjectsClientMock, + uiSettingsServiceMock, +} from 'src/core/server/mocks'; +import { + IEsSearchRequest, + IEsSearchResponse, + ISearchStrategy, + SearchStrategyDependencies, +} from 'src/plugins/data/server'; +import { InfraSource } from '../../lib/sources'; +import { createInfraSourcesMock } from '../../lib/sources/mocks'; +import { + logEntriesSearchRequestStateRT, + logEntriesSearchStrategyProvider, +} from './log_entries_search_strategy'; + +describe('LogEntries search strategy', () => { + it('handles initial search requests', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: true, + rawResponse: { + took: 0, + _shards: { total: 1, failed: 0, skipped: 0, successful: 0 }, + timed_out: false, + hits: { total: 0, max_score: 0, hits: [] }, + }, + }); + + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); + + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + + const response = await logEntriesSearchStrategy + .search( + { + params: { + sourceId: 'SOURCE_ID', + startTimestamp: 100, + endTimestamp: 200, + size: 3, + }, + }, + {}, + mockDependencies + ) + .toPromise(); + + expect(sourcesMock.getSourceConfiguration).toHaveBeenCalled(); + expect(esSearchStrategyMock.search).toHaveBeenCalledWith( + expect.objectContaining({ + params: expect.objectContaining({ + index: 'log-indices-*', + body: expect.objectContaining({ + fields: expect.arrayContaining(['event.dataset', 'message']), + }), + }), + }), + expect.anything(), + expect.anything() + ); + expect(response.id).toEqual(expect.any(String)); + expect(response.isRunning).toBe(true); + }); + + it('handles subsequent polling requests', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: false, + rawResponse: { + took: 1, + _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + timed_out: false, + hits: { + total: 0, + max_score: 0, + hits: [ + { + _id: 'HIT_ID', + _index: 'HIT_INDEX', + _type: '_doc', + _score: 0, + _source: null, + fields: { + '@timestamp': [1605116827143], + 'event.dataset': ['HIT_DATASET'], + MESSAGE_FIELD: ['HIT_MESSAGE'], + 'container.id': ['HIT_CONTAINER_ID'], + }, + sort: [1605116827143 as any, 1 as any], // incorrectly typed as string upstream + }, + ], + }, + }, + }); + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); + + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + const requestId = logEntriesSearchRequestStateRT.encode({ + esRequestId: 'ASYNC_REQUEST_ID', + }); + + const response = await logEntriesSearchStrategy + .search( + { + id: requestId, + params: { + sourceId: 'SOURCE_ID', + startTimestamp: 100, + endTimestamp: 200, + size: 3, + }, + }, + {}, + mockDependencies + ) + .toPromise(); + + expect(sourcesMock.getSourceConfiguration).toHaveBeenCalled(); + expect(esSearchStrategyMock.search).toHaveBeenCalled(); + expect(response.id).toEqual(requestId); + expect(response.isRunning).toBe(false); + expect(response.rawResponse.data.entries).toEqual([ + { + id: 'HIT_ID', + index: 'HIT_INDEX', + cursor: { + time: 1605116827143, + tiebreaker: 1, + }, + columns: [ + { + columnId: 'TIMESTAMP_COLUMN_ID', + timestamp: 1605116827143, + }, + { + columnId: 'DATASET_COLUMN_ID', + field: 'event.dataset', + value: ['HIT_DATASET'], + highlights: [], + }, + { + columnId: 'MESSAGE_COLUMN_ID', + message: [ + { + field: 'MESSAGE_FIELD', + value: ['HIT_MESSAGE'], + highlights: [], + }, + ], + }, + ], + context: { + 'container.id': 'HIT_CONTAINER_ID', + }, + }, + ]); + }); + + it('forwards errors from the underlying search strategy', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: false, + rawResponse: { + took: 1, + _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + timed_out: false, + hits: { total: 0, max_score: 0, hits: [] }, + }, + }); + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); + + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + + const response = logEntriesSearchStrategy.search( + { + id: logEntriesSearchRequestStateRT.encode({ esRequestId: 'UNKNOWN_ID' }), + params: { + sourceId: 'SOURCE_ID', + startTimestamp: 100, + endTimestamp: 200, + size: 3, + }, + }, + {}, + mockDependencies + ); + + await expect(response.toPromise()).rejects.toThrowError(ResponseError); + }); + + it('forwards cancellation to the underlying search strategy', async () => { + const esSearchStrategyMock = createEsSearchStrategyMock({ + id: 'ASYNC_REQUEST_ID', + isRunning: false, + rawResponse: { + took: 1, + _shards: { total: 1, failed: 0, skipped: 0, successful: 1 }, + timed_out: false, + hits: { total: 0, max_score: 0, hits: [] }, + }, + }); + const dataMock = createDataPluginMock(esSearchStrategyMock); + const sourcesMock = createInfraSourcesMock(); + sourcesMock.getSourceConfiguration.mockResolvedValue(createSourceConfigurationMock()); + const mockDependencies = createSearchStrategyDependenciesMock(); + + const logEntriesSearchStrategy = logEntriesSearchStrategyProvider({ + data: dataMock, + sources: sourcesMock, + }); + const requestId = logEntriesSearchRequestStateRT.encode({ + esRequestId: 'ASYNC_REQUEST_ID', + }); + + await logEntriesSearchStrategy.cancel?.(requestId, {}, mockDependencies); + + expect(esSearchStrategyMock.cancel).toHaveBeenCalled(); + }); +}); + +const createSourceConfigurationMock = (): InfraSource => ({ + id: 'SOURCE_ID', + origin: 'stored' as const, + configuration: { + name: 'SOURCE_NAME', + description: 'SOURCE_DESCRIPTION', + logAlias: 'log-indices-*', + metricAlias: 'metric-indices-*', + inventoryDefaultView: 'DEFAULT_VIEW', + metricsExplorerDefaultView: 'DEFAULT_VIEW', + logColumns: [ + { timestampColumn: { id: 'TIMESTAMP_COLUMN_ID' } }, + { + fieldColumn: { + id: 'DATASET_COLUMN_ID', + field: 'event.dataset', + }, + }, + { + messageColumn: { id: 'MESSAGE_COLUMN_ID' }, + }, + ], + fields: { + pod: 'POD_FIELD', + host: 'HOST_FIELD', + container: 'CONTAINER_FIELD', + message: ['MESSAGE_FIELD'], + timestamp: 'TIMESTAMP_FIELD', + tiebreaker: 'TIEBREAKER_FIELD', + }, + }, +}); + +const createEsSearchStrategyMock = (esSearchResponse: IEsSearchResponse) => ({ + search: jest.fn((esSearchRequest: IEsSearchRequest) => { + if (typeof esSearchRequest.id === 'string') { + if (esSearchRequest.id === esSearchResponse.id) { + return of(esSearchResponse); + } else { + return throwError( + new ResponseError({ + body: {}, + headers: {}, + meta: {} as any, + statusCode: 404, + warnings: [], + }) + ); + } + } else { + return of(esSearchResponse); + } + }), + cancel: jest.fn().mockResolvedValue(undefined), +}); + +const createSearchStrategyDependenciesMock = (): SearchStrategyDependencies => ({ + uiSettingsClient: uiSettingsServiceMock.createClient(), + esClient: elasticsearchServiceMock.createScopedClusterClient(), + savedObjectsClient: savedObjectsClientMock.create(), +}); + +// using the official data mock from within x-pack doesn't type-check successfully, +// because the `licensing` plugin modifies the `RequestHandlerContext` core type. +const createDataPluginMock = (esSearchStrategyMock: ISearchStrategy): any => ({ + search: { + getSearchStrategy: jest.fn().mockReturnValue(esSearchStrategyMock), + }, +}); diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts new file mode 100644 index 00000000000000..6ce3d4410a2dd2 --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_search_strategy.ts @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pick } from '@kbn/std'; +import * as rt from 'io-ts'; +import { combineLatest, concat, defer, forkJoin, of } from 'rxjs'; +import { concatMap, filter, map, shareReplay, take } from 'rxjs/operators'; +import type { + IEsSearchRequest, + IKibanaSearchRequest, + IKibanaSearchResponse, +} from '../../../../../../src/plugins/data/common'; +import type { + ISearchStrategy, + PluginStart as DataPluginStart, +} from '../../../../../../src/plugins/data/server'; +import { LogSourceColumnConfiguration } from '../../../common/http_api/log_sources'; +import { + getLogEntryCursorFromHit, + LogColumn, + LogEntry, + LogEntryAfterCursor, + logEntryAfterCursorRT, + LogEntryBeforeCursor, + logEntryBeforeCursorRT, + LogEntryContext, +} from '../../../common/log_entry'; +import { decodeOrThrow } from '../../../common/runtime_types'; +import { + LogEntriesSearchRequestParams, + logEntriesSearchRequestParamsRT, + LogEntriesSearchResponsePayload, + logEntriesSearchResponsePayloadRT, +} from '../../../common/search_strategies/log_entries/log_entries'; +import type { IInfraSources } from '../../lib/sources'; +import { + createAsyncRequestRTs, + createErrorFromShardFailure, + jsonFromBase64StringRT, +} from '../../utils/typed_search_strategy'; +import { + CompiledLogMessageFormattingRule, + compileFormattingRules, + getBuiltinRules, +} from './message'; +import { + createGetLogEntriesQuery, + getLogEntriesResponseRT, + getSortDirection, + LogEntryHit, +} from './queries/log_entries'; + +type LogEntriesSearchRequest = IKibanaSearchRequest; +type LogEntriesSearchResponse = IKibanaSearchResponse; + +export const logEntriesSearchStrategyProvider = ({ + data, + sources, +}: { + data: DataPluginStart; + sources: IInfraSources; +}): ISearchStrategy => { + const esSearchStrategy = data.search.getSearchStrategy('ese'); + + return { + search: (rawRequest, options, dependencies) => + defer(() => { + const request = decodeOrThrow(asyncRequestRT)(rawRequest); + + const sourceConfiguration$ = defer(() => + sources.getSourceConfiguration(dependencies.savedObjectsClient, request.params.sourceId) + ).pipe(take(1), shareReplay(1)); + + const messageFormattingRules$ = defer(() => + sourceConfiguration$.pipe( + map(({ configuration }) => + compileFormattingRules(getBuiltinRules(configuration.fields.message)) + ) + ) + ).pipe(take(1), shareReplay(1)); + + const recoveredRequest$ = of(request).pipe( + filter(asyncRecoveredRequestRT.is), + map(({ id: { esRequestId } }) => ({ id: esRequestId })) + ); + + const initialRequest$ = of(request).pipe( + filter(asyncInitialRequestRT.is), + concatMap(({ params }) => + forkJoin([sourceConfiguration$, messageFormattingRules$]).pipe( + map( + ([{ configuration }, messageFormattingRules]): IEsSearchRequest => { + return { + params: createGetLogEntriesQuery( + configuration.logAlias, + params.startTimestamp, + params.endTimestamp, + pickRequestCursor(params), + params.size + 1, + configuration.fields.timestamp, + configuration.fields.tiebreaker, + messageFormattingRules.requiredFields, + params.query, + params.highlightPhrase + ), + }; + } + ) + ) + ) + ); + + const searchResponse$ = concat(recoveredRequest$, initialRequest$).pipe( + take(1), + concatMap((esRequest) => esSearchStrategy.search(esRequest, options, dependencies)) + ); + + return combineLatest([searchResponse$, sourceConfiguration$, messageFormattingRules$]).pipe( + map(([esResponse, { configuration }, messageFormattingRules]) => { + const rawResponse = decodeOrThrow(getLogEntriesResponseRT)(esResponse.rawResponse); + + const entries = rawResponse.hits.hits + .slice(0, request.params.size) + .map(getLogEntryFromHit(configuration.logColumns, messageFormattingRules)); + + const sortDirection = getSortDirection(pickRequestCursor(request.params)); + + if (sortDirection === 'desc') { + entries.reverse(); + } + + const hasMore = rawResponse.hits.hits.length > entries.length; + const hasMoreBefore = sortDirection === 'desc' ? hasMore : undefined; + const hasMoreAfter = sortDirection === 'asc' ? hasMore : undefined; + + const { topCursor, bottomCursor } = getResponseCursors(entries); + + const errors = (rawResponse._shards.failures ?? []).map(createErrorFromShardFailure); + + return { + ...esResponse, + ...(esResponse.id + ? { id: logEntriesSearchRequestStateRT.encode({ esRequestId: esResponse.id }) } + : {}), + rawResponse: logEntriesSearchResponsePayloadRT.encode({ + data: { entries, topCursor, bottomCursor, hasMoreBefore, hasMoreAfter }, + errors, + }), + }; + }) + ); + }), + cancel: async (id, options, dependencies) => { + const { esRequestId } = decodeOrThrow(logEntriesSearchRequestStateRT)(id); + return await esSearchStrategy.cancel?.(esRequestId, options, dependencies); + }, + }; +}; + +// exported for tests +export const logEntriesSearchRequestStateRT = rt.string.pipe(jsonFromBase64StringRT).pipe( + rt.type({ + esRequestId: rt.string, + }) +); + +const { asyncInitialRequestRT, asyncRecoveredRequestRT, asyncRequestRT } = createAsyncRequestRTs( + logEntriesSearchRequestStateRT, + logEntriesSearchRequestParamsRT +); + +const getLogEntryFromHit = ( + columnDefinitions: LogSourceColumnConfiguration[], + messageFormattingRules: CompiledLogMessageFormattingRule +) => (hit: LogEntryHit): LogEntry => { + const cursor = getLogEntryCursorFromHit(hit); + return { + id: hit._id, + index: hit._index, + cursor, + columns: columnDefinitions.map( + (column): LogColumn => { + if ('timestampColumn' in column) { + return { + columnId: column.timestampColumn.id, + timestamp: cursor.time, + }; + } else if ('messageColumn' in column) { + return { + columnId: column.messageColumn.id, + message: messageFormattingRules.format(hit.fields, hit.highlight || {}), + }; + } else { + return { + columnId: column.fieldColumn.id, + field: column.fieldColumn.field, + value: hit.fields[column.fieldColumn.field] ?? [], + highlights: hit.highlight?.[column.fieldColumn.field] ?? [], + }; + } + } + ), + context: getContextFromHit(hit), + }; +}; + +const pickRequestCursor = ( + params: LogEntriesSearchRequestParams +): LogEntryAfterCursor | LogEntryBeforeCursor | null => { + if (logEntryAfterCursorRT.is(params)) { + return pick(params, ['after']); + } else if (logEntryBeforeCursorRT.is(params)) { + return pick(params, ['before']); + } + + return null; +}; + +const getContextFromHit = (hit: LogEntryHit): LogEntryContext => { + // Get all context fields, then test for the presence and type of the ones that go together + const containerId = hit.fields['container.id']?.[0]; + const hostName = hit.fields['host.name']?.[0]; + const logFilePath = hit.fields['log.file.path']?.[0]; + + if (typeof containerId === 'string') { + return { 'container.id': containerId }; + } + + if (typeof hostName === 'string' && typeof logFilePath === 'string') { + return { 'host.name': hostName, 'log.file.path': logFilePath }; + } + + return {}; +}; + +function getResponseCursors(entries: LogEntry[]) { + const hasEntries = entries.length > 0; + const topCursor = hasEntries ? entries[0].cursor : null; + const bottomCursor = hasEntries ? entries[entries.length - 1].cursor : null; + + return { topCursor, bottomCursor }; +} diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entries_service.ts b/x-pack/plugins/infra/server/services/log_entries/log_entries_service.ts index edd53be9db8411..9aba69428f2575 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entries_service.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entries_service.ts @@ -6,12 +6,18 @@ import { CoreSetup } from 'src/core/server'; import { LOG_ENTRY_SEARCH_STRATEGY } from '../../../common/search_strategies/log_entries/log_entry'; +import { LOG_ENTRIES_SEARCH_STRATEGY } from '../../../common/search_strategies/log_entries/log_entries'; +import { logEntriesSearchStrategyProvider } from './log_entries_search_strategy'; import { logEntrySearchStrategyProvider } from './log_entry_search_strategy'; import { LogEntriesServiceSetupDeps, LogEntriesServiceStartDeps } from './types'; export class LogEntriesService { public setup(core: CoreSetup, setupDeps: LogEntriesServiceSetupDeps) { core.getStartServices().then(([, startDeps]) => { + setupDeps.data.search.registerSearchStrategy( + LOG_ENTRIES_SEARCH_STRATEGY, + logEntriesSearchStrategyProvider({ ...setupDeps, ...startDeps }) + ); setupDeps.data.search.registerSearchStrategy( LOG_ENTRY_SEARCH_STRATEGY, logEntrySearchStrategyProvider({ ...setupDeps, ...startDeps }) diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts index 38626675f5ae7b..b3e1a31f73b7ab 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.test.ts @@ -121,7 +121,7 @@ describe('LogEntry search strategy', () => { expect(response.rawResponse.data).toEqual({ id: 'HIT_ID', index: 'HIT_INDEX', - key: { + cursor: { time: 1605116827143, tiebreaker: 1, }, diff --git a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts index a0dfe3d7176fd4..ab2b72055e4a41 100644 --- a/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts +++ b/x-pack/plugins/infra/server/services/log_entries/log_entry_search_strategy.ts @@ -119,6 +119,6 @@ const { asyncInitialRequestRT, asyncRecoveredRequestRT, asyncRequestRT } = creat const createLogEntryFromHit = (hit: LogEntryHit) => ({ id: hit._id, index: hit._index, - key: getLogEntryCursorFromHit(hit), + cursor: getLogEntryCursorFromHit(hit), fields: Object.entries(hit.fields).map(([field, value]) => ({ field, value })), }); diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_apache2.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_apache2.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_apache2.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_apache2.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_apache2.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_auditd.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_auditd.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_auditd.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_auditd.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_auditd.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_haproxy.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_haproxy.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_haproxy.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_haproxy.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_haproxy.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_icinga.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_icinga.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_icinga.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_icinga.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_icinga.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_iis.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_iis.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_iis.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_iis.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_iis.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_kafka.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_kafka.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_kafka.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_logstash.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_logstash.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_logstash.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_logstash.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_logstash.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mongodb.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mongodb.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mongodb.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mongodb.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mongodb.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mysql.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mysql.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mysql.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_mysql.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_mysql.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_nginx.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_nginx.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_nginx.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_nginx.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_nginx.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_osquery.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_osquery.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_osquery.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_osquery.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_osquery.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_redis.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_redis.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_redis.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_redis.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_system.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_system.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_system.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_system.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_traefik.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_traefik.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_traefik.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/filebeat_traefik.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/filebeat_traefik.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.test.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.test.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic_webserver.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic_webserver.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/generic_webserver.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/generic_webserver.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/helpers.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/helpers.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/helpers.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts b/x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/index.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/builtin_rules/index.ts rename to x-pack/plugins/infra/server/services/log_entries/message/builtin_rules/index.ts diff --git a/x-pack/plugins/infra/server/services/log_entries/message/index.ts b/x-pack/plugins/infra/server/services/log_entries/message/index.ts new file mode 100644 index 00000000000000..05126eea075afe --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/message/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './message'; +export { getBuiltinRules } from './builtin_rules'; diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts b/x-pack/plugins/infra/server/services/log_entries/message/message.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/message.ts rename to x-pack/plugins/infra/server/services/log_entries/message/message.ts diff --git a/x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts b/x-pack/plugins/infra/server/services/log_entries/message/rule_types.ts similarity index 100% rename from x-pack/plugins/infra/server/lib/domains/log_entries_domain/rule_types.ts rename to x-pack/plugins/infra/server/services/log_entries/message/rule_types.ts diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/common.ts b/x-pack/plugins/infra/server/services/log_entries/queries/common.ts new file mode 100644 index 00000000000000..f170fa337a8b9e --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/queries/common.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const createSortClause = ( + sortDirection: 'asc' | 'desc', + timestampField: string, + tiebreakerField: string +) => ({ + sort: { + [timestampField]: sortDirection, + [tiebreakerField]: sortDirection, + }, +}); + +export const createTimeRangeFilterClauses = ( + startTimestamp: number, + endTimestamp: number, + timestampField: string +) => [ + { + range: { + [timestampField]: { + gte: startTimestamp, + lte: endTimestamp, + format: 'epoch_millis', + }, + }, + }, +]; diff --git a/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts new file mode 100644 index 00000000000000..81476fa2b286e2 --- /dev/null +++ b/x-pack/plugins/infra/server/services/log_entries/queries/log_entries.ts @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import type { RequestParams } from '@elastic/elasticsearch'; +import * as rt from 'io-ts'; +import { + LogEntryAfterCursor, + logEntryAfterCursorRT, + LogEntryBeforeCursor, + logEntryBeforeCursorRT, +} from '../../../../common/log_entry'; +import { jsonArrayRT, JsonObject } from '../../../../common/typed_json'; +import { + commonHitFieldsRT, + commonSearchSuccessResponseFieldsRT, +} from '../../../utils/elasticsearch_runtime_types'; +import { createSortClause, createTimeRangeFilterClauses } from './common'; + +export const createGetLogEntriesQuery = ( + logEntriesIndex: string, + startTimestamp: number, + endTimestamp: number, + cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined, + size: number, + timestampField: string, + tiebreakerField: string, + fields: string[], + query?: JsonObject, + highlightTerm?: string +): RequestParams.AsyncSearchSubmit> => { + const sortDirection = getSortDirection(cursor); + const highlightQuery = createHighlightQuery(highlightTerm, fields); + + return { + index: logEntriesIndex, + allow_no_indices: true, + track_scores: false, + track_total_hits: false, + body: { + size, + query: { + bool: { + filter: [ + ...(query ? [query] : []), + ...(highlightQuery ? [highlightQuery] : []), + ...createTimeRangeFilterClauses(startTimestamp, endTimestamp, timestampField), + ], + }, + }, + fields, + _source: false, + ...createSortClause(sortDirection, timestampField, tiebreakerField), + ...createSearchAfterClause(cursor), + ...createHighlightClause(highlightQuery, fields), + }, + }; +}; + +export const getSortDirection = ( + cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined +): 'asc' | 'desc' => (logEntryBeforeCursorRT.is(cursor) ? 'desc' : 'asc'); + +const createSearchAfterClause = ( + cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined +): { search_after?: [number, number] } => { + if (logEntryBeforeCursorRT.is(cursor) && cursor.before !== 'last') { + return { + search_after: [cursor.before.time, cursor.before.tiebreaker], + }; + } else if (logEntryAfterCursorRT.is(cursor) && cursor.after !== 'first') { + return { + search_after: [cursor.after.time, cursor.after.tiebreaker], + }; + } + + return {}; +}; + +const createHighlightClause = (highlightQuery: JsonObject | undefined, fields: string[]) => + highlightQuery + ? { + highlight: { + boundary_scanner: 'word', + fields: fields.reduce( + (highlightFieldConfigs, fieldName) => ({ + ...highlightFieldConfigs, + [fieldName]: {}, + }), + {} + ), + fragment_size: 1, + number_of_fragments: 100, + post_tags: [''], + pre_tags: [''], + highlight_query: highlightQuery, + }, + } + : {}; + +const createHighlightQuery = ( + highlightTerm: string | undefined, + fields: string[] +): JsonObject | undefined => { + if (highlightTerm) { + return { + multi_match: { + fields, + lenient: true, + query: highlightTerm, + type: 'phrase', + }, + }; + } +}; + +export const logEntryHitRT = rt.intersection([ + commonHitFieldsRT, + rt.type({ + fields: rt.record(rt.string, jsonArrayRT), + sort: rt.tuple([rt.number, rt.number]), + }), + rt.partial({ + highlight: rt.record(rt.string, rt.array(rt.string)), + }), +]); + +export type LogEntryHit = rt.TypeOf; + +export const getLogEntriesResponseRT = rt.intersection([ + commonSearchSuccessResponseFieldsRT, + rt.type({ + hits: rt.type({ + hits: rt.array(logEntryHitRT), + }), + }), +]); + +export type GetLogEntriesResponse = rt.TypeOf; diff --git a/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts b/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts index 79d5e683444321..0e61e3aaa0754a 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/log_entries.ts @@ -6,21 +6,17 @@ import expect from '@kbn/expect'; import { v4 as uuidv4 } from 'uuid'; - -import { decodeOrThrow } from '../../../../plugins/infra/common/runtime_types'; - import { LOG_ENTRIES_PATH, logEntriesRequestRT, logEntriesResponseRT, } from '../../../../plugins/infra/common/http_api'; - import { - LogTimestampColumn, LogFieldColumn, LogMessageColumn, + LogTimestampColumn, } from '../../../../plugins/infra/common/log_entry'; - +import { decodeOrThrow } from '../../../../plugins/infra/common/runtime_types'; import { FtrProviderContext } from '../../ftr_provider_context'; const KEY_WITHIN_DATA_RANGE = { From 762abea14e68a1741c5fbea9896272784b4b2ab8 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Tue, 2 Feb 2021 16:16:25 +0100 Subject: [PATCH 02/12] migrate more core-owned plugins to tsproject ref (#89975) * migrate more plugins to tsproject ref * revert changes for xpack_legacy * fix IT --- .../integration_tests/team_assignment.test.js | 14 ++++++----- src/plugins/kibana_overview/tsconfig.json | 23 +++++++++++++++++++ test/tsconfig.json | 1 + tsconfig.json | 2 ++ tsconfig.refs.json | 1 + x-pack/plugins/code/tsconfig.json | 16 +++++++++++++ x-pack/test/tsconfig.json | 2 ++ x-pack/tsconfig.json | 3 +++ x-pack/tsconfig.refs.json | 13 ++++++----- 9 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 src/plugins/kibana_overview/tsconfig.json create mode 100644 x-pack/plugins/code/tsconfig.json diff --git a/src/dev/code_coverage/ingest_coverage/integration_tests/team_assignment.test.js b/src/dev/code_coverage/ingest_coverage/integration_tests/team_assignment.test.js index bf4c55871598e8..13ef28def40802 100644 --- a/src/dev/code_coverage/ingest_coverage/integration_tests/team_assignment.test.js +++ b/src/dev/code_coverage/ingest_coverage/integration_tests/team_assignment.test.js @@ -8,7 +8,6 @@ import { resolve } from 'path'; import execa from 'execa'; -import expect from '@kbn/expect'; import shell from 'shelljs'; const ROOT_DIR = resolve(__dirname, '../../../../..'); @@ -38,11 +37,14 @@ describe('Team Assignment', () => { describe(`when the codeowners file contains #CC#`, () => { it(`should strip the prefix and still drill down through the fs`, async () => { const { stdout } = await execa('grep', ['tre', teamAssignmentsPath], { cwd: ROOT_DIR }); - expect(stdout).to.be(`x-pack/plugins/code/jest.config.js kibana-tre -x-pack/plugins/code/server/config.ts kibana-tre -x-pack/plugins/code/server/index.ts kibana-tre -x-pack/plugins/code/server/plugin.test.ts kibana-tre -x-pack/plugins/code/server/plugin.ts kibana-tre`); + const lines = stdout.split('\n').filter((line) => !line.includes('/target')); + expect(lines).toEqual([ + 'x-pack/plugins/code/jest.config.js kibana-tre', + 'x-pack/plugins/code/server/config.ts kibana-tre', + 'x-pack/plugins/code/server/index.ts kibana-tre', + 'x-pack/plugins/code/server/plugin.test.ts kibana-tre', + 'x-pack/plugins/code/server/plugin.ts kibana-tre', + ]); }); }); }); diff --git a/src/plugins/kibana_overview/tsconfig.json b/src/plugins/kibana_overview/tsconfig.json new file mode 100644 index 00000000000000..ac3ac109cb35f0 --- /dev/null +++ b/src/plugins/kibana_overview/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": [ + "public/**/*", + "common/**/*", + ], + "references": [ + { "path": "../../core/tsconfig.json" }, + { "path": "../../plugins/navigation/tsconfig.json" }, + { "path": "../../plugins/data/tsconfig.json" }, + { "path": "../../plugins/home/tsconfig.json" }, + { "path": "../../plugins/newsfeed/tsconfig.json" }, + { "path": "../../plugins/usage_collection/tsconfig.json" }, + { "path": "../../plugins/kibana_react/tsconfig.json" }, + ] +} diff --git a/test/tsconfig.json b/test/tsconfig.json index 1dc58f7b25c241..c3acf94f8c2679 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -26,6 +26,7 @@ { "path": "../src/plugins/expressions/tsconfig.json" }, { "path": "../src/plugins/home/tsconfig.json" }, { "path": "../src/plugins/inspector/tsconfig.json" }, + { "path": "../src/plugins/kibana_overview/tsconfig.json" }, { "path": "../src/plugins/kibana_react/tsconfig.json" }, { "path": "../src/plugins/kibana_usage_collection/tsconfig.json" }, { "path": "../src/plugins/kibana_utils/tsconfig.json" }, diff --git a/tsconfig.json b/tsconfig.json index 21760919c89e93..f6e0fbc8d9e971 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,6 +24,7 @@ "src/plugins/input_control_vis/**/*", "src/plugins/inspector/**/*", "src/plugins/kibana_legacy/**/*", + "src/plugins/kibana_overview/**/*", "src/plugins/kibana_react/**/*", "src/plugins/kibana_usage_collection/**/*", "src/plugins/kibana_utils/**/*", @@ -84,6 +85,7 @@ { "path": "./src/plugins/home/tsconfig.json" }, { "path": "./src/plugins/inspector/tsconfig.json" }, { "path": "./src/plugins/kibana_legacy/tsconfig.json" }, + { "path": "./src/plugins/kibana_overview/tsconfig.json" }, { "path": "./src/plugins/kibana_react/tsconfig.json" }, { "path": "./src/plugins/kibana_usage_collection/tsconfig.json" }, { "path": "./src/plugins/kibana_utils/tsconfig.json" }, diff --git a/tsconfig.refs.json b/tsconfig.refs.json index 1d08e764709cad..17b1fc5dc1fe91 100644 --- a/tsconfig.refs.json +++ b/tsconfig.refs.json @@ -17,6 +17,7 @@ { "path": "./src/plugins/home/tsconfig.json" }, { "path": "./src/plugins/inspector/tsconfig.json" }, { "path": "./src/plugins/kibana_legacy/tsconfig.json" }, + { "path": "./src/plugins/kibana_overview/tsconfig.json" }, { "path": "./src/plugins/kibana_react/tsconfig.json" }, { "path": "./src/plugins/kibana_usage_collection/tsconfig.json" }, { "path": "./src/plugins/kibana_utils/tsconfig.json" }, diff --git a/x-pack/plugins/code/tsconfig.json b/x-pack/plugins/code/tsconfig.json new file mode 100644 index 00000000000000..9c0b0ed21330f0 --- /dev/null +++ b/x-pack/plugins/code/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": [ + "server/**/*" + ], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + ] +} diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index a9d76eea80d8fd..a1c0c272deb04e 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -21,6 +21,7 @@ { "path": "../../src/plugins/es_ui_shared/tsconfig.json" }, { "path": "../../src/plugins/expressions/tsconfig.json" }, { "path": "../../src/plugins/home/tsconfig.json" }, + { "path": "../../src/plugins/kibana_overview/tsconfig.json" }, { "path": "../../src/plugins/kibana_react/tsconfig.json" }, { "path": "../../src/plugins/kibana_usage_collection/tsconfig.json" }, { "path": "../../src/plugins/kibana_utils/tsconfig.json" }, @@ -40,6 +41,7 @@ { "path": "../plugins/actions/tsconfig.json" }, { "path": "../plugins/alerts/tsconfig.json" }, + { "path": "../plugins/code/tsconfig.json" }, { "path": "../plugins/console_extensions/tsconfig.json" }, { "path": "../plugins/data_enhanced/tsconfig.json" }, { "path": "../plugins/enterprise_search/tsconfig.json" }, diff --git a/x-pack/tsconfig.json b/x-pack/tsconfig.json index 7f64a552a51699..48283b3ac27472 100644 --- a/x-pack/tsconfig.json +++ b/x-pack/tsconfig.json @@ -9,6 +9,7 @@ "plugins/apm/scripts/**/*", "plugins/canvas/**/*", "plugins/console_extensions/**/*", + "plugins/code/**/*", "plugins/data_enhanced/**/*", "plugins/discover_enhanced/**/*", "plugins/dashboard_enhanced/**/*", @@ -67,6 +68,7 @@ { "path": "../src/plugins/home/tsconfig.json" }, { "path": "../src/plugins/inspector/tsconfig.json" }, { "path": "../src/plugins/kibana_legacy/tsconfig.json" }, + { "path": "../src/plugins/kibana_overview/tsconfig.json" }, { "path": "../src/plugins/kibana_react/tsconfig.json" }, { "path": "../src/plugins/kibana_usage_collection/tsconfig.json" }, { "path": "../src/plugins/kibana_utils/tsconfig.json" }, @@ -92,6 +94,7 @@ { "path": "./plugins/beats_management/tsconfig.json" }, { "path": "./plugins/canvas/tsconfig.json" }, { "path": "./plugins/cloud/tsconfig.json" }, + { "path": "./plugins/code/tsconfig.json" }, { "path": "./plugins/console_extensions/tsconfig.json" }, { "path": "./plugins/data_enhanced/tsconfig.json" }, { "path": "./plugins/discover_enhanced/tsconfig.json" }, diff --git a/x-pack/tsconfig.refs.json b/x-pack/tsconfig.refs.json index 43a488e8727cc0..23b06040f3ec36 100644 --- a/x-pack/tsconfig.refs.json +++ b/x-pack/tsconfig.refs.json @@ -1,11 +1,12 @@ { "include": [], "references": [ - { "path": "./plugins/actions/tsconfig.json"}, - { "path": "./plugins/alerts/tsconfig.json"}, + { "path": "./plugins/actions/tsconfig.json" }, + { "path": "./plugins/alerts/tsconfig.json" }, { "path": "./plugins/beats_management/tsconfig.json" }, { "path": "./plugins/canvas/tsconfig.json" }, { "path": "./plugins/cloud/tsconfig.json" }, + { "path": "./plugins/code/tsconfig.json" }, { "path": "./plugins/console_extensions/tsconfig.json" }, { "path": "./plugins/dashboard_enhanced/tsconfig.json" }, { "path": "./plugins/data_enhanced/tsconfig.json" }, @@ -13,7 +14,7 @@ { "path": "./plugins/embeddable_enhanced/tsconfig.json" }, { "path": "./plugins/encrypted_saved_objects/tsconfig.json" }, { "path": "./plugins/enterprise_search/tsconfig.json" }, - { "path": "./plugins/event_log/tsconfig.json"}, + { "path": "./plugins/event_log/tsconfig.json" }, { "path": "./plugins/features/tsconfig.json" }, { "path": "./plugins/global_search_bar/tsconfig.json" }, { "path": "./plugins/global_search_providers/tsconfig.json" }, @@ -31,14 +32,14 @@ { "path": "./plugins/searchprofiler/tsconfig.json" }, { "path": "./plugins/security/tsconfig.json" }, { "path": "./plugins/spaces/tsconfig.json" }, - { "path": "./plugins/stack_alerts/tsconfig.json"}, + { "path": "./plugins/stack_alerts/tsconfig.json" }, { "path": "./plugins/task_manager/tsconfig.json" }, { "path": "./plugins/telemetry_collection_xpack/tsconfig.json" }, { "path": "./plugins/translations/tsconfig.json" }, - { "path": "./plugins/triggers_actions_ui/tsconfig.json"}, + { "path": "./plugins/triggers_actions_ui/tsconfig.json" }, { "path": "./plugins/spaces/tsconfig.json" }, { "path": "./plugins/security/tsconfig.json" }, - { "path": "./plugins/stack_alerts/tsconfig.json"}, + { "path": "./plugins/stack_alerts/tsconfig.json" }, { "path": "./plugins/encrypted_saved_objects/tsconfig.json" }, { "path": "./plugins/beats_management/tsconfig.json" }, { "path": "./plugins/cloud/tsconfig.json" }, From 8ef8f3b49057ff1aec96a8c7274e3961b0335d5f Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 2 Feb 2021 16:18:55 +0100 Subject: [PATCH 03/12] [APM] use latency sum instead of avg for impact (#89990) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../get_service_dependencies/index.ts | 25 +++-- .../service_overview/dependencies/index.ts | 106 ++++++++++++++++-- 2 files changed, 111 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/index.ts b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/index.ts index 2b209f8f6a80a9..8d6b9bfc1a4e64 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/index.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/index.ts @@ -196,19 +196,26 @@ export async function getServiceDependencies({ }); const latencySums = metricsByResolvedAddress - .map((metrics) => metrics.latency.value) + .map( + (metric) => (metric.latency.value ?? 0) * (metric.throughput.value ?? 0) + ) .filter(isFiniteNumber); const minLatencySum = Math.min(...latencySums); const maxLatencySum = Math.max(...latencySums); - return metricsByResolvedAddress.map((metric) => ({ - ...metric, - impact: - metric.latency.value === null - ? 0 - : ((metric.latency.value - minLatencySum) / + return metricsByResolvedAddress.map((metric) => { + const impact = + isFiniteNumber(metric.latency.value) && + isFiniteNumber(metric.throughput.value) + ? ((metric.latency.value * metric.throughput.value - minLatencySum) / (maxLatencySum - minLatencySum)) * - 100, - })); + 100 + : 0; + + return { + ...metric, + impact, + }; + }); } diff --git a/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.ts b/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.ts index b3e7e0672fc7fe..fe32e4493b6e4c 100644 --- a/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.ts +++ b/x-pack/test/apm_api_integration/tests/service_overview/dependencies/index.ts @@ -6,7 +6,7 @@ import expect from '@kbn/expect'; import url from 'url'; -import { sortBy, pick, last } from 'lodash'; +import { sortBy, pick, last, omit } from 'lodash'; import { ValuesType } from 'utility-types'; import { registry } from '../../../common/registry'; import { Maybe } from '../../../../../plugins/apm/typings/common'; @@ -306,7 +306,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { response = await supertest.get( url.format({ - pathname: `/api/apm/services/opbeans-java/dependencies`, + pathname: `/api/apm/services/opbeans-python/dependencies`, query: { start, end, @@ -323,14 +323,41 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns at least one item', () => { expect(response.body.length).to.be.greaterThan(0); + + expectSnapshot( + omit(response.body[0], [ + 'errorRate.timeseries', + 'throughput.timeseries', + 'latency.timeseries', + ]) + ).toMatchInline(` + Object { + "errorRate": Object { + "value": 0, + }, + "impact": 1.97910470896139, + "latency": Object { + "value": 1043.99015586546, + }, + "name": "redis", + "spanSubtype": "redis", + "spanType": "db", + "throughput": Object { + "value": 40.6333333333333, + }, + "type": "external", + } + `); }); it('returns the right names', () => { const names = response.body.map((item) => item.name); expectSnapshot(names.sort()).toMatchInline(` Array [ - "opbeans-go", + "elasticsearch", + "opbeans-java", "postgresql", + "redis", ] `); }); @@ -342,7 +369,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { expectSnapshot(serviceNames.sort()).toMatchInline(` Array [ - "opbeans-go", + "opbeans-java", ] `); }); @@ -356,32 +383,89 @@ export default function ApiTest({ getService }: FtrProviderContext) { expectSnapshot(latencyValues).toMatchInline(` Array [ Object { - "latency": 38506.4285714286, - "name": "opbeans-go", + "latency": 2568.40816326531, + "name": "elasticsearch", + }, + Object { + "latency": 25593.875, + "name": "opbeans-java", }, Object { - "latency": 5908.77272727273, + "latency": 28885.3293963255, "name": "postgresql", }, + Object { + "latency": 1043.99015586546, + "name": "redis", + }, ] `); }); it('returns the right throughput values', () => { const throughputValues = sortBy( - response.body.map((item) => ({ name: item.name, latency: item.throughput.value })), + response.body.map((item) => ({ name: item.name, throughput: item.throughput.value })), 'name' ); expectSnapshot(throughputValues).toMatchInline(` Array [ Object { - "latency": 0.466666666666667, - "name": "opbeans-go", + "name": "elasticsearch", + "throughput": 13.0666666666667, + }, + Object { + "name": "opbeans-java", + "throughput": 0.533333333333333, }, Object { - "latency": 3.66666666666667, "name": "postgresql", + "throughput": 50.8, + }, + Object { + "name": "redis", + "throughput": 40.6333333333333, + }, + ] + `); + }); + + it('returns the right impact values', () => { + const impactValues = sortBy( + response.body.map((item) => ({ + name: item.name, + impact: item.impact, + latency: item.latency.value, + throughput: item.throughput.value, + })), + 'name' + ); + + expectSnapshot(impactValues).toMatchInline(` + Array [ + Object { + "impact": 1.36961744704522, + "latency": 2568.40816326531, + "name": "elasticsearch", + "throughput": 13.0666666666667, + }, + Object { + "impact": 0, + "latency": 25593.875, + "name": "opbeans-java", + "throughput": 0.533333333333333, + }, + Object { + "impact": 100, + "latency": 28885.3293963255, + "name": "postgresql", + "throughput": 50.8, + }, + Object { + "impact": 1.97910470896139, + "latency": 1043.99015586546, + "name": "redis", + "throughput": 40.6333333333333, }, ] `); From 3f97a04c6337e693dd1c3031ca43499e4fdf9f32 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 2 Feb 2021 16:47:13 +0100 Subject: [PATCH 04/12] [Form lib] UseField `onError` listener (#89895) * added callback for listening to field onerror events * added onError component integration test * address tslint issues Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../components/use_field.test.tsx | 51 +++++++++++++++++++ .../hook_form_lib/components/use_field.tsx | 4 +- .../forms/hook_form_lib/hooks/use_field.ts | 12 ++++- 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/plugins/es_ui_shared/static/forms/hook_form_lib/components/use_field.test.tsx b/src/plugins/es_ui_shared/static/forms/hook_form_lib/components/use_field.test.tsx index 36841f8b1d521e..4945b7a059e8c0 100644 --- a/src/plugins/es_ui_shared/static/forms/hook_form_lib/components/use_field.test.tsx +++ b/src/plugins/es_ui_shared/static/forms/hook_form_lib/components/use_field.test.tsx @@ -287,4 +287,55 @@ describe('', () => { expect(formHook?.getFormData()).toEqual({ name: 'myName' }); }); }); + + describe('change handlers', () => { + const onError = jest.fn(); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + const getTestComp = (fieldConfig: FieldConfig) => { + const TestComp = () => { + const { form } = useForm(); + + return ( +
+ + + ); + }; + return TestComp; + }; + + const setup = (fieldConfig: FieldConfig) => { + return registerTestBed(getTestComp(fieldConfig), { + memoryRouter: { wrapComponent: false }, + })() as TestBed; + }; + + it('calls onError when validation state changes', async () => { + const { + form: { setInputValue }, + } = setup({ + validations: [ + { + validator: ({ value }) => (value === '1' ? undefined : { message: 'oops!' }), + }, + ], + }); + + expect(onError).toBeCalledTimes(0); + await act(async () => { + setInputValue('myField', '0'); + }); + expect(onError).toBeCalledTimes(1); + expect(onError).toBeCalledWith(['oops!']); + await act(async () => { + setInputValue('myField', '1'); + }); + expect(onError).toBeCalledTimes(2); + expect(onError).toBeCalledWith(null); + }); + }); }); diff --git a/src/plugins/es_ui_shared/static/forms/hook_form_lib/components/use_field.tsx b/src/plugins/es_ui_shared/static/forms/hook_form_lib/components/use_field.tsx index 94c2bc42d28558..cc79ed24b5d0cb 100644 --- a/src/plugins/es_ui_shared/static/forms/hook_form_lib/components/use_field.tsx +++ b/src/plugins/es_ui_shared/static/forms/hook_form_lib/components/use_field.tsx @@ -20,6 +20,7 @@ export interface Props { componentProps?: Record; readDefaultValueOnForm?: boolean; onChange?: (value: I) => void; + onError?: (errors: string[] | null) => void; children?: (field: FieldHook) => JSX.Element | null; [key: string]: any; } @@ -33,6 +34,7 @@ function UseFieldComp(props: Props(props: Props(form, path, fieldConfig, onChange); + const field = useField(form, path, fieldConfig, onChange, onError); // Children prevails over anything else provided. if (children) { diff --git a/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts b/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts index c396f223e97fde..db7b0b2820a47d 100644 --- a/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts +++ b/src/plugins/es_ui_shared/static/forms/hook_form_lib/hooks/use_field.ts @@ -27,7 +27,8 @@ export const useField = ( form: FormHook, path: string, config: FieldConfig & InternalFieldConfig = {}, - valueChangeListener?: (value: I) => void + valueChangeListener?: (value: I) => void, + errorChangeListener?: (errors: string[] | null) => void ) => { const { type = FIELD_TYPES.TEXT, @@ -596,6 +597,15 @@ export const useField = ( }; }, [onValueChange]); + useEffect(() => { + if (!isMounted.current) { + return; + } + if (errorChangeListener) { + errorChangeListener(errors.length ? errors.map((error) => error.message) : null); + } + }, [errors, errorChangeListener]); + useEffect(() => { isMounted.current = true; From 33bf59038673d77156847638fe0e0c7bdaa58f75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20C=C3=B4t=C3=A9?= Date: Tue, 2 Feb 2021 11:13:55 -0500 Subject: [PATCH 05/12] Rename getProxyAgents to getCustomAgents (#89813) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../builtin_action_types/lib/axios_utils.test.ts | 4 ++-- .../server/builtin_action_types/lib/axios_utils.ts | 4 ++-- ...roxy_agents.test.ts => get_custom_agents.test.ts} | 10 +++++----- .../{get_proxy_agents.ts => get_custom_agents.ts} | 12 ++++++------ .../actions/server/builtin_action_types/slack.ts | 12 ++++++------ 5 files changed, 21 insertions(+), 21 deletions(-) rename x-pack/plugins/actions/server/builtin_action_types/lib/{get_proxy_agents.test.ts => get_custom_agents.test.ts} (81%) rename x-pack/plugins/actions/server/builtin_action_types/lib/{get_proxy_agents.ts => get_custom_agents.ts} (90%) diff --git a/x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.test.ts b/x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.test.ts index 23e16b7463914a..a4db25310c7933 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.test.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.test.ts @@ -10,7 +10,7 @@ import { Logger } from '../../../../../../src/core/server'; import { addTimeZoneToDate, request, patch, getErrorMessage } from './axios_utils'; import { loggingSystemMock } from '../../../../../../src/core/server/mocks'; import { actionsConfigMock } from '../../actions_config.mock'; -import { getProxyAgents } from './get_proxy_agents'; +import { getCustomAgents } from './get_custom_agents'; const logger = loggingSystemMock.create().get() as jest.Mocked; const configurationUtilities = actionsConfigMock.create(); @@ -66,7 +66,7 @@ describe('request', () => { proxyRejectUnauthorizedCertificates: true, proxyUrl: 'https://localhost:1212', }); - const { httpAgent, httpsAgent } = getProxyAgents(configurationUtilities, logger); + const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger); const res = await request({ axios, diff --git a/x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.ts b/x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.ts index a70a452737dc62..9a8c4e09ad5532 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.ts @@ -6,7 +6,7 @@ import { AxiosInstance, Method, AxiosResponse, AxiosBasicCredentials } from 'axios'; import { Logger } from '../../../../../../src/core/server'; -import { getProxyAgents } from './get_proxy_agents'; +import { getCustomAgents } from './get_custom_agents'; import { ActionsConfigurationUtilities } from '../../actions_config'; export const request = async ({ @@ -29,7 +29,7 @@ export const request = async ({ validateStatus?: (status: number) => boolean; auth?: AxiosBasicCredentials; }): Promise => { - const { httpAgent, httpsAgent } = getProxyAgents(configurationUtilities, logger); + const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger); return await axios(url, { ...rest, diff --git a/x-pack/plugins/actions/server/builtin_action_types/lib/get_proxy_agents.test.ts b/x-pack/plugins/actions/server/builtin_action_types/lib/get_custom_agents.test.ts similarity index 81% rename from x-pack/plugins/actions/server/builtin_action_types/lib/get_proxy_agents.test.ts rename to x-pack/plugins/actions/server/builtin_action_types/lib/get_custom_agents.test.ts index da2ad9bb3990dc..cc2f729a033a96 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/lib/get_proxy_agents.test.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/lib/get_custom_agents.test.ts @@ -8,12 +8,12 @@ import { Agent as HttpsAgent } from 'https'; import HttpProxyAgent from 'http-proxy-agent'; import { HttpsProxyAgent } from 'https-proxy-agent'; import { Logger } from '../../../../../../src/core/server'; -import { getProxyAgents } from './get_proxy_agents'; +import { getCustomAgents } from './get_custom_agents'; import { loggingSystemMock } from '../../../../../../src/core/server/mocks'; import { actionsConfigMock } from '../../actions_config.mock'; const logger = loggingSystemMock.create().get() as jest.Mocked; -describe('getProxyAgents', () => { +describe('getCustomAgents', () => { const configurationUtilities = actionsConfigMock.create(); test('get agents for valid proxy URL', () => { @@ -21,7 +21,7 @@ describe('getProxyAgents', () => { proxyUrl: 'https://someproxyhost', proxyRejectUnauthorizedCertificates: false, }); - const { httpAgent, httpsAgent } = getProxyAgents(configurationUtilities, logger); + const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger); expect(httpAgent instanceof HttpProxyAgent).toBeTruthy(); expect(httpsAgent instanceof HttpsProxyAgent).toBeTruthy(); }); @@ -31,13 +31,13 @@ describe('getProxyAgents', () => { proxyUrl: ':nope: not a valid URL', proxyRejectUnauthorizedCertificates: false, }); - const { httpAgent, httpsAgent } = getProxyAgents(configurationUtilities, logger); + const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger); expect(httpAgent).toBe(undefined); expect(httpsAgent instanceof HttpsAgent).toBeTruthy(); }); test('return default agents for undefined proxy options', () => { - const { httpAgent, httpsAgent } = getProxyAgents(configurationUtilities, logger); + const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger); expect(httpAgent).toBe(undefined); expect(httpsAgent instanceof HttpsAgent).toBeTruthy(); }); diff --git a/x-pack/plugins/actions/server/builtin_action_types/lib/get_proxy_agents.ts b/x-pack/plugins/actions/server/builtin_action_types/lib/get_custom_agents.ts similarity index 90% rename from x-pack/plugins/actions/server/builtin_action_types/lib/get_proxy_agents.ts rename to x-pack/plugins/actions/server/builtin_action_types/lib/get_custom_agents.ts index a49889570f4bf5..ad97dd5023f800 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/lib/get_proxy_agents.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/lib/get_custom_agents.ts @@ -11,17 +11,17 @@ import { HttpsProxyAgent } from 'https-proxy-agent'; import { Logger } from '../../../../../../src/core/server'; import { ActionsConfigurationUtilities } from '../../actions_config'; -interface GetProxyAgentsResponse { +interface GetCustomAgentsResponse { httpAgent: HttpAgent | undefined; httpsAgent: HttpsAgent | undefined; } -export function getProxyAgents( +export function getCustomAgents( configurationUtilities: ActionsConfigurationUtilities, logger: Logger -): GetProxyAgentsResponse { +): GetCustomAgentsResponse { const proxySettings = configurationUtilities.getProxySettings(); - const defaultResponse = { + const defaultAgents = { httpAgent: undefined, httpsAgent: new HttpsAgent({ rejectUnauthorized: configurationUtilities.isRejectUnauthorizedCertificatesEnabled(), @@ -29,7 +29,7 @@ export function getProxyAgents( }; if (!proxySettings) { - return defaultResponse; + return defaultAgents; } logger.debug(`Creating proxy agents for proxy: ${proxySettings.proxyUrl}`); @@ -38,7 +38,7 @@ export function getProxyAgents( proxyUrl = new URL(proxySettings.proxyUrl); } catch (err) { logger.warn(`invalid proxy URL "${proxySettings.proxyUrl}" ignored`); - return defaultResponse; + return defaultAgents; } const httpAgent = new HttpProxyAgent(proxySettings.proxyUrl); diff --git a/x-pack/plugins/actions/server/builtin_action_types/slack.ts b/x-pack/plugins/actions/server/builtin_action_types/slack.ts index 5d2c5a24b3edd1..9f0a4c44b3c547 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/slack.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/slack.ts @@ -22,7 +22,7 @@ import { ExecutorType, } from '../types'; import { ActionsConfigurationUtilities } from '../actions_config'; -import { getProxyAgents } from './lib/get_proxy_agents'; +import { getCustomAgents } from './lib/get_custom_agents'; export type SlackActionType = ActionType<{}, ActionTypeSecretsType, ActionParamsType, unknown>; export type SlackActionTypeExecutorOptions = ActionTypeExecutorOptions< @@ -130,10 +130,10 @@ async function slackExecutor( const { message } = params; const proxySettings = configurationUtilities.getProxySettings(); - const proxyAgents = getProxyAgents(configurationUtilities, logger); - const httpProxyAgent = webhookUrl.toLowerCase().startsWith('https') - ? proxyAgents.httpsAgent - : proxyAgents.httpAgent; + const customAgents = getCustomAgents(configurationUtilities, logger); + const agent = webhookUrl.toLowerCase().startsWith('https') + ? customAgents.httpsAgent + : customAgents.httpAgent; if (proxySettings) { logger.debug(`IncomingWebhook was called with proxyUrl ${proxySettings.proxyUrl}`); @@ -143,7 +143,7 @@ async function slackExecutor( // https://slack.dev/node-slack-sdk/webhook // node-slack-sdk use Axios inside :) const webhook = new IncomingWebhook(webhookUrl, { - agent: httpProxyAgent, + agent, }); result = await webhook.send(message); } catch (err) { From f317316fd4f3fa5bb6e1310eb547c86fa4e094b5 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Tue, 2 Feb 2021 10:17:48 -0600 Subject: [PATCH 06/12] Round start and end values (#89030) When getting the start and end times, use the d3 time scale `ticks` function to round the start and end times. Example from a query: Before: ```json { "range": { "@timestamp": { "gte": 1611262874814, "lte": 1611263774814, "format": "epoch_millis" } } }, ``` After: ```json { "range": { "@timestamp": { "gte": 1611263040000, "lte": 1611263880000, "format": "epoch_millis" } } }, ``` The `ticks` function makes it so the amount of rounding is proportional to the size of the time range, so shorter time ranges will be rounded less. Also fix a bug where invalid ranges in the query string were not handled correctly. Fixes #84530. --- .../app/TraceLink/trace_link.test.tsx | 10 +- .../shared/DatePicker/date_picker.test.tsx | 3 +- .../url_params_context/helpers.test.ts | 97 +++++++++++++++---- .../context/url_params_context/helpers.ts | 29 ++++-- .../mock_url_params_context_provider.tsx | 3 +- .../url_params_context.test.tsx | 54 +++++++---- .../url_params_context/url_params_context.tsx | 54 +++++------ .../plugins/apm/public/hooks/use_fetcher.tsx | 3 + 8 files changed, 176 insertions(+), 77 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/TraceLink/trace_link.test.tsx b/x-pack/plugins/apm/public/components/app/TraceLink/trace_link.test.tsx index c07e00ef387c9c..a33be140d9a361 100644 --- a/x-pack/plugins/apm/public/components/app/TraceLink/trace_link.test.tsx +++ b/x-pack/plugins/apm/public/components/app/TraceLink/trace_link.test.tsx @@ -60,12 +60,13 @@ describe('TraceLink', () => { describe('when no transaction is found', () => { it('renders a trace page', () => { jest.spyOn(urlParamsHooks, 'useUrlParams').mockReturnValue({ + rangeId: 0, + refreshTimeRange: jest.fn(), + uiFilters: {}, urlParams: { rangeFrom: 'now-24h', rangeTo: 'now', }, - refreshTimeRange: jest.fn(), - uiFilters: {}, }); jest.spyOn(hooks, 'useFetcher').mockReturnValue({ data: { transaction: undefined }, @@ -87,12 +88,13 @@ describe('TraceLink', () => { describe('transaction page', () => { beforeAll(() => { jest.spyOn(urlParamsHooks, 'useUrlParams').mockReturnValue({ + rangeId: 0, + refreshTimeRange: jest.fn(), + uiFilters: {}, urlParams: { rangeFrom: 'now-24h', rangeTo: 'now', }, - refreshTimeRange: jest.fn(), - uiFilters: {}, }); }); diff --git a/x-pack/plugins/apm/public/components/shared/DatePicker/date_picker.test.tsx b/x-pack/plugins/apm/public/components/shared/DatePicker/date_picker.test.tsx index 222c27cc7ed6d8..1bf3cee32f286e 100644 --- a/x-pack/plugins/apm/public/components/shared/DatePicker/date_picker.test.tsx +++ b/x-pack/plugins/apm/public/components/shared/DatePicker/date_picker.test.tsx @@ -31,8 +31,9 @@ function MockUrlParamsProvider({ return ( { - describe('getParsedDate', () => { - describe('given undefined', () => { - it('returns undefined', () => { - expect(helpers.getParsedDate(undefined)).toBeUndefined(); + describe('getDateRange', () => { + describe('with non-rounded dates', () => { + describe('one minute', () => { + it('rounds the values', () => { + expect( + helpers.getDateRange({ + state: {}, + rangeFrom: '2021-01-28T05:47:52.134Z', + rangeTo: '2021-01-28T05:48:55.304Z', + }) + ).toEqual({ + start: '2021-01-28T05:47:50.000Z', + end: '2021-01-28T05:49:00.000Z', + }); + }); }); - }); - - describe('given a parsable date', () => { - it('returns the parsed date', () => { - expect(helpers.getParsedDate('1970-01-01T00:00:00.000Z')).toEqual( - '1970-01-01T00:00:00.000Z' - ); + describe('one day', () => { + it('rounds the values', () => { + expect( + helpers.getDateRange({ + state: {}, + rangeFrom: '2021-01-27T05:46:07.377Z', + rangeTo: '2021-01-28T05:46:13.367Z', + }) + ).toEqual({ + start: '2021-01-27T03:00:00.000Z', + end: '2021-01-28T06:00:00.000Z', + }); + }); }); - }); - describe('given a non-parsable date', () => { - it('returns null', () => { - expect(helpers.getParsedDate('nope')).toEqual(null); + describe('one year', () => { + it('rounds the values', () => { + expect( + helpers.getDateRange({ + state: {}, + rangeFrom: '2020-01-28T05:52:36.290Z', + rangeTo: '2021-01-28T05:52:39.741Z', + }) + ).toEqual({ + start: '2020-01-01T00:00:00.000Z', + end: '2021-02-01T00:00:00.000Z', + }); + }); }); }); - }); - describe('getDateRange', () => { describe('when rangeFrom and rangeTo are not changed', () => { it('returns the previous state', () => { expect( @@ -52,6 +76,45 @@ describe('url_params_context helpers', () => { }); }); + describe('when rangeFrom or rangeTo are falsy', () => { + it('returns the previous state', () => { + // Disable console warning about not receiving a valid date for rangeFrom + jest.spyOn(console, 'warn').mockImplementationOnce(() => {}); + + expect( + helpers.getDateRange({ + state: { + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }, + rangeFrom: '', + rangeTo: 'now', + }) + ).toEqual({ + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }); + }); + }); + + describe('when the start or end are invalid', () => { + it('returns the previous state', () => { + expect( + helpers.getDateRange({ + state: { + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }, + rangeFrom: 'nope', + rangeTo: 'now', + }) + ).toEqual({ + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }); + }); + }); + describe('when rangeFrom or rangeTo have changed', () => { it('returns new state', () => { jest.spyOn(datemath, 'parse').mockReturnValue(moment(0).utc()); diff --git a/x-pack/plugins/apm/public/context/url_params_context/helpers.ts b/x-pack/plugins/apm/public/context/url_params_context/helpers.ts index bff2ef5deb86cd..0be11d440aecb2 100644 --- a/x-pack/plugins/apm/public/context/url_params_context/helpers.ts +++ b/x-pack/plugins/apm/public/context/url_params_context/helpers.ts @@ -4,15 +4,16 @@ * you may not use this file except in compliance with the Elastic License. */ -import { compact, pickBy } from 'lodash'; import datemath from '@elastic/datemath'; +import { scaleUtc } from 'd3-scale'; +import { compact, pickBy } from 'lodash'; import { IUrlParams } from './types'; -export function getParsedDate(rawDate?: string, opts = {}) { +function getParsedDate(rawDate?: string, options = {}) { if (rawDate) { - const parsed = datemath.parse(rawDate, opts); - if (parsed) { - return parsed.toISOString(); + const parsed = datemath.parse(rawDate, options); + if (parsed && parsed.isValid()) { + return parsed.toDate(); } } } @@ -26,13 +27,27 @@ export function getDateRange({ rangeFrom?: string; rangeTo?: string; }) { + // If the previous state had the same range, just return that instead of calculating a new range. if (state.rangeFrom === rangeFrom && state.rangeTo === rangeTo) { return { start: state.start, end: state.end }; } + const start = getParsedDate(rangeFrom); + const end = getParsedDate(rangeTo, { roundUp: true }); + + // `getParsedDate` will return undefined for invalid or empty dates. We return + // the previous state if either date is undefined. + if (!start || !end) { + return { start: state.start, end: state.end }; + } + + // Calculate ticks for the time ranges to produce nicely rounded values. + const ticks = scaleUtc().domain([start, end]).nice().ticks(); + + // Return the first and last tick values. return { - start: getParsedDate(rangeFrom), - end: getParsedDate(rangeTo, { roundUp: true }), + start: ticks[0].toISOString(), + end: ticks[ticks.length - 1].toISOString(), }; } diff --git a/x-pack/plugins/apm/public/context/url_params_context/mock_url_params_context_provider.tsx b/x-pack/plugins/apm/public/context/url_params_context/mock_url_params_context_provider.tsx index b593cbd57a9a9a..1e546599ee8a30 100644 --- a/x-pack/plugins/apm/public/context/url_params_context/mock_url_params_context_provider.tsx +++ b/x-pack/plugins/apm/public/context/url_params_context/mock_url_params_context_provider.tsx @@ -30,8 +30,9 @@ export function MockUrlParamsContextProvider({ return ( ) { } describe('UrlParamsContext', () => { - beforeEach(() => { + beforeAll(() => { moment.tz.setDefault('Etc/GMT'); }); - afterEach(() => { + afterAll(() => { moment.tz.setDefault(''); }); @@ -50,8 +49,11 @@ describe('UrlParamsContext', () => { const wrapper = mountParams(location); const params = getDataFromOutput(wrapper); - expect(params.start).toEqual('2010-03-15T12:00:00.000Z'); - expect(params.end).toEqual('2010-04-10T12:00:00.000Z'); + + expect([params.start, params.end]).toEqual([ + '2010-03-15T00:00:00.000Z', + '2010-04-11T00:00:00.000Z', + ]); }); it('should update param values if location has changed', () => { @@ -66,8 +68,11 @@ describe('UrlParamsContext', () => { // force an update wrapper.setProps({ abc: 123 }); const params = getDataFromOutput(wrapper); - expect(params.start).toEqual('2009-03-15T12:00:00.000Z'); - expect(params.end).toEqual('2009-04-10T12:00:00.000Z'); + + expect([params.start, params.end]).toEqual([ + '2009-03-15T00:00:00.000Z', + '2009-04-11T00:00:00.000Z', + ]); }); it('should parse relative time ranges on mount', () => { @@ -76,13 +81,20 @@ describe('UrlParamsContext', () => { search: '?rangeFrom=now-1d%2Fd&rangeTo=now-1d%2Fd&transactionId=UPDATED', } as Location; + const nowSpy = jest.spyOn(Date, 'now').mockReturnValue(0); + const wrapper = mountParams(location); // force an update wrapper.setProps({ abc: 123 }); const params = getDataFromOutput(wrapper); - expect(params.start).toEqual(getParsedDate('now-1d/d')); - expect(params.end).toEqual(getParsedDate('now-1d/d', { roundUp: true })); + + expect([params.start, params.end]).toEqual([ + '1969-12-31T00:00:00.000Z', + '1970-01-01T00:00:00.000Z', + ]); + + nowSpy.mockRestore(); }); it('should refresh the time range with new values', async () => { @@ -130,8 +142,11 @@ describe('UrlParamsContext', () => { expect(calls.length).toBe(2); const params = getDataFromOutput(wrapper); - expect(params.start).toEqual('2005-09-20T12:00:00.000Z'); - expect(params.end).toEqual('2005-10-21T12:00:00.000Z'); + + expect([params.start, params.end]).toEqual([ + '2005-09-19T00:00:00.000Z', + '2005-10-23T00:00:00.000Z', + ]); }); it('should refresh the time range with new values if time range is relative', async () => { @@ -177,7 +192,10 @@ describe('UrlParamsContext', () => { await waitFor(() => {}); const params = getDataFromOutput(wrapper); - expect(params.start).toEqual('2000-06-14T00:00:00.000Z'); - expect(params.end).toEqual('2000-06-14T23:59:59.999Z'); + + expect([params.start, params.end]).toEqual([ + '2000-06-14T00:00:00.000Z', + '2000-06-15T00:00:00.000Z', + ]); }); }); diff --git a/x-pack/plugins/apm/public/context/url_params_context/url_params_context.tsx b/x-pack/plugins/apm/public/context/url_params_context/url_params_context.tsx index 0a3f8459ff0023..f66a45db261a70 100644 --- a/x-pack/plugins/apm/public/context/url_params_context/url_params_context.tsx +++ b/x-pack/plugins/apm/public/context/url_params_context/url_params_context.tsx @@ -4,28 +4,25 @@ * you may not use this file except in compliance with the Elastic License. */ +import { mapValues } from 'lodash'; import React, { createContext, - useMemo, useCallback, + useMemo, useRef, useState, } from 'react'; import { withRouter } from 'react-router-dom'; -import { uniqueId, mapValues } from 'lodash'; -import { IUrlParams } from './types'; -import { getParsedDate } from './helpers'; -import { resolveUrlParams } from './resolve_url_params'; -import { UIFilters } from '../../../typings/ui_filters'; -import { - localUIFilterNames, - - // eslint-disable-next-line @kbn/eslint/no-restricted-paths -} from '../../../server/lib/ui_filters/local_ui_filters/config'; +import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; +import { LocalUIFilterName } from '../../../common/ui_filter'; import { pickKeys } from '../../../common/utils/pick_keys'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { localUIFilterNames } from '../../../server/lib/ui_filters/local_ui_filters/config'; +import { UIFilters } from '../../../typings/ui_filters'; import { useDeepObjectIdentity } from '../../hooks/useDeepObjectIdentity'; -import { LocalUIFilterName } from '../../../common/ui_filter'; -import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; +import { getDateRange } from './helpers'; +import { resolveUrlParams } from './resolve_url_params'; +import { IUrlParams } from './types'; interface TimeRange { rangeFrom: string; @@ -49,9 +46,10 @@ function useUiFilters(params: IUrlParams): UIFilters { const defaultRefresh = (_time: TimeRange) => {}; const UrlParamsContext = createContext({ - urlParams: {} as IUrlParams, + rangeId: 0, refreshTimeRange: defaultRefresh, uiFilters: {} as UIFilters, + urlParams: {} as IUrlParams, }); const UrlParamsProvider: React.ComponentClass<{}> = withRouter( @@ -60,7 +58,8 @@ const UrlParamsProvider: React.ComponentClass<{}> = withRouter( const { start, end, rangeFrom, rangeTo } = refUrlParams.current; - const [, forceUpdate] = useState(''); + // Counter to force an update in useFetcher when the refresh button is clicked. + const [rangeId, setRangeId] = useState(0); const urlParams = useMemo( () => @@ -75,28 +74,25 @@ const UrlParamsProvider: React.ComponentClass<{}> = withRouter( refUrlParams.current = urlParams; - const refreshTimeRange = useCallback( - (timeRange: TimeRange) => { - refUrlParams.current = { - ...refUrlParams.current, - start: getParsedDate(timeRange.rangeFrom), - end: getParsedDate(timeRange.rangeTo, { roundUp: true }), - }; - - forceUpdate(uniqueId()); - }, - [forceUpdate] - ); + const refreshTimeRange = useCallback((timeRange: TimeRange) => { + refUrlParams.current = { + ...refUrlParams.current, + ...getDateRange({ state: {}, ...timeRange }), + }; + + setRangeId((prevRangeId) => prevRangeId + 1); + }, []); const uiFilters = useUiFilters(urlParams); const contextValue = useMemo(() => { return { - urlParams, + rangeId, refreshTimeRange, + urlParams, uiFilters, }; - }, [urlParams, refreshTimeRange, uiFilters]); + }, [rangeId, refreshTimeRange, uiFilters, urlParams]); return ( diff --git a/x-pack/plugins/apm/public/hooks/use_fetcher.tsx b/x-pack/plugins/apm/public/hooks/use_fetcher.tsx index 27427d80adc962..0da96691be9571 100644 --- a/x-pack/plugins/apm/public/hooks/use_fetcher.tsx +++ b/x-pack/plugins/apm/public/hooks/use_fetcher.tsx @@ -13,6 +13,7 @@ import { AutoAbortedAPMClient, } from '../services/rest/createCallApmApi'; import { useApmPluginContext } from '../context/apm_plugin/use_apm_plugin_context'; +import { useUrlParams } from '../context/url_params_context/use_url_params'; export enum FETCH_STATUS { LOADING = 'loading', @@ -75,6 +76,7 @@ export function useFetcher( status: FETCH_STATUS.NOT_INITIATED, }); const [counter, setCounter] = useState(0); + const { rangeId } = useUrlParams(); useEffect(() => { let controller: AbortController = new AbortController(); @@ -157,6 +159,7 @@ export function useFetcher( }, [ counter, preservePreviousData, + rangeId, showToastOnError, ...fnDeps, /* eslint-enable react-hooks/exhaustive-deps */ From 047dd29747eb7817782f059c312e87c1e32c9df8 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Tue, 2 Feb 2021 17:32:37 +0100 Subject: [PATCH 07/12] [Discover] Adapt default column behavior (#89826) --- .../discover/public/application/angular/discover.js | 13 ++----------- .../public/application/components/discover.tsx | 13 ++++++++++--- test/functional/apps/discover/_shared_links.ts | 2 +- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index dcf86babaa5e16..8370e6659554d6 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -414,18 +414,9 @@ function discoverController($route, $scope, Promise) { setBreadcrumbsTitle(savedSearch, chrome); - function removeSourceFromColumns(columns) { - return columns.filter((col) => col !== '_source'); - } - function getDefaultColumns() { - const columns = [...savedSearch.columns]; - - if ($scope.useNewFieldsApi) { - return removeSourceFromColumns(columns); - } - if (columns.length > 0) { - return columns; + if (savedSearch.columns.length > 0) { + return [...savedSearch.columns]; } return [...config.get(DEFAULT_COLUMNS_SETTING)]; } diff --git a/src/plugins/discover/public/application/components/discover.tsx b/src/plugins/discover/public/application/components/discover.tsx index 5653ef4f574356..e87ff82156799a 100644 --- a/src/plugins/discover/public/application/components/discover.tsx +++ b/src/plugins/discover/public/application/components/discover.tsx @@ -7,7 +7,7 @@ */ import './discover.scss'; -import React, { useState, useRef } from 'react'; +import React, { useState, useRef, useMemo } from 'react'; import { EuiButtonEmpty, EuiButtonIcon, @@ -102,6 +102,13 @@ export function Discover({ const contentCentered = resultState === 'uninitialized'; const isLegacy = services.uiSettings.get('doc_table:legacy'); const useNewFieldsApi = !services.uiSettings.get(SEARCH_FIELDS_FROM_SOURCE); + + const columns = useMemo(() => { + if (!state.columns) { + return []; + } + return useNewFieldsApi ? state.columns.filter((col) => col !== '_source') : state.columns; + }, [state, useNewFieldsApi]); return ( @@ -127,7 +134,7 @@ export function Discover({ {isLegacy && rows && rows.length && ( Date: Tue, 2 Feb 2021 09:55:09 -0700 Subject: [PATCH 08/12] [ml] migrate file_data_visualizer/import route to file_upload plugin (#89640) * migrate file_upload plugin to maps_file_upload * update plugins list * migrate ml import endpoint * migrate ml telemetry to file_upload plugin * add fileUpload plugin to ml * add TS project * update ML to use file_upload endpoint * move types to file_upload plugin * ignore error * clean up * i18n clean-up * remove schemas from ml * remove usageCollection from ml * node scripts/build_plugin_list_docs * update telemety collector * revert changes to ingestPipeline schema * change name of TELEMETRY_DOC_ID to unique value * remove ImportFile from ml/server/routes/apidoc.json * fix typo in x=pack/tsconfig.json Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- docs/developer/plugin-list.asciidoc | 4 + src/plugins/telemetry/schema/oss_plugins.json | 6 +- .../common/constants.ts} | 0 x-pack/plugins/file_upload/common/index.ts | 8 ++ x-pack/plugins/file_upload/common/types.ts | 54 +++++++++++ x-pack/plugins/file_upload/jest.config.js | 11 +++ x-pack/plugins/file_upload/kibana.json | 8 ++ .../file_upload/server/error_wrapper.ts | 23 +++++ .../server}/import_data.ts | 11 +-- x-pack/plugins/file_upload/server/index.ts | 9 ++ x-pack/plugins/file_upload/server/plugin.ts | 24 +++++ x-pack/plugins/file_upload/server/routes.ts | 85 ++++++++++++++++++ x-pack/plugins/file_upload/server/schemas.ts | 24 +++++ .../server}/telemetry/index.ts | 2 +- .../server}/telemetry/internal_repository.ts | 0 .../server}/telemetry/mappings.ts | 4 +- .../server}/telemetry/telemetry.test.ts | 2 +- .../server}/telemetry/telemetry.ts | 10 +-- .../server/telemetry/usage_collector.ts} | 21 +++-- x-pack/plugins/file_upload/tsconfig.json | 15 ++++ .../public/util/indexing_service.js | 2 +- .../server/routes/file_upload.js | 2 +- .../ml/common/types/file_datavisualizer.ts | 49 ---------- x-pack/plugins/ml/kibana.json | 2 +- .../components/combined_fields/utils.ts | 7 +- .../file_error_callouts.tsx | 2 +- .../import_view/importer/importer.ts | 2 +- .../import_view/importer/message_importer.ts | 6 +- .../file_based/components/utils/utils.ts | 2 +- .../services/ml_api_service/datavisualizer.ts | 4 +- x-pack/plugins/ml/public/plugin.ts | 3 - .../ml/server/lib/register_settings.ts | 2 +- .../data_frame_analytics/analytics_manager.ts | 2 +- .../models/file_data_visualizer/index.ts | 2 - x-pack/plugins/ml/server/plugin.ts | 2 - x-pack/plugins/ml/server/routes/apidoc.json | 1 - .../ml/server/routes/file_data_visualizer.ts | 89 +------------------ .../schemas/file_data_visualizer_schema.ts | 17 ---- x-pack/plugins/ml/server/types.ts | 2 - .../schema/xpack_plugins.json | 28 +++--- x-pack/tsconfig.json | 2 + x-pack/tsconfig.refs.json | 1 + 42 files changed, 330 insertions(+), 220 deletions(-) rename x-pack/plugins/{ml/common/constants/file_datavisualizer.ts => file_upload/common/constants.ts} (100%) create mode 100644 x-pack/plugins/file_upload/common/index.ts create mode 100644 x-pack/plugins/file_upload/common/types.ts create mode 100644 x-pack/plugins/file_upload/jest.config.js create mode 100644 x-pack/plugins/file_upload/kibana.json create mode 100644 x-pack/plugins/file_upload/server/error_wrapper.ts rename x-pack/plugins/{ml/server/models/file_data_visualizer => file_upload/server}/import_data.ts (95%) create mode 100644 x-pack/plugins/file_upload/server/index.ts create mode 100644 x-pack/plugins/file_upload/server/plugin.ts create mode 100644 x-pack/plugins/file_upload/server/routes.ts create mode 100644 x-pack/plugins/file_upload/server/schemas.ts rename x-pack/plugins/{ml/server/lib => file_upload/server}/telemetry/index.ts (82%) rename x-pack/plugins/{ml/server/lib => file_upload/server}/telemetry/internal_repository.ts (100%) rename x-pack/plugins/{ml/server/lib => file_upload/server}/telemetry/mappings.ts (86%) rename x-pack/plugins/{ml/server/lib => file_upload/server}/telemetry/telemetry.test.ts (97%) rename x-pack/plugins/{ml/server/lib => file_upload/server}/telemetry/telemetry.ts (89%) rename x-pack/plugins/{ml/server/lib/telemetry/ml_usage_collector.ts => file_upload/server/telemetry/usage_collector.ts} (62%) create mode 100644 x-pack/plugins/file_upload/tsconfig.json diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 0ab1c89c1d8f75..215a4f3a4ebb42 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -370,6 +370,10 @@ and actions. |The features plugin enhance Kibana with a per-feature privilege system. +|{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload] +|WARNING: Missing README. + + |{kib-repo}blob/{branch}/x-pack/plugins/fleet/README.md[fleet] |Fleet needs to have Elasticsearch API keys enabled, and also to have TLS enabled on kibana, (if you want to run Kibana without TLS you can provide the following config flag --xpack.fleet.agents.tlsCheckDisabled=false) diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 950fdf9405b754..14cd7141ac9e21 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -4097,6 +4097,9 @@ "xpackDashboardMode:roles": { "type": "keyword" }, + "securitySolution:ipReputationLinks": { + "type": "keyword" + }, "visualize:enableLabs": { "type": "boolean" }, @@ -4115,9 +4118,6 @@ "visualization:tileMap:maxPrecision": { "type": "long" }, - "securitySolution:ipReputationLinks": { - "type": "keyword" - }, "csv:separator": { "type": "keyword" }, diff --git a/x-pack/plugins/ml/common/constants/file_datavisualizer.ts b/x-pack/plugins/file_upload/common/constants.ts similarity index 100% rename from x-pack/plugins/ml/common/constants/file_datavisualizer.ts rename to x-pack/plugins/file_upload/common/constants.ts diff --git a/x-pack/plugins/file_upload/common/index.ts b/x-pack/plugins/file_upload/common/index.ts new file mode 100644 index 00000000000000..6c1725d61c0597 --- /dev/null +++ b/x-pack/plugins/file_upload/common/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './constants'; +export * from './types'; diff --git a/x-pack/plugins/file_upload/common/types.ts b/x-pack/plugins/file_upload/common/types.ts new file mode 100644 index 00000000000000..229983f1c535af --- /dev/null +++ b/x-pack/plugins/file_upload/common/types.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface ImportResponse { + success: boolean; + id: string; + index?: string; + pipelineId?: string; + docCount: number; + failures: ImportFailure[]; + error?: any; + ingestError?: boolean; +} + +export interface ImportFailure { + item: number; + reason: string; + doc: ImportDoc; +} + +export interface Doc { + message: string; +} + +export type ImportDoc = Doc | string; + +export interface Settings { + pipeline?: string; + index: string; + body: any[]; + [key: string]: any; +} + +export interface Mappings { + _meta?: { + created_by: string; + }; + properties: { + [key: string]: any; + }; +} + +export interface IngestPipelineWrapper { + id: string; + pipeline: IngestPipeline; +} + +export interface IngestPipeline { + description: string; + processors: any[]; +} diff --git a/x-pack/plugins/file_upload/jest.config.js b/x-pack/plugins/file_upload/jest.config.js new file mode 100644 index 00000000000000..6a042a4cc5c1e1 --- /dev/null +++ b/x-pack/plugins/file_upload/jest.config.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/file_upload'], +}; diff --git a/x-pack/plugins/file_upload/kibana.json b/x-pack/plugins/file_upload/kibana.json new file mode 100644 index 00000000000000..7ca024174ec6a1 --- /dev/null +++ b/x-pack/plugins/file_upload/kibana.json @@ -0,0 +1,8 @@ +{ + "id": "fileUpload", + "version": "8.0.0", + "kibanaVersion": "kibana", + "server": true, + "ui": false, + "requiredPlugins": ["usageCollection"] +} diff --git a/x-pack/plugins/file_upload/server/error_wrapper.ts b/x-pack/plugins/file_upload/server/error_wrapper.ts new file mode 100644 index 00000000000000..fb41d30e34faeb --- /dev/null +++ b/x-pack/plugins/file_upload/server/error_wrapper.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { boomify, isBoom } from '@hapi/boom'; +import { ResponseError, CustomHttpResponseOptions } from 'kibana/server'; + +export function wrapError(error: any): CustomHttpResponseOptions { + const boom = isBoom(error) + ? error + : boomify(error, { statusCode: error.status ?? error.statusCode }); + const statusCode = boom.output.statusCode; + return { + body: { + message: boom, + ...(statusCode !== 500 && error.body ? { attributes: { body: error.body } } : {}), + }, + headers: boom.output.headers as { [key: string]: string }, + statusCode, + }; +} diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts similarity index 95% rename from x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts rename to x-pack/plugins/file_upload/server/import_data.ts index 26dba7c2f00c14..1eb495d6570c2f 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts +++ b/x-pack/plugins/file_upload/server/import_data.ts @@ -5,19 +5,20 @@ */ import { IScopedClusterClient } from 'kibana/server'; -import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { INDEX_META_DATA_CREATED_BY } from '../common/constants'; import { ImportResponse, ImportFailure, Settings, Mappings, IngestPipelineWrapper, -} from '../../../common/types/file_datavisualizer'; -import { InputData } from './file_data_visualizer'; +} from '../common'; + +export type InputData = any[]; export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { async function importData( - id: string, + id: string | undefined, index: string, settings: Settings, mappings: Mappings, @@ -77,7 +78,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { } catch (error) { return { success: false, - id, + id: id!, index: createdIndex, pipelineId: createdPipelineId, error: error.body !== undefined ? error.body : error, diff --git a/x-pack/plugins/file_upload/server/index.ts b/x-pack/plugins/file_upload/server/index.ts new file mode 100644 index 00000000000000..44a208b7924bc7 --- /dev/null +++ b/x-pack/plugins/file_upload/server/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FileUploadPlugin } from './plugin'; + +export const plugin = () => new FileUploadPlugin(); diff --git a/x-pack/plugins/file_upload/server/plugin.ts b/x-pack/plugins/file_upload/server/plugin.ts new file mode 100644 index 00000000000000..eea3239e52d1ce --- /dev/null +++ b/x-pack/plugins/file_upload/server/plugin.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CoreSetup, CoreStart, Plugin } from 'src/core/server'; +import { fileUploadRoutes } from './routes'; +import { initFileUploadTelemetry } from './telemetry'; +import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; + +interface SetupDeps { + usageCollection: UsageCollectionSetup; +} + +export class FileUploadPlugin implements Plugin { + async setup(coreSetup: CoreSetup, plugins: SetupDeps) { + fileUploadRoutes(coreSetup.http.createRouter()); + + initFileUploadTelemetry(coreSetup, plugins.usageCollection); + } + + start(core: CoreStart) {} +} diff --git a/x-pack/plugins/file_upload/server/routes.ts b/x-pack/plugins/file_upload/server/routes.ts new file mode 100644 index 00000000000000..c98f413caba643 --- /dev/null +++ b/x-pack/plugins/file_upload/server/routes.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter, IScopedClusterClient } from 'kibana/server'; +import { MAX_FILE_SIZE_BYTES, IngestPipelineWrapper, Mappings, Settings } from '../common'; +import { wrapError } from './error_wrapper'; +import { InputData, importDataProvider } from './import_data'; + +import { updateTelemetry } from './telemetry'; +import { importFileBodySchema, importFileQuerySchema } from './schemas'; + +function importData( + client: IScopedClusterClient, + id: string | undefined, + index: string, + settings: Settings, + mappings: Mappings, + ingestPipeline: IngestPipelineWrapper, + data: InputData +) { + const { importData: importDataFunc } = importDataProvider(client); + return importDataFunc(id, index, settings, mappings, ingestPipeline, data); +} + +/** + * Routes for the file upload. + */ +export function fileUploadRoutes(router: IRouter) { + /** + * @apiGroup FileDataVisualizer + * + * @api {post} /api/file_upload/import Import file data + * @apiName ImportFile + * @apiDescription Imports file data into elasticsearch index. + * + * @apiSchema (query) importFileQuerySchema + * @apiSchema (body) importFileBodySchema + */ + router.post( + { + path: '/api/file_upload/import', + validate: { + query: importFileQuerySchema, + body: importFileBodySchema, + }, + options: { + body: { + accepts: ['application/json'], + maxBytes: MAX_FILE_SIZE_BYTES, + }, + tags: ['access:ml:canFindFileStructure'], + }, + }, + async (context, request, response) => { + try { + const { id } = request.query; + const { index, data, settings, mappings, ingestPipeline } = request.body; + + // `id` being `undefined` tells us that this is a new import due to create a new index. + // follow-up import calls to just add additional data will include the `id` of the created + // index, we'll ignore those and don't increment the counter. + if (id === undefined) { + await updateTelemetry(); + } + + const result = await importData( + context.core.elasticsearch.client, + id, + index, + settings, + mappings, + // @ts-expect-error + ingestPipeline, + data + ); + return response.ok({ body: result }); + } catch (e) { + return response.customError(wrapError(e)); + } + } + ); +} diff --git a/x-pack/plugins/file_upload/server/schemas.ts b/x-pack/plugins/file_upload/server/schemas.ts new file mode 100644 index 00000000000000..79db26cdb8c05b --- /dev/null +++ b/x-pack/plugins/file_upload/server/schemas.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; + +export const importFileQuerySchema = schema.object({ + id: schema.maybe(schema.string()), +}); + +export const importFileBodySchema = schema.object({ + index: schema.string(), + data: schema.arrayOf(schema.any()), + settings: schema.maybe(schema.any()), + /** Mappings */ + mappings: schema.any(), + /** Ingest pipeline definition */ + ingestPipeline: schema.object({ + id: schema.maybe(schema.string()), + pipeline: schema.maybe(schema.any()), + }), +}); diff --git a/x-pack/plugins/ml/server/lib/telemetry/index.ts b/x-pack/plugins/file_upload/server/telemetry/index.ts similarity index 82% rename from x-pack/plugins/ml/server/lib/telemetry/index.ts rename to x-pack/plugins/file_upload/server/telemetry/index.ts index b5ec80daf17878..92d8ab425a7731 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/index.ts +++ b/x-pack/plugins/file_upload/server/telemetry/index.ts @@ -4,5 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ -export { initMlTelemetry } from './ml_usage_collector'; +export { initFileUploadTelemetry } from './usage_collector'; export { updateTelemetry } from './telemetry'; diff --git a/x-pack/plugins/ml/server/lib/telemetry/internal_repository.ts b/x-pack/plugins/file_upload/server/telemetry/internal_repository.ts similarity index 100% rename from x-pack/plugins/ml/server/lib/telemetry/internal_repository.ts rename to x-pack/plugins/file_upload/server/telemetry/internal_repository.ts diff --git a/x-pack/plugins/ml/server/lib/telemetry/mappings.ts b/x-pack/plugins/file_upload/server/telemetry/mappings.ts similarity index 86% rename from x-pack/plugins/ml/server/lib/telemetry/mappings.ts rename to x-pack/plugins/file_upload/server/telemetry/mappings.ts index 5aaf9f8c79dc0a..3d22bcb4162fd9 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/mappings.ts +++ b/x-pack/plugins/file_upload/server/telemetry/mappings.ts @@ -7,13 +7,13 @@ import { SavedObjectsType } from 'src/core/server'; import { TELEMETRY_DOC_ID } from './telemetry'; -export const mlTelemetryMappingsType: SavedObjectsType = { +export const telemetryMappingsType: SavedObjectsType = { name: TELEMETRY_DOC_ID, hidden: false, namespaceType: 'agnostic', mappings: { properties: { - file_data_visualizer: { + file_upload: { properties: { index_creation_count: { type: 'long', diff --git a/x-pack/plugins/ml/server/lib/telemetry/telemetry.test.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts similarity index 97% rename from x-pack/plugins/ml/server/lib/telemetry/telemetry.test.ts rename to x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts index f41c4fda93a54f..2ad36338f49283 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/telemetry.test.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts @@ -34,7 +34,7 @@ describe('ml plugin telemetry', () => { it('should update existing telemetry', async () => { const internalRepo = mockInit({ attributes: { - file_data_visualizer: { + file_upload: { index_creation_count: 2, }, }, diff --git a/x-pack/plugins/ml/server/lib/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts similarity index 89% rename from x-pack/plugins/ml/server/lib/telemetry/telemetry.ts rename to x-pack/plugins/file_upload/server/telemetry/telemetry.ts index 06577d69371019..aac45d5d0f8713 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -9,10 +9,10 @@ import { ISavedObjectsRepository } from 'kibana/server'; import { getInternalRepository } from './internal_repository'; -export const TELEMETRY_DOC_ID = 'ml-telemetry'; +export const TELEMETRY_DOC_ID = 'file-upload-usage-collection-telemetry'; export interface Telemetry { - file_data_visualizer: { + file_upload: { index_creation_count: number; }; } @@ -23,7 +23,7 @@ export interface TelemetrySavedObject { export function initTelemetry(): Telemetry { return { - file_data_visualizer: { + file_upload: { index_creation_count: 0, }, }; @@ -74,8 +74,8 @@ export async function updateTelemetry(internalRepo?: ISavedObjectsRepository) { function incrementCounts(telemetry: Telemetry) { return { - file_data_visualizer: { - index_creation_count: telemetry.file_data_visualizer.index_creation_count + 1, + file_upload: { + index_creation_count: telemetry.file_upload.index_creation_count + 1, }, }; } diff --git a/x-pack/plugins/ml/server/lib/telemetry/ml_usage_collector.ts b/x-pack/plugins/file_upload/server/telemetry/usage_collector.ts similarity index 62% rename from x-pack/plugins/ml/server/lib/telemetry/ml_usage_collector.ts rename to x-pack/plugins/file_upload/server/telemetry/usage_collector.ts index 35c6936598c406..4a1334ef53d951 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/ml_usage_collector.ts +++ b/x-pack/plugins/file_upload/server/telemetry/usage_collector.ts @@ -8,23 +8,26 @@ import { CoreSetup } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { getTelemetry, initTelemetry, Telemetry } from './telemetry'; -import { mlTelemetryMappingsType } from './mappings'; +import { telemetryMappingsType } from './mappings'; import { setInternalRepository } from './internal_repository'; -export function initMlTelemetry(coreSetup: CoreSetup, usageCollection: UsageCollectionSetup) { - coreSetup.savedObjects.registerType(mlTelemetryMappingsType); - registerMlUsageCollector(usageCollection); +export function initFileUploadTelemetry( + coreSetup: CoreSetup, + usageCollection: UsageCollectionSetup +) { + coreSetup.savedObjects.registerType(telemetryMappingsType); + registerUsageCollector(usageCollection); coreSetup.getStartServices().then(([core]) => { setInternalRepository(core.savedObjects.createInternalRepository); }); } -function registerMlUsageCollector(usageCollection: UsageCollectionSetup): void { - const mlUsageCollector = usageCollection.makeUsageCollector({ - type: 'mlTelemetry', +function registerUsageCollector(usageCollectionSetup: UsageCollectionSetup): void { + const usageCollector = usageCollectionSetup.makeUsageCollector({ + type: 'fileUpload', isReady: () => true, schema: { - file_data_visualizer: { + file_upload: { index_creation_count: { type: 'long' }, }, }, @@ -38,5 +41,5 @@ function registerMlUsageCollector(usageCollection: UsageCollectionSetup): void { }, }); - usageCollection.registerCollector(mlUsageCollector); + usageCollectionSetup.registerCollector(usageCollector); } diff --git a/x-pack/plugins/file_upload/tsconfig.json b/x-pack/plugins/file_upload/tsconfig.json new file mode 100644 index 00000000000000..f985a4599d5fed --- /dev/null +++ b/x-pack/plugins/file_upload/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": ["common/**/*", "public/**/*", "server/**/*"], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../../../src/plugins/usage_collection/tsconfig.json" } + ] +} diff --git a/x-pack/plugins/maps_file_upload/public/util/indexing_service.js b/x-pack/plugins/maps_file_upload/public/util/indexing_service.js index 28cdb602455b52..14d02ce881cda6 100644 --- a/x-pack/plugins/maps_file_upload/public/util/indexing_service.js +++ b/x-pack/plugins/maps_file_upload/public/util/indexing_service.js @@ -119,7 +119,7 @@ async function writeToIndex(indexingDetails) { const { appName, index, data, settings, mappings, ingestPipeline } = indexingDetails; return await httpService({ - url: `/api/fileupload/import`, + url: `/api/maps/fileupload/import`, method: 'POST', ...(query ? { query } : {}), data: { diff --git a/x-pack/plugins/maps_file_upload/server/routes/file_upload.js b/x-pack/plugins/maps_file_upload/server/routes/file_upload.js index 3935d4ca5fe8ef..0323f23a51df55 100644 --- a/x-pack/plugins/maps_file_upload/server/routes/file_upload.js +++ b/x-pack/plugins/maps_file_upload/server/routes/file_upload.js @@ -9,7 +9,7 @@ import { updateTelemetry } from '../telemetry/telemetry'; import { MAX_BYTES } from '../../common/constants/file_import'; import { schema } from '@kbn/config-schema'; -export const IMPORT_ROUTE = '/api/fileupload/import'; +export const IMPORT_ROUTE = '/api/maps/fileupload/import'; export const querySchema = schema.maybe( schema.object({ diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts index b1967cfe83f3c0..8ba30111c4c8c1 100644 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ b/x-pack/plugins/ml/common/types/file_datavisualizer.ts @@ -68,52 +68,3 @@ export interface FindFileStructureResponse { timestamp_field?: string; should_trim_fields?: boolean; } - -export interface ImportResponse { - success: boolean; - id: string; - index?: string; - pipelineId?: string; - docCount: number; - failures: ImportFailure[]; - error?: any; - ingestError?: boolean; -} - -export interface ImportFailure { - item: number; - reason: string; - doc: ImportDoc; -} - -export interface Doc { - message: string; -} - -export type ImportDoc = Doc | string; - -export interface Settings { - pipeline?: string; - index: string; - body: any[]; - [key: string]: any; -} - -export interface Mappings { - _meta?: { - created_by: string; - }; - properties: { - [key: string]: any; - }; -} - -export interface IngestPipelineWrapper { - id: string; - pipeline: IngestPipeline; -} - -export interface IngestPipeline { - description: string; - processors: any[]; -} diff --git a/x-pack/plugins/ml/kibana.json b/x-pack/plugins/ml/kibana.json index 1c47512e0b3de0..ede6b8abbd09c7 100644 --- a/x-pack/plugins/ml/kibana.json +++ b/x-pack/plugins/ml/kibana.json @@ -10,8 +10,8 @@ "data", "cloud", "features", + "fileUpload", "licensing", - "usageCollection", "share", "embeddable", "uiActions", diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts index 1cc513e778b2f6..25d5373b6dc7c9 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts @@ -8,11 +8,8 @@ import { i18n } from '@kbn/i18n'; import { cloneDeep } from 'lodash'; import uuid from 'uuid/v4'; import { CombinedField } from './types'; -import { - FindFileStructureResponse, - IngestPipeline, - Mappings, -} from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { IngestPipeline, Mappings } from '../../../../../../../file_upload/common'; const COMMON_LAT_NAMES = ['latitude', 'lat']; const COMMON_LON_NAMES = ['longitude', 'long', 'lon']; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx index d869676e488276..0c853493293cab 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx @@ -11,7 +11,7 @@ import { EuiCallOut, EuiSpacer, EuiButtonEmpty, EuiHorizontalRule } from '@elast import numeral from '@elastic/numeral'; import { ErrorResponse } from '../../../../../../common/types/errors'; -import { FILE_SIZE_DISPLAY_FORMAT } from '../../../../../../common/constants/file_datavisualizer'; +import { FILE_SIZE_DISPLAY_FORMAT } from '../../../../../../../file_upload/common'; interface FileTooLargeProps { fileSize: number; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts index 718587ad15ad58..ab0e83846661fe 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts @@ -15,7 +15,7 @@ import { Mappings, Settings, IngestPipeline, -} from '../../../../../../../common/types/file_datavisualizer'; +} from '../../../../../../../../file_upload/common'; const CHUNK_SIZE = 5000; const MAX_CHUNK_CHAR_COUNT = 1000000; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts index 65be24d9e7be4a..a74249ea758a8d 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts @@ -5,10 +5,8 @@ */ import { Importer, ImportConfig, CreateDocsResponse } from './importer'; -import { - Doc, - FindFileStructureResponse, -} from '../../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer'; +import { Doc } from '../../../../../../../../file_upload/common'; export class MessageImporter extends Importer { private _excludeLinesRegex: RegExp | null; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts index 781f400180b107..ce15fb9a03fca0 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts @@ -14,7 +14,7 @@ import { MAX_FILE_SIZE_BYTES, ABSOLUTE_MAX_FILE_SIZE_BYTES, FILE_SIZE_DISPLAY_FORMAT, -} from '../../../../../../common/constants/file_datavisualizer'; +} from '../../../../../../../file_upload/common'; import { getUiSettings } from '../../../../util/dependency_cache'; import { FILE_DATA_VISUALIZER_MAX_FILE_SIZE } from '../../../../../../common/constants/settings'; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts index 20332546d9cdec..27d9b78725befe 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts @@ -7,7 +7,7 @@ import { http } from '../http_service'; import { basePath } from './index'; -import { ImportResponse } from '../../../../common/types/file_datavisualizer'; +import { ImportResponse } from '../../../../../file_upload/common'; export const fileDatavisualizer = { analyzeFile(file: string, params: Record = {}) { @@ -45,7 +45,7 @@ export const fileDatavisualizer = { }); return http({ - path: `${basePath()}/file_data_visualizer/import`, + path: `/api/file_upload/import`, method: 'POST', query, body, diff --git a/x-pack/plugins/ml/public/plugin.ts b/x-pack/plugins/ml/public/plugin.ts index 3ba79e0eb9187b..ef3de1a5ce65c0 100644 --- a/x-pack/plugins/ml/public/plugin.ts +++ b/x-pack/plugins/ml/public/plugin.ts @@ -21,7 +21,6 @@ import type { SharePluginStart, UrlGeneratorContract, } from 'src/plugins/share/public'; -import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import type { DataPublicPluginStart } from 'src/plugins/data/public'; import type { HomePublicPluginSetup } from 'src/plugins/home/public'; import type { IndexPatternManagementSetup } from 'src/plugins/index_pattern_management/public'; @@ -60,7 +59,6 @@ export interface MlSetupDependencies { security?: SecurityPluginSetup; licensing: LicensingPluginSetup; management?: ManagementSetup; - usageCollection: UsageCollectionSetup; licenseManagement?: LicenseManagementUIPluginSetup; home?: HomePublicPluginSetup; embeddable: EmbeddableSetup; @@ -102,7 +100,6 @@ export class MlPlugin implements Plugin { security: pluginsSetup.security, licensing: pluginsSetup.licensing, management: pluginsSetup.management, - usageCollection: pluginsSetup.usageCollection, licenseManagement: pluginsSetup.licenseManagement, home: pluginsSetup.home, embeddable: { ...pluginsSetup.embeddable, ...pluginsStart.embeddable }, diff --git a/x-pack/plugins/ml/server/lib/register_settings.ts b/x-pack/plugins/ml/server/lib/register_settings.ts index 0cdaaadf7f172e..4db2f806334800 100644 --- a/x-pack/plugins/ml/server/lib/register_settings.ts +++ b/x-pack/plugins/ml/server/lib/register_settings.ts @@ -14,7 +14,7 @@ import { DEFAULT_AD_RESULTS_TIME_FILTER, DEFAULT_ENABLE_AD_RESULTS_TIME_FILTER, } from '../../common/constants/settings'; -import { MAX_FILE_SIZE } from '../../common/constants/file_datavisualizer'; +import { MAX_FILE_SIZE } from '../../../file_upload/common'; export function registerKibanaSettings(coreSetup: CoreSetup) { coreSetup.uiSettings.register({ diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts index ea41fb3ae427b8..3f9587749d33dc 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts @@ -18,7 +18,7 @@ import { DataFrameAnalyticsStats, MapElements, } from '../../../common/types/data_frame_analytics'; -import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { INDEX_META_DATA_CREATED_BY } from '../../../../file_upload/common'; import { getAnalysisType } from '../../../common/util/analytics_utils'; import { ExtendAnalyticsMapArgs, diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts index f8a27fdcd7e1ae..aa699694e52a31 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts @@ -5,5 +5,3 @@ */ export { fileDataVisualizerProvider, InputData } from './file_data_visualizer'; - -export { importDataProvider } from './import_data'; diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index e48983c1c53656..3c82f2131e25f3 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -23,7 +23,6 @@ import { SpacesPluginSetup } from '../../spaces/server'; import { PLUGIN_ID } from '../common/constants/app'; import { MlCapabilities } from '../common/types/capabilities'; -import { initMlTelemetry } from './lib/telemetry'; import { initMlServerLog } from './lib/log'; import { initSampleDataSets } from './lib/sample_data_sets'; @@ -190,7 +189,6 @@ export class MlServerPlugin trainedModelsRoutes(routeInit); initMlServerLog({ log: this.log }); - initMlTelemetry(coreSetup, plugins.usageCollection); return { ...createSharedServices( diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index 5dc9a3107af868..1b2eb612fda1c6 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -43,7 +43,6 @@ "FileDataVisualizer", "AnalyzeFile", - "ImportFile", "ResultsService", "GetAnomaliesTableData", diff --git a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts index c4c449a9e2cb44..9ee19efef13f52 100644 --- a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts @@ -5,28 +5,13 @@ */ import { schema } from '@kbn/config-schema'; -import { IScopedClusterClient } from 'kibana/server'; -import { MAX_FILE_SIZE_BYTES } from '../../common/constants/file_datavisualizer'; -import { - InputOverrides, - Settings, - IngestPipelineWrapper, - Mappings, -} from '../../common/types/file_datavisualizer'; +import { MAX_FILE_SIZE_BYTES } from '../../../file_upload/common'; +import { InputOverrides } from '../../common/types/file_datavisualizer'; import { wrapError } from '../client/error_wrapper'; -import { - InputData, - fileDataVisualizerProvider, - importDataProvider, -} from '../models/file_data_visualizer'; +import { InputData, fileDataVisualizerProvider } from '../models/file_data_visualizer'; import { RouteInitialization } from '../types'; -import { updateTelemetry } from '../lib/telemetry'; -import { - analyzeFileQuerySchema, - importFileBodySchema, - importFileQuerySchema, -} from './schemas/file_data_visualizer_schema'; +import { analyzeFileQuerySchema } from './schemas/file_data_visualizer_schema'; import type { MlClient } from '../lib/ml_client'; function analyzeFiles(mlClient: MlClient, data: InputData, overrides: InputOverrides) { @@ -34,19 +19,6 @@ function analyzeFiles(mlClient: MlClient, data: InputData, overrides: InputOverr return analyzeFile(data, overrides); } -function importData( - client: IScopedClusterClient, - id: string, - index: string, - settings: Settings, - mappings: Mappings, - ingestPipeline: IngestPipelineWrapper, - data: InputData -) { - const { importData: importDataFunc } = importDataProvider(client); - return importDataFunc(id, index, settings, mappings, ingestPipeline, data); -} - /** * Routes for the file data visualizer. */ @@ -84,57 +56,4 @@ export function fileDataVisualizerRoutes({ router, routeGuard }: RouteInitializa } }) ); - - /** - * @apiGroup FileDataVisualizer - * - * @api {post} /api/ml/file_data_visualizer/import Import file data - * @apiName ImportFile - * @apiDescription Imports file data into elasticsearch index. - * - * @apiSchema (query) importFileQuerySchema - * @apiSchema (body) importFileBodySchema - */ - router.post( - { - path: '/api/ml/file_data_visualizer/import', - validate: { - query: importFileQuerySchema, - body: importFileBodySchema, - }, - options: { - body: { - accepts: ['application/json'], - maxBytes: MAX_FILE_SIZE_BYTES, - }, - tags: ['access:ml:canFindFileStructure'], - }, - }, - routeGuard.basicLicenseAPIGuard(async ({ client, request, response }) => { - try { - const { id } = request.query; - const { index, data, settings, mappings, ingestPipeline } = request.body; - - // `id` being `undefined` tells us that this is a new import due to create a new index. - // follow-up import calls to just add additional data will include the `id` of the created - // index, we'll ignore those and don't increment the counter. - if (id === undefined) { - await updateTelemetry(); - } - - const result = await importData( - client, - id, - index, - settings, - mappings, - ingestPipeline, - data - ); - return response.ok({ body: result }); - } catch (e) { - return response.customError(wrapError(e)); - } - }) - ); } diff --git a/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts b/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts index 9a80cf795cabfc..685f06f839ee38 100644 --- a/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts @@ -24,20 +24,3 @@ export const analyzeFileQuerySchema = schema.maybe( timestamp_format: schema.maybe(schema.string()), }) ); - -export const importFileQuerySchema = schema.object({ - id: schema.maybe(schema.string()), -}); - -export const importFileBodySchema = schema.object({ - index: schema.maybe(schema.string()), - data: schema.arrayOf(schema.any()), - settings: schema.maybe(schema.any()), - /** Mappings */ - mappings: schema.any(), - /** Ingest pipeline definition */ - ingestPipeline: schema.object({ - id: schema.maybe(schema.string()), - pipeline: schema.maybe(schema.any()), - }), -}); diff --git a/x-pack/plugins/ml/server/types.ts b/x-pack/plugins/ml/server/types.ts index 780a4284312e71..f3e3ee22f38176 100644 --- a/x-pack/plugins/ml/server/types.ts +++ b/x-pack/plugins/ml/server/types.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import type { HomeServerPluginSetup } from 'src/plugins/home/server'; import type { IRouter } from 'kibana/server'; import type { CloudSetup } from '../../cloud/server'; @@ -43,7 +42,6 @@ export interface PluginsSetup { licensing: LicensingPluginSetup; security?: SecurityPluginSetup; spaces?: SpacesPluginSetup; - usageCollection: UsageCollectionSetup; } export interface PluginsStart { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 4ca373d9260b7b..c1674f8a926699 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -1771,10 +1771,14 @@ } } }, - "fileUploadTelemetry": { + "fileUpload": { "properties": { - "filesUploadedTotalCount": { - "type": "long" + "file_upload": { + "properties": { + "index_creation_count": { + "type": "long" + } + } } } }, @@ -2215,6 +2219,13 @@ } } }, + "fileUploadTelemetry": { + "properties": { + "filesUploadedTotalCount": { + "type": "long" + } + } + }, "maps": { "properties": { "settings": { @@ -2308,17 +2319,6 @@ } } }, - "mlTelemetry": { - "properties": { - "file_data_visualizer": { - "properties": { - "index_creation_count": { - "type": "long" - } - } - } - } - }, "monitoring": { "properties": { "hasMonitoringData": { diff --git a/x-pack/tsconfig.json b/x-pack/tsconfig.json index 48283b3ac27472..c0eed6e89b60ef 100644 --- a/x-pack/tsconfig.json +++ b/x-pack/tsconfig.json @@ -17,6 +17,7 @@ "plugins/global_search_providers/**/*", "plugins/graph/**/*", "plugins/features/**/*", + "plugins/file_upload/**/*", "plugins/embeddable_enhanced/**/*", "plugins/event_log/**/*", "plugins/enterprise_search/**/*", @@ -103,6 +104,7 @@ { "path": "./plugins/enterprise_search/tsconfig.json" }, { "path": "./plugins/event_log/tsconfig.json" }, { "path": "./plugins/features/tsconfig.json" }, + { "path": "./plugins/file_upload/tsconfig.json" }, { "path": "./plugins/global_search_bar/tsconfig.json" }, { "path": "./plugins/global_search_providers/tsconfig.json" }, { "path": "./plugins/global_search/tsconfig.json" }, diff --git a/x-pack/tsconfig.refs.json b/x-pack/tsconfig.refs.json index 23b06040f3ec36..fe2d4873b7cec6 100644 --- a/x-pack/tsconfig.refs.json +++ b/x-pack/tsconfig.refs.json @@ -16,6 +16,7 @@ { "path": "./plugins/enterprise_search/tsconfig.json" }, { "path": "./plugins/event_log/tsconfig.json" }, { "path": "./plugins/features/tsconfig.json" }, + { "path": "./plugins/file_upload/tsconfig.json" }, { "path": "./plugins/global_search_bar/tsconfig.json" }, { "path": "./plugins/global_search_providers/tsconfig.json" }, { "path": "./plugins/global_search/tsconfig.json" }, From 802ac60fee4741e27741ab634007b6ca859144cd Mon Sep 17 00:00:00 2001 From: igoristic Date: Tue, 2 Feb 2021 12:04:53 -0500 Subject: [PATCH 09/12] Made imports static (#89935) --- x-pack/plugins/monitoring/public/plugin.ts | 26 +++++++++------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/monitoring/public/plugin.ts b/x-pack/plugins/monitoring/public/plugin.ts index 52f8d07f4fdb65..65c0c4d915f9cc 100644 --- a/x-pack/plugins/monitoring/public/plugin.ts +++ b/x-pack/plugins/monitoring/public/plugin.ts @@ -27,6 +27,14 @@ import { ALERT_DETAILS, } from '../common/constants'; +import { createCpuUsageAlertType } from './alerts/cpu_usage_alert'; +import { createMissingMonitoringDataAlertType } from './alerts/missing_monitoring_data_alert'; +import { createLegacyAlertTypes } from './alerts/legacy_alert'; +import { createDiskUsageAlertType } from './alerts/disk_usage_alert'; +import { createThreadPoolRejectionsAlertType } from './alerts/thread_pool_rejections_alert'; +import { createMemoryUsageAlertType } from './alerts/memory_usage_alert'; +import { createCCRReadExceptionsAlertType } from './alerts/ccr_read_exceptions_alert'; + interface MonitoringSetupPluginDependencies { home?: HomePublicPluginSetup; cloud?: { isCloudEnabled: boolean }; @@ -72,7 +80,7 @@ export class MonitoringPlugin }); } - await this.registerAlertsAsync(plugins); + this.registerAlerts(plugins); const app: App = { id, @@ -135,19 +143,7 @@ export class MonitoringPlugin ]; } - private registerAlertsAsync = async (plugins: MonitoringSetupPluginDependencies) => { - const { createCpuUsageAlertType } = await import('./alerts/cpu_usage_alert'); - const { createMissingMonitoringDataAlertType } = await import( - './alerts/missing_monitoring_data_alert' - ); - const { createLegacyAlertTypes } = await import('./alerts/legacy_alert'); - const { createDiskUsageAlertType } = await import('./alerts/disk_usage_alert'); - const { createThreadPoolRejectionsAlertType } = await import( - './alerts/thread_pool_rejections_alert' - ); - const { createMemoryUsageAlertType } = await import('./alerts/memory_usage_alert'); - const { createCCRReadExceptionsAlertType } = await import('./alerts/ccr_read_exceptions_alert'); - + private registerAlerts(plugins: MonitoringSetupPluginDependencies) { const { triggersActionsUi: { alertTypeRegistry }, } = plugins; @@ -172,5 +168,5 @@ export class MonitoringPlugin for (const legacyAlertType of legacyAlertTypes) { alertTypeRegistry.register(legacyAlertType); } - }; + } } From ad67ee551e4b31c9ecc3ad588e3593c05e7e8db7 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Tue, 2 Feb 2021 10:15:13 -0700 Subject: [PATCH 10/12] [SearchSource] Combine sort and parent fields when serializing (#89808) * [SearchSource] Combine sort and parent fields when serializing * fix docs * add link to issue ; * fix destructive recursion Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- ...gins-data-public.searchsource.getfields.md | 47 +---- ...public.searchsource.getserializedfields.md | 9 +- ...plugin-plugins-data-public.searchsource.md | 4 +- .../field_formats/field_formats_registry.ts | 16 +- .../search_source/normalize_sort_request.ts | 5 + .../search_source/search_source.test.ts | 173 +++++++++++++++++- .../search/search_source/search_source.ts | 49 ++++- .../__snapshots__/tabify_docs.test.ts.snap | 98 +++++++++- .../data/common/search/tabify/index.ts | 2 + .../data/common/search/tabify/tabify_docs.ts | 14 +- src/plugins/data/public/public.api.md | 24 +-- 11 files changed, 358 insertions(+), 83 deletions(-) diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getfields.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getfields.md index faff901bfc167d..b0ccedb819c95d 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getfields.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getfields.md @@ -9,45 +9,16 @@ returns all search source fields Signature: ```typescript -getFields(): { - type?: string | undefined; - query?: import("../..").Query | undefined; - filter?: Filter | Filter[] | (() => Filter | Filter[] | undefined) | undefined; - sort?: Record | Record[] | undefined; - highlight?: any; - highlightAll?: boolean | undefined; - aggs?: any; - from?: number | undefined; - size?: number | undefined; - source?: string | boolean | string[] | undefined; - version?: boolean | undefined; - fields?: SearchFieldValue[] | undefined; - fieldsFromSource?: string | boolean | string[] | undefined; - index?: import("../..").IndexPattern | undefined; - searchAfter?: import("./types").EsQuerySearchAfter | undefined; - timeout?: string | undefined; - terminate_after?: number | undefined; - }; +getFields(recurse?: boolean): SearchSourceFields; ``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| recurse | boolean | | + Returns: -`{ - type?: string | undefined; - query?: import("../..").Query | undefined; - filter?: Filter | Filter[] | (() => Filter | Filter[] | undefined) | undefined; - sort?: Record | Record[] | undefined; - highlight?: any; - highlightAll?: boolean | undefined; - aggs?: any; - from?: number | undefined; - size?: number | undefined; - source?: string | boolean | string[] | undefined; - version?: boolean | undefined; - fields?: SearchFieldValue[] | undefined; - fieldsFromSource?: string | boolean | string[] | undefined; - index?: import("../..").IndexPattern | undefined; - searchAfter?: import("./types").EsQuerySearchAfter | undefined; - timeout?: string | undefined; - terminate_after?: number | undefined; - }` +`SearchSourceFields` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getserializedfields.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getserializedfields.md index 3f58a76b24cd08..19bd4a7888bf24 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getserializedfields.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.getserializedfields.md @@ -9,8 +9,15 @@ serializes search source fields (which can later be passed to [ISearchStartSearc Signature: ```typescript -getSerializedFields(): SearchSourceFields; +getSerializedFields(recurse?: boolean): SearchSourceFields; ``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| recurse | boolean | | + Returns: `SearchSourceFields` diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md index 2af9cc14e36689..3250561c8b82e9 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md @@ -35,12 +35,12 @@ export declare class SearchSource | [fetch(options)](./kibana-plugin-plugins-data-public.searchsource.fetch.md) | | Fetch this source and reject the returned Promise on error | | [fetch$(options)](./kibana-plugin-plugins-data-public.searchsource.fetch_.md) | | Fetch this source from Elasticsearch, returning an observable over the response(s) | | [getField(field, recurse)](./kibana-plugin-plugins-data-public.searchsource.getfield.md) | | Gets a single field from the fields | -| [getFields()](./kibana-plugin-plugins-data-public.searchsource.getfields.md) | | returns all search source fields | +| [getFields(recurse)](./kibana-plugin-plugins-data-public.searchsource.getfields.md) | | returns all search source fields | | [getId()](./kibana-plugin-plugins-data-public.searchsource.getid.md) | | returns search source id | | [getOwnField(field)](./kibana-plugin-plugins-data-public.searchsource.getownfield.md) | | Get the field from our own fields, don't traverse up the chain | | [getParent()](./kibana-plugin-plugins-data-public.searchsource.getparent.md) | | Get the parent of this SearchSource {undefined\|searchSource} | | [getSearchRequestBody()](./kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md) | | Returns body contents of the search request, often referred as query DSL. | -| [getSerializedFields()](./kibana-plugin-plugins-data-public.searchsource.getserializedfields.md) | | serializes search source fields (which can later be passed to [ISearchStartSearchSource](./kibana-plugin-plugins-data-public.isearchstartsearchsource.md)) | +| [getSerializedFields(recurse)](./kibana-plugin-plugins-data-public.searchsource.getserializedfields.md) | | serializes search source fields (which can later be passed to [ISearchStartSearchSource](./kibana-plugin-plugins-data-public.isearchstartsearchsource.md)) | | [onRequestStart(handler)](./kibana-plugin-plugins-data-public.searchsource.onrequeststart.md) | | Add a handler that will be notified whenever requests start | | [removeField(field)](./kibana-plugin-plugins-data-public.searchsource.removefield.md) | | remove field | | [serialize()](./kibana-plugin-plugins-data-public.searchsource.serialize.md) | | Serializes the instance to a JSON string and a set of referenced objects. Use this method to get a representation of the search source which can be stored in a saved object.The references returned by this function can be mixed with other references in the same object, however make sure there are no name-collisions. The references will be named kibanaSavedObjectMeta.searchSourceJSON.index and kibanaSavedObjectMeta.searchSourceJSON.filter[<number>].meta.index.Using createSearchSource, the instance can be re-created. | diff --git a/src/plugins/data/common/field_formats/field_formats_registry.ts b/src/plugins/data/common/field_formats/field_formats_registry.ts index aa6cf4500c933e..b5cc1b944094c4 100644 --- a/src/plugins/data/common/field_formats/field_formats_registry.ts +++ b/src/plugins/data/common/field_formats/field_formats_registry.ts @@ -23,6 +23,7 @@ import { FormatFactory } from './utils'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../kbn_field_types/types'; import { UI_SETTINGS } from '../constants'; import { FieldFormatNotFoundError } from '../field_formats'; +import { SerializedFieldFormat } from '../../../expressions/common/types'; export class FieldFormatsRegistry { protected fieldFormats: Map = new Map(); @@ -30,7 +31,20 @@ export class FieldFormatsRegistry { protected metaParamsOptions: Record = {}; protected getConfig?: FieldFormatsGetConfigFn; // overriden on the public contract - public deserialize: FormatFactory = () => { + public deserialize: FormatFactory = (mapping?: SerializedFieldFormat) => { + if (!mapping) { + return new (FieldFormat.from(identity))(); + } + + const { id, params = {} } = mapping; + if (id) { + const Format = this.getType(id); + + if (Format) { + return new Format(params, this.getConfig); + } + } + return new (FieldFormat.from(identity))(); }; diff --git a/src/plugins/data/common/search/search_source/normalize_sort_request.ts b/src/plugins/data/common/search/search_source/normalize_sort_request.ts index 7f1cbbd7f2da66..7461b6c1788f80 100644 --- a/src/plugins/data/common/search/search_source/normalize_sort_request.ts +++ b/src/plugins/data/common/search/search_source/normalize_sort_request.ts @@ -49,6 +49,11 @@ function normalize( } } + // FIXME: for unknown reason on the server this setting is serialized + // https://github.com/elastic/kibana/issues/89902 + if (typeof defaultSortOptions === 'string') { + defaultSortOptions = JSON.parse(defaultSortOptions); + } // Don't include unmapped_type for _score field // eslint-disable-next-line @typescript-eslint/naming-convention const { unmapped_type, ...otherSortOptions } = defaultSortOptions; diff --git a/src/plugins/data/common/search/search_source/search_source.test.ts b/src/plugins/data/common/search/search_source/search_source.test.ts index c2a4beb9b61a52..49fb1fa62f4907 100644 --- a/src/plugins/data/common/search/search_source/search_source.test.ts +++ b/src/plugins/data/common/search/search_source/search_source.test.ts @@ -80,6 +80,175 @@ describe('SearchSource', () => { }); }); + describe('#getFields()', () => { + test('gets the value for the property', () => { + searchSource.setField('aggs', 5); + expect(searchSource.getFields()).toMatchInlineSnapshot(` + Object { + "aggs": 5, + } + `); + }); + + test('recurses parents to get the entire filters: plain object filter', () => { + const RECURSE = true; + + const parent = new SearchSource({}, searchSourceDependencies); + parent.setField('filter', [ + { + meta: { + index: 'd180cae0-60c3-11eb-8569-bd1f5ed24bc9', + params: {}, + alias: null, + disabled: false, + negate: false, + }, + query: { + range: { + '@date': { + gte: '2016-01-27T18:11:05.010Z', + lte: '2021-01-27T18:11:05.010Z', + format: 'strict_date_optional_time', + }, + }, + }, + }, + ]); + searchSource.setParent(parent); + searchSource.setField('aggs', 5); + expect(searchSource.getFields(RECURSE)).toMatchInlineSnapshot(` + Object { + "aggs": 5, + "filter": Array [ + Object { + "meta": Object { + "alias": null, + "disabled": false, + "index": "d180cae0-60c3-11eb-8569-bd1f5ed24bc9", + "negate": false, + "params": Object {}, + }, + "query": Object { + "range": Object { + "@date": Object { + "format": "strict_date_optional_time", + "gte": "2016-01-27T18:11:05.010Z", + "lte": "2021-01-27T18:11:05.010Z", + }, + }, + }, + }, + ], + } + `); + + // calling twice gives the same result: no searchSources in the hierarchy were modified + expect(searchSource.getFields(RECURSE)).toMatchInlineSnapshot(` + Object { + "aggs": 5, + "filter": Array [ + Object { + "meta": Object { + "alias": null, + "disabled": false, + "index": "d180cae0-60c3-11eb-8569-bd1f5ed24bc9", + "negate": false, + "params": Object {}, + }, + "query": Object { + "range": Object { + "@date": Object { + "format": "strict_date_optional_time", + "gte": "2016-01-27T18:11:05.010Z", + "lte": "2021-01-27T18:11:05.010Z", + }, + }, + }, + }, + ], + } + `); + }); + + test('recurses parents to get the entire filters: function filter', () => { + const RECURSE = true; + + const parent = new SearchSource({}, searchSourceDependencies); + parent.setField('filter', () => ({ + meta: { + index: 'd180cae0-60c3-11eb-8569-bd1f5ed24bc9', + params: {}, + alias: null, + disabled: false, + negate: false, + }, + query: { + range: { + '@date': { + gte: '2016-01-27T18:11:05.010Z', + lte: '2021-01-27T18:11:05.010Z', + format: 'strict_date_optional_time', + }, + }, + }, + })); + searchSource.setParent(parent); + searchSource.setField('aggs', 5); + expect(searchSource.getFields(RECURSE)).toMatchInlineSnapshot(` + Object { + "aggs": 5, + "filter": Array [ + Object { + "meta": Object { + "alias": null, + "disabled": false, + "index": "d180cae0-60c3-11eb-8569-bd1f5ed24bc9", + "negate": false, + "params": Object {}, + }, + "query": Object { + "range": Object { + "@date": Object { + "format": "strict_date_optional_time", + "gte": "2016-01-27T18:11:05.010Z", + "lte": "2021-01-27T18:11:05.010Z", + }, + }, + }, + }, + ], + } + `); + + // calling twice gives the same result: no double-added filters + expect(searchSource.getFields(RECURSE)).toMatchInlineSnapshot(` + Object { + "aggs": 5, + "filter": Array [ + Object { + "meta": Object { + "alias": null, + "disabled": false, + "index": "d180cae0-60c3-11eb-8569-bd1f5ed24bc9", + "negate": false, + "params": Object {}, + }, + "query": Object { + "range": Object { + "@date": Object { + "format": "strict_date_optional_time", + "gte": "2016-01-27T18:11:05.010Z", + "lte": "2021-01-27T18:11:05.010Z", + }, + }, + }, + }, + ], + } + `); + }); + }); + describe('#removeField()', () => { test('remove property', () => { searchSource = new SearchSource({}, searchSourceDependencies); @@ -619,13 +788,13 @@ describe('SearchSource', () => { expect(JSON.parse(searchSourceJSON).from).toEqual(123456); }); - test('should omit sort and size', () => { + test('should omit size but not sort', () => { searchSource.setField('highlightAll', true); searchSource.setField('from', 123456); searchSource.setField('sort', { field: SortDirection.asc }); searchSource.setField('size', 200); const { searchSourceJSON } = searchSource.serialize(); - expect(Object.keys(JSON.parse(searchSourceJSON))).toEqual(['highlightAll', 'from']); + expect(Object.keys(JSON.parse(searchSourceJSON))).toEqual(['highlightAll', 'from', 'sort']); }); test('should serialize filters', () => { diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index bb60f0d7b4ad48..36c0aab6bc190e 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -172,7 +172,49 @@ export class SearchSource { /** * returns all search source fields */ - getFields() { + getFields(recurse = false): SearchSourceFields { + let thisFilter = this.fields.filter; // type is single value, array, or function + if (thisFilter) { + if (typeof thisFilter === 'function') { + thisFilter = thisFilter() || []; // type is single value or array + } + + if (Array.isArray(thisFilter)) { + thisFilter = [...thisFilter]; + } else { + thisFilter = [thisFilter]; + } + } else { + thisFilter = []; + } + + if (recurse) { + const parent = this.getParent(); + if (parent) { + const parentFields = parent.getFields(recurse); + + let parentFilter = parentFields.filter; // type is single value, array, or function + if (parentFilter) { + if (typeof parentFilter === 'function') { + parentFilter = parentFilter() || []; // type is single value or array + } + + if (Array.isArray(parentFilter)) { + thisFilter.push(...parentFilter); + } else { + thisFilter.push(parentFilter); + } + } + + // add combined filters to the fields + const thisFields = { + ...this.fields, + filter: thisFilter, + }; + + return { ...parentFields, ...thisFields }; + } + } return { ...this.fields }; } @@ -605,9 +647,8 @@ export class SearchSource { /** * serializes search source fields (which can later be passed to {@link ISearchStartSearchSource}) */ - public getSerializedFields() { - const { filter: originalFilters, ...searchSourceFields } = omit(this.getFields(), [ - 'sort', + public getSerializedFields(recurse = false) { + const { filter: originalFilters, ...searchSourceFields } = omit(this.getFields(recurse), [ 'size', ]); let serializedSearchSourceFields: SearchSourceFields = { diff --git a/src/plugins/data/common/search/tabify/__snapshots__/tabify_docs.test.ts.snap b/src/plugins/data/common/search/tabify/__snapshots__/tabify_docs.test.ts.snap index d5ddaa31b8ac3b..6cc191a67633c4 100644 --- a/src/plugins/data/common/search/tabify/__snapshots__/tabify_docs.test.ts.snap +++ b/src/plugins/data/common/search/tabify/__snapshots__/tabify_docs.test.ts.snap @@ -26,23 +26,38 @@ Object { "name": "invalidMapping", }, Object { - "id": "nested.field", + "id": "nested", + "meta": Object { + "field": "nested", + "index": "test-index", + "params": undefined, + "type": "object", + }, + "name": "nested", + }, + Object { + "id": "sourceTest", "meta": Object { - "field": "nested.field", + "field": "sourceTest", "index": "test-index", "params": Object { "id": "number", }, "type": "number", }, - "name": "nested.field", + "name": "sourceTest", }, ], "rows": Array [ Object { "fieldTest": 123, "invalidMapping": 345, - "nested.field": 123, + "nested": Array [ + Object { + "field": 123, + }, + ], + "sourceTest": 123, }, ], "type": "datatable", @@ -52,6 +67,38 @@ Object { exports[`tabifyDocs converts source if option is set 1`] = ` Object { "columns": Array [ + Object { + "id": "fieldTest", + "meta": Object { + "field": "fieldTest", + "index": "test-index", + "params": Object { + "id": "number", + }, + "type": "number", + }, + "name": "fieldTest", + }, + Object { + "id": "invalidMapping", + "meta": Object { + "field": "invalidMapping", + "index": "test-index", + "params": undefined, + "type": "number", + }, + "name": "invalidMapping", + }, + Object { + "id": "nested", + "meta": Object { + "field": "nested", + "index": "test-index", + "params": undefined, + "type": "object", + }, + "name": "nested", + }, Object { "id": "sourceTest", "meta": Object { @@ -67,6 +114,13 @@ Object { ], "rows": Array [ Object { + "fieldTest": 123, + "invalidMapping": 345, + "nested": Array [ + Object { + "field": 123, + }, + ], "sourceTest": 123, }, ], @@ -109,6 +163,18 @@ Object { }, "name": "nested", }, + Object { + "id": "sourceTest", + "meta": Object { + "field": "sourceTest", + "index": "test-index", + "params": Object { + "id": "number", + }, + "type": "number", + }, + "name": "sourceTest", + }, ], "rows": Array [ Object { @@ -119,6 +185,7 @@ Object { "field": 123, }, ], + "sourceTest": 123, }, ], "type": "datatable", @@ -149,21 +216,36 @@ Object { "name": "invalidMapping", }, Object { - "id": "nested.field", + "id": "nested", + "meta": Object { + "field": "nested", + "index": undefined, + "params": undefined, + "type": "object", + }, + "name": "nested", + }, + Object { + "id": "sourceTest", "meta": Object { - "field": "nested.field", + "field": "sourceTest", "index": undefined, "params": undefined, "type": "number", }, - "name": "nested.field", + "name": "sourceTest", }, ], "rows": Array [ Object { "fieldTest": 123, "invalidMapping": 345, - "nested.field": 123, + "nested": Array [ + Object { + "field": 123, + }, + ], + "sourceTest": 123, }, ], "type": "datatable", diff --git a/src/plugins/data/common/search/tabify/index.ts b/src/plugins/data/common/search/tabify/index.ts index 08d54316d9d918..9c650061fb0134 100644 --- a/src/plugins/data/common/search/tabify/index.ts +++ b/src/plugins/data/common/search/tabify/index.ts @@ -26,6 +26,8 @@ export const tabify = ( ); }; +export { tabifyDocs }; + export { tabifyAggResponse } from './tabify'; export { tabifyGetColumns } from './get_columns'; diff --git a/src/plugins/data/common/search/tabify/tabify_docs.ts b/src/plugins/data/common/search/tabify/tabify_docs.ts index d66be3c5748fe0..7d4d0fad20730d 100644 --- a/src/plugins/data/common/search/tabify/tabify_docs.ts +++ b/src/plugins/data/common/search/tabify/tabify_docs.ts @@ -12,9 +12,9 @@ import { IndexPattern } from '../../index_patterns/index_patterns'; import { Datatable, DatatableColumn, DatatableColumnType } from '../../../../expressions/common'; export function flattenHit( - hit: Record, + hit: SearchResponse['hits']['hits'][0], indexPattern?: IndexPattern, - shallow: boolean = false + params?: TabifyDocsOptions ) { const flat = {} as Record; @@ -24,7 +24,7 @@ export function flattenHit( const field = indexPattern?.fields.getByName(key); - if (!shallow) { + if (params?.shallow === false) { const isNestedField = field?.type === 'nested'; if (Array.isArray(val) && !isNestedField) { val.forEach((v) => isPlainObject(v) && flatten(v, key + '.')); @@ -52,7 +52,10 @@ export function flattenHit( } } - flatten(hit); + flatten(hit.fields); + if (params?.source !== false && hit._source) { + flatten(hit._source as Record); + } return flat; } @@ -70,8 +73,7 @@ export const tabifyDocs = ( const rows = esResponse.hits.hits .map((hit) => { - const toConvert = params.source ? hit._source : hit.fields; - const flat = flattenHit(toConvert, index, params.shallow); + const flat = flattenHit(hit, index, params); for (const [key, value] of Object.entries(flat)) { const field = index?.fields.getByName(key); const fieldName = field?.name || key; diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 4f197dd43a83ef..78947feb88c20e 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -2368,30 +2368,12 @@ export class SearchSource { // @deprecated fetch(options?: ISearchOptions): Promise>; getField(field: K, recurse?: boolean): SearchSourceFields[K]; - getFields(): { - type?: string | undefined; - query?: import("../..").Query | undefined; - filter?: Filter | Filter[] | (() => Filter | Filter[] | undefined) | undefined; - sort?: Record | Record[] | undefined; - highlight?: any; - highlightAll?: boolean | undefined; - aggs?: any; - from?: number | undefined; - size?: number | undefined; - source?: string | boolean | string[] | undefined; - version?: boolean | undefined; - fields?: SearchFieldValue[] | undefined; - fieldsFromSource?: string | boolean | string[] | undefined; - index?: import("../..").IndexPattern | undefined; - searchAfter?: import("./types").EsQuerySearchAfter | undefined; - timeout?: string | undefined; - terminate_after?: number | undefined; - }; + getFields(recurse?: boolean): SearchSourceFields; getId(): string; getOwnField(field: K): SearchSourceFields[K]; getParent(): SearchSource | undefined; getSearchRequestBody(): Promise; - getSerializedFields(): SearchSourceFields; + getSerializedFields(recurse?: boolean): SearchSourceFields; // Warning: (ae-incompatible-release-tags) The symbol "history" is marked as @public, but its signature references "SearchRequest" which is marked as @internal // // (undocumented) @@ -2415,6 +2397,7 @@ export class SearchSource { export interface SearchSourceFields { // (undocumented) aggs?: any; + // Warning: (ae-forgotten-export) The symbol "SearchFieldValue" needs to be exported by the entry point index.d.ts fields?: SearchFieldValue[]; // @deprecated fieldsFromSource?: NameList; @@ -2607,7 +2590,6 @@ export const UI_SETTINGS: { // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:138:7 - (ae-forgotten-export) The symbol "FieldAttrSet" needs to be exported by the entry point index.d.ts // src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts:169:7 - (ae-forgotten-export) The symbol "RuntimeField" needs to be exported by the entry point index.d.ts // src/plugins/data/common/search/aggs/types.ts:139:51 - (ae-forgotten-export) The symbol "AggTypesRegistryStart" needs to be exported by the entry point index.d.ts -// src/plugins/data/common/search/search_source/search_source.ts:187:7 - (ae-forgotten-export) The symbol "SearchFieldValue" needs to be exported by the entry point index.d.ts // src/plugins/data/public/field_formats/field_formats_service.ts:56:3 - (ae-forgotten-export) The symbol "FormatFactory" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:55:23 - (ae-forgotten-export) The symbol "FILTERS" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:55:23 - (ae-forgotten-export) The symbol "getDisplayValueFromFilter" needs to be exported by the entry point index.d.ts From 4940a1cbd9d305718223942bbd95ef69dfede150 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Tue, 2 Feb 2021 11:38:20 -0600 Subject: [PATCH 11/12] TypeScript project references for Observability plugin (#89320) References #80508. References #81003. --- .../annotations/create_annotations_client.ts | 2 +- .../server/utils/unwrap_es_response.ts | 4 ++-- x-pack/plugins/observability/tsconfig.json | 22 +++++++++++++++++++ x-pack/test/tsconfig.json | 1 + x-pack/tsconfig.json | 18 +++++++-------- x-pack/tsconfig.refs.json | 17 ++++---------- 6 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugins/observability/tsconfig.json diff --git a/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts b/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts index 76890cbd587e98..3ab645be28c60e 100644 --- a/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts +++ b/x-pack/plugins/observability/server/lib/annotations/create_annotations_client.ts @@ -37,7 +37,7 @@ interface IndexDocumentResponse { result: string; } -interface GetResponse { +export interface GetResponse { _id: string; _index: string; _source: Annotation; diff --git a/x-pack/plugins/observability/server/utils/unwrap_es_response.ts b/x-pack/plugins/observability/server/utils/unwrap_es_response.ts index 418ceeb64cc879..f9be40e49553c7 100644 --- a/x-pack/plugins/observability/server/utils/unwrap_es_response.ts +++ b/x-pack/plugins/observability/server/utils/unwrap_es_response.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { PromiseValueType } from '../../../apm/typings/common'; +import type { UnwrapPromise } from '@kbn/utility-types'; export function unwrapEsResponse>( responsePromise: T -): Promise['body']> { +): Promise['body']> { return responsePromise.then((res) => res.body); } diff --git a/x-pack/plugins/observability/tsconfig.json b/x-pack/plugins/observability/tsconfig.json new file mode 100644 index 00000000000000..62aecc1e0899f1 --- /dev/null +++ b/x-pack/plugins/observability/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": ["common/**/*", "public/**/*", "server/**/*", "typings/**/*"], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../../../src/plugins/data/tsconfig.json" }, + { "path": "../../../src/plugins/home/tsconfig.json" }, + { "path": "../../../src/plugins/kibana_react/tsconfig.json" }, + { "path": "../../../src/plugins/kibana_utils/tsconfig.json" }, + { "path": "../../../src/plugins/usage_collection/tsconfig.json" }, + { "path": "../alerts/tsconfig.json" }, + { "path": "../licensing/tsconfig.json" }, + { "path": "../translations/tsconfig.json" } + ] +} diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index a1c0c272deb04e..6039cd330c1493 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -64,6 +64,7 @@ { "path": "../plugins/cloud/tsconfig.json" }, { "path": "../plugins/saved_objects_tagging/tsconfig.json" }, { "path": "../plugins/global_search_bar/tsconfig.json" }, + { "path": "../plugins/observability/tsconfig.json" }, { "path": "../plugins/ingest_pipelines/tsconfig.json" }, { "path": "../plugins/license_management/tsconfig.json" }, { "path": "../plugins/snapshot_restore/tsconfig.json" }, diff --git a/x-pack/tsconfig.json b/x-pack/tsconfig.json index c0eed6e89b60ef..56420b503dd5cb 100644 --- a/x-pack/tsconfig.json +++ b/x-pack/tsconfig.json @@ -26,6 +26,7 @@ "plugins/maps/**/*", "plugins/maps_file_upload/**/*", "plugins/maps_legacy_licensing/**/*", + "plugins/observability/**/*", "plugins/reporting/**/*", "plugins/searchprofiler/**/*", "plugins/security_solution/cypress/**/*", @@ -67,6 +68,7 @@ { "path": "../src/plugins/es_ui_shared/tsconfig.json" }, { "path": "../src/plugins/expressions/tsconfig.json" }, { "path": "../src/plugins/home/tsconfig.json" }, + { "path": "../src/plugins/index_pattern_management/tsconfig.json" }, { "path": "../src/plugins/inspector/tsconfig.json" }, { "path": "../src/plugins/kibana_legacy/tsconfig.json" }, { "path": "../src/plugins/kibana_overview/tsconfig.json" }, @@ -89,9 +91,8 @@ { "path": "../src/plugins/ui_actions/tsconfig.json" }, { "path": "../src/plugins/url_forwarding/tsconfig.json" }, { "path": "../src/plugins/usage_collection/tsconfig.json" }, - { "path": "../src/plugins/index_pattern_management/tsconfig.json" }, - { "path": "./plugins/actions/tsconfig.json"}, - { "path": "./plugins/alerts/tsconfig.json"}, + { "path": "./plugins/actions/tsconfig.json" }, + { "path": "./plugins/alerts/tsconfig.json" }, { "path": "./plugins/beats_management/tsconfig.json" }, { "path": "./plugins/canvas/tsconfig.json" }, { "path": "./plugins/cloud/tsconfig.json" }, @@ -109,28 +110,25 @@ { "path": "./plugins/global_search_providers/tsconfig.json" }, { "path": "./plugins/global_search/tsconfig.json" }, { "path": "./plugins/graph/tsconfig.json" }, + { "path": "./plugins/grokdebugger/tsconfig.json" }, + { "path": "./plugins/ingest_pipelines/tsconfig.json" }, { "path": "./plugins/lens/tsconfig.json" }, { "path": "./plugins/license_management/tsconfig.json" }, { "path": "./plugins/licensing/tsconfig.json" }, { "path": "./plugins/maps_file_upload/tsconfig.json" }, { "path": "./plugins/maps_legacy_licensing/tsconfig.json" }, { "path": "./plugins/maps/tsconfig.json" }, + { "path": "./plugins/observability/tsconfig.json" }, { "path": "./plugins/painless_lab/tsconfig.json" }, { "path": "./plugins/saved_objects_tagging/tsconfig.json" }, { "path": "./plugins/searchprofiler/tsconfig.json" }, { "path": "./plugins/security/tsconfig.json" }, + { "path": "./plugins/snapshot_restore/tsconfig.json" }, { "path": "./plugins/spaces/tsconfig.json" }, { "path": "./plugins/stack_alerts/tsconfig.json" }, { "path": "./plugins/task_manager/tsconfig.json" }, { "path": "./plugins/telemetry_collection_xpack/tsconfig.json" }, { "path": "./plugins/translations/tsconfig.json" }, - { "path": "./plugins/triggers_actions_ui/tsconfig.json"}, - { "path": "./plugins/stack_alerts/tsconfig.json"}, - { "path": "./plugins/snapshot_restore/tsconfig.json" }, - { "path": "./plugins/grokdebugger/tsconfig.json" }, - { "path": "./plugins/ingest_pipelines/tsconfig.json"}, - { "path": "./plugins/license_management/tsconfig.json" }, - { "path": "./plugins/painless_lab/tsconfig.json" }, { "path": "./plugins/triggers_actions_ui/tsconfig.json" }, { "path": "./plugins/ui_actions_enhanced/tsconfig.json" }, { "path": "./plugins/watcher/tsconfig.json" } diff --git a/x-pack/tsconfig.refs.json b/x-pack/tsconfig.refs.json index fe2d4873b7cec6..82b52c959b6d96 100644 --- a/x-pack/tsconfig.refs.json +++ b/x-pack/tsconfig.refs.json @@ -21,36 +21,27 @@ { "path": "./plugins/global_search_providers/tsconfig.json" }, { "path": "./plugins/global_search/tsconfig.json" }, { "path": "./plugins/graph/tsconfig.json" }, + { "path": "./plugins/grokdebugger/tsconfig.json" }, + { "path": "./plugins/ingest_pipelines/tsconfig.json" }, { "path": "./plugins/lens/tsconfig.json" }, { "path": "./plugins/license_management/tsconfig.json" }, { "path": "./plugins/licensing/tsconfig.json" }, { "path": "./plugins/maps_file_upload/tsconfig.json" }, { "path": "./plugins/maps_legacy_licensing/tsconfig.json" }, { "path": "./plugins/maps/tsconfig.json" }, + { "path": "./plugins/observability/tsconfig.json" }, { "path": "./plugins/painless_lab/tsconfig.json" }, { "path": "./plugins/reporting/tsconfig.json" }, { "path": "./plugins/saved_objects_tagging/tsconfig.json" }, { "path": "./plugins/searchprofiler/tsconfig.json" }, { "path": "./plugins/security/tsconfig.json" }, + { "path": "./plugins/snapshot_restore/tsconfig.json" }, { "path": "./plugins/spaces/tsconfig.json" }, { "path": "./plugins/stack_alerts/tsconfig.json" }, { "path": "./plugins/task_manager/tsconfig.json" }, { "path": "./plugins/telemetry_collection_xpack/tsconfig.json" }, { "path": "./plugins/translations/tsconfig.json" }, { "path": "./plugins/triggers_actions_ui/tsconfig.json" }, - { "path": "./plugins/spaces/tsconfig.json" }, - { "path": "./plugins/security/tsconfig.json" }, - { "path": "./plugins/stack_alerts/tsconfig.json" }, - { "path": "./plugins/encrypted_saved_objects/tsconfig.json" }, - { "path": "./plugins/beats_management/tsconfig.json" }, - { "path": "./plugins/cloud/tsconfig.json" }, - { "path": "./plugins/saved_objects_tagging/tsconfig.json" }, - { "path": "./plugins/global_search_bar/tsconfig.json" }, - { "path": "./plugins/snapshot_restore/tsconfig.json" }, - { "path": "./plugins/grokdebugger/tsconfig.json" }, - { "path": "./plugins/ingest_pipelines/tsconfig.json" }, - { "path": "./plugins/license_management/tsconfig.json" }, - { "path": "./plugins/painless_lab/tsconfig.json" }, { "path": "./plugins/ui_actions_enhanced/tsconfig.json" }, { "path": "./plugins/watcher/tsconfig.json" } ] From 977fc6c464115a875131312e7d8f2e6366a51eda Mon Sep 17 00:00:00 2001 From: Constance Date: Tue, 2 Feb 2021 09:41:02 -0800 Subject: [PATCH 12/12] [App Search] DRY helper for encoding/decoding routes that can have special characters in params (#89811) * Add encodePathParam helper + update components that need it - Primarily document URLs & analytics queries (which uses generateEnginePath) * Add useDecodedParams helper + update components that need it - Documents titles & Analytics queries * [Misc] Change popout icon to eye - Feedback from Davey - the pages don't open in a new window, so shouldn't use the popout icon - Not strictly related but since we're touching these links anyway, I'm shoving it in (sorry) * Remove document detail decode test - now handled/tested by useDecodedParams helper * Add new generateEncodedPath helper - Should be used in place of generatePath * Update all instances of generatePath to generateEncodedPath for consistency across the App Search codebase * Fix failing tests due to extra encodeURI() done by generatePath * Add missing branch test for analytics query titles Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../__mocks__/react_router_history.mock.ts | 20 +++++--- .../app_search/__mocks__/engine_logic.mock.ts | 4 +- .../analytics_tables/shared_columns.tsx | 2 +- .../analytics/views/query_detail.test.tsx | 11 ++++- .../analytics/views/query_detail.tsx | 7 +-- .../documents/document_detail.test.tsx | 6 --- .../components/documents/document_detail.tsx | 8 +-- .../app_search/components/engine/utils.ts | 4 +- .../components/engines/engines_table.tsx | 4 +- .../app_search/components/result/result.tsx | 6 +-- .../utils/encode_path_params/index.test.ts | 49 +++++++++++++++++++ .../utils/encode_path_params/index.ts | 34 +++++++++++++ 12 files changed, 125 insertions(+), 30 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/utils/encode_path_params/index.test.ts create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/utils/encode_path_params/index.ts diff --git a/x-pack/plugins/enterprise_search/public/applications/__mocks__/react_router_history.mock.ts b/x-pack/plugins/enterprise_search/public/applications/__mocks__/react_router_history.mock.ts index 1516aa9096eca0..d3c11d33fdbf7e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/__mocks__/react_router_history.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/__mocks__/react_router_history.mock.ts @@ -24,12 +24,20 @@ export const mockLocation = { state: {}, }; -jest.mock('react-router-dom', () => ({ - ...(jest.requireActual('react-router-dom') as object), - useHistory: jest.fn(() => mockHistory), - useLocation: jest.fn(() => mockLocation), - useParams: jest.fn(() => ({})), -})); +jest.mock('react-router-dom', () => { + const originalModule = jest.requireActual('react-router-dom'); + return { + ...originalModule, + useHistory: jest.fn(() => mockHistory), + useLocation: jest.fn(() => mockLocation), + useParams: jest.fn(() => ({})), + // Note: RR's generatePath() opinionatedly encodeURI()s paths (although this doesn't actually + // show up/affect the final browser URL). Since we already have a generateEncodedPath helper & + // RR is removing this behavior in history 5.0+, I'm mocking tests to remove the extra encoding + // for now to make reading generateEncodedPath URLs a little less of a pain + generatePath: jest.fn((path, params) => decodeURI(originalModule.generatePath(path, params))), + }; +}); /** * For example usage, @see public/applications/shared/react_router_helpers/eui_link.test.tsx diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/engine_logic.mock.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/engine_logic.mock.ts index 6326a41c1d2ca4..edc87d7025c5d0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/engine_logic.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/engine_logic.mock.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { generatePath } from 'react-router-dom'; +import { generateEncodedPath } from '../utils/encode_path_params'; export const mockEngineValues = { engineName: 'some-engine', @@ -12,7 +12,7 @@ export const mockEngineValues = { }; export const mockGenerateEnginePath = jest.fn((path, pathParams = {}) => - generatePath(path, { engineName: mockEngineValues.engineName, ...pathParams }) + generateEncodedPath(path, { engineName: mockEngineValues.engineName, ...pathParams }) ); jest.mock('../components/engine', () => ({ diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/components/analytics_tables/shared_columns.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/components/analytics_tables/shared_columns.tsx index 16743405e0b5e1..8546ee428b6815 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/components/analytics_tables/shared_columns.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/components/analytics_tables/shared_columns.tsx @@ -56,7 +56,7 @@ export const ACTIONS_COLUMN = { { defaultMessage: 'View query analytics' } ), type: 'icon', - icon: 'popout', + icon: 'eye', color: 'primary', onClick: (item: Query | RecentQuery) => { const { navigateToUrl } = KibanaLogic.values; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.test.tsx index 7705d342ecdce5..42f13a0631a0aa 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.test.tsx @@ -13,6 +13,7 @@ import { shallow } from 'enzyme'; import { SetAppSearchChrome as SetPageChrome } from '../../../../shared/kibana_chrome'; +import { AnalyticsLayout } from '../analytics_layout'; import { AnalyticsCards, AnalyticsChart, QueryClicksTable } from '../components'; import { QueryDetail } from './'; @@ -20,7 +21,7 @@ describe('QueryDetail', () => { const mockBreadcrumbs = ['Engines', 'some-engine', 'Analytics']; beforeEach(() => { - (useParams as jest.Mock).mockReturnValueOnce({ query: 'some-query' }); + (useParams as jest.Mock).mockReturnValue({ query: 'some-query' }); setMockValues({ totalQueriesForQuery: 100, @@ -31,6 +32,7 @@ describe('QueryDetail', () => { it('renders', () => { const wrapper = shallow(); + expect(wrapper.find(AnalyticsLayout).prop('title')).toEqual('"some-query"'); expect(wrapper.find(SetPageChrome).prop('trail')).toEqual([ 'Engines', 'some-engine', @@ -43,4 +45,11 @@ describe('QueryDetail', () => { expect(wrapper.find(AnalyticsChart)).toHaveLength(1); expect(wrapper.find(QueryClicksTable)).toHaveLength(1); }); + + it('renders empty "" search titles correctly', () => { + (useParams as jest.Mock).mockReturnValue({ query: '""' }); + const wrapper = shallow(); + + expect(wrapper.find(AnalyticsLayout).prop('title')).toEqual('""'); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.tsx index d5d864f35f6819..0ec81f5caa9357 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.tsx @@ -5,7 +5,6 @@ */ import React from 'react'; -import { useParams } from 'react-router-dom'; import { useValues } from 'kea'; import { i18n } from '@kbn/i18n'; @@ -13,6 +12,7 @@ import { EuiSpacer } from '@elastic/eui'; import { SetAppSearchChrome as SetPageChrome } from '../../../../shared/kibana_chrome'; import { BreadcrumbTrail } from '../../../../shared/kibana_chrome/generate_breadcrumbs'; +import { useDecodedParams } from '../../../utils/encode_path_params'; import { AnalyticsLayout } from '../analytics_layout'; import { AnalyticsSection, QueryClicksTable } from '../components'; @@ -27,14 +27,15 @@ interface Props { breadcrumbs: BreadcrumbTrail; } export const QueryDetail: React.FC = ({ breadcrumbs }) => { - const { query } = useParams() as { query: string }; + const { query } = useDecodedParams(); + const queryTitle = query === '""' ? query : `"${query}"`; const { totalQueriesForQuery, queriesPerDayForQuery, startDate, topClicksForQuery } = useValues( AnalyticsLogic ); return ( - + { expect(actions.deleteDocument).toHaveBeenCalledWith('1'); }); - - it('correctly decodes document IDs', () => { - (useParams as jest.Mock).mockReturnValueOnce({ documentId: 'hello%20world%20%26%3F!' }); - const wrapper = shallow(); - expect(wrapper.find('h1').text()).toEqual('Document: hello world &?!'); - }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/documents/document_detail.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/documents/document_detail.tsx index 1be7e6c53d3431..3fadda6165c9b3 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/documents/document_detail.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/documents/document_detail.tsx @@ -23,6 +23,7 @@ import { i18n } from '@kbn/i18n'; import { Loading } from '../../../shared/loading'; import { SetAppSearchChrome as SetPageChrome } from '../../../shared/kibana_chrome'; import { FlashMessages } from '../../../shared/flash_messages'; +import { useDecodedParams } from '../../utils/encode_path_params'; import { ResultFieldValue } from '../result'; import { DocumentDetailLogic } from './document_detail_logic'; @@ -43,6 +44,7 @@ export const DocumentDetail: React.FC = ({ engineBreadcrumb }) => { const { deleteDocument, getDocumentDetails, setFields } = useActions(DocumentDetailLogic); const { documentId } = useParams() as { documentId: string }; + const { documentId: documentTitle } = useDecodedParams(); useEffect(() => { getDocumentDetails(documentId); @@ -74,13 +76,11 @@ export const DocumentDetail: React.FC = ({ engineBreadcrumb }) => { return ( <> - + -

{DOCUMENT_DETAIL_TITLE(decodeURIComponent(documentId))}

+

{DOCUMENT_DETAIL_TITLE(documentTitle)}

 diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/utils.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/utils.ts index b7efcbb6e6b27a..8e197eb402ea7b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/utils.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/utils.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { generatePath } from 'react-router-dom'; +import { generateEncodedPath } from '../../utils/encode_path_params'; import { EngineLogic } from './'; @@ -13,5 +13,5 @@ import { EngineLogic } from './'; */ export const generateEnginePath = (path: string, pathParams: object = {}) => { const { engineName } = EngineLogic.values; - return generatePath(path, { engineName, ...pathParams }); + return generateEncodedPath(path, { engineName, ...pathParams }); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/engines_table.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/engines_table.tsx index a9455b4a2306ab..34bf0fe1b3a04e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/engines_table.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/engines_table.tsx @@ -5,7 +5,6 @@ */ import React from 'react'; -import { generatePath } from 'react-router-dom'; import { useActions } from 'kea'; import { EuiBasicTable, EuiBasicTableColumn } from '@elastic/eui'; import { FormattedMessage, FormattedDate, FormattedNumber } from '@kbn/i18n/react'; @@ -13,6 +12,7 @@ import { i18n } from '@kbn/i18n'; import { TelemetryLogic } from '../../../shared/telemetry'; import { EuiLinkTo } from '../../../shared/react_router_helpers'; +import { generateEncodedPath } from '../../utils/encode_path_params'; import { ENGINE_PATH } from '../../routes'; import { ENGINES_PAGE_SIZE } from '../../../../../common/constants'; @@ -41,7 +41,7 @@ export const EnginesTable: React.FC = ({ const { sendAppSearchTelemetry } = useActions(TelemetryLogic); const engineLinkProps = (engineName: string) => ({ - to: generatePath(ENGINE_PATH, { engineName }), + to: generateEncodedPath(ENGINE_PATH, { engineName }), onClick: () => sendAppSearchTelemetry({ action: 'clicked', diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/result/result.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/result/result.tsx index a3935bb782f906..ff8b373f1bee33 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/result/result.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/result/result.tsx @@ -5,7 +5,6 @@ */ import React, { useState, useMemo } from 'react'; -import { generatePath } from 'react-router-dom'; import classNames from 'classnames'; import './result.scss'; @@ -14,6 +13,7 @@ import { EuiPanel, EuiIcon } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { ReactRouterHelper } from '../../../shared/react_router_helpers/eui_components'; +import { generateEncodedPath } from '../../utils/encode_path_params'; import { ENGINE_DOCUMENT_DETAIL_PATH } from '../../routes'; import { Schema } from '../../../shared/types'; @@ -52,7 +52,7 @@ export const Result: React.FC = ({ if (schemaForTypeHighlights) return schemaForTypeHighlights[fieldName]; }; - const documentLink = generatePath(ENGINE_DOCUMENT_DETAIL_PATH, { + const documentLink = generateEncodedPath(ENGINE_DOCUMENT_DETAIL_PATH, { engineName: resultMeta.engine, documentId: resultMeta.id, }); @@ -135,7 +135,7 @@ export const Result: React.FC = ({ { defaultMessage: 'Visit document details' } )} > - + )} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/utils/encode_path_params/index.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/utils/encode_path_params/index.test.ts new file mode 100644 index 00000000000000..f311909bdf5bec --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/utils/encode_path_params/index.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import '../../../__mocks__/react_router_history.mock'; +import { useParams } from 'react-router-dom'; + +import { encodePathParams, generateEncodedPath, useDecodedParams } from './'; + +describe('encodePathParams', () => { + it('encodeURIComponent()s all object values', () => { + const params = { + someValue: 'hello world???', + anotherValue: 'test!@#$%^&*[]/|;:"<>~`', + }; + expect(encodePathParams(params)).toEqual({ + someValue: 'hello%20world%3F%3F%3F', + anotherValue: 'test!%40%23%24%25%5E%26*%5B%5D%2F%7C%3B%3A%22%3C%3E~%60', + }); + }); +}); + +describe('generateEncodedPath', () => { + it('generates a react router path with encoded path parameters', () => { + expect( + generateEncodedPath('/values/:someValue/:anotherValue/new', { + someValue: 'hello world???', + anotherValue: 'test!@#$%^&*[]/|;:"<>~`', + }) + ).toEqual( + '/values/hello%20world%3F%3F%3F/test!%40%23%24%25%5E%26*%5B%5D%2F%7C%3B%3A%22%3C%3E~%60/new' + ); + }); +}); + +describe('useDecodedParams', () => { + it('decodeURIComponent()s all object values from useParams()', () => { + (useParams as jest.Mock).mockReturnValue({ + someValue: 'hello%20world%3F%3F%3F', + anotherValue: 'test!%40%23%24%25%5E%26*%5B%5D%2F%7C%3B%3A%22%3C%3E~%60', + }); + expect(useDecodedParams()).toEqual({ + someValue: 'hello world???', + anotherValue: 'test!@#$%^&*[]/|;:"<>~`', + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/utils/encode_path_params/index.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/utils/encode_path_params/index.ts new file mode 100644 index 00000000000000..c8934ba47fe454 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/utils/encode_path_params/index.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { generatePath, useParams } from 'react-router-dom'; + +type PathParams = Record; + +export const encodePathParams = (pathParams: PathParams) => { + const encodedParams: PathParams = {}; + + Object.entries(pathParams).map(([key, value]) => { + encodedParams[key] = encodeURIComponent(value); + }); + + return encodedParams; +}; + +export const generateEncodedPath = (path: string, pathParams: PathParams) => { + return generatePath(path, encodePathParams(pathParams)); +}; + +export const useDecodedParams = () => { + const decodedParams: PathParams = {}; + + const params = useParams(); + Object.entries(params).map(([key, value]) => { + decodedParams[key] = decodeURIComponent(value as string); + }); + + return decodedParams; +};