From 3caf266cf8353a25b32de77ebbedc494d12cf468 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 14 May 2024 21:07:18 +0200 Subject: [PATCH] [Dataset quality] Warning for datasets not supporting _ignored aggregation (#183183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://github.com/elastic/kibana/issues/179227. ## 📝 Summary This PR adds a warning to main page and flyout whenever a dataStream has indices that doesn't support `_ignored` aggregation ## 🎥 Demo https://github.com/elastic/kibana/assets/1313018/c6d5fd81-d9a9-4fcc-92d0-8e65b996df9c #### Flyout https://github.com/elastic/kibana/assets/1313018/2972e104-8b10-413a-a430-3efc5252b757 --- .../dataset_quality/common/api_types.ts | 9 ++ .../common/data_streams_stats/types.ts | 5 + .../common/utils/dataset_name.test.ts | 26 ++-- .../common/utils/dataset_name.ts | 10 +- .../dataset_quality/dataset_quality.tsx | 4 + .../dataset_quality/warnings/warnings.tsx | 97 ++++++++++++++ .../public/components/flyout/flyout.tsx | 2 + .../flyout/flyout_summary/flyout_summary.tsx | 76 ++++++++++- .../hooks/use_dataset_quality_flyout.tsx | 2 + .../hooks/use_dataset_quality_warnings.ts | 23 ++++ .../data_streams_stats_client.ts | 35 ++++- .../services/data_streams_stats/types.ts | 7 +- .../src/defaults.ts | 1 + .../src/notifications.ts | 9 ++ .../src/state_machine.ts | 122 ++++++++++++++++++ .../dataset_quality_controller/src/types.ts | 12 ++ .../get_non_aggregatable_data_streams.ts | 53 ++++++++ .../server/routes/data_streams/routes.ts | 30 +++++ .../utils/create_dataset_quality_es_client.ts | 5 +- 19 files changed, 503 insertions(+), 25 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/warnings/warnings.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index c953d4589c614f..85490520b2a849 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -119,3 +119,12 @@ export type DataStreamsEstimatedDataInBytes = rt.TypeOf; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index 0c9456c2d72576..bde4e9e2ba9dd7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -43,6 +43,11 @@ export type GetDataStreamDetailsParams = GetDataStreamDetailsPathParams & export type GetDataStreamDetailsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>; +export type GetNonAggregatableDataStreamsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/non_aggregatable`>['params']['query']; +export type GetNonAggregatableDataStreamsResponse = + APIReturnType<`GET /internal/dataset_quality/data_streams/non_aggregatable`>; + export type GetDataStreamsEstimatedDataInBytesParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/estimated_data`>['params']; export type GetDataStreamsEstimatedDataInBytesResponse = diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts index 9244c9e65f13c7..8ffcbfe5657fa0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts @@ -55,27 +55,33 @@ describe('dataset_name', () => { }); describe('extractIndexNameFromBackingIndex', () => { - it('returns the correct index name if backing index provieded', () => { + it('returns the correct index name if backing index provided', () => { expect( - extractIndexNameFromBackingIndex( - '.ds-logs-apm.app.adservice-default-2024.04.29-000001', - 'logs' - ) + extractIndexNameFromBackingIndex('.ds-logs-apm.app.adservice-default-2024.04.29-000001') ).toEqual('logs-apm.app.adservice-default'); }); it('returns the correct index name if index name is passed', () => { - expect(extractIndexNameFromBackingIndex('logs-nginx.access-default', 'logs')).toEqual( + expect(extractIndexNameFromBackingIndex('logs-nginx.access-default')).toEqual( 'logs-nginx.access-default' ); }); + it('returns the correct index name if backing index contains _', () => { + expect( + extractIndexNameFromBackingIndex('.ds-logs-elastic_agent-default-2024.04.29-000001') + ).toEqual('logs-elastic_agent-default'); + }); + + it('returns the correct index name if backing index contains only -', () => { + expect( + extractIndexNameFromBackingIndex('.ds-logs-generic-pods-default-2024.04.29-000001') + ).toEqual('logs-generic-pods-default'); + }); + it('handles different types', () => { expect( - extractIndexNameFromBackingIndex( - '.ds-metrics-apm.app.adservice-default-2024.04.29-000001', - 'metrics' - ) + extractIndexNameFromBackingIndex('.ds-metrics-apm.app.adservice-default-2024.04.29-000001') ).toEqual('metrics-apm.app.adservice-default'); }); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts index 8914a4d893b6df..eaca58ded64049 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts @@ -39,14 +39,8 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => { }; }; -export const extractIndexNameFromBackingIndex = ( - indexString: string, - type: DataStreamType -): string => { - const pattern: RegExp = new RegExp( - `(?:\\.ds-)?(${type}-(?:[^-.]+(?:\\.[^.]+)+)-[^-]+)-\\d{4}\\.\\d{2}\\.\\d{2}-\\d{6}` - ); - +export const extractIndexNameFromBackingIndex = (indexString: string): string => { + const pattern = /.ds-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/; const match = indexString.match(pattern); return match ? match[1] : indexString; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/dataset_quality.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/dataset_quality.tsx index ad343f912e4ffb..3c2e896371a6c2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/dataset_quality.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/dataset_quality.tsx @@ -56,6 +56,7 @@ export const createDatasetQuality = ({ }; const Header = dynamic(() => import('./header')); +const Warnings = dynamic(() => import('./warnings/warnings')); const Table = dynamic(() => import('./table/table')); const Filters = dynamic(() => import('./filters/filters')); const SummaryPanel = dynamic(() => import('./summary_panel/summary_panel')); @@ -66,6 +67,9 @@ function DatasetQuality() {
+ + + diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/warnings/warnings.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/warnings/warnings.tsx new file mode 100644 index 00000000000000..94f99b5386ef7b --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/warnings/warnings.tsx @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiAccordion, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiLink } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import React from 'react'; +import { useDatasetQualityWarnings } from '../../../hooks/use_dataset_quality_warnings'; + +const nonAggregatableWarningTitle = i18n.translate('xpack.datasetQuality.nonAggregatable.title', { + defaultMessage: 'Your request may take longer to complete', +}); + +const nonAggregatableWarningDescription = (nonAggregatableDatasets: string[]) => ( + +
    + {nonAggregatableDatasets.map((dataset) => ( +
  • {dataset}
    • + ))} +
+ + ), + }} + /> + ), + howToFixIt: ( + + {i18n.translate('xpack.datasetQuality.nonAggregatableDatasets.link.title', { + defaultMessage: 'rollover', + })} + + ), + }} + /> + ), + }} + /> + ), + }} + /> +); + +// Allow for lazy loading +// eslint-disable-next-line import/no-default-export +export default function Warnings() { + const { loading, nonAggregatableDatasets } = useDatasetQualityWarnings(); + + return ( + + {!loading && nonAggregatableDatasets.length > 0 && ( + + +

{nonAggregatableWarningDescription(nonAggregatableDatasets)}

+ + + )} + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx index b99ca3f3b0004c..7543322a5dafcb 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx @@ -34,6 +34,7 @@ export default function Flyout({ dataset, closeFlyout }: FlyoutProps) { dataStreamStat, dataStreamSettings, dataStreamDetails, + isNonAggregatable, fieldFormats, timeRange, loadingState, @@ -60,6 +61,7 @@ export default function Flyout({ dataset, closeFlyout }: FlyoutProps) { dataStreamDetails={dataStreamDetails} dataStreamDetailsLoading={loadingState.dataStreamDetailsLoading} timeRange={timeRange} + isNonAggregatable={isNonAggregatable} /> diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx index 7ba5f315e607f1..c3e38d5a9940af 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx @@ -6,8 +6,19 @@ */ import React, { useCallback, useState } from 'react'; -import { OnRefreshProps, OnTimeChangeProps, EuiSpacer } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { + OnRefreshProps, + OnTimeChangeProps, + EuiSpacer, + EuiFlexGroup, + EuiFlexItem, + EuiCallOut, + EuiLink, + EuiCode, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; import { DegradedDocs } from '../degraded_docs_trend/degraded_docs'; import { DataStreamDetails } from '../../../../common/api_types'; import { DEFAULT_TIME_RANGE, DEFAULT_DATEPICKER_REFRESH } from '../../../../common/constants'; @@ -16,10 +27,60 @@ import { FlyoutDataset, TimeRangeConfig } from '../../../state_machines/dataset_ import { FlyoutSummaryHeader } from './flyout_summary_header'; import { FlyoutSummaryKpis, FlyoutSummaryKpisLoading } from './flyout_summary_kpis'; +const nonAggregatableWarningTitle = i18n.translate('xpack.datasetQuality.nonAggregatable.title', { + defaultMessage: 'Your request may take longer to complete', +}); + +const nonAggregatableWarningDescription = (dataset: string) => ( + + {dataset} + + ), + howToFixIt: ( + + {i18n.translate( + 'xpack.datasetQuality.flyout.nonAggregatableDatasets.link.title', + { + defaultMessage: 'rollover', + } + )} + + ), + }} + /> + ), + }} + /> + ), + }} + /> +); + export function FlyoutSummary({ dataStream, dataStreamStat, dataStreamDetails, + isNonAggregatable, dataStreamDetailsLoading, timeRange = { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_DATEPICKER_REFRESH }, }: { @@ -28,6 +89,7 @@ export function FlyoutSummary({ dataStreamDetails?: DataStreamDetails; dataStreamDetailsLoading: boolean; timeRange?: TimeRangeConfig; + isNonAggregatable?: boolean; }) { const { service } = useDatasetQualityContext(); const [lastReloadTime, setLastReloadTime] = useState(Date.now()); @@ -72,6 +134,18 @@ export function FlyoutSummary({ return ( <> + {isNonAggregatable && ( + + + +

{nonAggregatableWarningDescription(dataStream)}

+ + + + )} { datasetDetails: dataStreamDetails, insightsTimeRange, breakdownField, + isNonAggregatable, } = useSelector(service, (state) => state.context.flyout) ?? {}; const { timeRange } = useSelector(service, (state) => state.context.filters); @@ -36,6 +37,7 @@ export const useDatasetQualityFlyout = () => { dataStreamStat, dataStreamSettings, dataStreamDetails, + isNonAggregatable, fieldFormats, timeRange: insightsTimeRange ?? timeRange, breakdownField, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts new file mode 100644 index 00000000000000..ddb116dc483065 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useSelector } from '@xstate/react'; +import { useDatasetQualityContext } from '../components/dataset_quality/context'; + +export function useDatasetQualityWarnings() { + const { service } = useDatasetQualityContext(); + + const nonAggregatableDatasets = useSelector( + service, + (state) => state.context.nonAggregatableDatasets + ); + + const isNonAggregatableDatasetsLoading = useSelector(service, (state) => + state.matches('nonAggregatableDatasets.fetching') + ); + + return { loading: isNonAggregatableDatasetsLoading, nonAggregatableDatasets }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index d908c8665c1f4a..791c4208fdc215 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -7,26 +7,29 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; -import { Integration } from '../../../common/data_streams_stats/integration'; import { getDataStreamsDegradedDocsStatsResponseRt, - getDataStreamsStatsResponseRt, getDataStreamsEstimatedDataInBytesResponseRt, + getDataStreamsStatsResponseRt, getIntegrationsResponseRt, + getNonAggregatableDatasetsRt, } from '../../../common/api_types'; import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, GetDataStreamsDegradedDocsStatsResponse, + GetDataStreamsEstimatedDataInBytesParams, + GetDataStreamsEstimatedDataInBytesResponse, GetDataStreamsStatsError, GetDataStreamsStatsQuery, GetDataStreamsStatsResponse, - GetDataStreamsEstimatedDataInBytesParams, - GetDataStreamsEstimatedDataInBytesResponse, GetIntegrationsParams, + GetNonAggregatableDataStreamsParams, + GetNonAggregatableDataStreamsResponse, IntegrationsResponse, } from '../../../common/data_streams_stats'; +import { Integration } from '../../../common/data_streams_stats/integration'; import { IDataStreamsStatsClient } from './types'; export class DataStreamsStatsClient implements IDataStreamsStatsClient { @@ -78,6 +81,30 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { return degradedDocs; } + public async getNonAggregatableDatasets(params: GetNonAggregatableDataStreamsParams) { + const response = await this.http + .get( + '/internal/dataset_quality/data_streams/non_aggregatable', + { + query: { + ...params, + type: DEFAULT_DATASET_TYPE, + }, + } + ) + .catch((error) => { + throw new GetDataStreamsStatsError(`Failed to fetch non aggregatable datasets: ${error}`); + }); + + const nonAggregatableDatasets = decodeOrThrow( + getNonAggregatableDatasetsRt, + (message: string) => + new GetDataStreamsStatsError(`Failed to fetch non aggregatable datasets: ${message}`) + )(response); + + return nonAggregatableDatasets; + } + public async getDataStreamsEstimatedDataInBytes( params: GetDataStreamsEstimatedDataInBytesParams ) { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts index bfd36db4e83752..1450fe1e27f781 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts @@ -10,10 +10,12 @@ import { DataStreamDegradedDocsStatServiceResponse, DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, - GetDataStreamsStatsQuery, GetDataStreamsEstimatedDataInBytesParams, GetDataStreamsEstimatedDataInBytesResponse, + GetDataStreamsStatsQuery, GetIntegrationsParams, + GetNonAggregatableDataStreamsParams, + GetNonAggregatableDataStreamsResponse, IntegrationsResponse, } from '../../../common/data_streams_stats'; @@ -36,4 +38,7 @@ export interface IDataStreamsStatsClient { params: GetDataStreamsEstimatedDataInBytesParams ): Promise; getIntegrations(params: GetIntegrationsParams['query']): Promise; + getNonAggregatableDatasets( + params: GetNonAggregatableDataStreamsParams + ): Promise; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts index da376fa8ca2525..b1bcd6b884156e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts @@ -41,4 +41,5 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { }, flyout: {}, datasets: [], + nonAggregatableDatasets: [], }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts index 3d88e6645fe479..70da9d3d74e70f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts @@ -44,6 +44,15 @@ export const fetchDegradedStatsFailedNotifier = (toasts: IToasts, error: Error) }); }; +export const fetchNonAggregatableDatasetsFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.fetchNonAggregatableDatasetsFailed', { + defaultMessage: "We couldn't get non aggregatable datasets information.", + }), + text: error.message, + }); +}; + export const fetchIntegrationDashboardsFailedNotifier = (toasts: IToasts, error: Error) => { toasts.addDanger({ title: i18n.translate('xpack.datasetQuality.fetchIntegrationDashboardsFailed', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 7d54e268bb5df4..176cce468b5b69 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -17,6 +17,8 @@ import { DataStreamDetails, GetDataStreamsStatsQuery, GetIntegrationsParams, + GetNonAggregatableDataStreamsParams, + GetNonAggregatableDataStreamsResponse, } from '../../../../common/data_streams_stats'; import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat'; import { DataStreamType } from '../../../../common/types'; @@ -32,11 +34,13 @@ import { fetchIntegrationDashboardsFailedNotifier, fetchIntegrationsFailedNotifier, noDatasetSelected, + fetchNonAggregatableDatasetsFailedNotifier, } from './notifications'; import { DatasetQualityControllerContext, DatasetQualityControllerEvent, DatasetQualityControllerTypeState, + DefaultDatasetQualityControllerState, FlyoutDataset, } from './types'; @@ -173,12 +177,69 @@ export const createPureDatasetQualityControllerStateMachine = ( }, }, }, + nonAggregatableDatasets: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadNonAggregatableDatasets', + onDone: { + target: 'loaded', + actions: ['storeNonAggregatableDatasets'], + }, + onError: { + target: 'loaded', + actions: ['notifyFetchNonAggregatableDatasetsFailed'], + }, + }, + }, + loaded: {}, + }, + on: { + UPDATE_TIME_RANGE: { + target: 'nonAggregatableDatasets.fetching', + }, + REFRESH_DATA: { + target: 'nonAggregatableDatasets.fetching', + }, + }, + }, flyout: { initial: 'closed', states: { initializing: { type: 'parallel', states: { + nonAggregatableDataset: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDatasetIsNonAggregatable', + onDone: { + target: 'done', + actions: ['storeDatasetIsNonAggregatable'], + }, + onError: { + target: 'done', + actions: ['notifyFetchNonAggregatableDatasetsFailed'], + }, + }, + }, + done: { + on: { + UPDATE_INSIGHTS_TIME_RANGE: { + target: 'fetching', + actions: ['storeFlyoutOptions'], + }, + SELECT_DATASET: { + target: 'fetching', + actions: ['storeFlyoutOptions'], + }, + }, + }, + }, + }, dataStreamSettings: { initial: 'fetching', states: { @@ -402,6 +463,18 @@ export const createPureDatasetQualityControllerStateMachine = ( } : {}; }), + storeNonAggregatableDatasets: assign( + ( + _context: DefaultDatasetQualityControllerState, + event: DoneInvokeEvent + ) => { + return 'data' in event + ? { + nonAggregatableDatasets: event.data.datasets, + } + : {}; + } + ), storeDataStreamSettings: assign((context, event) => { return 'data' in event ? { @@ -422,6 +495,21 @@ export const createPureDatasetQualityControllerStateMachine = ( } : {}; }), + storeDatasetIsNonAggregatable: assign( + ( + context: DefaultDatasetQualityControllerState, + event: DoneInvokeEvent + ) => { + return 'data' in event + ? { + flyout: { + ...context.flyout, + isNonAggregatable: !event.data.aggregatable, + }, + } + : {}; + } + ), storeIntegrations: assign((_context, event) => { return 'data' in event ? { @@ -484,6 +572,8 @@ export const createDatasetQualityControllerStateMachine = ({ fetchDatasetStatsFailedNotifier(toasts, event.data), notifyFetchDegradedStatsFailed: (_context, event: DoneInvokeEvent) => fetchDegradedStatsFailedNotifier(toasts, event.data), + notifyFetchNonAggregatableDatasetsFailed: (_context, event: DoneInvokeEvent) => + fetchNonAggregatableDatasetsFailedNotifier(toasts, event.data), notifyFetchDatasetSettingsFailed: (_context, event: DoneInvokeEvent) => fetchDatasetSettingsFailedNotifier(toasts, event.data), notifyFetchDatasetDetailsFailed: (_context, event: DoneInvokeEvent) => @@ -514,6 +604,15 @@ export const createDatasetQualityControllerStateMachine = ({ type: context.type as GetIntegrationsParams['query']['type'], }); }, + loadNonAggregatableDatasets: (context) => { + const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); + + return dataStreamStatsClient.getNonAggregatableDatasets({ + type: context.type as GetNonAggregatableDataStreamsParams['type'], + start, + end, + }); + }, loadDataStreamSettings: (context) => { if (!context.flyout.dataset) { fetchDatasetSettingsFailedNotifier(toasts, new Error(noDatasetSelected)); @@ -566,6 +665,29 @@ export const createDatasetQualityControllerStateMachine = ({ ? dataStreamDetailsClient.getIntegrationDashboards({ integration: integration.name }) : Promise.resolve({}); }, + loadDatasetIsNonAggregatable: async (context) => { + if (!context.flyout.dataset || !context.flyout.insightsTimeRange) { + fetchDatasetDetailsFailedNotifier(toasts, new Error(noDatasetSelected)); + + return Promise.resolve({}); + } + + const { type, name: dataset, namespace } = context.flyout.dataset; + const { startDate: start, endDate: end } = getDateISORange( + context.flyout.insightsTimeRange + ); + + return dataStreamStatsClient.getNonAggregatableDatasets({ + type: context.type as GetNonAggregatableDataStreamsParams['type'], + start, + end, + dataStream: dataStreamPartsToIndexName({ + type: type as DataStreamType, + dataset, + namespace, + }), + }); + }, }, }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 10c08f8fa0bfcf..3c23dc9a918bf9 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -20,6 +20,7 @@ import { IntegrationsResponse, DataStreamStat, DataStreamStatType, + GetNonAggregatableDataStreamsResponse, } from '../../../../common/data_streams_stats'; export type FlyoutDataset = Omit< @@ -61,6 +62,7 @@ export interface WithFlyoutOptions { datasetDetails?: DataStreamDetails; insightsTimeRange?: TimeRangeConfig; breakdownField?: string; + isNonAggregatable?: boolean; }; } @@ -76,6 +78,10 @@ export interface WithDegradedDocs { degradedDocStats: DegradedDocsStat[]; } +export interface WithNonAggregatableDatasets { + nonAggregatableDatasets: string[]; +} + export interface WithDatasets { datasets: DataStreamStat[]; } @@ -90,6 +96,7 @@ export type DefaultDatasetQualityControllerState = { type: string } & WithTableO WithFlyoutOptions & WithDatasets & WithFilters & + WithNonAggregatableDatasets & Partial; type DefaultDatasetQualityStateContext = DefaultDatasetQualityControllerState & @@ -120,6 +127,10 @@ export type DatasetQualityControllerTypeState = value: 'integrations.fetching'; context: DefaultDatasetQualityStateContext; } + | { + value: 'nonAggregatableDatasets.fetching'; + context: DefaultDatasetQualityStateContext; + } | { value: 'flyout.initializing.dataStreamSettings.fetching'; context: DefaultDatasetQualityStateContext; @@ -189,6 +200,7 @@ export type DatasetQualityControllerEvent = query: string; } | DoneInvokeEvent + | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts new file mode 100644 index 00000000000000..a776bed1fbdde0 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { rangeQuery } from '@kbn/observability-plugin/server/utils/queries'; +import { extractIndexNameFromBackingIndex } from '../../../common/utils'; +import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; +import { _IGNORED } from '../../../common/es_fields'; +import { DataStreamType } from '../../../common/types'; +import { createDatasetQualityESClient } from '../../utils'; + +export async function getNonAggregatableDataStreams({ + esClient, + type = DEFAULT_DATASET_TYPE, + start, + end, + dataStream, +}: { + esClient: ElasticsearchClient; + type?: DataStreamType; + start: number; + end: number; + dataStream?: string; +}) { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const response = await datasetQualityESClient.fieldCaps({ + index: dataStream ?? `${type}-*`, + fields: [_IGNORED], + index_filter: { + ...rangeQuery(start, end)[0], + }, + }); + + const ignoredField = response.fields._ignored?._ignored; + + const nonAggregatableIndices = ignoredField?.non_aggregatable_indices ?? []; + + const nonAggregatableDatasets = new Set( + (Array.isArray(nonAggregatableIndices) ? nonAggregatableIndices : [nonAggregatableIndices]).map( + extractIndexNameFromBackingIndex + ) + ); + + return { + aggregatable: ignoredField?.aggregatable ?? true, + datasets: Array.from(nonAggregatableDatasets), + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 3ad5a103f43138..c703998e2160a8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -13,6 +13,7 @@ import { DataStreamSettings, DataStreamStat, DegradedDocs, + NonAggregatableDatasets, } from '../../../common/api_types'; import { indexNameToDataStreamParts } from '../../../common/utils'; import { rangeRt, typeRt } from '../../types/default_api_types'; @@ -22,6 +23,7 @@ import { getDataStreams } from './get_data_streams'; import { getDataStreamsStats } from './get_data_streams_stats'; import { getDegradedDocsPaginated } from './get_degraded_docs'; import { getEstimatedDataInBytes } from './get_estimated_data_in_bytes'; +import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', @@ -95,6 +97,33 @@ const degradedDocsRoute = createDatasetQualityServerRoute({ }, }); +const nonAggregatableDatasetsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/non_aggregatable', + params: t.type({ + query: t.intersection([ + rangeRt, + typeRt, + t.partial({ + dataStream: t.string, + }), + ]), + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const coreContext = await context.core; + + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + return await getNonAggregatableDataStreams({ + esClient, + ...params.query, + }); + }, +}); + const dataStreamSettingsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', params: t.type({ @@ -200,6 +229,7 @@ const estimatedDataInBytesRoute = createDatasetQualityServerRoute({ export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, + ...nonAggregatableDatasetsRoute, ...dataStreamDetailsRoute, ...dataStreamSettingsRoute, ...estimatedDataInBytesRoute, diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts index d2c95ebbf0dbf3..414c313ee373c1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts @@ -7,7 +7,7 @@ import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import { ElasticsearchClient } from '@kbn/core/server'; -import { Indices } from '@elastic/elasticsearch/lib/api/types'; +import { FieldCapsRequest, FieldCapsResponse, Indices } from '@elastic/elasticsearch/lib/api/types'; type DatasetQualityESSearchParams = ESSearchRequest & { size: number; @@ -32,5 +32,8 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) { searches: searches.map((search) => [index, search]).flat(), }) as Promise; }, + async fieldCaps(params: FieldCapsRequest): Promise { + return esClient.fieldCaps(params) as Promise; + }, }; }