[Dataset quality] Warning for datasets not supporting _ignored aggreg…

…ation (#183183) Closes #179227. ## 📝 Summary This PR adds a warning to main page and flyout whenever a dataStream has indices that doesn't support `_ignored` aggregation ## 🎥 Demo https://github.com/elastic/kibana/assets/1313018/c6d5fd81-d9a9-4fcc-92d0-8e65b996df9c #### Flyout https://github.com/elastic/kibana/assets/1313018/2972e104-8b10-413a-a430-3efc5252b757
elastic · May 14, 2024 · 3caf266 · 3caf266
1 parent e2dd5ee
commit 3caf266
Show file tree

Hide file tree

Showing 19 changed files with 503 additions and 25 deletions.
diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts
@@ -119,3 +119,12 @@ export type DataStreamsEstimatedDataInBytes = rt.TypeOf<typeof dataStreamsEstima
 export const getDataStreamsEstimatedDataInBytesResponseRt = rt.exact(
   dataStreamsEstimatedDataInBytesRT
 );
+
+export const getNonAggregatableDatasetsRt = rt.exact(
+  rt.type({
+    aggregatable: rt.boolean,
+    datasets: rt.array(rt.string),
+  })
+);
+
+export type NonAggregatableDatasets = rt.TypeOf<typeof getNonAggregatableDatasetsRt>;
diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts
@@ -43,6 +43,11 @@ export type GetDataStreamDetailsParams = GetDataStreamDetailsPathParams &
 export type GetDataStreamDetailsResponse =
   APIReturnType<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>;
 
+export type GetNonAggregatableDataStreamsParams =
+  APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/non_aggregatable`>['params']['query'];
+export type GetNonAggregatableDataStreamsResponse =
+  APIReturnType<`GET /internal/dataset_quality/data_streams/non_aggregatable`>;
+
 export type GetDataStreamsEstimatedDataInBytesParams =
   APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/estimated_data`>['params'];
 export type GetDataStreamsEstimatedDataInBytesResponse =

diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts
@@ -55,27 +55,33 @@ describe('dataset_name', () => {
   });
 
   describe('extractIndexNameFromBackingIndex', () => {
-    it('returns the correct index name if backing index provieded', () => {
+    it('returns the correct index name if backing index provided', () => {
       expect(
-        extractIndexNameFromBackingIndex(
-          '.ds-logs-apm.app.adservice-default-2024.04.29-000001',
-          'logs'
-        )
+        extractIndexNameFromBackingIndex('.ds-logs-apm.app.adservice-default-2024.04.29-000001')
       ).toEqual('logs-apm.app.adservice-default');
     });
 
     it('returns the correct index name if index name is passed', () => {
-      expect(extractIndexNameFromBackingIndex('logs-nginx.access-default', 'logs')).toEqual(
+      expect(extractIndexNameFromBackingIndex('logs-nginx.access-default')).toEqual(
         'logs-nginx.access-default'
       );
     });
 
+    it('returns the correct index name if backing index contains _', () => {
+      expect(
+        extractIndexNameFromBackingIndex('.ds-logs-elastic_agent-default-2024.04.29-000001')
+      ).toEqual('logs-elastic_agent-default');
+    });
+
+    it('returns the correct index name if backing index contains only -', () => {
+      expect(
+        extractIndexNameFromBackingIndex('.ds-logs-generic-pods-default-2024.04.29-000001')
+      ).toEqual('logs-generic-pods-default');
+    });
+
     it('handles different types', () => {
       expect(
-        extractIndexNameFromBackingIndex(
-          '.ds-metrics-apm.app.adservice-default-2024.04.29-000001',
-          'metrics'
-        )
+        extractIndexNameFromBackingIndex('.ds-metrics-apm.app.adservice-default-2024.04.29-000001')
       ).toEqual('metrics-apm.app.adservice-default');
     });
   });

diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts b/x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts
@@ -39,14 +39,8 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => {
   };
 };
 
-export const extractIndexNameFromBackingIndex = (
-  indexString: string,
-  type: DataStreamType
-): string => {
-  const pattern: RegExp = new RegExp(
-    `(?:\\.ds-)?(${type}-(?:[^-.]+(?:\\.[^.]+)+)-[^-]+)-\\d{4}\\.\\d{2}\\.\\d{2}-\\d{6}`
-  );
-
+export const extractIndexNameFromBackingIndex = (indexString: string): string => {
+  const pattern = /.ds-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
   const match = indexString.match(pattern);
 
   return match ? match[1] : indexString;

diff --git a/...ervability_solution/dataset_quality/public/components/dataset_quality/dataset_quality.tsx b/...ervability_solution/dataset_quality/public/components/dataset_quality/dataset_quality.tsx
@@ -56,6 +56,7 @@ export const createDatasetQuality = ({
 };
 
 const Header = dynamic(() => import('./header'));
+const Warnings = dynamic(() => import('./warnings/warnings'));
 const Table = dynamic(() => import('./table/table'));
 const Filters = dynamic(() => import('./filters/filters'));
 const SummaryPanel = dynamic(() => import('./summary_panel/summary_panel'));
@@ -66,6 +67,9 @@ function DatasetQuality() {
       <EuiFlexItem grow={false}>
         <Header />
       </EuiFlexItem>
+      <EuiFlexItem grow={false}>
+        <Warnings />
+      </EuiFlexItem>
       <EuiFlexItem grow={false}>
         <Filters />
       </EuiFlexItem>

diff --git a/...vability_solution/dataset_quality/public/components/dataset_quality/warnings/warnings.tsx b/...vability_solution/dataset_quality/public/components/dataset_quality/warnings/warnings.tsx
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiAccordion, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiLink } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n-react';
+import React from 'react';
+import { useDatasetQualityWarnings } from '../../../hooks/use_dataset_quality_warnings';
+
+const nonAggregatableWarningTitle = i18n.translate('xpack.datasetQuality.nonAggregatable.title', {
+  defaultMessage: 'Your request may take longer to complete',
+});
+
+const nonAggregatableWarningDescription = (nonAggregatableDatasets: string[]) => (
+  <FormattedMessage
+    id="xpack.datasetQuality.nonAggregatable.description"
+    defaultMessage="{description}"
+    values={{
+      description: (
+        <FormattedMessage
+          id="xpack.datasetQuality.nonAggregatable.warning"
+          defaultMessage="Some of your datasets do not support _ignored aggregation and may cause delays when querying data. {howToFixIt} {showDatasets}"
+          values={{
+            showDatasets: (
+              <FormattedMessage
+                id="xpack.datasetQuality.nonAggregatable.warning.description."
+                defaultMessage="{accordion}"
+                values={{
+                  accordion: (
+                    <EuiAccordion
+                      style={{ marginTop: '12px ' }}
+                      buttonContent={i18n.translate(
+                        'xpack.datasetQuality.nonAggregatable.showAffectedDatasets',
+                        {
+                          defaultMessage: 'Show affected datasets',
+                        }
+                      )}
+                      id={''}
+                    >
+                      <ul>
+                        {nonAggregatableDatasets.map((dataset) => (
+                          <li key={dataset}>{dataset}</li>
+                        ))}
+                      </ul>
+                    </EuiAccordion>
+                  ),
+                }}
+              />
+            ),
+            howToFixIt: (
+              <FormattedMessage
+                id="xpack.datasetQuality.nonAggregatable.howToFixIt"
+                defaultMessage="Manually {rolloverLink} these datasets to prevent future delays."
+                values={{
+                  rolloverLink: (
+                    <EuiLink
+                      external
+                      target="_blank"
+                      data-test-subj="datasetQualityNonAggregatableHowToFixItLink"
+                      href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-rollover-index.html"
+                    >
+                      {i18n.translate('xpack.datasetQuality.nonAggregatableDatasets.link.title', {
+                        defaultMessage: 'rollover',
+                      })}
+                    </EuiLink>
+                  ),
+                }}
+              />
+            ),
+          }}
+        />
+      ),
+    }}
+  />
+);
+
+// Allow for lazy loading
+// eslint-disable-next-line import/no-default-export
+export default function Warnings() {
+  const { loading, nonAggregatableDatasets } = useDatasetQualityWarnings();
+
+  return (
+    <EuiFlexGroup data-test-subj="datasetQualityWarningsContainer" gutterSize="s" wrap>
+      {!loading && nonAggregatableDatasets.length > 0 && (
+        <EuiFlexItem>
+          <EuiCallOut title={nonAggregatableWarningTitle} color="warning" iconType="warning">
+            <p>{nonAggregatableWarningDescription(nonAggregatableDatasets)}</p>
+          </EuiCallOut>
+        </EuiFlexItem>
+      )}
+    </EuiFlexGroup>
+  );
+}
diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx
@@ -34,6 +34,7 @@ export default function Flyout({ dataset, closeFlyout }: FlyoutProps) {
     dataStreamStat,
     dataStreamSettings,
     dataStreamDetails,
+    isNonAggregatable,
     fieldFormats,
     timeRange,
     loadingState,
@@ -60,6 +61,7 @@ export default function Flyout({ dataset, closeFlyout }: FlyoutProps) {
                 dataStreamDetails={dataStreamDetails}
                 dataStreamDetailsLoading={loadingState.dataStreamDetailsLoading}
                 timeRange={timeRange}
+                isNonAggregatable={isNonAggregatable}
               />
             </EuiPanel>
 

diff --git a/...ility_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx b/...ility_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx
@@ -6,8 +6,19 @@
  */
 
 import React, { useCallback, useState } from 'react';
-import { OnRefreshProps, OnTimeChangeProps, EuiSpacer } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import {
+  OnRefreshProps,
+  OnTimeChangeProps,
+  EuiSpacer,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiCallOut,
+  EuiLink,
+  EuiCode,
+} from '@elastic/eui';
 
+import { FormattedMessage } from '@kbn/i18n-react';
 import { DegradedDocs } from '../degraded_docs_trend/degraded_docs';
 import { DataStreamDetails } from '../../../../common/api_types';
 import { DEFAULT_TIME_RANGE, DEFAULT_DATEPICKER_REFRESH } from '../../../../common/constants';
@@ -16,10 +27,60 @@ import { FlyoutDataset, TimeRangeConfig } from '../../../state_machines/dataset_
 import { FlyoutSummaryHeader } from './flyout_summary_header';
 import { FlyoutSummaryKpis, FlyoutSummaryKpisLoading } from './flyout_summary_kpis';
 
+const nonAggregatableWarningTitle = i18n.translate('xpack.datasetQuality.nonAggregatable.title', {
+  defaultMessage: 'Your request may take longer to complete',
+});
+
+const nonAggregatableWarningDescription = (dataset: string) => (
+  <FormattedMessage
+    id="xpack.datasetQuality.flyout.nonAggregatable.description"
+    defaultMessage="{description}"
+    values={{
+      description: (
+        <FormattedMessage
+          id="xpack.datasetQuality.flyout.nonAggregatable.warning"
+          defaultMessage="{dataset}does not support _ignored aggregation and may cause delays when querying data. {howToFixIt}"
+          values={{
+            dataset: (
+              <EuiCode language="json" transparentBackground>
+                {dataset}
+              </EuiCode>
+            ),
+            howToFixIt: (
+              <FormattedMessage
+                id="xpack.datasetQuality.flyout.nonAggregatable.howToFixIt"
+                defaultMessage="Manually {rolloverLink} this dataset to prevent future delays."
+                values={{
+                  rolloverLink: (
+                    <EuiLink
+                      external
+                      target="_blank"
+                      data-test-subj="datasetQualityFlyoutNonAggregatableHowToFixItLink"
+                      href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-rollover-index.html"
+                    >
+                      {i18n.translate(
+                        'xpack.datasetQuality.flyout.nonAggregatableDatasets.link.title',
+                        {
+                          defaultMessage: 'rollover',
+                        }
+                      )}
+                    </EuiLink>
+                  ),
+                }}
+              />
+            ),
+          }}
+        />
+      ),
+    }}
+  />
+);
+
 export function FlyoutSummary({
   dataStream,
   dataStreamStat,
   dataStreamDetails,
+  isNonAggregatable,
   dataStreamDetailsLoading,
   timeRange = { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_DATEPICKER_REFRESH },
 }: {
@@ -28,6 +89,7 @@ export function FlyoutSummary({
   dataStreamDetails?: DataStreamDetails;
   dataStreamDetailsLoading: boolean;
   timeRange?: TimeRangeConfig;
+  isNonAggregatable?: boolean;
 }) {
   const { service } = useDatasetQualityContext();
   const [lastReloadTime, setLastReloadTime] = useState<number>(Date.now());
@@ -72,6 +134,18 @@ export function FlyoutSummary({
 
   return (
     <>
+      {isNonAggregatable && (
+        <EuiFlexGroup
+          data-test-subj="datasetQualityFlyoutNonAggregatableWarning"
+          style={{ marginBottom: '24px' }}
+        >
+          <EuiFlexItem>
+            <EuiCallOut title={nonAggregatableWarningTitle} color="warning" iconType="warning">
+              <p>{nonAggregatableWarningDescription(dataStream)}</p>
+            </EuiCallOut>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      )}
       <FlyoutSummaryHeader
         timeRange={timeRange}
         onTimeChange={handleTimeChange}

diff --git a/...lugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx b/...lugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx
@@ -22,6 +22,7 @@ export const useDatasetQualityFlyout = () => {
     datasetDetails: dataStreamDetails,
     insightsTimeRange,
     breakdownField,
+    isNonAggregatable,
   } = useSelector(service, (state) => state.context.flyout) ?? {};
 
   const { timeRange } = useSelector(service, (state) => state.context.filters);
@@ -36,6 +37,7 @@ export const useDatasetQualityFlyout = () => {
     dataStreamStat,
     dataStreamSettings,
     dataStreamDetails,
+    isNonAggregatable,
     fieldFormats,
     timeRange: insightsTimeRange ?? timeRange,
     breakdownField,

diff --git a/...ugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts b/...ugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { useSelector } from '@xstate/react';
+import { useDatasetQualityContext } from '../components/dataset_quality/context';
+
+export function useDatasetQualityWarnings() {
+  const { service } = useDatasetQualityContext();
+
+  const nonAggregatableDatasets = useSelector(
+    service,
+    (state) => state.context.nonAggregatableDatasets
+  );
+
+  const isNonAggregatableDatasetsLoading = useSelector(service, (state) =>
+    state.matches('nonAggregatableDatasets.fetching')
+  );
+
+  return { loading: isNonAggregatableDatasetsLoading, nonAggregatableDatasets };
+}