[ML] Add Anomaly charts embeddables to Dashboard from Anomaly Explorer page

x-pack/plugins/ml/common/index.ts

@@ -10,6 +10,7 @@ export { ChartData } from './types/field_histograms';
 export { ANOMALY_SEVERITY, ANOMALY_THRESHOLD, SEVERITY_COLORS } from './constants/anomalies';
 export { getSeverityColor, getSeverityType } from './util/anomaly_utils';
 export { isPopulatedObject } from './util/object_utils';
-export { isRuntimeMappings } from './util/runtime_field_utils';
 export { composeValidators, patternValidator } from './util/validators';
+export { isRuntimeMappings, isRuntimeField } from './util/runtime_field_utils';
 export { extractErrorMessage } from './util/errors';
+export type { RuntimeMappings } from './types/fields';

x-pack/plugins/ml/common/types/fields.ts

@@ -28,7 +28,7 @@ export interface Field {
   aggregatable?: boolean;
   aggIds?: AggId[];
   aggs?: Aggregation[];
-  runtimeField?: RuntimeField;
+  runtimeField?: estypes.RuntimeField;
 }
 
 export interface Aggregation {
@@ -108,17 +108,4 @@ export interface AggCardinality {
 
 export type RollupFields = Record<FieldId, [Record<'agg', ES_AGGREGATION>]>;
 
-// Replace this with import once #88995 is merged
-export const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
-export type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
-
-export interface RuntimeField {
-  type: RuntimeType;
-  script?:
-    | string
-    | {
-        source: string;
-      };
-}
-
 export type RuntimeMappings = estypes.RuntimeFields;

x-pack/plugins/ml/common/util/runtime_field_utils.ts

@@ -4,14 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import { estypes } from '@elastic/elasticsearch';
 import { isPopulatedObject } from './object_utils';
 import { RUNTIME_FIELD_TYPES } from '../../../../../src/plugins/data/common';
-import type { RuntimeField, RuntimeMappings } from '../types/fields';
+import type { RuntimeMappings } from '../types/fields';
 
 type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
 
-export function isRuntimeField(arg: unknown): arg is RuntimeField {
+export function isRuntimeField(arg: unknown): arg is estypes.RuntimeField {
   return (
     ((isPopulatedObject(arg, ['type']) && Object.keys(arg).length === 1) ||
       (isPopulatedObject(arg, ['type', 'script']) &&

x-pack/plugins/ml/public/application/components/data_grid/common.ts

@@ -18,6 +18,7 @@ import { i18n } from '@kbn/i18n';
 
 import { CoreSetup } from 'src/core/public';
 
+import type { estypes } from '@elastic/elasticsearch';
 import {
   IndexPattern,
   IFieldType,
@@ -49,7 +50,7 @@ import { getNestedProperty } from '../../util/object_utils';
 import { mlFieldFormatService } from '../../services/field_format_service';
 
 import { DataGridItem, IndexPagination, RenderCellValue } from './types';
-import { RuntimeMappings, RuntimeField } from '../../../../common/types/fields';
+import { RuntimeMappings } from '../../../../common/types/fields';
 import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils';
 
 export const INIT_MAX_COLUMNS = 10;
@@ -179,7 +180,7 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results
 export const NON_AGGREGATABLE = 'non-aggregatable';
 
 export const getDataGridSchemaFromESFieldType = (
-  fieldType: ES_FIELD_TYPES | undefined | RuntimeField['type']
+  fieldType: ES_FIELD_TYPES | undefined | estypes.RuntimeField['type']
 ): string | undefined => {
   // Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json']
   // To fall back to the default string schema it needs to be undefined.

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/form_options_validation.ts

@@ -6,8 +6,8 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { estypes } from '@elastic/elasticsearch';
 import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
-import { RuntimeType } from '../../../../../../../../../../src/plugins/data/common';
 import { EVENT_RATE_FIELD_ID } from '../../../../../../../common/types/fields';
 import { ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics';
 import { AnalyticsJobType } from '../../../analytics_management/hooks/use_create_analytics_form/state';
@@ -18,7 +18,7 @@ export const CATEGORICAL_TYPES = new Set(['ip', 'keyword']);
 // Regression supports numeric fields. Classification supports categorical, numeric, and boolean.
 export const shouldAddAsDepVarOption = (
   fieldId: string,
-  fieldType: ES_FIELD_TYPES | RuntimeType,
+  fieldType: ES_FIELD_TYPES | estypes.RuntimeField['type'],
   jobType: AnalyticsJobType
 ) => {
   if (fieldId === EVENT_RATE_FIELD_ID) return false;

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts

@@ -13,7 +13,7 @@ import { CoreSetup } from 'src/core/public';
 
 import { IndexPattern } from '../../../../../../../../../src/plugins/data/public';
 import { isRuntimeMappings } from '../../../../../../common/util/runtime_field_utils';
-import { RuntimeMappings, RuntimeField } from '../../../../../../common/types/fields';
+import { RuntimeMappings } from '../../../../../../common/types/fields';
 import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../../common/constants/field_histograms';
 
 import { DataLoader } from '../../../../datavisualizer/index_based/data_loader';
@@ -44,7 +44,7 @@ interface MLEuiDataGridColumn extends EuiDataGridColumn {
 function getRuntimeFieldColumns(runtimeMappings: RuntimeMappings) {
   return Object.keys(runtimeMappings).map((id) => {
     const field = runtimeMappings[id];
-    const schema = getDataGridSchemaFromESFieldType(field.type as RuntimeField['type']);
+    const schema = getDataGridSchemaFromESFieldType(field.type as estypes.RuntimeField['type']);
     return { id, schema, isExpandable: schema !== 'boolean', isRuntimeFieldColumn: true };
   });
 }
@@ -64,7 +64,7 @@ export const useIndexData = (
       const field = indexPattern.fields.getByName(id);
       const isRuntimeFieldColumn = field?.runtimeField !== undefined;
       const schema = isRuntimeFieldColumn
-        ? getDataGridSchemaFromESFieldType(field?.type as RuntimeField['type'])
+        ? getDataGridSchemaFromESFieldType(field?.type as estypes.RuntimeField['type'])
         : getDataGridSchemaFromKibanaFieldType(field);
       return {
         id,

x-pack/plugins/ml/public/application/explorer/actions/load_explorer_data.ts

@@ -10,7 +10,7 @@ import { isEqual } from 'lodash';
 import useObservable from 'react-use/lib/useObservable';
 
 import { forkJoin, of, Observable, Subject } from 'rxjs';
-import { mergeMap, switchMap, tap } from 'rxjs/operators';
+import { mergeMap, switchMap, tap, map } from 'rxjs/operators';
 
 import { useCallback, useMemo } from 'react';
 import { explorerService } from '../explorer_dashboard_service';
@@ -21,7 +21,6 @@ import {
   getSelectionTimeRange,
   loadAnnotationsTableData,
   loadAnomaliesTableData,
-  loadDataForCharts,
   loadFilteredTopInfluencers,
   loadTopInfluencers,
   AppStateSelectedCells,
@@ -36,8 +35,9 @@ import { ANOMALY_SWIM_LANE_HARD_LIMIT } from '../explorer_constants';
 import { TimefilterContract } from '../../../../../../../src/plugins/data/public';
 import { AnomalyExplorerChartsService } from '../../services/anomaly_explorer_charts_service';
 import { CombinedJob } from '../../../../common/types/anomaly_detection_jobs';
-import { mlJobService } from '../../services/job_service';
 import { InfluencersFilterQuery } from '../../../../common/types/es_client';
+import { ExplorerChartsData } from '../explorer_charts/explorer_charts_container_service';
+import { mlJobService } from '../../services/job_service';
 
 // Memoize the data fetching methods.
 // wrapWithLastRefreshArg() wraps any given function and preprends a `lastRefresh` argument
@@ -58,7 +58,6 @@ const memoize = <T extends (...a: any[]) => any>(func: T, context?: any) => {
 const memoizedLoadAnnotationsTableData = memoize<typeof loadAnnotationsTableData>(
   loadAnnotationsTableData
 );
-const memoizedLoadDataForCharts = memoize<typeof loadDataForCharts>(loadDataForCharts);
 const memoizedLoadFilteredTopInfluencers = memoize<typeof loadFilteredTopInfluencers>(
   loadFilteredTopInfluencers
 );
@@ -96,7 +95,7 @@ export const isLoadExplorerDataConfig = (arg: any): arg is LoadExplorerDataConfi
 const loadExplorerDataProvider = (
   mlResultsService: MlResultsService,
   anomalyTimelineService: AnomalyTimelineService,
-  anomalyExplorerService: AnomalyExplorerChartsService,
+  anomalyExplorerChartsService: AnomalyExplorerChartsService,
   timefilter: TimefilterContract
 ) => {
   const memoizedLoadOverallData = memoize(
@@ -108,8 +107,8 @@ const loadExplorerDataProvider = (
     anomalyTimelineService
   );
   const memoizedAnomalyDataChange = memoize(
-    anomalyExplorerService.getAnomalyData,
-    anomalyExplorerService
+    anomalyExplorerChartsService.getAnomalyData,
+    anomalyExplorerChartsService
   );
 
   return (config: LoadExplorerDataConfig): Observable<Partial<ExplorerState>> => {
@@ -160,9 +159,7 @@ const loadExplorerDataProvider = (
         swimlaneBucketInterval.asSeconds(),
         bounds
       ),
-      anomalyChartRecords: memoizedLoadDataForCharts(
-        lastRefresh,
-        mlResultsService,
+      anomalyChartRecords: anomalyExplorerChartsService.loadDataForCharts$(
         jobIds,
         timerange.earliestMs,
         timerange.latestMs,
@@ -214,42 +211,30 @@ const loadExplorerDataProvider = (
       // show the view-by loading indicator
       // and pass on the data we already fetched.
       tap(explorerService.setViewBySwimlaneLoading),
-      // Trigger a side-effect to update the charts.
-      tap(({ anomalyChartRecords, topFieldValues }) => {
-        if (selectedCells !== undefined && Array.isArray(anomalyChartRecords)) {
-          memoizedAnomalyDataChange(
-            lastRefresh,
-            explorerService,
-            combinedJobRecords,
-            swimlaneContainerWidth,
-            anomalyChartRecords,
-            timerange.earliestMs,
-            timerange.latestMs,
-            timefilter,
-            tableSeverity
-          );
-        } else {
-          memoizedAnomalyDataChange(
-            lastRefresh,
-            explorerService,
-            combinedJobRecords,
-            swimlaneContainerWidth,
-            [],
-            timerange.earliestMs,
-            timerange.latestMs,
-            timefilter,
-            tableSeverity
-          );
-        }
-      }),
-      // Load view-by swimlane data and filtered top influencers.
-      // mergeMap is used to have access to the already fetched data and act on it in arg #1.
-      // In arg #2 of mergeMap we combine the data and pass it on in the action format
-      // which can be consumed by explorerReducer() later on.
+      tap(explorerService.setChartsDataLoading),
       mergeMap(
-        ({ anomalyChartRecords, influencers, overallState, topFieldValues }) =>
+        ({
+          anomalyChartRecords,
+          influencers,
+          overallState,
+          topFieldValues,
+          annotationsData,
+          tableData,
+        }) =>
           forkJoin({
-            influencers:
+            anomalyChartsData: memoizedAnomalyDataChange(
+              lastRefresh,
+              combinedJobRecords,
+              swimlaneContainerWidth,
+              selectedCells !== undefined && Array.isArray(anomalyChartRecords)
+                ? anomalyChartRecords
+                : [],
+              timerange.earliestMs,
+              timerange.latestMs,
+              timefilter,
+              tableSeverity
+            ),
+            filteredTopInfluencers:
               (selectionInfluencers.length > 0 || influencersFilterQuery !== undefined) &&
               anomalyChartRecords !== undefined &&
               anomalyChartRecords.length > 0
@@ -280,24 +265,26 @@ const loadExplorerDataProvider = (
               swimlaneContainerWidth,
               influencersFilterQuery
             ),
-          }),
-        (
-          { annotationsData, overallState, tableData },
-          { influencers, viewBySwimlaneState }
-        ): Partial<ExplorerState> => {
-          return {
-            annotations: annotationsData,
-            influencers: influencers as any,
-            loading: false,
-            viewBySwimlaneDataLoading: false,
-            overallSwimlaneData: overallState,
-            viewBySwimlaneData: viewBySwimlaneState as any,
-            tableData,
-            swimlaneLimit: isViewBySwimLaneData(viewBySwimlaneState)
-              ? viewBySwimlaneState.cardinality
-              : undefined,
-          };
-        }
+          }).pipe(
+            tap(({ anomalyChartsData }) => {
+              explorerService.setCharts(anomalyChartsData as ExplorerChartsData);
+            }),
+            map(({ viewBySwimlaneState, filteredTopInfluencers }) => {
+              return {
+                annotations: annotationsData,
+                influencers: filteredTopInfluencers as any,
+                loading: false,
+                viewBySwimlaneDataLoading: false,
+                anomalyChartsDataLoading: false,
+                overallSwimlaneData: overallState,
+                viewBySwimlaneData: viewBySwimlaneState as any,
+                tableData,
+                swimlaneLimit: isViewBySwimLaneData(viewBySwimlaneState)
+                  ? viewBySwimlaneState.cardinality
+                  : undefined,
+              };
+            })
+          )
       )
     );
   };
@@ -319,15 +306,15 @@ export const useExplorerData = (): [Partial<ExplorerState> | undefined, (d: any)
       uiSettings,
       mlResultsService
     );
-    const anomalyExplorerService = new AnomalyExplorerChartsService(
+    const anomalyExplorerChartsService = new AnomalyExplorerChartsService(
       timefilter,
       mlApiServices,
       mlResultsService
     );
     return loadExplorerDataProvider(
       mlResultsService,
       anomalyTimelineService,
-      anomalyExplorerService,
+      anomalyExplorerChartsService,
       timefilter
     );
   }, []);