[ML] Add better UI support for runtime fields Transforms

x-pack/plugins/ml/common/types/feature_importance.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import { isPopulatedObject } from '../util/object_utils';
+
 export type FeatureImportanceClassName = string | number | boolean;
 
 export interface ClassFeatureImportance {
@@ -87,7 +89,7 @@ export function isClassificationFeatureImportanceBaseline(
   baselineData: any
 ): baselineData is ClassificationFeatureImportanceBaseline {
   return (
-    typeof baselineData === 'object' &&
+    isPopulatedObject(baselineData) &&
     baselineData.hasOwnProperty('classes') &&
     Array.isArray(baselineData.classes)
   );
@@ -96,5 +98,5 @@ export function isClassificationFeatureImportanceBaseline(
 export function isRegressionFeatureImportanceBaseline(
   baselineData: any
 ): baselineData is RegressionFeatureImportanceBaseline {
-  return typeof baselineData === 'object' && baselineData.hasOwnProperty('baseline');
+  return isPopulatedObject(baselineData) && baselineData.hasOwnProperty('baseline');
 }

x-pack/plugins/ml/common/types/fields.ts

@@ -103,3 +103,18 @@ export interface ScriptAggCardinality {
 export interface AggCardinality {
   cardinality: FieldAggCardinality | ScriptAggCardinality;
 }
+
+// Replace this with import once #88995 is merged
+const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
+type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
+
+export interface RuntimeField {
+  type: RuntimeType;
+  script:
+    | string
+    | {
+        source: string;
+      };
+}
+
+export type RuntimeMappings = Record<string, RuntimeField>;

x-pack/plugins/ml/common/util/datafeed_utils.ts

@@ -20,7 +20,7 @@ export const getDatafeedAggregations = (
 };
 
 export const getAggregationBucketsName = (aggregations: any): string | undefined => {
-  if (typeof aggregations === 'object') {
+  if (aggregations !== null && typeof aggregations === 'object') {
     const keys = Object.keys(aggregations);
     return keys.length > 0 ? keys[0] : undefined;
   }

x-pack/plugins/ml/common/util/job_utils.ts

@@ -28,6 +28,7 @@ import {
   getDatafeedAggregations,
 } from './datafeed_utils';
 import { findAggField } from './validation_utils';
+import { isPopulatedObject } from './object_utils';
 
 export interface ValidationResults {
   valid: boolean;
@@ -51,17 +52,9 @@ export function calculateDatafeedFrequencyDefaultSeconds(bucketSpanSeconds: numb
 }
 
 export function hasRuntimeMappings(job: CombinedJob): boolean {
-  const hasDatafeed =
-    typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
+  const hasDatafeed = isPopulatedObject(job.datafeed_config);
   if (hasDatafeed) {
-    const runtimeMappings =
-      typeof job.datafeed_config.runtime_mappings === 'object'
-        ? Object.keys(job.datafeed_config.runtime_mappings)
-        : undefined;
-
-    if (Array.isArray(runtimeMappings) && runtimeMappings.length > 0) {
-      return true;
-    }
+    return isPopulatedObject(job.datafeed_config.runtime_mappings);
   }
   return false;
 }
@@ -114,7 +107,11 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex
     // If the datafeed uses script fields, we can only plot the time series if
     // model plot is enabled. Without model plot it will be very difficult or impossible
     // to invert to a reverse search of the underlying metric data.
-    if (isSourceDataChartable === true && typeof job.datafeed_config?.script_fields === 'object') {
+    if (
+      isSourceDataChartable === true &&
+      job.datafeed_config?.script_fields !== null &&
+      typeof job.datafeed_config?.script_fields === 'object'
+    ) {
       // Perform extra check to see if the detector is using a scripted field.
       const scriptFields = Object.keys(job.datafeed_config.script_fields);
       isSourceDataChartable =
@@ -123,8 +120,7 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex
         scriptFields.indexOf(dtr.over_field_name!) === -1;
     }
 
-    const hasDatafeed =
-      typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
+    const hasDatafeed = isPopulatedObject(job.datafeed_config);
     if (hasDatafeed) {
       // We cannot plot the source data for some specific aggregation configurations
       const aggs = getDatafeedAggregations(job.datafeed_config);

x-pack/plugins/ml/common/util/object_utils.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const isPopulatedObject = <T = Record<string, any>>(arg: any): arg is T => {
+  return typeof arg === 'object' && arg !== null && Object.keys(arg).length > 0;
+};

x-pack/plugins/ml/common/util/validation_utils.ts

@@ -45,7 +45,7 @@ export function findAggField(
       value = returnParent === true ? aggs : aggs[k];
       return true;
     }
-    if (aggs.hasOwnProperty(k) && typeof aggs[k] === 'object') {
+    if (aggs.hasOwnProperty(k) && aggs[k] !== null && typeof aggs[k] === 'object') {
       value = findAggField(aggs[k], fieldName, returnParent);
       return value !== undefined;
     }

x-pack/plugins/ml/public/application/components/data_grid/common.ts

@@ -44,6 +44,9 @@ import { getNestedProperty } from '../../util/object_utils';
 import { mlFieldFormatService } from '../../services/field_format_service';
 
 import { DataGridItem, IndexPagination, RenderCellValue } from './types';
+import type { RuntimeField } from '../../../../../../../src/plugins/data/common/index_patterns';
+import { RuntimeMappings } from '../../../../common/types/fields';
+import { isPopulatedObject } from '../../../../common/util/object_utils';
 
 export const INIT_MAX_COLUMNS = 10;
 
@@ -82,6 +85,37 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str
   return indexPatternFields;
 };
 
+/**
+ * Return a map of runtime_mappings for each of the index pattern field provided
+ * to provide in ES search queries
+ * @param indexPatternFields
+ * @param indexPattern
+ * @param clonedRuntimeMappings
+ */
+export const getRuntimeFieldsMapping = (
+  indexPatternFields: string[] | undefined,
+  indexPattern: IndexPattern | undefined,
+  clonedRuntimeMappings?: RuntimeMappings
+) => {
+  if (!Array.isArray(indexPatternFields) || indexPattern === undefined) return {};
+  const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields;
+  let combinedRuntimeMappings: RuntimeMappings = {};
+
+  if (isPopulatedObject(ipRuntimeMappings)) {
+    indexPatternFields.forEach((ipField) => {
+      if (ipRuntimeMappings.hasOwnProperty(ipField)) {
+        combinedRuntimeMappings[ipField] = ipRuntimeMappings[ipField];
+      }
+    });
+  }
+  if (isPopulatedObject(clonedRuntimeMappings)) {
+    combinedRuntimeMappings = { ...combinedRuntimeMappings, ...clonedRuntimeMappings };
+  }
+  return Object.keys(combinedRuntimeMappings).length > 0
+    ? { runtime_mappings: combinedRuntimeMappings }
+    : {};
+};
+
 export interface FieldTypes {
   [key: string]: ES_FIELD_TYPES;
 }
@@ -131,6 +165,45 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results
 };
 
 export const NON_AGGREGATABLE = 'non-aggregatable';
+
+export const getDataGridSchemaFromESFieldType = (
+  fieldType: ES_FIELD_TYPES | undefined | RuntimeField['type']
+): string | undefined => {
+  // Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json']
+  // To fall back to the default string schema it needs to be undefined.
+  let schema;
+
+  switch (fieldType) {
+    case ES_FIELD_TYPES.GEO_POINT:
+    case ES_FIELD_TYPES.GEO_SHAPE:
+      schema = 'json';
+      break;
+    case ES_FIELD_TYPES.BOOLEAN:
+      schema = 'boolean';
+      break;
+    case ES_FIELD_TYPES.DATE:
+    case ES_FIELD_TYPES.DATE_NANOS:
+      schema = 'datetime';
+      break;
+    case ES_FIELD_TYPES.BYTE:
+    case ES_FIELD_TYPES.DOUBLE:
+    case ES_FIELD_TYPES.FLOAT:
+    case ES_FIELD_TYPES.HALF_FLOAT:
+    case ES_FIELD_TYPES.INTEGER:
+    case ES_FIELD_TYPES.LONG:
+    case ES_FIELD_TYPES.SCALED_FLOAT:
+    case ES_FIELD_TYPES.SHORT:
+      schema = 'numeric';
+      break;
+    // keep schema undefined for text based columns
+    case ES_FIELD_TYPES.KEYWORD:
+    case ES_FIELD_TYPES.TEXT:
+      break;
+  }
+
+  return schema;
+};
+
 export const getDataGridSchemaFromKibanaFieldType = (
   field: IFieldType | undefined
 ): string | undefined => {

x-pack/plugins/ml/public/application/components/data_grid/index.ts

@@ -7,8 +7,10 @@
 
 export {
   getDataGridSchemasFromFieldTypes,
+  getDataGridSchemaFromESFieldType,
   getDataGridSchemaFromKibanaFieldType,
   getFieldsFromKibanaIndexPattern,
+  getRuntimeFieldsMapping,
   multiColumnSortFactory,
   showDataGridColumnChartErrorMessageToast,
   useRenderCellValue,

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx

@@ -8,6 +8,7 @@
 import React, { FC, Fragment, useEffect, useMemo, useRef, useState } from 'react';
 import {
   EuiBadge,
+  EuiCallOut,
   EuiComboBox,
   EuiComboBoxOptionOption,
   EuiFormRow,
@@ -19,6 +20,7 @@ import {
 import { i18n } from '@kbn/i18n';
 import { debounce } from 'lodash';
 
+import { FormattedMessage } from '@kbn/i18n/react';
 import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
 import { useMlContext } from '../../../../../contexts/ml';
 
@@ -314,6 +316,15 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
     };
   }, [jobType, dependentVariable, trainingPercent, JSON.stringify(includes), jobConfigQueryString]);
 
+  const unsupportedRuntimeFields = useMemo(
+    () =>
+      currentIndexPattern.fields
+        .getAll()
+        .filter((f) => f.runtimeField)
+        .map((f) => `'${f.displayName}'`),
+    [currentIndexPattern.fields]
+  );
+
   return (
     <Fragment>
       <Messages messages={requestMessages} />
@@ -445,6 +456,37 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
       >
         <Fragment />
       </EuiFormRow>
+      {Array.isArray(unsupportedRuntimeFields) && unsupportedRuntimeFields.length > 0 && (
+        <>
+          <EuiCallOut size="s" color="warning">
+            <FormattedMessage
+              id="xpack.ml.dataframe.analytics.create.unsupportedRuntimeFieldsCallout"
+              defaultMessage="The runtime {runtimeFieldsCount, plural, one {field} other {fields}} {unsupportedRuntimeFields} {extraCountMsg} are not supported for analysis."
+              values={{
+                runtimeFieldsCount: unsupportedRuntimeFields.length,
+                extraCountMsg:
+                  unsupportedRuntimeFields.length - 5 > 0 ? (
+                    <FormattedMessage
+                      id="xpack.ml.dataframe.analytics.create.extraUnsupportedRuntimeFieldsMsg"
+                      defaultMessage="and {count} more"
+                      values={{
+                        count: unsupportedRuntimeFields.length - 5,
+                      }}
+                    />
+                  ) : (
+                    ''
+                  ),
+                unsupportedRuntimeFields:
+                  unsupportedRuntimeFields.length > 5
+                    ? unsupportedRuntimeFields.slice(0, 5).join(', ')
+                    : unsupportedRuntimeFields.join(', '),
+              }}
+            />
+          </EuiCallOut>
+          <EuiSpacer />
+        </>
+      )}
+
       <AnalysisFieldsTable
         dependentVariable={dependentVariable}
         includes={includes}

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts

@@ -30,6 +30,7 @@ import type { SearchResponse7 } from '../../../../../../common/types/es_client';
 import { extractErrorMessage } from '../../../../../../common/util/errors';
 import { INDEX_STATUS } from '../../../common/analytics';
 import { ml } from '../../../../services/ml_api_service';
+import { getRuntimeFieldsMapping } from '../../../../components/data_grid/common';
 
 type IndexSearchResponse = SearchResponse7;
 
@@ -38,7 +39,9 @@ export const useIndexData = (
   query: any,
   toastNotifications: CoreSetup['notifications']['toasts']
 ): UseIndexDataReturnType => {
-  const indexPatternFields = getFieldsFromKibanaIndexPattern(indexPattern);
+  const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [
+    indexPattern,
+  ]);
 
   // EuiDataGrid State
   const columns: EuiDataGridColumn[] = [
@@ -75,7 +78,6 @@ export const useIndexData = (
       s[column.id] = { order: column.direction };
       return s;
     }, {} as EsSorting);
-
     const esSearchRequest = {
       index: indexPattern.title,
       body: {
@@ -86,6 +88,7 @@ export const useIndexData = (
         fields: ['*'],
         _source: false,
         ...(Object.keys(sort).length > 0 ? { sort } : {}),
+        ...getRuntimeFieldsMapping(indexPatternFields, indexPattern),
       },
     };
 
@@ -105,7 +108,7 @@ export const useIndexData = (
   useEffect(() => {
     getIndexData();
     // custom comparison
-  }, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]);
+  }, [indexPattern.title, indexPatternFields, JSON.stringify([query, pagination, sortingColumns])]);
 
   const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [
     indexPattern,

x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js

@@ -19,6 +19,7 @@ import { stringMatch } from '../../../util/string_utils';
 import { JOB_STATE, DATAFEED_STATE } from '../../../../../common/constants/states';
 import { parseInterval } from '../../../../../common/util/parse_interval';
 import { mlCalendarService } from '../../../services/calendar_service';
+import { isPopulatedObject } from '../../../../../common/util/object_utils';
 
 export function loadFullJob(jobId) {
   return new Promise((resolve, reject) => {
@@ -379,7 +380,7 @@ export function checkForAutoStartDatafeed() {
     mlJobService.tempJobCloningObjects.datafeed = undefined;
     mlJobService.tempJobCloningObjects.createdBy = undefined;
 
-    const hasDatafeed = typeof datafeed === 'object' && Object.keys(datafeed).length > 0;
+    const hasDatafeed = isPopulatedObject(datafeed);
     const datafeedId = hasDatafeed ? datafeed.datafeed_id : '';
     return {
       id: job.job_id,

x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx

@@ -21,6 +21,7 @@ import { CombinedJob } from '../../../../../../../../common/types/anomaly_detect
 import { MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor';
 import { mlJobService } from '../../../../../../services/job_service';
 import { ML_DATA_PREVIEW_COUNT } from '../../../../../../../../common/util/job_utils';
+import { isPopulatedObject } from '../../../../../../../../common/util/object_utils';
 
 export const DatafeedPreview: FC<{
   combinedJob: CombinedJob | null;
@@ -64,7 +65,7 @@ export const DatafeedPreview: FC<{
         const resp = await mlJobService.searchPreview(combinedJob);
         let data = resp.hits.hits;
         // the first item under aggregations can be any name
-        if (typeof resp.aggregations === 'object' && Object.keys(resp.aggregations).length > 0) {
+        if (isPopulatedObject(resp.aggregations)) {
           const accessor = Object.keys(resp.aggregations)[0];
           data = resp.aggregations[accessor].buckets.slice(0, ML_DATA_PREVIEW_COUNT);
         }

x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts

@@ -24,6 +24,7 @@ import { findAggField } from '../../../../common/util/validation_utils';
 import { getDatafeedAggregations } from '../../../../common/util/datafeed_utils';
 import { aggregationTypeTransform } from '../../../../common/util/anomaly_utils';
 import { ES_AGGREGATION } from '../../../../common/constants/aggregation_types';
+import { isPopulatedObject } from '../../../../common/util/object_utils';
 
 interface ResultResponse {
   success: boolean;
@@ -175,7 +176,7 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) {
         // when the field is an aggregation field, because the field doesn't actually exist in the indices
         // we need to pass all the sub aggs from the original datafeed config
         // so that we can access the aggregated field
-        if (typeof aggFields === 'object' && Object.keys(aggFields).length > 0) {
+        if (isPopulatedObject(aggFields)) {
           // first item under aggregations can be any name, not necessarily 'buckets'
           const accessor = Object.keys(aggFields)[0];
           const tempAggs = { ...(aggFields[accessor].aggs ?? aggFields[accessor].aggregations) };