[ML] Add better UI support for runtime fields Transforms

x-pack/plugins/ml/public/application/components/data_grid/common.ts

@@ -44,6 +44,7 @@ import { getNestedProperty } from '../../util/object_utils';
 import { mlFieldFormatService } from '../../services/field_format_service';
 
 import { DataGridItem, IndexPagination, RenderCellValue } from './types';
+import type { RuntimeField } from '../../../../../../../src/plugins/data/common/index_patterns';
 
 export const INIT_MAX_COLUMNS = 10;
 
@@ -82,6 +83,37 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str
   return indexPatternFields;
 };
 
+/**
+ * Return a map of runtime_mappings for each of the index pattern field provided
+ * to provide in ES search queries
+ * @param indexPatternFields
+ * @param indexPattern
+ * @param clonedRuntimeMappings
+ */
+export const getRuntimeFieldsMapping = (
+  indexPatternFields: string[],
+  indexPattern: IndexPattern,
+  clonedRuntimeMappings?: { [key: string]: RuntimeField }
+) => {
+  if (!Array.isArray(indexPatternFields) || indexPattern === undefined) return {};
+  const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields;
+  let combinedRuntimeMappings: { [ipField: string]: any } = {};
+
+  if (typeof ipRuntimeMappings === 'object' && Object.keys(ipRuntimeMappings).length > 0) {
+    indexPatternFields.forEach((ipField) => {
+      if (ipRuntimeMappings.hasOwnProperty(ipField)) {
+        combinedRuntimeMappings[ipField] = ipRuntimeMappings[ipField];
+      }
+    });
+  }
+  if (typeof clonedRuntimeMappings === 'object') {
+    combinedRuntimeMappings = { ...combinedRuntimeMappings, ...clonedRuntimeMappings };
+  }
+  return Object.keys(combinedRuntimeMappings).length > 0
+    ? { runtime_mappings: combinedRuntimeMappings }
+    : {};
+};
+
 export interface FieldTypes {
   [key: string]: ES_FIELD_TYPES;
 }
@@ -131,6 +163,45 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results
 };
 
 export const NON_AGGREGATABLE = 'non-aggregatable';
+
+export const getDataGridSchemaFromESFieldType = (
+  fieldType: ES_FIELD_TYPES | undefined | RuntimeField['type']
+): string | undefined => {
+  // Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json']
+  // To fall back to the default string schema it needs to be undefined.
+  let schema;
+
+  switch (fieldType) {
+    case ES_FIELD_TYPES.GEO_POINT:
+    case ES_FIELD_TYPES.GEO_SHAPE:
+      schema = 'json';
+      break;
+    case ES_FIELD_TYPES.BOOLEAN:
+      schema = 'boolean';
+      break;
+    case ES_FIELD_TYPES.DATE:
+    case ES_FIELD_TYPES.DATE_NANOS:
+      schema = 'datetime';
+      break;
+    case ES_FIELD_TYPES.BYTE:
+    case ES_FIELD_TYPES.DOUBLE:
+    case ES_FIELD_TYPES.FLOAT:
+    case ES_FIELD_TYPES.HALF_FLOAT:
+    case ES_FIELD_TYPES.INTEGER:
+    case ES_FIELD_TYPES.LONG:
+    case ES_FIELD_TYPES.SCALED_FLOAT:
+    case ES_FIELD_TYPES.SHORT:
+      schema = 'numeric';
+      break;
+    // keep schema undefined for text based columns
+    case ES_FIELD_TYPES.KEYWORD:
+    case ES_FIELD_TYPES.TEXT:
+      break;
+  }
+
+  return schema;
+};
+
 export const getDataGridSchemaFromKibanaFieldType = (
   field: IFieldType | undefined
 ): string | undefined => {

x-pack/plugins/ml/public/application/components/data_grid/index.ts

@@ -7,8 +7,10 @@
 
 export {
   getDataGridSchemasFromFieldTypes,
+  getDataGridSchemaFromESFieldType,
   getDataGridSchemaFromKibanaFieldType,
   getFieldsFromKibanaIndexPattern,
+  getRuntimeFieldsMapping,
   multiColumnSortFactory,
   showDataGridColumnChartErrorMessageToast,
   useRenderCellValue,

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx

@@ -8,6 +8,7 @@
 import React, { FC, Fragment, useEffect, useMemo, useRef, useState } from 'react';
 import {
   EuiBadge,
+  EuiCallOut,
   EuiComboBox,
   EuiComboBoxOptionOption,
   EuiFormRow,
@@ -19,6 +20,7 @@ import {
 import { i18n } from '@kbn/i18n';
 import { debounce } from 'lodash';
 
+import { FormattedMessage } from '@kbn/i18n/react';
 import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
 import { useMlContext } from '../../../../../contexts/ml';
 
@@ -314,6 +316,15 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
     };
   }, [jobType, dependentVariable, trainingPercent, JSON.stringify(includes), jobConfigQueryString]);
 
+  const unsupportedRuntimeFields = useMemo(
+    () =>
+      currentIndexPattern.fields
+        .getAll()
+        .filter((f) => f.runtimeField)
+        .map((f) => `'${f.displayName}'`),
+    [currentIndexPattern.fields]
+  );
+
   return (
     <Fragment>
       <Messages messages={requestMessages} />
@@ -445,6 +456,26 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
       >
         <Fragment />
       </EuiFormRow>
+      {unsupportedRuntimeFields && unsupportedRuntimeFields.length > 0 && (
+        <>
+          <EuiCallOut size="s" color="warning">
+            <FormattedMessage
+              id="xpack.ml.dataframe.analytics.create.unsupportedRuntimeFields"
+              defaultMessage="The runtime fields {unsupportedRuntimeFields} are not supported for analysis."
+              values={{
+                unsupportedRuntimeFields:
+                  unsupportedRuntimeFields.length > 5
+                    ? `${unsupportedRuntimeFields.slice(0, 5).join(', ')} ... (and ${
+                        unsupportedRuntimeFields.length
+                      } more)`
+                    : unsupportedRuntimeFields.join(', '),
+              }}
+            />
+          </EuiCallOut>
+          <EuiSpacer />
+        </>
+      )}
+
       <AnalysisFieldsTable
         dependentVariable={dependentVariable}
         includes={includes}

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts

@@ -30,6 +30,7 @@ import type { SearchResponse7 } from '../../../../../../common/types/es_client';
 import { extractErrorMessage } from '../../../../../../common/util/errors';
 import { INDEX_STATUS } from '../../../common/analytics';
 import { ml } from '../../../../services/ml_api_service';
+import { getRuntimeFieldsMapping } from '../../../../components/data_grid/common';
 
 type IndexSearchResponse = SearchResponse7;
 
@@ -38,7 +39,9 @@ export const useIndexData = (
   query: any,
   toastNotifications: CoreSetup['notifications']['toasts']
 ): UseIndexDataReturnType => {
-  const indexPatternFields = getFieldsFromKibanaIndexPattern(indexPattern);
+  const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [
+    indexPattern,
+  ]);
 
   // EuiDataGrid State
   const columns: EuiDataGridColumn[] = [
@@ -75,7 +78,6 @@ export const useIndexData = (
       s[column.id] = { order: column.direction };
       return s;
     }, {} as EsSorting);
-
     const esSearchRequest = {
       index: indexPattern.title,
       body: {
@@ -86,6 +88,7 @@ export const useIndexData = (
         fields: ['*'],
         _source: false,
         ...(Object.keys(sort).length > 0 ? { sort } : {}),
+        ...getRuntimeFieldsMapping(indexPatternFields, indexPattern),
       },
     };
 
@@ -105,7 +108,7 @@ export const useIndexData = (
   useEffect(() => {
     getIndexData();
     // custom comparison
-  }, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]);
+  }, [indexPattern.title, indexPatternFields, JSON.stringify([query, pagination, sortingColumns])]);
 
   const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [
     indexPattern,

x-pack/plugins/transform/common/api_schemas/transforms.ts

@@ -58,6 +58,7 @@ export const settingsSchema = schema.object({
 });
 
 export const sourceSchema = schema.object({
+  runtime_mappings: schema.maybe(schema.recordOf(schema.string(), schema.any())),
   index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
   query: schema.maybe(schema.recordOf(schema.string(), schema.any())),
 });

x-pack/plugins/transform/public/app/common/request.ts

@@ -57,17 +57,45 @@ export function isDefaultQuery(query: PivotQuery): boolean {
   return isSimpleQuery(query) && query.query_string.query === '*';
 }
 
+export function getCombinedRuntimeMappings(
+  indexPattern: IndexPattern | undefined,
+  runtimeMappings?: StepDefineExposedState['runtimeMappings']
+): StepDefineExposedState['runtimeMappings'] | undefined {
+  let combinedRuntimeMappings = {};
+
+  // Use runtime field mappings defined inline from API
+  if (typeof runtimeMappings === 'object') {
+    combinedRuntimeMappings = { ...combinedRuntimeMappings, ...runtimeMappings };
+  }
+
+  // And runtime field mappings defined by index pattern
+  if (indexPattern !== undefined) {
+    const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields;
+    combinedRuntimeMappings = { ...combinedRuntimeMappings, ...ipRuntimeMappings };
+  }
+
+  if (
+    typeof combinedRuntimeMappings === 'object' &&
+    Object.keys(combinedRuntimeMappings).length > 0
+  ) {
+    return combinedRuntimeMappings;
+  }
+  return undefined;
+}
+
 export function getPreviewTransformRequestBody(
   indexPatternTitle: IndexPattern['title'],
   query: PivotQuery,
-  partialRequest?: StepDefineExposedState['previewRequest'] | undefined
+  partialRequest?: StepDefineExposedState['previewRequest'] | undefined,
+  runtimeMappings?: object
 ): PostTransformsPreviewRequestSchema {
   const index = indexPatternTitle.split(',').map((name: string) => name.trim());
 
   return {
     source: {
       index,
       ...(!isDefaultQuery(query) && !isMatchAllQuery(query) ? { query } : {}),
+      ...(typeof runtimeMappings === 'object' ? { runtime_mappings: runtimeMappings } : {}),
     },
     ...(partialRequest ?? {}),
   };
@@ -95,7 +123,8 @@ export const getCreateTransformRequestBody = (
   ...getPreviewTransformRequestBody(
     indexPatternTitle,
     getPivotQuery(pivotState.searchQuery),
-    pivotState.previewRequest
+    pivotState.previewRequest,
+    pivotState.runtimeMappings
   ),
   // conditionally add optional description
   ...(transformDetailsState.transformDescription !== ''

x-pack/plugins/transform/public/app/hooks/use_index_data.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { useEffect } from 'react';
+import { useEffect, useMemo } from 'react';
 
 import { EuiDataGridColumn } from '@elastic/eui';
 
@@ -21,17 +21,20 @@ import { SearchItems } from './use_search_items';
 import { useApi } from './use_api';
 
 import { useAppDependencies, useToastNotifications } from '../app_dependencies';
+import type { StepDefineExposedState } from '../sections/create_transform/components/step_define/common';
 
 export const useIndexData = (
   indexPattern: SearchItems['indexPattern'],
-  query: PivotQuery
+  query: PivotQuery,
+  combinedRuntimeMappings?: StepDefineExposedState['runtimeMappings']
 ): UseIndexDataReturnType => {
   const api = useApi();
   const toastNotifications = useToastNotifications();
   const {
     ml: {
       getFieldType,
       getDataGridSchemaFromKibanaFieldType,
+      getDataGridSchemaFromESFieldType,
       getFieldsFromKibanaIndexPattern,
       showDataGridColumnChartErrorMessageToast,
       useDataGrid,
@@ -41,16 +44,42 @@ export const useIndexData = (
     },
   } = useAppDependencies();
 
-  const indexPatternFields = getFieldsFromKibanaIndexPattern(indexPattern);
+  const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [
+    indexPattern,
+    getFieldsFromKibanaIndexPattern,
+  ]);
+
+  const columns: EuiDataGridColumn[] = useMemo(() => {
+    let result: Array<{ id: string; schema: string | undefined }> = [];
+
+    // Get the the runtime fields that are defined from API field and index patterns
+    if (combinedRuntimeMappings !== undefined) {
+      result = Object.keys(combinedRuntimeMappings).map((fieldName) => {
+        const field = combinedRuntimeMappings[fieldName];
+        const schema = getDataGridSchemaFromESFieldType(field.type);
+        return { id: fieldName, schema };
+      });
+    }
 
-  // EuiDataGrid State
-  const columns: EuiDataGridColumn[] = [
-    ...indexPatternFields.map((id) => {
+    // Combine the runtime field that are defined from API field
+    indexPatternFields.forEach((id) => {
       const field = indexPattern.fields.getByName(id);
-      const schema = getDataGridSchemaFromKibanaFieldType(field);
-      return { id, schema };
-    }),
-  ];
+      if (!field?.runtimeField) {
+        const schema = getDataGridSchemaFromKibanaFieldType(field);
+        result.push({ id, schema });
+      }
+    });
+
+    return result.sort((a, b) => a.id.localeCompare(b.id));
+  }, [
+    indexPatternFields,
+    indexPattern.fields,
+    combinedRuntimeMappings,
+    getDataGridSchemaFromESFieldType,
+    getDataGridSchemaFromKibanaFieldType,
+  ]);
+
+  // EuiDataGrid State
 
   const dataGrid = useDataGrid(columns);
 
@@ -92,9 +121,12 @@ export const useIndexData = (
         from: pagination.pageIndex * pagination.pageSize,
         size: pagination.pageSize,
         ...(Object.keys(sort).length > 0 ? { sort } : {}),
+        ...(typeof combinedRuntimeMappings === 'object' &&
+        Object.keys(combinedRuntimeMappings).length > 0
+          ? { runtime_mappings: combinedRuntimeMappings }
+          : {}),
       },
     };
-
     const resp = await api.esSearch(esSearchRequest);
 
     if (!isEsSearchResponse(resp)) {
@@ -134,7 +166,12 @@ export const useIndexData = (
     fetchDataGridData();
     // custom comparison
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]);
+  }, [
+    indexPattern.title,
+    indexPatternFields,
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    JSON.stringify([query, pagination, sortingColumns, combinedRuntimeMappings]),
+  ]);
 
   useEffect(() => {
     if (chartsVisible) {