diff --git a/x-pack/plugins/ml/common/types/feature_importance.ts b/x-pack/plugins/ml/common/types/feature_importance.ts index 2e45c3cd4d8c44..964ce8c3257838 100644 --- a/x-pack/plugins/ml/common/types/feature_importance.ts +++ b/x-pack/plugins/ml/common/types/feature_importance.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { isPopulatedObject } from '../util/object_utils'; + export type FeatureImportanceClassName = string | number | boolean; export interface ClassFeatureImportance { @@ -87,7 +89,7 @@ export function isClassificationFeatureImportanceBaseline( baselineData: any ): baselineData is ClassificationFeatureImportanceBaseline { return ( - typeof baselineData === 'object' && + isPopulatedObject(baselineData) && baselineData.hasOwnProperty('classes') && Array.isArray(baselineData.classes) ); @@ -96,5 +98,5 @@ export function isClassificationFeatureImportanceBaseline( export function isRegressionFeatureImportanceBaseline( baselineData: any ): baselineData is RegressionFeatureImportanceBaseline { - return typeof baselineData === 'object' && baselineData.hasOwnProperty('baseline'); + return isPopulatedObject(baselineData) && baselineData.hasOwnProperty('baseline'); } diff --git a/x-pack/plugins/ml/common/types/fields.ts b/x-pack/plugins/ml/common/types/fields.ts index ae157cef5735fc..581ce861e8331c 100644 --- a/x-pack/plugins/ml/common/types/fields.ts +++ b/x-pack/plugins/ml/common/types/fields.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ES_FIELD_TYPES, RuntimeField } from '../../../../../src/plugins/data/common'; +import { ES_FIELD_TYPES } from '../../../../../src/plugins/data/common'; import { ML_JOB_AGGREGATION, KIBANA_AGGREGATION, @@ -106,4 +106,18 @@ export interface AggCardinality { } export type RollupFields = Record]>; + +// Replace this with import once #88995 is merged +const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const; +type RuntimeType = typeof RUNTIME_FIELD_TYPES[number]; + +export interface RuntimeField { + type: RuntimeType; + script: + | string + | { + source: string; + }; +} + export type RuntimeMappings = Record; diff --git a/x-pack/plugins/ml/common/util/datafeed_utils.ts b/x-pack/plugins/ml/common/util/datafeed_utils.ts index fa1a940ba5492c..c0579ce947992a 100644 --- a/x-pack/plugins/ml/common/util/datafeed_utils.ts +++ b/x-pack/plugins/ml/common/util/datafeed_utils.ts @@ -20,7 +20,7 @@ export const getDatafeedAggregations = ( }; export const getAggregationBucketsName = (aggregations: any): string | undefined => { - if (typeof aggregations === 'object') { + if (aggregations !== null && typeof aggregations === 'object') { const keys = Object.keys(aggregations); return keys.length > 0 ? keys[0] : undefined; } diff --git a/x-pack/plugins/ml/common/util/job_utils.ts b/x-pack/plugins/ml/common/util/job_utils.ts index 711103b499ec90..ab56726e160f7c 100644 --- a/x-pack/plugins/ml/common/util/job_utils.ts +++ b/x-pack/plugins/ml/common/util/job_utils.ts @@ -28,6 +28,7 @@ import { getDatafeedAggregations, } from './datafeed_utils'; import { findAggField } from './validation_utils'; +import { isPopulatedObject } from './object_utils'; export interface ValidationResults { valid: boolean; @@ -51,17 +52,9 @@ export function calculateDatafeedFrequencyDefaultSeconds(bucketSpanSeconds: numb } export function hasRuntimeMappings(job: CombinedJob): boolean { - const hasDatafeed = - typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0; + const hasDatafeed = isPopulatedObject(job.datafeed_config); if (hasDatafeed) { - const runtimeMappings = - typeof job.datafeed_config.runtime_mappings === 'object' - ? Object.keys(job.datafeed_config.runtime_mappings) - : undefined; - - if (Array.isArray(runtimeMappings) && runtimeMappings.length > 0) { - return true; - } + return isPopulatedObject(job.datafeed_config.runtime_mappings); } return false; } @@ -114,7 +107,11 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex // If the datafeed uses script fields, we can only plot the time series if // model plot is enabled. Without model plot it will be very difficult or impossible // to invert to a reverse search of the underlying metric data. - if (isSourceDataChartable === true && typeof job.datafeed_config?.script_fields === 'object') { + if ( + isSourceDataChartable === true && + job.datafeed_config?.script_fields !== null && + typeof job.datafeed_config?.script_fields === 'object' + ) { // Perform extra check to see if the detector is using a scripted field. const scriptFields = Object.keys(job.datafeed_config.script_fields); isSourceDataChartable = @@ -123,8 +120,7 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex scriptFields.indexOf(dtr.over_field_name!) === -1; } - const hasDatafeed = - typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0; + const hasDatafeed = isPopulatedObject(job.datafeed_config); if (hasDatafeed) { // We cannot plot the source data for some specific aggregation configurations const aggs = getDatafeedAggregations(job.datafeed_config); diff --git a/x-pack/plugins/ml/common/util/object_utils.ts b/x-pack/plugins/ml/common/util/object_utils.ts new file mode 100644 index 00000000000000..4bbd0c1c2810fe --- /dev/null +++ b/x-pack/plugins/ml/common/util/object_utils.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const isPopulatedObject = >(arg: any): arg is T => { + return typeof arg === 'object' && arg !== null && Object.keys(arg).length > 0; +}; diff --git a/x-pack/plugins/ml/common/util/validation_utils.ts b/x-pack/plugins/ml/common/util/validation_utils.ts index 7f0208e726ab0b..66084f83ea87d1 100644 --- a/x-pack/plugins/ml/common/util/validation_utils.ts +++ b/x-pack/plugins/ml/common/util/validation_utils.ts @@ -45,7 +45,7 @@ export function findAggField( value = returnParent === true ? aggs : aggs[k]; return true; } - if (aggs.hasOwnProperty(k) && typeof aggs[k] === 'object') { + if (aggs.hasOwnProperty(k) && aggs[k] !== null && typeof aggs[k] === 'object') { value = findAggField(aggs[k], fieldName, returnParent); return value !== undefined; } diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts index 2805a28996acc1..069c13df2470f7 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts @@ -48,6 +48,9 @@ import { getNestedProperty } from '../../util/object_utils'; import { mlFieldFormatService } from '../../services/field_format_service'; import { DataGridItem, IndexPagination, RenderCellValue } from './types'; +import type { RuntimeField } from '../../../../../../../src/plugins/data/common/index_patterns'; +import { RuntimeMappings } from '../../../../common/types/fields'; +import { isPopulatedObject } from '../../../../common/util/object_utils'; export const INIT_MAX_COLUMNS = 10; @@ -86,6 +89,37 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str return indexPatternFields; }; +/** + * Return a map of runtime_mappings for each of the index pattern field provided + * to provide in ES search queries + * @param indexPatternFields + * @param indexPattern + * @param clonedRuntimeMappings + */ +export const getRuntimeFieldsMapping = ( + indexPatternFields: string[] | undefined, + indexPattern: IndexPattern | undefined, + clonedRuntimeMappings?: RuntimeMappings +) => { + if (!Array.isArray(indexPatternFields) || indexPattern === undefined) return {}; + const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields; + let combinedRuntimeMappings: RuntimeMappings = {}; + + if (isPopulatedObject(ipRuntimeMappings)) { + indexPatternFields.forEach((ipField) => { + if (ipRuntimeMappings.hasOwnProperty(ipField)) { + combinedRuntimeMappings[ipField] = ipRuntimeMappings[ipField]; + } + }); + } + if (isPopulatedObject(clonedRuntimeMappings)) { + combinedRuntimeMappings = { ...combinedRuntimeMappings, ...clonedRuntimeMappings }; + } + return Object.keys(combinedRuntimeMappings).length > 0 + ? { runtime_mappings: combinedRuntimeMappings } + : {}; +}; + export interface FieldTypes { [key: string]: ES_FIELD_TYPES; } @@ -135,6 +169,45 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results }; export const NON_AGGREGATABLE = 'non-aggregatable'; + +export const getDataGridSchemaFromESFieldType = ( + fieldType: ES_FIELD_TYPES | undefined | RuntimeField['type'] +): string | undefined => { + // Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json'] + // To fall back to the default string schema it needs to be undefined. + let schema; + + switch (fieldType) { + case ES_FIELD_TYPES.GEO_POINT: + case ES_FIELD_TYPES.GEO_SHAPE: + schema = 'json'; + break; + case ES_FIELD_TYPES.BOOLEAN: + schema = 'boolean'; + break; + case ES_FIELD_TYPES.DATE: + case ES_FIELD_TYPES.DATE_NANOS: + schema = 'datetime'; + break; + case ES_FIELD_TYPES.BYTE: + case ES_FIELD_TYPES.DOUBLE: + case ES_FIELD_TYPES.FLOAT: + case ES_FIELD_TYPES.HALF_FLOAT: + case ES_FIELD_TYPES.INTEGER: + case ES_FIELD_TYPES.LONG: + case ES_FIELD_TYPES.SCALED_FLOAT: + case ES_FIELD_TYPES.SHORT: + schema = 'numeric'; + break; + // keep schema undefined for text based columns + case ES_FIELD_TYPES.KEYWORD: + case ES_FIELD_TYPES.TEXT: + break; + } + + return schema; +}; + export const getDataGridSchemaFromKibanaFieldType = ( field: IFieldType | undefined ): string | undefined => { diff --git a/x-pack/plugins/ml/public/application/components/data_grid/index.ts b/x-pack/plugins/ml/public/application/components/data_grid/index.ts index ccd2f3f56e45df..79a8d65f9905a2 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/index.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/index.ts @@ -7,8 +7,10 @@ export { getDataGridSchemasFromFieldTypes, + getDataGridSchemaFromESFieldType, getDataGridSchemaFromKibanaFieldType, getFieldsFromKibanaIndexPattern, + getRuntimeFieldsMapping, multiColumnSortFactory, showDataGridColumnChartErrorMessageToast, useRenderCellValue, diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_details.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_details.tsx index f92d391ecd4a95..ef88c363e3e279 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_details.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_details.tsx @@ -69,9 +69,16 @@ export const ConfigurationStepDetails: FC = ({ setCurrentStep, state }) = }), description: includes.length > MAX_INCLUDES_LENGTH - ? `${includes.slice(0, MAX_INCLUDES_LENGTH).join(', ')} ... (and ${ - includes.length - MAX_INCLUDES_LENGTH - } more)` + ? i18n.translate( + 'xpack.ml.dataframe.analytics.create.configDetails.includedFieldsAndMoreDescription', + { + defaultMessage: '{includedFields} ... (and {extraCount} more)', + values: { + extraCount: includes.length - MAX_INCLUDES_LENGTH, + includedFields: includes.slice(0, MAX_INCLUDES_LENGTH).join(', '), + }, + } + ) : includes.join(', '), }, ]; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx index 6ad874d3abd6c9..0432094c30c500 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx @@ -8,6 +8,7 @@ import React, { FC, Fragment, useEffect, useMemo, useRef, useState } from 'react'; import { EuiBadge, + EuiCallOut, EuiComboBox, EuiComboBoxOptionOption, EuiFormRow, @@ -19,6 +20,7 @@ import { import { i18n } from '@kbn/i18n'; import { debounce } from 'lodash'; +import { FormattedMessage } from '@kbn/i18n/react'; import { newJobCapsService } from '../../../../../services/new_job_capabilities_service'; import { useMlContext } from '../../../../../contexts/ml'; @@ -62,6 +64,8 @@ const requiredFieldsErrorText = i18n.translate( } ); +const maxRuntimeFieldsDisplayCount = 5; + export const ConfigurationStepForm: FC = ({ actions, state, @@ -314,6 +318,15 @@ export const ConfigurationStepForm: FC = ({ }; }, [jobType, dependentVariable, trainingPercent, JSON.stringify(includes), jobConfigQueryString]); + const unsupportedRuntimeFields = useMemo( + () => + currentIndexPattern.fields + .getAll() + .filter((f) => f.runtimeField) + .map((f) => `'${f.displayName}'`), + [currentIndexPattern.fields] + ); + return ( @@ -445,6 +458,36 @@ export const ConfigurationStepForm: FC = ({ > + {Array.isArray(unsupportedRuntimeFields) && unsupportedRuntimeFields.length > 0 && ( + <> + + 0 ? ( + + ) : ( + '' + ), + unsupportedRuntimeFields: unsupportedRuntimeFields + .slice(0, maxRuntimeFieldsDisplayCount) + .join(', '), + }} + /> + + + + )} + { - const indexPatternFields = getFieldsFromKibanaIndexPattern(indexPattern); + const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [ + indexPattern, + ]); // EuiDataGrid State const columns: EuiDataGridColumn[] = [ @@ -75,7 +78,6 @@ export const useIndexData = ( s[column.id] = { order: column.direction }; return s; }, {} as EsSorting); - const esSearchRequest = { index: indexPattern.title, body: { @@ -86,6 +88,7 @@ export const useIndexData = ( fields: ['*'], _source: false, ...(Object.keys(sort).length > 0 ? { sort } : {}), + ...getRuntimeFieldsMapping(indexPatternFields, indexPattern), }, }; @@ -105,7 +108,7 @@ export const useIndexData = ( useEffect(() => { getIndexData(); // custom comparison - }, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]); + }, [indexPattern.title, indexPatternFields, JSON.stringify([query, pagination, sortingColumns])]); const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [ indexPattern, diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js index 98d8b5eaf912a7..5b8fa5c672c6e0 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js @@ -19,6 +19,7 @@ import { stringMatch } from '../../../util/string_utils'; import { JOB_STATE, DATAFEED_STATE } from '../../../../../common/constants/states'; import { parseInterval } from '../../../../../common/util/parse_interval'; import { mlCalendarService } from '../../../services/calendar_service'; +import { isPopulatedObject } from '../../../../../common/util/object_utils'; export function loadFullJob(jobId) { return new Promise((resolve, reject) => { @@ -379,7 +380,7 @@ export function checkForAutoStartDatafeed() { mlJobService.tempJobCloningObjects.datafeed = undefined; mlJobService.tempJobCloningObjects.createdBy = undefined; - const hasDatafeed = typeof datafeed === 'object' && Object.keys(datafeed).length > 0; + const hasDatafeed = isPopulatedObject(datafeed); const datafeedId = hasDatafeed ? datafeed.datafeed_id : ''; return { id: job.job_id, diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx index 6afc1122fcdab1..916a25271c63b8 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/datafeed_preview_flyout/datafeed_preview.tsx @@ -21,6 +21,7 @@ import { CombinedJob } from '../../../../../../../../common/types/anomaly_detect import { MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor'; import { mlJobService } from '../../../../../../services/job_service'; import { ML_DATA_PREVIEW_COUNT } from '../../../../../../../../common/util/job_utils'; +import { isPopulatedObject } from '../../../../../../../../common/util/object_utils'; export const DatafeedPreview: FC<{ combinedJob: CombinedJob | null; @@ -64,7 +65,7 @@ export const DatafeedPreview: FC<{ const resp = await mlJobService.searchPreview(combinedJob); let data = resp.hits.hits; // the first item under aggregations can be any name - if (typeof resp.aggregations === 'object' && Object.keys(resp.aggregations).length > 0) { + if (isPopulatedObject(resp.aggregations)) { const accessor = Object.keys(resp.aggregations)[0]; data = resp.aggregations[accessor].buckets.slice(0, ML_DATA_PREVIEW_COUNT); } diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index ec1d36a1ced4c2..a8ae42658f3689 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -24,6 +24,7 @@ import { findAggField } from '../../../../common/util/validation_utils'; import { getDatafeedAggregations } from '../../../../common/util/datafeed_utils'; import { aggregationTypeTransform } from '../../../../common/util/anomaly_utils'; import { ES_AGGREGATION } from '../../../../common/constants/aggregation_types'; +import { isPopulatedObject } from '../../../../common/util/object_utils'; interface ResultResponse { success: boolean; @@ -175,7 +176,7 @@ export function resultsServiceRxProvider(mlApiServices: MlApiServices) { // when the field is an aggregation field, because the field doesn't actually exist in the indices // we need to pass all the sub aggs from the original datafeed config // so that we can access the aggregated field - if (typeof aggFields === 'object' && Object.keys(aggFields).length > 0) { + if (isPopulatedObject(aggFields)) { // first item under aggregations can be any name, not necessarily 'buckets' const accessor = Object.keys(aggFields)[0]; const tempAggs = { ...(aggFields[accessor].aggs ?? aggFields[accessor].aggregations) }; diff --git a/x-pack/plugins/ml/public/shared.ts b/x-pack/plugins/ml/public/shared.ts index a0107ce8e049c5..7fb27f889c4173 100644 --- a/x-pack/plugins/ml/public/shared.ts +++ b/x-pack/plugins/ml/public/shared.ts @@ -17,8 +17,8 @@ export * from '../common/types/audit_message'; export * from '../common/util/anomaly_utils'; export * from '../common/util/errors'; export * from '../common/util/validators'; +export * from '../common/util/date_utils'; export * from './application/formatters/metric_change_description'; export * from './application/components/data_grid'; export * from './application/data_frame_analytics/common'; -export * from '../common/util/date_utils'; diff --git a/x-pack/plugins/ml/server/models/job_service/jobs.ts b/x-pack/plugins/ml/server/models/job_service/jobs.ts index 0af8f1e1ec1cab..dc2c04540ef21d 100644 --- a/x-pack/plugins/ml/server/models/job_service/jobs.ts +++ b/x-pack/plugins/ml/server/models/job_service/jobs.ts @@ -39,6 +39,7 @@ import { } from '../../../common/util/job_utils'; import { groupsProvider } from './groups'; import type { MlClient } from '../../lib/ml_client'; +import { isPopulatedObject } from '../../../common/util/object_utils'; interface Results { [id: string]: { @@ -172,8 +173,7 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) { }); const jobs = fullJobsList.map((job) => { - const hasDatafeed = - typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0; + const hasDatafeed = isPopulatedObject(job.datafeed_config); const dataCounts = job.data_counts; const errorMessage = getSingleMetricViewerJobErrorMessage(job); @@ -233,8 +233,7 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) { const jobs = fullJobsList.map((job) => { jobsMap[job.job_id] = job.groups || []; - const hasDatafeed = - typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0; + const hasDatafeed = isPopulatedObject(job.datafeed_config); const timeRange: { to?: number; from?: number } = {}; const dataCounts = job.data_counts; diff --git a/x-pack/plugins/transform/common/api_schemas/transforms.ts b/x-pack/plugins/transform/common/api_schemas/transforms.ts index f9dedf0acb56ac..3d8d7ef4d8ae3a 100644 --- a/x-pack/plugins/transform/common/api_schemas/transforms.ts +++ b/x-pack/plugins/transform/common/api_schemas/transforms.ts @@ -64,7 +64,30 @@ export const settingsSchema = schema.object({ docs_per_second: schema.maybe(schema.nullable(schema.number())), }); +export const runtimeMappingsSchema = schema.maybe( + schema.recordOf( + schema.string(), + schema.object({ + type: schema.oneOf([ + schema.literal('keyword'), + schema.literal('long'), + schema.literal('double'), + schema.literal('date'), + schema.literal('ip'), + schema.literal('boolean'), + ]), + script: schema.oneOf([ + schema.string(), + schema.object({ + source: schema.string(), + }), + ]), + }) + ) +); + export const sourceSchema = schema.object({ + runtime_mappings: runtimeMappingsSchema, index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]), query: schema.maybe(schema.recordOf(schema.string(), schema.any())), }); diff --git a/x-pack/plugins/transform/public/app/common/pivot_group_by.ts b/x-pack/plugins/transform/public/app/common/pivot_group_by.ts index 18b0c7dde819f2..4f23b8aa4d86fd 100644 --- a/x-pack/plugins/transform/public/app/common/pivot_group_by.ts +++ b/x-pack/plugins/transform/public/app/common/pivot_group_by.ts @@ -10,6 +10,7 @@ import { Dictionary } from '../../../common/types/common'; import { EsFieldName } from '../../../common/types/fields'; import { GenericAgg } from '../../../common/types/pivot_group_by'; import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common'; +import { PivotAggsConfigWithUiSupport } from './pivot_aggs'; export enum PIVOT_SUPPORTED_GROUP_BY_AGGS { DATE_HISTOGRAM = 'date_histogram', @@ -117,3 +118,7 @@ export function getEsAggFromGroupByConfig(groupByConfig: GroupByConfigBase): Gen [agg]: esAgg, }; } + +export function isPivotAggConfigWithUiSupport(arg: any): arg is PivotAggsConfigWithUiSupport { + return arg.hasOwnProperty('agg') && arg.hasOwnProperty('field'); +} diff --git a/x-pack/plugins/transform/public/app/common/request.test.ts b/x-pack/plugins/transform/public/app/common/request.test.ts index fa39419c254ba9..13e7c0a9feb7a8 100644 --- a/x-pack/plugins/transform/public/app/common/request.test.ts +++ b/x-pack/plugins/transform/public/app/common/request.test.ts @@ -27,6 +27,7 @@ import { PivotQuery, } from './request'; import { LatestFunctionConfigUI } from '../../../common/types/transform'; +import { RuntimeField } from '../../../../../../src/plugins/data/common/index_patterns'; const simpleQuery: PivotQuery = { query_string: { query: 'airline:AAL' } }; @@ -168,6 +169,9 @@ describe('Transform: Common', () => { validationStatus: { isValid: true, }, + runtimeMappings: undefined, + runtimeMappingsUpdated: false, + isRuntimeMappingsEditorEnabled: false, }; const transformDetailsState: StepDetailsExposedState = { continuousModeDateField: 'the-continuous-mode-date-field', @@ -212,6 +216,85 @@ describe('Transform: Common', () => { }); }); + test('getCreateTransformRequestBody() with runtime mappings', () => { + const runtimeMappings = { + rt_bytes_bigger: { + type: 'double', + script: { + source: "emit(doc['bytes'].value * 2.0)", + }, + } as RuntimeField, + }; + + const pivotState: StepDefineExposedState = { + aggList: { 'the-agg-name': aggsAvg }, + groupByList: { 'the-group-by-name': groupByTerms }, + isAdvancedPivotEditorEnabled: false, + isAdvancedSourceEditorEnabled: false, + sourceConfigUpdated: false, + searchLanguage: 'kuery', + searchString: 'the-query', + searchQuery: 'the-search-query', + valid: true, + transformFunction: 'pivot', + latestConfig: {} as LatestFunctionConfigUI, + previewRequest: { + pivot: { + aggregations: { 'the-agg-agg-name': { avg: { field: 'the-agg-field' } } }, + group_by: { 'the-group-by-agg-name': { terms: { field: 'the-group-by-field' } } }, + }, + }, + validationStatus: { + isValid: true, + }, + runtimeMappings, + runtimeMappingsUpdated: false, + isRuntimeMappingsEditorEnabled: false, + }; + const transformDetailsState: StepDetailsExposedState = { + continuousModeDateField: 'the-continuous-mode-date-field', + continuousModeDelay: 'the-continuous-mode-delay', + createIndexPattern: false, + isContinuousModeEnabled: false, + isRetentionPolicyEnabled: false, + retentionPolicyDateField: '', + retentionPolicyMaxAge: '', + transformId: 'the-transform-id', + transformDescription: 'the-transform-description', + transformFrequency: '1m', + transformSettingsMaxPageSearchSize: 100, + transformSettingsDocsPerSecond: 400, + destinationIndex: 'the-destination-index', + touched: true, + valid: true, + }; + + const request = getCreateTransformRequestBody( + 'the-index-pattern-title', + pivotState, + transformDetailsState + ); + + expect(request).toEqual({ + description: 'the-transform-description', + dest: { index: 'the-destination-index' }, + frequency: '1m', + pivot: { + aggregations: { 'the-agg-agg-name': { avg: { field: 'the-agg-field' } } }, + group_by: { 'the-group-by-agg-name': { terms: { field: 'the-group-by-field' } } }, + }, + settings: { + max_page_search_size: 100, + docs_per_second: 400, + }, + source: { + index: ['the-index-pattern-title'], + query: { query_string: { default_operator: 'AND', query: 'the-search-query' } }, + runtime_mappings: runtimeMappings, + }, + }); + }); + test('getCreateTransformSettingsRequestBody() with multiple settings', () => { const transformDetailsState: Partial = { transformSettingsDocsPerSecond: 400, diff --git a/x-pack/plugins/transform/public/app/common/request.ts b/x-pack/plugins/transform/public/app/common/request.ts index 82faa802138161..e4cfd0a874f0f5 100644 --- a/x-pack/plugins/transform/public/app/common/request.ts +++ b/x-pack/plugins/transform/public/app/common/request.ts @@ -20,6 +20,7 @@ import type { import type { SavedSearchQuery } from '../hooks/use_search_items'; import type { StepDefineExposedState } from '../sections/create_transform/components/step_define'; import type { StepDetailsExposedState } from '../sections/create_transform/components/step_details'; +import { isPopulatedObject } from './utils/object_utils'; export interface SimpleQuery { query_string: { @@ -57,10 +58,34 @@ export function isDefaultQuery(query: PivotQuery): boolean { return isSimpleQuery(query) && query.query_string.query === '*'; } +export function getCombinedRuntimeMappings( + indexPattern: IndexPattern | undefined, + runtimeMappings?: StepDefineExposedState['runtimeMappings'] +): StepDefineExposedState['runtimeMappings'] | undefined { + let combinedRuntimeMappings = {}; + + // Use runtime field mappings defined inline from API + if (isPopulatedObject(runtimeMappings)) { + combinedRuntimeMappings = { ...combinedRuntimeMappings, ...runtimeMappings }; + } + + // And runtime field mappings defined by index pattern + if (indexPattern !== undefined) { + const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields; + combinedRuntimeMappings = { ...combinedRuntimeMappings, ...ipRuntimeMappings }; + } + + if (isPopulatedObject(combinedRuntimeMappings)) { + return combinedRuntimeMappings; + } + return undefined; +} + export function getPreviewTransformRequestBody( indexPatternTitle: IndexPattern['title'], query: PivotQuery, - partialRequest?: StepDefineExposedState['previewRequest'] | undefined + partialRequest?: StepDefineExposedState['previewRequest'] | undefined, + runtimeMappings?: StepDefineExposedState['runtimeMappings'] ): PostTransformsPreviewRequestSchema { const index = indexPatternTitle.split(',').map((name: string) => name.trim()); @@ -68,6 +93,7 @@ export function getPreviewTransformRequestBody( source: { index, ...(!isDefaultQuery(query) && !isMatchAllQuery(query) ? { query } : {}), + ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}), }, ...(partialRequest ?? {}), }; @@ -95,7 +121,8 @@ export const getCreateTransformRequestBody = ( ...getPreviewTransformRequestBody( indexPatternTitle, getPivotQuery(pivotState.searchQuery), - pivotState.previewRequest + pivotState.previewRequest, + pivotState.runtimeMappings ), // conditionally add optional description ...(transformDetailsState.transformDescription !== '' diff --git a/x-pack/plugins/transform/public/app/common/utils/object_utils.ts b/x-pack/plugins/transform/public/app/common/utils/object_utils.ts new file mode 100644 index 00000000000000..4bbd0c1c2810fe --- /dev/null +++ b/x-pack/plugins/transform/public/app/common/utils/object_utils.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const isPopulatedObject = >(arg: any): arg is T => { + return typeof arg === 'object' && arg !== null && Object.keys(arg).length > 0; +}; diff --git a/x-pack/plugins/transform/public/app/hooks/use_index_data.test.tsx b/x-pack/plugins/transform/public/app/hooks/use_index_data.test.tsx index d7a760503a00cb..bd361afac2d8d7 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_index_data.test.tsx +++ b/x-pack/plugins/transform/public/app/hooks/use_index_data.test.tsx @@ -25,6 +25,7 @@ jest.mock('./use_api'); import { useAppDependencies } from '../__mocks__/app_dependencies'; import { MlSharedContext } from '../__mocks__/shared_context'; +import { RuntimeField } from '../../../../../../src/plugins/data/common/index_patterns'; const query: SimpleQuery = { query_string: { @@ -33,13 +34,21 @@ const query: SimpleQuery = { }, }; +const runtimeMappings = { + rt_bytes_bigger: { + type: 'double', + script: { + source: "emit(doc['bytes'].value * 2.0)", + }, + } as RuntimeField, +}; + describe('Transform: useIndexData()', () => { test('indexPattern set triggers loading', async () => { const mlShared = await getMlSharedImports(); const wrapper: FC = ({ children }) => ( {children} ); - const { result, waitForNextUpdate } = renderHook( () => useIndexData( @@ -48,7 +57,8 @@ describe('Transform: useIndexData()', () => { title: 'the-title', fields: [], } as unknown) as SearchItems['indexPattern'], - query + query, + runtimeMappings ), { wrapper } ); @@ -77,7 +87,7 @@ describe('Transform: with useIndexData()', () => { ml: { DataGrid }, } = useAppDependencies(); const props = { - ...useIndexData(indexPattern, { match_all: {} }), + ...useIndexData(indexPattern, { match_all: {} }, runtimeMappings), copyToClipboard: 'the-copy-to-clipboard-code', copyToClipboardDescription: 'the-copy-to-clipboard-description', dataTestSubj: 'the-data-test-subj', diff --git a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts index ff2d5d2a8d71c5..abc63d886dbcc3 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; @@ -21,10 +21,12 @@ import { SearchItems } from './use_search_items'; import { useApi } from './use_api'; import { useAppDependencies, useToastNotifications } from '../app_dependencies'; +import type { StepDefineExposedState } from '../sections/create_transform/components/step_define/common'; export const useIndexData = ( indexPattern: SearchItems['indexPattern'], - query: PivotQuery + query: PivotQuery, + combinedRuntimeMappings?: StepDefineExposedState['runtimeMappings'] ): UseIndexDataReturnType => { const api = useApi(); const toastNotifications = useToastNotifications(); @@ -32,6 +34,7 @@ export const useIndexData = ( ml: { getFieldType, getDataGridSchemaFromKibanaFieldType, + getDataGridSchemaFromESFieldType, getFieldsFromKibanaIndexPattern, showDataGridColumnChartErrorMessageToast, useDataGrid, @@ -43,14 +46,37 @@ export const useIndexData = ( const indexPatternFields = getFieldsFromKibanaIndexPattern(indexPattern); - // EuiDataGrid State - const columns: EuiDataGridColumn[] = [ - ...indexPatternFields.map((id) => { + const columns: EuiDataGridColumn[] = useMemo(() => { + let result: Array<{ id: string; schema: string | undefined }> = []; + + // Get the the runtime fields that are defined from API field and index patterns + if (combinedRuntimeMappings !== undefined) { + result = Object.keys(combinedRuntimeMappings).map((fieldName) => { + const field = combinedRuntimeMappings[fieldName]; + const schema = getDataGridSchemaFromESFieldType(field.type); + return { id: fieldName, schema }; + }); + } + + // Combine the runtime field that are defined from API field + indexPatternFields.forEach((id) => { const field = indexPattern.fields.getByName(id); - const schema = getDataGridSchemaFromKibanaFieldType(field); - return { id, schema }; - }), - ]; + if (!field?.runtimeField) { + const schema = getDataGridSchemaFromKibanaFieldType(field); + result.push({ id, schema }); + } + }); + + return result.sort((a, b) => a.id.localeCompare(b.id)); + }, [ + indexPatternFields, + indexPattern.fields, + combinedRuntimeMappings, + getDataGridSchemaFromESFieldType, + getDataGridSchemaFromKibanaFieldType, + ]); + + // EuiDataGrid State const dataGrid = useDataGrid(columns); @@ -92,9 +118,12 @@ export const useIndexData = ( from: pagination.pageIndex * pagination.pageSize, size: pagination.pageSize, ...(Object.keys(sort).length > 0 ? { sort } : {}), + ...(typeof combinedRuntimeMappings === 'object' && + Object.keys(combinedRuntimeMappings).length > 0 + ? { runtime_mappings: combinedRuntimeMappings } + : {}), }, }; - const resp = await api.esSearch(esSearchRequest); if (!isEsSearchResponse(resp)) { @@ -134,7 +163,17 @@ export const useIndexData = ( fetchDataGridData(); // custom comparison // eslint-disable-next-line react-hooks/exhaustive-deps - }, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]); + }, [ + indexPattern.title, + // eslint-disable-next-line react-hooks/exhaustive-deps + JSON.stringify([ + query, + pagination, + sortingColumns, + indexPatternFields, + combinedRuntimeMappings, + ]), + ]); useEffect(() => { if (chartsVisible) { diff --git a/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts b/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts index 673d8d38aa8fd1..62b3a077df5e61 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_pivot_data.ts @@ -71,7 +71,8 @@ export const usePivotData = ( indexPatternTitle: SearchItems['indexPattern']['title'], query: PivotQuery, validationStatus: StepDefineExposedState['validationStatus'], - requestPayload: StepDefineExposedState['previewRequest'] + requestPayload: StepDefineExposedState['previewRequest'], + combinedRuntimeMappings?: StepDefineExposedState['runtimeMappings'] ): UseIndexDataReturnType => { const [ previewMappingsProperties, @@ -79,7 +80,13 @@ export const usePivotData = ( ] = useState({}); const api = useApi(); const { - ml: { formatHumanReadableDateTimeSeconds, multiColumnSortFactory, useDataGrid, INDEX_STATUS }, + ml: { + getDataGridSchemaFromESFieldType, + formatHumanReadableDateTimeSeconds, + multiColumnSortFactory, + useDataGrid, + INDEX_STATUS, + }, } = useAppDependencies(); // Filters mapping properties of type `object`, which get returned for nested field parents. @@ -97,38 +104,7 @@ export const usePivotData = ( // EuiDataGrid State const columns: EuiDataGridColumn[] = columnKeys.map((id) => { const field = previewMappingsProperties[id]; - - // Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json'] - // To fall back to the default string schema it needs to be undefined. - let schema; - - switch (field?.type) { - case ES_FIELD_TYPES.GEO_POINT: - case ES_FIELD_TYPES.GEO_SHAPE: - schema = 'json'; - break; - case ES_FIELD_TYPES.BOOLEAN: - schema = 'boolean'; - break; - case ES_FIELD_TYPES.DATE: - case ES_FIELD_TYPES.DATE_NANOS: - schema = 'datetime'; - break; - case ES_FIELD_TYPES.BYTE: - case ES_FIELD_TYPES.DOUBLE: - case ES_FIELD_TYPES.FLOAT: - case ES_FIELD_TYPES.HALF_FLOAT: - case ES_FIELD_TYPES.INTEGER: - case ES_FIELD_TYPES.LONG: - case ES_FIELD_TYPES.SCALED_FLOAT: - case ES_FIELD_TYPES.SHORT: - schema = 'numeric'; - break; - // keep schema undefined for text based columns - case ES_FIELD_TYPES.KEYWORD: - case ES_FIELD_TYPES.TEXT: - break; - } + const schema = getDataGridSchemaFromESFieldType(field?.type); return { id, schema }; }); @@ -159,7 +135,12 @@ export const usePivotData = ( setNoDataMessage(''); setStatus(INDEX_STATUS.LOADING); - const previewRequest = getPreviewTransformRequestBody(indexPatternTitle, query, requestPayload); + const previewRequest = getPreviewTransformRequestBody( + indexPatternTitle, + query, + requestPayload, + combinedRuntimeMappings + ); const resp = await api.getTransformsPreview(previewRequest); if (!isPostTransformsPreviewResponseSchema(resp)) { @@ -196,11 +177,7 @@ export const usePivotData = ( getPreviewData(); // custom comparison /* eslint-disable react-hooks/exhaustive-deps */ - }, [ - indexPatternTitle, - JSON.stringify([requestPayload, query]), - /* eslint-enable react-hooks/exhaustive-deps */ - ]); + }, [indexPatternTitle, JSON.stringify([requestPayload, query, combinedRuntimeMappings])]); if (sortingColumns.length > 0) { tableItems.sort(multiColumnSortFactory(sortingColumns)); diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor/advanced_runtime_mappings_editor.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor/advanced_runtime_mappings_editor.tsx new file mode 100644 index 00000000000000..087bae97e287ef --- /dev/null +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor/advanced_runtime_mappings_editor.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEqual } from 'lodash'; +import React, { memo, FC } from 'react'; + +import { EuiCodeEditor } from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; + +import { StepDefineFormHook } from '../step_define'; + +export const AdvancedRuntimeMappingsEditor: FC = memo( + ({ + actions: { + convertToJson, + setAdvancedRuntimeMappingsConfig, + setRuntimeMappingsEditorApplyButtonEnabled, + }, + state: { advancedEditorRuntimeMappingsLastApplied, advancedRuntimeMappingsConfig, xJsonMode }, + }) => { + return ( + { + setAdvancedRuntimeMappingsConfig(d); + + // Disable the "Apply"-Button if the config hasn't changed. + if (advancedEditorRuntimeMappingsLastApplied === d) { + setRuntimeMappingsEditorApplyButtonEnabled(false); + return; + } + + // Try to parse the string passed on from the editor. + // If parsing fails, the "Apply"-Button will be disabled + try { + JSON.parse(convertToJson(d)); + setRuntimeMappingsEditorApplyButtonEnabled(true); + } catch (e) { + setRuntimeMappingsEditorApplyButtonEnabled(false); + } + }} + setOptions={{ + fontSize: '12px', + }} + theme="textmate" + aria-label={i18n.translate('xpack.transform.stepDefineForm.advancedEditorAriaLabel', { + defaultMessage: 'Advanced pivot editor', + })} + /> + ); + }, + (prevProps, nextProps) => isEqual(pickProps(prevProps), pickProps(nextProps)) +); + +function pickProps(props: StepDefineFormHook['runtimeMappingsEditor']) { + return [ + props.state.advancedEditorRuntimeMappingsLastApplied, + props.state.advancedRuntimeMappingsConfig, + ]; +} diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor_switch/advanced_runtime_mappings_editor_switch.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor_switch/advanced_runtime_mappings_editor_switch.tsx new file mode 100644 index 00000000000000..be297c10a8f88c --- /dev/null +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor_switch/advanced_runtime_mappings_editor_switch.tsx @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC } from 'react'; +import { EuiSwitch } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { StepDefineFormHook } from '../step_define'; + +export const AdvancedRuntimeMappingsEditorSwitch: FC< + StepDefineFormHook['runtimeMappingsEditor'] +> = (props) => { + const { + actions: { setRuntimeMappingsUpdated, toggleRuntimeMappingsEditor }, + state: { isRuntimeMappingsEditorEnabled }, + } = props; + + // If switching to KQL after updating via editor - reset search + const toggleEditorHandler = (reset = false) => { + if (reset === true) { + setRuntimeMappingsUpdated(false); + } + toggleRuntimeMappingsEditor(reset); + }; + + return ( + toggleEditorHandler()} + data-test-subj="transformAdvancedRuntimeMappingsEditorSwitch" + /> + ); +}; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor_switch/index.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor_switch/index.ts new file mode 100644 index 00000000000000..89a05690cab52d --- /dev/null +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_editor_switch/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { AdvancedRuntimeMappingsEditorSwitch } from './advanced_runtime_mappings_editor_switch'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_settings/advanced_runtime_mappings_settings.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_settings/advanced_runtime_mappings_settings.tsx new file mode 100644 index 00000000000000..f3c121a86cdc1b --- /dev/null +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_settings/advanced_runtime_mappings_settings.tsx @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC } from 'react'; +import { + EuiButton, + EuiButtonIcon, + EuiCopy, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiSpacer, + EuiText, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { StepDefineFormHook } from '../step_define'; +import { AdvancedRuntimeMappingsEditor } from '../advanced_runtime_mappings_editor/advanced_runtime_mappings_editor'; +import { AdvancedRuntimeMappingsEditorSwitch } from '../advanced_runtime_mappings_editor_switch'; +import { + isPivotGroupByConfigWithUiSupport, + PivotAggsConfigWithUiSupport, +} from '../../../../common'; +import { isPivotAggConfigWithUiSupport } from '../../../../common/pivot_group_by'; + +const advancedEditorsSidebarWidth = '220px'; +const COPY_TO_CLIPBOARD_RUNTIME_MAPPINGS = i18n.translate( + 'xpack.transform.indexPreview.copyRuntimeMappingsClipboardTooltip', + { + defaultMessage: 'Copy Dev Console statement of the runtime mappings to the clipboard.', + } +); + +export const AdvancedRuntimeMappingsSettings: FC = (props) => { + const { + actions: { applyRuntimeMappingsEditorChanges }, + state: { + runtimeMappings, + advancedRuntimeMappingsConfig, + isRuntimeMappingsEditorApplyButtonEnabled, + isRuntimeMappingsEditorEnabled, + }, + } = props.runtimeMappingsEditor; + const { + actions: { deleteAggregation, deleteGroupBy }, + state: { groupByList, aggList }, + } = props.pivotConfig; + + const applyChanges = () => { + const nextConfig = JSON.parse(advancedRuntimeMappingsConfig); + const previousConfig = runtimeMappings; + + applyRuntimeMappingsEditorChanges(); + + // If the user updates the name of the runtime mapping fields + // delete any groupBy or aggregation associated with the deleted field + Object.keys(groupByList).forEach((groupByKey) => { + const groupBy = groupByList[groupByKey]; + if ( + isPivotGroupByConfigWithUiSupport(groupBy) && + previousConfig?.hasOwnProperty(groupBy.field) && + !nextConfig.hasOwnProperty(groupBy.field) + ) { + deleteGroupBy(groupByKey); + } + }); + Object.keys(aggList).forEach((aggName) => { + const agg = aggList[aggName] as PivotAggsConfigWithUiSupport; + if ( + isPivotAggConfigWithUiSupport(agg) && + agg.field !== undefined && + previousConfig?.hasOwnProperty(agg.field) && + !nextConfig.hasOwnProperty(agg.field) + ) { + deleteAggregation(aggName); + } + }); + }; + return ( + <> + + + + + + {runtimeMappings !== undefined && Object.keys(runtimeMappings).length > 0 ? ( + + ) : ( + + )} + + {isRuntimeMappingsEditorEnabled && ( + <> + + + + )} + + + + + + + + + + + + {(copy: () => void) => ( + + )} + + + + + + {isRuntimeMappingsEditorEnabled && ( + + + + {i18n.translate( + 'xpack.transform.stepDefineForm.advancedRuntimeMappingsEditorHelpText', + { + defaultMessage: + 'The advanced editor allows you to edit the runtime mappings of the transform configuration.', + } + )} + + + + {i18n.translate( + 'xpack.transform.stepDefineForm.advancedSourceEditorApplyButtonText', + { + defaultMessage: 'Apply changes', + } + )} + + + )} + + + + + + + ); +}; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_settings/index.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_settings/index.ts new file mode 100644 index 00000000000000..69b3bc36a559e9 --- /dev/null +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/advanced_runtime_mappings_settings/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { AdvancedRuntimeMappingsSettings } from './advanced_runtime_mappings_settings'; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx index 807830d749892d..34832ec968e296 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_create/step_create_form.tsx @@ -46,6 +46,9 @@ import { PutTransformsLatestRequestSchema, PutTransformsPivotRequestSchema, } from '../../../../../../common/api_schemas/transforms'; +import type { RuntimeField } from '../../../../../../../../../src/plugins/data/common/index_patterns'; +import { isPopulatedObject } from '../../../../common/utils/object_utils'; +import { isLatestTransform } from '../../../../../../common/types/transform'; export interface StepDetailsExposedState { created: boolean; @@ -189,12 +192,19 @@ export const StepCreateForm: FC = React.memo( const createKibanaIndexPattern = async () => { setLoading(true); const indexPatternName = transformConfig.dest.index; + const runtimeMappings = transformConfig.source.runtime_mappings as Record< + string, + RuntimeField + >; try { const newIndexPattern = await indexPatterns.createAndSave( { title: indexPatternName, timeFieldName, + ...(isPopulatedObject(runtimeMappings) && isLatestTransform(transformConfig) + ? { runtimeFieldMap: runtimeMappings } + : {}), }, false, true diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts index 77b60b6f5966af..6298874a203666 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/apply_transform_config_to_define_state.ts @@ -30,12 +30,19 @@ import { TRANSFORM_FUNCTION } from '../../../../../../../common/constants'; import { StepDefineFormProps } from '../step_define_form'; import { validateLatestConfig } from '../hooks/use_latest_function_config'; import { validatePivotConfig } from '../hooks/use_pivot_config'; +import { getCombinedRuntimeMappings } from '../../../../../common/request'; export function applyTransformConfigToDefineState( state: StepDefineExposedState, transformConfig?: TransformBaseConfig, indexPattern?: StepDefineFormProps['searchItems']['indexPattern'] ): StepDefineExposedState { + // apply runtime mappings from both the index pattern and inline configurations + state.runtimeMappings = getCombinedRuntimeMappings( + indexPattern, + transformConfig?.source?.runtime_mappings + ); + if (transformConfig === undefined) { return state; } @@ -107,6 +114,5 @@ export function applyTransformConfigToDefineState( // applying a transform config to wizard state will always result in a valid configuration state.valid = true; - return state; } diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/common.test.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/common.test.ts index deaaddc44ba7ab..fcdbac8c7ff39c 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/common.test.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/common.test.ts @@ -8,6 +8,7 @@ import { getPivotDropdownOptions } from '../common'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; import { FilterAggForm } from './filter_agg/components'; +import type { RuntimeField } from '../../../../../../../../../../src/plugins/data/common/index_patterns'; describe('Transform: Define Pivot Common', () => { test('getPivotDropdownOptions()', () => { @@ -109,5 +110,169 @@ describe('Transform: Define Pivot Common', () => { }, }, }); + + const runtimeMappings = { + rt_bytes_bigger: { + type: 'double', + script: { + source: "emit(doc['bytes'].value * 2.0)", + }, + } as RuntimeField, + }; + const optionsWithRuntimeFields = getPivotDropdownOptions(indexPattern, runtimeMappings); + expect(optionsWithRuntimeFields).toMatchObject({ + aggOptions: [ + { + label: ' the-f[i]e>ld ', + options: [ + { label: 'avg( the-f[i]e>ld )' }, + { label: 'cardinality( the-f[i]e>ld )' }, + { label: 'max( the-f[i]e>ld )' }, + { label: 'min( the-f[i]e>ld )' }, + { label: 'percentiles( the-f[i]e>ld )' }, + { label: 'sum( the-f[i]e>ld )' }, + { label: 'value_count( the-f[i]e>ld )' }, + { label: 'filter( the-f[i]e>ld )' }, + ], + }, + { + label: 'rt_bytes_bigger', + options: [ + { label: 'avg(rt_bytes_bigger)' }, + { label: 'cardinality(rt_bytes_bigger)' }, + { label: 'max(rt_bytes_bigger)' }, + { label: 'min(rt_bytes_bigger)' }, + { label: 'percentiles(rt_bytes_bigger)' }, + { label: 'sum(rt_bytes_bigger)' }, + { label: 'value_count(rt_bytes_bigger)' }, + { label: 'filter(rt_bytes_bigger)' }, + ], + }, + ], + aggOptionsData: { + 'avg( the-f[i]e>ld )': { + agg: 'avg', + aggName: 'the-field.avg', + dropDownName: 'avg( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + }, + 'cardinality( the-f[i]e>ld )': { + agg: 'cardinality', + aggName: 'the-field.cardinality', + dropDownName: 'cardinality( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + }, + 'max( the-f[i]e>ld )': { + agg: 'max', + aggName: 'the-field.max', + dropDownName: 'max( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + }, + 'min( the-f[i]e>ld )': { + agg: 'min', + aggName: 'the-field.min', + dropDownName: 'min( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + }, + 'percentiles( the-f[i]e>ld )': { + agg: 'percentiles', + aggName: 'the-field.percentiles', + dropDownName: 'percentiles( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + percents: [1, 5, 25, 50, 75, 95, 99], + }, + 'sum( the-f[i]e>ld )': { + agg: 'sum', + aggName: 'the-field.sum', + dropDownName: 'sum( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + }, + 'value_count( the-f[i]e>ld )': { + agg: 'value_count', + aggName: 'the-field.value_count', + dropDownName: 'value_count( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + }, + 'filter( the-f[i]e>ld )': { + agg: 'filter', + aggName: 'the-field.filter', + dropDownName: 'filter( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + isSubAggsSupported: true, + AggFormComponent: FilterAggForm, + }, + 'avg(rt_bytes_bigger)': { + agg: 'avg', + aggName: 'rt_bytes_bigger.avg', + dropDownName: 'avg(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + }, + 'cardinality(rt_bytes_bigger)': { + agg: 'cardinality', + aggName: 'rt_bytes_bigger.cardinality', + dropDownName: 'cardinality(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + }, + 'max(rt_bytes_bigger)': { + agg: 'max', + aggName: 'rt_bytes_bigger.max', + dropDownName: 'max(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + }, + 'min(rt_bytes_bigger)': { + agg: 'min', + aggName: 'rt_bytes_bigger.min', + dropDownName: 'min(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + }, + 'percentiles(rt_bytes_bigger)': { + agg: 'percentiles', + aggName: 'rt_bytes_bigger.percentiles', + dropDownName: 'percentiles(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + percents: [1, 5, 25, 50, 75, 95, 99], + }, + 'sum(rt_bytes_bigger)': { + agg: 'sum', + aggName: 'rt_bytes_bigger.sum', + dropDownName: 'sum(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + }, + 'value_count(rt_bytes_bigger)': { + agg: 'value_count', + aggName: 'rt_bytes_bigger.value_count', + dropDownName: 'value_count(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + }, + 'filter(rt_bytes_bigger)': { + agg: 'filter', + aggName: 'rt_bytes_bigger.filter', + dropDownName: 'filter(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + isSubAggsSupported: true, + AggFormComponent: FilterAggForm, + }, + }, + groupByOptions: [ + { label: 'histogram( the-f[i]e>ld )' }, + { label: 'histogram(rt_bytes_bigger)' }, + ], + groupByOptionsData: { + 'histogram( the-f[i]e>ld )': { + agg: 'histogram', + aggName: 'the-field', + dropDownName: 'histogram( the-f[i]e>ld )', + field: ' the-f[i]e>ld ', + interval: '10', + }, + 'histogram(rt_bytes_bigger)': { + agg: 'histogram', + aggName: 'rt_bytes_bigger', + dropDownName: 'histogram(rt_bytes_bigger)', + field: 'rt_bytes_bigger', + interval: '10', + }, + }, + }); }); }); diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_agg_form.test.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_agg_form.test.tsx index dae8f61aaa4dff..7f9c4256f77557 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_agg_form.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_agg_form.test.tsx @@ -10,11 +10,23 @@ import React from 'react'; import { I18nProvider } from '@kbn/i18n/react'; import { FilterAggForm } from './filter_agg_form'; import { CreateTransformWizardContext } from '../../../../wizard/wizard'; -import { KBN_FIELD_TYPES } from '../../../../../../../../../../../../src/plugins/data/common'; +import { + KBN_FIELD_TYPES, + RuntimeField, +} from '../../../../../../../../../../../../src/plugins/data/common'; import { IndexPattern } from '../../../../../../../../../../../../src/plugins/data/public'; import { FilterTermForm } from './filter_term_form'; describe('FilterAggForm', () => { + const runtimeMappings = { + rt_bytes_bigger: { + type: 'double', + script: { + source: "emit(doc['bytes'].value * 2.0)", + }, + } as RuntimeField, + }; + const indexPattern = ({ fields: { getByName: jest.fn((fieldName: string) => { @@ -37,7 +49,7 @@ describe('FilterAggForm', () => { const { getByLabelText, findByTestId, container } = render( - + @@ -62,7 +74,7 @@ describe('FilterAggForm', () => { const { findByTestId } = render( - + @@ -90,7 +102,7 @@ describe('FilterAggForm', () => { const { rerender, findByTestId } = render( - + @@ -99,7 +111,7 @@ describe('FilterAggForm', () => { // re-render the same component with different props rerender( - + @@ -127,7 +139,7 @@ describe('FilterAggForm', () => { const { findByTestId, container } = render( - + { - const { indexPattern } = useContext(CreateTransformWizardContext); + const { indexPattern, runtimeMappings } = useContext(CreateTransformWizardContext); - const filterAggsOptions = useMemo(() => getSupportedFilterAggs(selectedField, indexPattern!), [ - indexPattern, - selectedField, - ]); + const filterAggsOptions = useMemo( + () => getSupportedFilterAggs(selectedField, indexPattern!, runtimeMappings), + [indexPattern, selectedField, runtimeMappings] + ); useUpdateEffect(() => { // reset filter agg on field change diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_range_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_range_form.tsx index 2e9ad761d3b790..67c904946d302b 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_range_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_range_form.tsx @@ -17,6 +17,7 @@ import { import { FormattedMessage } from '@kbn/i18n/react'; import { FilterAggConfigRange } from '../types'; +const BUTTON_SIZE = 40; /** * Form component for the range filter aggregation for number type fields. */ @@ -45,7 +46,7 @@ export const FilterRangeForm: FilterAggConfigRange['aggTypeConfig']['FilterAggFo return ( <> - + { updateConfig({ includeFrom: e.target.checked }); }} @@ -94,13 +96,14 @@ export const FilterRangeForm: FilterAggConfigRange['aggTypeConfig']['FilterAggFo step="any" append={ { updateConfig({ includeTo: !includeTo }); }} fill={includeTo} > - {includeTo ? '≤' : '<'}s + {includeTo ? '≤' : '<'} } /> diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_term_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_term_form.tsx index ad06cfb31a62f1..f2db6167c163c6 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_term_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/filter_agg/components/filter_term_form.tsx @@ -26,7 +26,7 @@ export const FilterTermForm: FilterAggConfigTerm['aggTypeConfig']['FilterAggForm selectedField, }) => { const api = useApi(); - const { indexPattern } = useContext(CreateTransformWizardContext); + const { indexPattern, runtimeMappings } = useContext(CreateTransformWizardContext); const toastNotifications = useToastNotifications(); const [options, setOptions] = useState([]); @@ -38,6 +38,7 @@ export const FilterTermForm: FilterAggConfigTerm['aggTypeConfig']['FilterAggForm const esSearchRequest = { index: indexPattern!.title, body: { + ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}), query: { wildcard: { [selectedField!]: { diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_step_define_state.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_step_define_state.ts index d3b1df41b3cfbd..c75da651f79d0d 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_step_define_state.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_default_step_define_state.ts @@ -30,5 +30,8 @@ export function getDefaultStepDefineState(searchItems: SearchItems): StepDefineE isValid: false, }, previewRequest: undefined, + runtimeMappings: undefined, + runtimeMappingsUpdated: false, + isRuntimeMappingsEditorEnabled: false, }; } diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_pivot_dropdown_options.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_pivot_dropdown_options.ts index 6845d096a2e022..c88b604989680a 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_pivot_dropdown_options.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/get_pivot_dropdown_options.ts @@ -7,6 +7,7 @@ import { EuiComboBoxOptionOption } from '@elastic/eui'; import { + ES_FIELD_TYPES, IndexPattern, KBN_FIELD_TYPES, } from '../../../../../../../../../../src/plugins/data/public'; @@ -24,11 +25,40 @@ import { import { getDefaultAggregationConfig } from './get_default_aggregation_config'; import { getDefaultGroupByConfig } from './get_default_group_by_config'; -import { Field } from './types'; +import type { Field, StepDefineExposedState } from './types'; +import { isPopulatedObject } from '../../../../../common/utils/object_utils'; const illegalEsAggNameChars = /[[\]>]/g; -export function getPivotDropdownOptions(indexPattern: IndexPattern) { +export function getKibanaFieldTypeFromEsType(type: string): KBN_FIELD_TYPES { + switch (type) { + case ES_FIELD_TYPES.FLOAT: + case ES_FIELD_TYPES.HALF_FLOAT: + case ES_FIELD_TYPES.SCALED_FLOAT: + case ES_FIELD_TYPES.DOUBLE: + case ES_FIELD_TYPES.INTEGER: + case ES_FIELD_TYPES.LONG: + case ES_FIELD_TYPES.SHORT: + case ES_FIELD_TYPES.UNSIGNED_LONG: + return KBN_FIELD_TYPES.NUMBER; + + case ES_FIELD_TYPES.DATE: + case ES_FIELD_TYPES.DATE_NANOS: + return KBN_FIELD_TYPES.DATE; + + case ES_FIELD_TYPES.KEYWORD: + case ES_FIELD_TYPES.STRING: + return KBN_FIELD_TYPES.STRING; + + default: + return type as KBN_FIELD_TYPES; + } +} + +export function getPivotDropdownOptions( + indexPattern: IndexPattern, + runtimeMappings?: StepDefineExposedState['runtimeMappings'] +) { // The available group by options const groupByOptions: EuiComboBoxOptionOption[] = []; const groupByOptionsData: PivotGroupByConfigWithUiSupportDict = {}; @@ -38,11 +68,26 @@ export function getPivotDropdownOptions(indexPattern: IndexPattern) { const aggOptionsData: PivotAggsConfigWithUiSupportDict = {}; const ignoreFieldNames = ['_id', '_index', '_type']; - const fields = indexPattern.fields - .filter((field) => field.aggregatable === true && !ignoreFieldNames.includes(field.name)) + const indexPatternFields = indexPattern.fields + .filter( + (field) => + field.aggregatable === true && !ignoreFieldNames.includes(field.name) && !field.runtimeField + ) .map((field): Field => ({ name: field.name, type: field.type as KBN_FIELD_TYPES })); - fields.forEach((field) => { + // Support for runtime_mappings that are defined by queries + let runtimeFields: Field[] = []; + if (isPopulatedObject(runtimeMappings)) { + runtimeFields = Object.keys(runtimeMappings).map((fieldName) => { + const field = runtimeMappings[fieldName]; + return { name: fieldName, type: getKibanaFieldTypeFromEsType(field.type) }; + }); + } + + const sortByLabel = (a: Field, b: Field) => a.name.localeCompare(b.name); + + const combinedFields = [...indexPatternFields, ...runtimeFields].sort(sortByLabel); + combinedFields.forEach((field) => { // Group by const availableGroupByAggs: [] = getNestedProperty(pivotGroupByFieldSupport, field.type); diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts index d1325e4af5ce74..cdba7a3f5482c9 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/common/types.ts @@ -9,7 +9,11 @@ import { KBN_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/ import { EsFieldName } from '../../../../../../../common/types/fields'; -import { PivotAggsConfigDict, PivotGroupByConfigDict } from '../../../../../common'; +import { + PivotAggsConfigDict, + PivotGroupByConfigDict, + PivotGroupByConfigWithUiSupportDict, +} from '../../../../../common'; import { SavedSearchQuery } from '../../../../../hooks/use_search_items'; import { QUERY_LANGUAGE } from './constants'; @@ -30,10 +34,24 @@ export interface Field { type: KBN_FIELD_TYPES; } +// Replace this with import once #88995 is merged +const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const; +type RuntimeType = typeof RUNTIME_FIELD_TYPES[number]; + +export interface RuntimeField { + type: RuntimeType; + script: + | string + | { + source: string; + }; +} + +export type RuntimeMappings = Record; export interface StepDefineExposedState { transformFunction: TransformFunction; aggList: PivotAggsConfigDict; - groupByList: PivotGroupByConfigDict; + groupByList: PivotGroupByConfigDict | PivotGroupByConfigWithUiSupportDict; latestConfig: LatestFunctionConfigUI; isAdvancedPivotEditorEnabled: boolean; isAdvancedSourceEditorEnabled: boolean; @@ -47,6 +65,9 @@ export interface StepDefineExposedState { * Undefined when the form is incomplete or invalid */ previewRequest: { latest: LatestFunctionConfig } | { pivot: PivotConfigDefinition } | undefined; + runtimeMappings?: RuntimeMappings; + runtimeMappingsUpdated: boolean; + isRuntimeMappingsEditorEnabled: boolean; } export function isPivotPartialRequest(arg: any): arg is { pivot: PivotConfigDefinition } { diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_runtime_mappings_editor.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_runtime_mappings_editor.ts new file mode 100644 index 00000000000000..9bb5f91ae03c79 --- /dev/null +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_advanced_runtime_mappings_editor.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useState } from 'react'; +import { XJsonMode } from '@kbn/ace'; +import { StepDefineExposedState } from '../common'; +import { XJson } from '../../../../../../../../../../src/plugins/es_ui_shared/public'; + +const { useXJsonMode } = XJson; +const xJsonMode = new XJsonMode(); + +export const useAdvancedRuntimeMappingsEditor = (defaults: StepDefineExposedState) => { + const stringifiedRuntimeMappings = JSON.stringify(defaults.runtimeMappings, null, 2); + + // Advanced editor for source config state + const [runtimeMappingsUpdated, setRuntimeMappingsUpdated] = useState( + defaults.runtimeMappingsUpdated + ); + const [runtimeMappings, setRuntimeMappings] = useState(defaults.runtimeMappings); + + const [ + isRuntimeMappingsEditorSwitchModalVisible, + setRuntimeMappingsEditorSwitchModalVisible, + ] = useState(false); + + const [isRuntimeMappingsEditorEnabled, setRuntimeMappingsEditorEnabled] = useState( + defaults.isRuntimeMappingsEditorEnabled + ); + + const [ + isRuntimeMappingsEditorApplyButtonEnabled, + setRuntimeMappingsEditorApplyButtonEnabled, + ] = useState(false); + + const [ + advancedEditorRuntimeMappingsLastApplied, + setAdvancedEditorRuntimeMappingsLastApplied, + ] = useState(stringifiedRuntimeMappings); + + const [advancedEditorRuntimeMappings, setAdvancedEditorRuntimeMappings] = useState( + stringifiedRuntimeMappings + ); + + const { + convertToJson, + setXJson: setAdvancedRuntimeMappingsConfig, + xJson: advancedRuntimeMappingsConfig, + } = useXJsonMode(stringifiedRuntimeMappings ?? ''); + + const applyRuntimeMappingsEditorChanges = () => { + const parsedRuntimeMappings = JSON.parse(advancedRuntimeMappingsConfig); + const prettySourceConfig = JSON.stringify(parsedRuntimeMappings, null, 2); + setRuntimeMappingsUpdated(true); + setRuntimeMappings(parsedRuntimeMappings); + setAdvancedEditorRuntimeMappings(prettySourceConfig); + setAdvancedEditorRuntimeMappingsLastApplied(prettySourceConfig); + setRuntimeMappingsEditorApplyButtonEnabled(false); + }; + + // If switching to KQL after updating via editor - reset search + const toggleRuntimeMappingsEditor = (reset = false) => { + if (reset === true) { + setRuntimeMappingsUpdated(false); + } + if (isRuntimeMappingsEditorEnabled === false) { + setAdvancedEditorRuntimeMappingsLastApplied(advancedEditorRuntimeMappings); + } + + setRuntimeMappingsEditorEnabled(!isRuntimeMappingsEditorEnabled); + setRuntimeMappingsEditorApplyButtonEnabled(false); + }; + + return { + actions: { + applyRuntimeMappingsEditorChanges, + setRuntimeMappingsEditorApplyButtonEnabled, + setRuntimeMappingsEditorEnabled, + setAdvancedEditorRuntimeMappings, + setAdvancedEditorRuntimeMappingsLastApplied, + setRuntimeMappingsEditorSwitchModalVisible, + setRuntimeMappingsUpdated, + toggleRuntimeMappingsEditor, + convertToJson, + setAdvancedRuntimeMappingsConfig, + }, + state: { + advancedEditorRuntimeMappings, + advancedEditorRuntimeMappingsLastApplied, + isRuntimeMappingsEditorApplyButtonEnabled, + isRuntimeMappingsEditorEnabled, + isRuntimeMappingsEditorSwitchModalVisible, + runtimeMappingsUpdated, + advancedRuntimeMappingsConfig, + xJsonMode, + runtimeMappings, + }, + }; +}; diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts index ecc8bf673d93a8..d52bd3f5bf7060 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_latest_function_config.ts @@ -32,19 +32,28 @@ export const latestConfigMapper = { * Provides available options for unique_key and sort fields * @param indexPattern * @param aggConfigs + * @param runtimeMappings */ function getOptions( indexPattern: StepDefineFormProps['searchItems']['indexPattern'], - aggConfigs: AggConfigs + aggConfigs: AggConfigs, + runtimeMappings?: StepDefineExposedState['runtimeMappings'] ) { const aggConfig = aggConfigs.aggs[0]; const param = aggConfig.type.params.find((p) => p.type === 'field'); const filteredIndexPatternFields = param - ? ((param as unknown) as FieldParamType).getAvailableFields(aggConfig) + ? ((param as unknown) as FieldParamType) + .getAvailableFields(aggConfig) + // runtimeMappings may already include runtime fields defined by the index pattern + .filter((ip) => ip.runtimeField === undefined) : []; const ignoreFieldNames = new Set(['_source', '_type', '_index', '_id', '_version', '_score']); + const runtimeFieldsOptions = runtimeMappings + ? Object.keys(runtimeMappings).map((k) => ({ label: k, value: k })) + : []; + const uniqueKeyOptions: Array> = filteredIndexPatternFields .filter((v) => !ignoreFieldNames.has(v.name)) .map((v) => ({ @@ -52,7 +61,16 @@ function getOptions( value: v.name, })); - const sortFieldOptions: Array> = indexPattern.fields + const runtimeFieldsSortOptions: Array> = runtimeMappings + ? Object.entries(runtimeMappings) + .filter(([fieldName, fieldMapping]) => fieldMapping.type === 'date') + .map(([fieldName, fieldMapping]) => ({ + label: fieldName, + value: fieldName, + })) + : []; + + const indexPatternFieldsSortOptions: Array> = indexPattern.fields // The backend API for `latest` allows all field types for sort but the UI will be limited to `date`. .filter((v) => !ignoreFieldNames.has(v.name) && v.sortable && v.type === 'date') .map((v) => ({ @@ -60,7 +78,15 @@ function getOptions( value: v.name, })); - return { uniqueKeyOptions, sortFieldOptions }; + const sortByLabel = (a: EuiComboBoxOptionOption, b: EuiComboBoxOptionOption) => + a.label.localeCompare(b.label); + + return { + uniqueKeyOptions: [...uniqueKeyOptions, ...runtimeFieldsOptions].sort(sortByLabel), + sortFieldOptions: [...indexPatternFieldsSortOptions, ...runtimeFieldsSortOptions].sort( + sortByLabel + ), + }; } /** @@ -86,7 +112,8 @@ export function validateLatestConfig(config?: LatestFunctionConfig) { export function useLatestFunctionConfig( defaults: StepDefineExposedState['latestConfig'], - indexPattern: StepDefineFormProps['searchItems']['indexPattern'] + indexPattern: StepDefineFormProps['searchItems']['indexPattern'], + runtimeMappings: StepDefineExposedState['runtimeMappings'] ): { config: LatestFunctionConfigUI; uniqueKeyOptions: Array>; @@ -104,8 +131,8 @@ export function useLatestFunctionConfig( const { uniqueKeyOptions, sortFieldOptions } = useMemo(() => { const aggConfigs = data.search.aggs.createAggConfigs(indexPattern, [{ type: 'terms' }]); - return getOptions(indexPattern, aggConfigs); - }, [indexPattern, data.search.aggs]); + return getOptions(indexPattern, aggConfigs, runtimeMappings); + }, [indexPattern, data.search.aggs, runtimeMappings]); const updateLatestFunctionConfig = useCallback( (update) => diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts index 1748f6f8fd4873..a02d3bafac9848 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_pivot_config.ts @@ -115,8 +115,8 @@ export const usePivotConfig = ( const toastNotifications = useToastNotifications(); const { aggOptions, aggOptionsData, groupByOptions, groupByOptionsData } = useMemo( - () => getPivotDropdownOptions(indexPattern), - [indexPattern] + () => getPivotDropdownOptions(indexPattern, defaults.runtimeMappings), + [defaults.runtimeMappings, indexPattern] ); // The list of selected aggregations diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts index c2f01db05ff3e7..0ceea070df1b66 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/hooks/use_step_define_form.ts @@ -19,6 +19,7 @@ import { usePivotConfig } from './use_pivot_config'; import { useSearchBar } from './use_search_bar'; import { useLatestFunctionConfig } from './use_latest_function_config'; import { TRANSFORM_FUNCTION } from '../../../../../../../common/constants'; +import { useAdvancedRuntimeMappingsEditor } from './use_advanced_runtime_mappings_editor'; export type StepDefineFormHook = ReturnType; @@ -30,12 +31,18 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi const searchBar = useSearchBar(defaults, indexPattern); const pivotConfig = usePivotConfig(defaults, indexPattern); - const latestFunctionConfig = useLatestFunctionConfig(defaults.latestConfig, indexPattern); + + const latestFunctionConfig = useLatestFunctionConfig( + defaults.latestConfig, + indexPattern, + defaults?.runtimeMappings + ); const previewRequest = getPreviewTransformRequestBody( indexPattern.title, searchBar.state.pivotQuery, - pivotConfig.state.requestPayload + pivotConfig.state.requestPayload, + defaults?.runtimeMappings ); // pivot config hook @@ -44,12 +51,17 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi // source config hook const advancedSourceEditor = useAdvancedSourceEditor(defaults, previewRequest); + // runtime mappings config hook + const runtimeMappingsEditor = useAdvancedRuntimeMappingsEditor(defaults); + useEffect(() => { + const runtimeMappings = runtimeMappingsEditor.state.runtimeMappings; if (!advancedSourceEditor.state.isAdvancedSourceEditorEnabled) { const previewRequestUpdate = getPreviewTransformRequestBody( indexPattern.title, searchBar.state.pivotQuery, - pivotConfig.state.requestPayload + pivotConfig.state.requestPayload, + runtimeMappings ); const stringifiedSourceConfigUpdate = JSON.stringify( @@ -60,7 +72,6 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi advancedSourceEditor.actions.setAdvancedEditorSourceConfig(stringifiedSourceConfigUpdate); } - onChange({ transformFunction, latestConfig: latestFunctionConfig.config, @@ -84,6 +95,9 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi transformFunction === TRANSFORM_FUNCTION.PIVOT ? pivotConfig.state.requestPayload : latestFunctionConfig.requestPayload, + runtimeMappings, + runtimeMappingsUpdated: runtimeMappingsEditor.state.runtimeMappingsUpdated, + isRuntimeMappingsEditorEnabled: runtimeMappingsEditor.state.isRuntimeMappingsEditorEnabled, }); // custom comparison /* eslint-disable react-hooks/exhaustive-deps */ @@ -92,9 +106,13 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi JSON.stringify(advancedSourceEditor.state), pivotConfig.state, JSON.stringify(searchBar.state), + JSON.stringify([ + runtimeMappingsEditor.state.runtimeMappings, + runtimeMappingsEditor.state.runtimeMappingsUpdated, + runtimeMappingsEditor.state.isRuntimeMappingsEditorEnabled, + ]), latestFunctionConfig.config, transformFunction, - /* eslint-enable react-hooks/exhaustive-deps */ ]); return { @@ -102,6 +120,7 @@ export const useStepDefineForm = ({ overrides, onChange, searchItems }: StepDefi setTransformFunction, advancedPivotEditor, advancedSourceEditor, + runtimeMappingsEditor, pivotConfig, latestFunctionConfig, searchBar, diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx index a5d9310e586e66..1ddb9aa61045ba 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx @@ -57,6 +57,7 @@ import { getAggConfigFromEsAgg } from '../../../../common/pivot_aggs'; import { TransformFunctionSelector } from './transform_function_selector'; import { TRANSFORM_FUNCTION } from '../../../../../../common/constants'; import { LatestFunctionForm } from './latest_function_form'; +import { AdvancedRuntimeMappingsSettings } from '../advanced_runtime_mappings_settings'; export interface StepDefineFormProps { overrides?: StepDefineExposedState; @@ -67,7 +68,6 @@ export interface StepDefineFormProps { export const StepDefineForm: FC = React.memo((props) => { const { searchItems } = props; const { indexPattern } = searchItems; - const { ml: { DataGrid }, } = useAppDependencies(); @@ -87,11 +87,14 @@ export const StepDefineForm: FC = React.memo((props) => { const pivotQuery = stepDefineForm.searchBar.state.pivotQuery; const indexPreviewProps = { - ...useIndexData(indexPattern, stepDefineForm.searchBar.state.pivotQuery), + ...useIndexData( + indexPattern, + stepDefineForm.searchBar.state.pivotQuery, + stepDefineForm.runtimeMappingsEditor.state.runtimeMappings + ), dataTestSubj: 'transformIndexPreview', toastNotifications, }; - const { requestPayload, validationStatus } = stepDefineForm.transformFunction === TRANSFORM_FUNCTION.PIVOT ? stepDefineForm.pivotConfig.state @@ -102,7 +105,8 @@ export const StepDefineForm: FC = React.memo((props) => { pivotQuery, stepDefineForm.transformFunction === TRANSFORM_FUNCTION.PIVOT ? stepDefineForm.pivotConfig.state.requestPayload - : stepDefineForm.latestFunctionConfig.requestPayload + : stepDefineForm.latestFunctionConfig.requestPayload, + stepDefineForm.runtimeMappingsEditor.state.runtimeMappings ); const copyToClipboardSource = getIndexDevConsoleStatement(pivotQuery, indexPattern.title); @@ -122,7 +126,13 @@ export const StepDefineForm: FC = React.memo((props) => { ); const pivotPreviewProps = { - ...usePivotData(indexPattern.title, pivotQuery, validationStatus, requestPayload), + ...usePivotData( + indexPattern.title, + pivotQuery, + validationStatus, + requestPayload, + stepDefineForm.runtimeMappingsEditor.state.runtimeMappings + ), dataTestSubj: 'transformPivotPreview', title: i18n.translate('xpack.transform.pivotPreview.transformPreviewTitle', { defaultMessage: 'Transform preview', @@ -273,7 +283,7 @@ export const StepDefineForm: FC = React.memo((props) => { defaultMessage: 'The advanced editor allows you to edit the source query clause of the transform configuration.', } - )}{' '} + )} {i18n.translate( 'xpack.transform.stepDefineForm.advancedEditorHelpTextLink', @@ -304,6 +314,9 @@ export const StepDefineForm: FC = React.memo((props) => { + + + diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx index 614965c8a3efe3..27e25596c980fa 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_summary.tsx @@ -37,6 +37,7 @@ interface Props { export const StepDefineSummary: FC = ({ formState: { + runtimeMappings, searchString, searchQuery, groupByList, @@ -57,14 +58,16 @@ export const StepDefineSummary: FC = ({ const previewRequest = getPreviewTransformRequestBody( searchItems.indexPattern.title, pivotQuery, - partialPreviewRequest + partialPreviewRequest, + runtimeMappings ); const pivotPreviewProps = usePivotData( searchItems.indexPattern.title, pivotQuery, validationStatus, - partialPreviewRequest + partialPreviewRequest, + runtimeMappings ); const isModifiedQuery = diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx index 1fa16e26565b65..0d39ec77d059fb 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx @@ -118,7 +118,8 @@ export const StepDetailsForm: FC = React.memo( const previewRequest = getPreviewTransformRequestBody( searchItems.indexPattern.title, pivotQuery, - partialPreviewRequest + partialPreviewRequest, + stepDefineState.runtimeMappings ); const transformPreview = await api.getTransformsPreview(previewRequest); diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx index 9837ace2720725..5ae464affa0164 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/wizard/wizard.tsx @@ -32,6 +32,7 @@ import { } from '../step_details'; import { WizardNav } from '../wizard_nav'; import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; +import type { RuntimeMappings } from '../step_define/common/types'; enum KBN_MANAGEMENT_PAGE_CLASSNAME { DEFAULT_BODY = 'mgtPage__body', @@ -89,8 +90,12 @@ interface WizardProps { searchItems: SearchItems; } -export const CreateTransformWizardContext = createContext<{ indexPattern: IndexPattern | null }>({ +export const CreateTransformWizardContext = createContext<{ + indexPattern: IndexPattern | null; + runtimeMappings: RuntimeMappings | undefined; +}>({ indexPattern: null, + runtimeMappings: undefined, }); export const Wizard: FC = React.memo(({ cloneConfig, searchItems }) => { @@ -239,7 +244,9 @@ export const Wizard: FC = React.memo(({ cloneConfig, searchItems }) const stepsConfig = [stepDefine, stepDetails, stepCreate]; return ( - + ); diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx index 2ee558d449c9ac..87ae90afdf9c97 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_preview_pane.tsx @@ -29,7 +29,7 @@ export const ExpandedRowPreviewPane: FC = ({ transf } = useAppDependencies(); const toastNotifications = useToastNotifications(); - const { searchQuery, validationStatus, previewRequest } = useMemo( + const { searchQuery, validationStatus, previewRequest, runtimeMappings } = useMemo( () => applyTransformConfigToDefineState( getDefaultStepDefineState({} as SearchItems), @@ -48,7 +48,8 @@ export const ExpandedRowPreviewPane: FC = ({ transf indexPatternTitle, pivotQuery, validationStatus, - previewRequest + previewRequest, + runtimeMappings ); return (