[Fleet] Support dynamic_template mappings from object field

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -483,6 +483,7 @@ export type PackageAssetReference = Pick<SavedObjectReference, 'id'> & {
 
 export interface IndexTemplateMappings {
   properties: any;
+  dynamic_templates?: any;
 }
 
 // This is an index template v2, see https://github.com/elastic/elasticsearch/issues/53101

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Field } from '../../fields/field';
+
+const DEFAULT_SCALING_FACTOR = 1000;
+
+interface Properties {
+  [key: string]: any;
+}
+
+export function getDefaultProperties(field: Field): Properties {
+  const properties: Properties = {};
+
+  if (field.index !== undefined) {
+    properties.index = field.index;
+  }
+  if (field.doc_values !== undefined) {
+    properties.doc_values = field.doc_values;
+  }
+  if (field.copy_to) {
+    properties.copy_to = field.copy_to;
+  }
+
+  return properties;
+}
+
+export function scaledFloat(field: Field): Properties {
+  const fieldProps = getDefaultProperties(field);
+  fieldProps.type = 'scaled_float';
+  fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR;
+  if (field.metric_type) {
+    fieldProps.time_series_metric = field.metric_type;
+  }
+
+  return fieldProps;
+}
+
+export function histogram(field: Field): Properties {
+  const fieldProps = getDefaultProperties(field);
+  fieldProps.type = 'histogram';
+
+  return fieldProps;
+}

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -137,6 +137,17 @@ describe('EPM template', () => {
     expect(mappings).toMatchSnapshot(path.basename(ymlPath));
   });
 
+  it('tests loading cockroachdb_dynamic_templates.yml', () => {
+    const ymlPath = path.join(__dirname, '../../fields/tests/cockroachdb_dynamic_templates.yml');
+    const fieldsYML = readFileSync(ymlPath, 'utf-8');
+    const fields: Field[] = safeLoad(fieldsYML);
+    const processedFields = processFields(fields);
+
+    const mappings = generateMappings(processedFields);
+
+    expect(mappings).toMatchSnapshot(path.basename(ymlPath));
+  });
+
   it('tests processing long field with index false', () => {
     const longWithIndexFalseYml = `
 - name: longIndexFalse

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -24,6 +24,8 @@ import {
 import { getESAssetMetadata } from '../meta';
 import { retryTransientEsErrors } from '../retry';
 
+import { getDefaultProperties, histogram, scaledFloat } from './mappings';
+
 interface Properties {
   [key: string]: any;
 }
@@ -40,7 +42,7 @@ export interface CurrentDataStream {
   replicated: boolean;
   indexTemplate: IndexTemplate;
 }
-const DEFAULT_SCALING_FACTOR = 1000;
+
 const DEFAULT_IGNORE_ABOVE = 1024;
 
 // see discussion in https://github.com/elastic/kibana/issues/88307
@@ -103,6 +105,55 @@ export function getTemplate({
  * @param fields
  */
 export function generateMappings(fields: Field[]): IndexTemplateMappings {
+  const dynamicTemplates: Array<{ [k: string]: Properties }> = [];
+  const dynamicTemplateNames = new Set<string>();
+
+  const { properties } = _generateMappings(fields, {
+    addDynamicMapping: (arg1: any) => {
+      const name = arg1.path;
+      if (dynamicTemplateNames.has(name)) {
+        return;
+      }
+
+      const dynamicTemplate: Properties = {
+        mapping: arg1.properties,
+      };
+
+      if (arg1.matchingType) {
+        dynamicTemplate.match_mapping_type = arg1.matchingType;
+      }
+
+      if (arg1.pathMatch) {
+        dynamicTemplate.path_match = arg1.pathMatch;
+      }
+      dynamicTemplateNames.add(name);
+      dynamicTemplates.push({ [arg1.path]: dynamicTemplate });
+    },
+  });
+
+  return dynamicTemplates.length
+    ? {
+        properties,
+        dynamic_templates: dynamicTemplates,
+      }
+    : { properties };
+}
+
+/**
+ * Generate mapping takes the given nested fields array and creates the Elasticsearch
+ * mapping properties out of it.
+ *
+ * This assumes that all fields with dotted.names have been expanded in a previous step.
+ *
+ * @param fields
+ */
+function _generateMappings(
+  fields: Field[],
+  ctx: {
+    addDynamicMapping: any;
+    groupFieldName?: string;
+  }
+): IndexTemplateMappings {
   const props: Properties = {};
   // TODO: this can happen when the fields property in fields.yml is present but empty
   // Maybe validation should be moved to fields/field.ts
@@ -115,11 +166,24 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
 
       switch (type) {
         case 'group':
-          fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) };
+          fieldProps = {
+            ..._generateMappings(field.fields!, {
+              ...ctx,
+              groupFieldName: ctx.groupFieldName
+                ? `${ctx.groupFieldName}.${field.name}`
+                : field.name,
+            }),
+            ...generateDynamicAndEnabled(field),
+          };
           break;
         case 'group-nested':
           fieldProps = {
-            ...generateMappings(field.fields!),
+            ..._generateMappings(field.fields!, {
+              ...ctx,
+              groupFieldName: ctx.groupFieldName
+                ? `${ctx.groupFieldName}.${field.name}`
+                : field.name,
+            }),
             ...generateNestedProps(field),
             type: 'nested',
           };
@@ -128,11 +192,7 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
           fieldProps.type = 'long';
           break;
         case 'scaled_float':
-          fieldProps.type = 'scaled_float';
-          fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR;
-          if (field.metric_type) {
-            fieldProps.time_series_metric = field.metric_type;
-          }
+          fieldProps = scaledFloat(field);
           break;
         case 'text':
           const textMapping = generateTextMapping(field);
@@ -163,6 +223,46 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
           break;
         case 'object':
           fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' };
+          const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
+          const pathMatch = path.includes('*') ? path : `${path}.*`;
+
+          let dynProperties: Properties = getDefaultProperties(field);
+          let matchingType: string | undefined;
+          switch (field.object_type) {
+            case 'histogram':
+              dynProperties = histogram(field);
+              matchingType = field.object_type_mapping_type ?? '*';
+              break;
+            case 'text':
+              dynProperties.type = field.object_type;
+              matchingType = field.object_type_mapping_type ?? 'string';
+              break;
+            case 'keyword':
+              dynProperties.type = field.object_type;
+              matchingType = field.object_type_mapping_type ?? 'string';
+              break;
+            case 'byte':
+            case 'double':
+            case 'float':
+            case 'long':
+            case 'short':
+            case 'boolean':
+              dynProperties = {
+                type: field.object_type,
+              };
+              matchingType = field.object_type_mapping_type ?? field.object_type;
+            default:
+              break;
+          }
+
+          if (dynProperties && matchingType) {
+            ctx.addDynamicMapping({
+              path,
+              pathMatch,
+              matchingType,
+              properties: dynProperties,
+            });
+          }
           break;
         case 'nested':
           fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' };
@@ -295,22 +395,6 @@ function generateWildcardMapping(field: Field): IndexTemplateMapping {
   return mapping;
 }
 
-function getDefaultProperties(field: Field): Properties {
-  const properties: Properties = {};
-
-  if (field.index !== undefined) {
-    properties.index = field.index;
-  }
-  if (field.doc_values !== undefined) {
-    properties.doc_values = field.doc_values;
-  }
-  if (field.copy_to) {
-    properties.copy_to = field.copy_to;
-  }
-
-  return properties;
-}
-
 /**
  * Generates the template name out of the given information
  */

x-pack/plugins/fleet/server/services/epm/fields/field.ts

@@ -30,6 +30,7 @@ export interface Field {
   search_analyzer?: string;
   ignore_above?: number;
   object_type?: string;
+  object_type_mapping_type?: string;
   scaling_factor?: number;
   dynamic?: 'strict' | boolean;
   include_in_parent?: boolean;