Skip to content

Commit

Permalink
Introduce runtime section in mappings (#62906)
Browse files Browse the repository at this point in the history 
The runtime section is at the same level as the existing properties section. Its purpose is to hold runtime fields only. With the introduction of the runtime section, a runtime field can be defined by specifying its type (previously called runtime_type) and script.

```
PUT /my-index/_mappings
{
    "runtime" : {
        "day_of_week" : {
            "type" : "keyword",
            "script" : {
                "source" : "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
            }
        }
    },
    "properties" : {
        "timestamp" : {
            "type" : "date"
        }
    }
}
```

Fields defined in the runtime section can be updated at any time as they are not present in the lucene index. They get replaced entirely when they get updated.

Thanks to the introduction of the runtime section, runtime fields override existing mapped fields defined with the same name, similarly to runtime fields defined in the search request.

Relates to #59332
  • Loading branch information
@javanna
javanna authored Nov 12, 2020
1 parent db15c4d commit 3101293
Show file tree
Hide file tree
Showing 66 changed files with 1,845 additions and 1,508 deletions.
2 changes: 2 additions & 0 deletions .../elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ private static Map<String, FieldMappingMetadata> findFieldMappings(Predicate<Str
if (documentMapper == null) {
return Collections.emptyMap();
}
//TODO the logic here needs to be reworked to also include runtime fields. Though matching is against mappers rather
// than field types, and runtime fields are mixed with ordinary fields in FieldTypeLookup
Map<String, FieldMappingMetadata> fieldMappings = new HashMap<>();
final MappingLookup mappingLookup = documentMapper.mappers();
for (String field : request.fields()) {
Expand Down
22 changes: 11 additions & 11 deletions server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,15 +278,7 @@ public final FieldMapper merge(Mapper mergeWith) {
Conflicts conflicts = new Conflicts(name());
builder.merge((FieldMapper) mergeWith, conflicts);
conflicts.check();
return builder.build(parentPath(name()));
}

private static ContentPath parentPath(String name) {
int endPos = name.lastIndexOf(".");
if (endPos == -1) {
return new ContentPath(0);
}
return new ContentPath(name.substring(0, endPos));
return builder.build(Builder.parentPath(name()));
}

protected void checkIncomingMergeType(FieldMapper mergeWith) {
Expand Down Expand Up @@ -482,7 +474,7 @@ public List<String> copyToFields() {
/**
* Serializes a parameter
*/
protected interface Serializer<T> {
public interface Serializer<T> {
void serialize(XContentBuilder builder, String name, T value) throws IOException;
}

Expand Down Expand Up @@ -931,7 +923,7 @@ protected String buildFullName(ContentPath contentPath) {
/**
* Writes the current builder parameter values as XContent
*/
protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
for (Parameter<?> parameter : getParameters()) {
parameter.toXContent(builder, includeDefaults);
}
Expand Down Expand Up @@ -1011,6 +1003,14 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
validate();
}

protected static ContentPath parentPath(String name) {
int endPos = name.lastIndexOf(".");
if (endPos == -1) {
return new ContentPath(0);
}
return new ContentPath(name.substring(0, endPos));
}

// These parameters were previously *always* parsed by TypeParsers#parseField(), even if they
// made no sense; if we've got here, that means that they're not declared on a current mapper,
// and so we emit a deprecation warning rather than failing a previously working mapping.
Expand Down
9 changes: 7 additions & 2 deletions server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
* An immutable container for looking up {@link MappedFieldType}s by their name.
*/
final class FieldTypeLookup {

private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();

/**
Expand All @@ -47,7 +46,8 @@ final class FieldTypeLookup {
private final DynamicKeyFieldTypeLookup dynamicKeyLookup;

FieldTypeLookup(Collection<FieldMapper> fieldMappers,
Collection<FieldAliasMapper> fieldAliasMappers) {
Collection<FieldAliasMapper> fieldAliasMappers,
Collection<RuntimeFieldType> runtimeFieldTypes) {
Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();

for (FieldMapper fieldMapper : fieldMappers) {
Expand Down Expand Up @@ -77,6 +77,11 @@ final class FieldTypeLookup {
fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path));
}

for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) {
//this will override concrete fields with runtime fields that have the same name
fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType);
}

this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
}

Expand Down
10 changes: 8 additions & 2 deletions server/src/main/java/org/elasticsearch/index/mapper/Mapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ class ParserContext {

private final Function<String, SimilarityProvider> similarityLookupService;
private final Function<String, TypeParser> typeParsers;
private final Function<String, RuntimeFieldType.Parser> runtimeTypeParsers;
private final Version indexVersionCreated;
private final Supplier<QueryShardContext> queryShardContextSupplier;
private final DateFormatter dateFormatter;
Expand All @@ -69,6 +70,7 @@ class ParserContext {

public ParserContext(Function<String, SimilarityProvider> similarityLookupService,
Function<String, TypeParser> typeParsers,
Function<String, RuntimeFieldType.Parser> runtimeTypeParsers,
Version indexVersionCreated,
Supplier<QueryShardContext> queryShardContextSupplier,
DateFormatter dateFormatter,
Expand All @@ -78,6 +80,7 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
BooleanSupplier idFieldDataEnabled) {
this.similarityLookupService = similarityLookupService;
this.typeParsers = typeParsers;
this.runtimeTypeParsers = runtimeTypeParsers;
this.indexVersionCreated = indexVersionCreated;
this.queryShardContextSupplier = queryShardContextSupplier;
this.dateFormatter = dateFormatter;
Expand Down Expand Up @@ -132,6 +135,8 @@ public DateFormatter getDateFormatter() {

protected Function<String, TypeParser> typeParsers() { return typeParsers; }

protected Function<String, RuntimeFieldType.Parser> runtimeTypeParsers() { return runtimeTypeParsers; }

protected Function<String, SimilarityProvider> similarityLookupService() { return similarityLookupService; }

/**
Expand All @@ -147,8 +152,9 @@ public ParserContext createMultiFieldContext(ParserContext in) {

static class MultiFieldParserContext extends ParserContext {
MultiFieldParserContext(ParserContext in) {
super(in.similarityLookupService, in.typeParsers, in.indexVersionCreated, in.queryShardContextSupplier,
in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, in.idFieldDataEnabled);
super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
in.idFieldDataEnabled);
}

@Override
Expand Down
4 changes: 2 additions & 2 deletions server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
this.mapperRegistry = mapperRegistry;
Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction =
dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get,
indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings,
idFieldDataEnabled);
mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter,
scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled);
this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction);
Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers =
mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated());
Expand Down
18 changes: 9 additions & 9 deletions server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import java.util.stream.Stream;

public final class MappingLookup {

/** Full field name to mapper */
private final Map<String, Mapper> fieldMappers;
private final Map<String, ObjectMapper> objectMappers;
Expand All @@ -50,24 +49,24 @@ public static MappingLookup fromMapping(Mapping mapping) {
newFieldMappers.add(metadataMapper);
}
}
collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers);
return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length);
for (Mapper child : mapping.root) {
collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers);
}
return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers,
mapping.root.runtimeFieldTypes(), mapping.metadataMappers.length);
}

private static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers,
Collection<FieldMapper> fieldMappers,
Collection<FieldAliasMapper> fieldAliasMappers) {
if (mapper instanceof RootObjectMapper) {
// root mapper isn't really an object mapper
} else if (mapper instanceof ObjectMapper) {
if (mapper instanceof ObjectMapper) {
objectMappers.add((ObjectMapper)mapper);
} else if (mapper instanceof FieldMapper) {
fieldMappers.add((FieldMapper)mapper);
} else if (mapper instanceof FieldAliasMapper) {
fieldAliasMappers.add((FieldAliasMapper) mapper);
} else {
throw new IllegalStateException("Unrecognized mapper type [" +
mapper.getClass().getSimpleName() + "].");
throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "].");
}

for (Mapper child : mapper) {
Expand All @@ -78,6 +77,7 @@ private static void collect(Mapper mapper, Collection<ObjectMapper> objectMapper
public MappingLookup(Collection<FieldMapper> mappers,
Collection<ObjectMapper> objectMappers,
Collection<FieldAliasMapper> aliasMappers,
Collection<RuntimeFieldType> runtimeFieldTypes,
int metadataFieldCount) {
Map<String, Mapper> fieldMappers = new HashMap<>();
Map<String, Analyzer> indexAnalyzers = new HashMap<>();
Expand Down Expand Up @@ -114,7 +114,7 @@ public MappingLookup(Collection<FieldMapper> mappers,
}
}

this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers);
this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFieldTypes);

this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);
Expand Down
59 changes: 45 additions & 14 deletions server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.index.mapper;

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Strings;
Expand All @@ -34,11 +35,14 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;

import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
Expand All @@ -62,6 +66,7 @@ public static class Builder extends ObjectMapper.Builder {
protected Explicit<DateFormatter[]> dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
protected Explicit<Boolean> dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false);
protected Explicit<Boolean> numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false);
protected final Map<String, RuntimeFieldType> runtimeFieldTypes = new HashMap<>();

public Builder(String name, Version indexCreatedVersion) {
super(name, indexCreatedVersion);
Expand All @@ -83,6 +88,11 @@ public RootObjectMapper.Builder add(Mapper.Builder builder) {
return this;
}

public RootObjectMapper.Builder addRuntime(RuntimeFieldType runtimeFieldType) {
this.runtimeFieldTypes.put(runtimeFieldType.name(), runtimeFieldType);
return this;
}

@Override
public RootObjectMapper build(ContentPath contentPath) {
return (RootObjectMapper) super.build(contentPath);
Expand All @@ -92,7 +102,7 @@ public RootObjectMapper build(ContentPath contentPath) {
protected ObjectMapper createMapper(String name, String fullPath, Explicit<Boolean> enabled, Nested nested, Dynamic dynamic,
Map<String, Mapper> mappers, Version indexCreatedVersion) {
assert !nested.isNested();
return new RootObjectMapper(name, enabled, dynamic, mappers,
return new RootObjectMapper(name, enabled, dynamic, mappers, runtimeFieldTypes,
dynamicDateTimeFormatters,
dynamicTemplates,
dateDetection, numericDetection, indexCreatedVersion);
Expand Down Expand Up @@ -127,7 +137,7 @@ private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean pare
}
}

public static class TypeParser extends ObjectMapper.TypeParser {
static final class TypeParser extends ObjectMapper.TypeParser {

@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
Expand All @@ -146,8 +156,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
}

@SuppressWarnings("unchecked")
protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode,
ParserContext parserContext) {
private boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) {
if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
if (fieldNode instanceof List) {
List<DateFormatter> formatters = new ArrayList<>();
Expand Down Expand Up @@ -190,10 +199,8 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
String templateName = entry.getKey();
Map<String, Object> templateParams = (Map<String, Object>) entry.getValue();
DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams);
if (template != null) {
validateDynamicTemplate(parserContext, template);
templates.add(template);
}
validateDynamicTemplate(parserContext, template);
templates.add(template);
}
builder.dynamicTemplates(templates);
return true;
Expand All @@ -203,6 +210,13 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
} else if (fieldName.equals("numeric_detection")) {
builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true);
return true;
} else if (fieldName.equals("runtime")) {
if (fieldNode instanceof Map) {
RuntimeFieldType.parseRuntimeFields((Map<String, Object>) fieldNode, parserContext, builder::addRuntime);
return true;
} else {
throw new ElasticsearchParseException("runtime must be a map type");
}
}
return false;
}
Expand All @@ -212,11 +226,14 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
private Explicit<Boolean> dateDetection;
private Explicit<Boolean> numericDetection;
private Explicit<DynamicTemplate[]> dynamicTemplates;
private final Map<String, RuntimeFieldType> runtimeFieldTypes;

RootObjectMapper(String name, Explicit<Boolean> enabled, Dynamic dynamic, Map<String, Mapper> mappers,
Map<String, RuntimeFieldType> runtimeFieldTypes,
Explicit<DateFormatter[]> dynamicDateTimeFormatters, Explicit<DynamicTemplate[]> dynamicTemplates,
Explicit<Boolean> dateDetection, Explicit<Boolean> numericDetection, Version indexCreatedVersion) {
super(name, name, enabled, Nested.NO, dynamic, mappers, indexCreatedVersion);
this.runtimeFieldTypes = runtimeFieldTypes;
this.dynamicTemplates = dynamicTemplates;
this.dynamicDateTimeFormatters = dynamicDateTimeFormatters;
this.dateDetection = dateDetection;
Expand All @@ -236,23 +253,26 @@ public ObjectMapper mappingUpdate(Mapper mapper) {
return update;
}

public boolean dateDetection() {
boolean dateDetection() {
return this.dateDetection.value();
}

public boolean numericDetection() {
boolean numericDetection() {
return this.numericDetection.value();
}

public DateFormatter[] dynamicDateTimeFormatters() {
DateFormatter[] dynamicDateTimeFormatters() {
return dynamicDateTimeFormatters.value();
}

public DynamicTemplate[] dynamicTemplates() {
DynamicTemplate[] dynamicTemplates() {
return dynamicTemplates.value();
}

@SuppressWarnings("rawtypes")
Collection<RuntimeFieldType> runtimeFieldTypes() {
return runtimeFieldTypes.values();
}

public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) {
return findTemplateBuilder(context, name, matchType, null);
}
Expand All @@ -268,7 +288,6 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat
* @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format
* @return a mapper builder, or null if there is no template for such a field
*/
@SuppressWarnings("rawtypes")
private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) {
DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType);
if (dynamicTemplate == null) {
Expand Down Expand Up @@ -331,6 +350,8 @@ protected void doMerge(ObjectMapper mergeWith, MergeReason reason) {
this.dynamicTemplates = mergeWithObject.dynamicTemplates;
}
}

this.runtimeFieldTypes.putAll(mergeWithObject.runtimeFieldTypes);
}

@Override
Expand Down Expand Up @@ -361,6 +382,16 @@ protected void doXContent(XContentBuilder builder, ToXContent.Params params) thr
if (numericDetection.explicit() || includeDefaults) {
builder.field("numeric_detection", numericDetection.value());
}

if (runtimeFieldTypes.size() > 0) {
builder.startObject("runtime");
List<RuntimeFieldType> sortedRuntimeFieldTypes = runtimeFieldTypes.values().stream().sorted(
Comparator.comparing(RuntimeFieldType::name)).collect(Collectors.toList());
for (RuntimeFieldType fieldType : sortedRuntimeFieldTypes) {
fieldType.toXContent(builder, params);
}
builder.endObject();
}
}

private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext,
Expand Down
Loading

0 comments on commit 3101293

Please sign in to comment.