diff --git a/docs/changelog/98808.yaml b/docs/changelog/98808.yaml new file mode 100644 index 0000000000000..100b0ec39cbb4 --- /dev/null +++ b/docs/changelog/98808.yaml @@ -0,0 +1,6 @@ +pr: 98808 +summary: Set default index mode for `TimeSeries` to `null` +area: Aggregations +type: enhancement +issues: + - 97429 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index bfca9c79bc6fd..21f30b6828f36 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -415,6 +415,37 @@ nested fields: type: long time_series_metric: gauge +--- +"Synthetic source": + - skip: + version: " - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on" + + - do: + indices.create: + index: tsdb-synthetic + body: + settings: + number_of_replicas: 0 + mode: time_series + routing_path: [field1] + time_series: + start_time: 2000-01-01T00:00:00Z + end_time: 2099-12-31T23:59:59Z + mappings: + properties: + field1: + type: keyword + time_series_dimension: true + field2: + type: long + time_series_metric: gauge + + - do: + indices.get_mapping: {} + + - match: {tsdb-synthetic.mappings._source.mode: synthetic} + --- regular source: - skip: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 2372210160c80..afc5cf20cae9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import java.util.List; @@ -30,18 +31,30 @@ public static DocumentMapper createEmpty(MapperService mapperService) { ); MetadataFieldMapper[] metadata = mapperService.getMetadataMappers().values().toArray(new MetadataFieldMapper[0]); Mapping mapping = new Mapping(root, metadata, null); - return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent()); + return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent(), IndexVersion.current()); } - DocumentMapper(DocumentParser documentParser, Mapping mapping, CompressedXContent source) { + DocumentMapper(DocumentParser documentParser, Mapping mapping, CompressedXContent source, IndexVersion version) { this.documentParser = documentParser; this.type = mapping.getRoot().name(); this.mappingLookup = MappingLookup.fromMapping(mapping); this.mappingSource = source; - assert mapping.toCompressedXContent().equals(source) + + assert mapping.toCompressedXContent().equals(source) || isSyntheticSourceMalformed(source, version) : "provided source [" + source + "] differs from mapping [" + mapping.toCompressedXContent() + "]"; } + /** + * Indexes built at v.8.7 were missing an explicit entry for synthetic_source. + * This got restored in v.8.10 to avoid confusion. The change is only restricted to mapping printout, it has no + * functional effect as the synthetic source already applies. + */ + boolean isSyntheticSourceMalformed(CompressedXContent source, IndexVersion version) { + return sourceMapper().isSynthetic() + && source.string().contains("\"_source\":{\"mode\":\"synthetic\"}") == false + && version.onOrBefore(IndexVersion.V_8_10_0); + } + public Mapping mapping() { return mappingLookup.getMapping(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 4e4e09fb8f882..d19e0bc716a9a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -348,13 +348,15 @@ boolean assertNoUpdateRequired(final IndexMetadata newIndexMetadata) { Mapping newMapping = parseMapping(mapping.type(), mapping.source()); final CompressedXContent currentSource = this.mapper.mappingSource(); final CompressedXContent newSource = newMapping.toCompressedXContent(); - if (Objects.equals(currentSource, newSource) == false) { + if (Objects.equals(currentSource, newSource) == false + && mapper.isSyntheticSourceMalformed(currentSource, indexVersionCreated) == false) { throw new IllegalStateException( "expected current mapping [" + currentSource + "] to be the same as new mapping [" + newSource + "]" ); } } return true; + } public void merge(IndexMetadata indexMetadata, MergeReason reason) { @@ -386,7 +388,7 @@ public DocumentMapper merge(String type, CompressedXContent mappingSource, Merge } private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, CompressedXContent mappingSource) { - DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource); + DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource, indexVersionCreated); newMapper.validate(indexSettings, reason != MergeReason.MAPPING_RECOVERY); return newMapper; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 2a1c5d8886212..ee709be35bc82 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.Source; @@ -61,6 +62,18 @@ private enum Mode { IndexMode.TIME_SERIES ); + /* + * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below + * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. + */ + private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES + ); + public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; @@ -86,10 +99,15 @@ public static class Builder extends MetadataFieldMapper.Builder { .setMergeValidator( (previous, current, conflicts) -> (previous.value() == current.value()) || (previous.value() && current.value() == false) ); + + /* + * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic + * source mode. + */ private final Parameter mode = new Parameter<>( "mode", true, - () -> getIndexMode() == IndexMode.TIME_SERIES ? Mode.SYNTHETIC : null, + () -> null, (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), m -> toType(m).enabled.explicit() ? null : toType(m).mode, (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), @@ -136,10 +154,11 @@ private boolean isDefault() { @Override public SourceFieldMapper build() { - if (enabled.getValue().explicit() && mode.get() != null) { + if (enabled.getValue().explicit()) { if (indexMode == IndexMode.TIME_SERIES) { throw new MapperParsingException("Time series indices only support synthetic source"); - } else { + } + if (mode.get() != null) { throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters"); } } @@ -165,7 +184,9 @@ private IndexMode getIndexMode() { } public static final TypeParser PARSER = new ConfigurableTypeParser( - c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT, + c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES + ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT + : DEFAULT, c -> new Builder(c.getIndexSettings().getMode()) ); @@ -217,7 +238,7 @@ private SourceFieldMapper(Mode mode, Explicit enabled, String[] include this.sourceFilter = buildSourceFilter(includes, excludes); this.includes = includes; this.excludes = excludes; - if (this.sourceFilter != null && mode == Mode.SYNTHETIC) { + if (this.sourceFilter != null && (mode == Mode.SYNTHETIC || indexMode == IndexMode.TIME_SERIES)) { throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source"); } this.complete = stored() && sourceFilter == null; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 19bc4bc3947f1..a86a417a37632 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -69,7 +69,7 @@ public void testAddFields() throws Exception { assertThat(stage1.mappers().getMapper("obj1.prop1"), nullValue()); // but merged should DocumentParser documentParser = new DocumentParser(null, null, () -> DocumentParsingObserver.EMPTY_INSTANCE); - DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent()); + DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent(), IndexVersion.current()); assertThat(mergedMapper.mappers().getMapper("age"), notNullValue()); assertThat(mergedMapper.mappers().getMapper("obj1.prop1"), notNullValue()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 5c421ee10083f..a3706b7ddab18 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.CompositeFieldScript; @@ -2580,7 +2581,12 @@ same name need to be part of the same mappings (hence the same document). If th // merge without going through toXContent and reparsing, otherwise the potential leaf path issue gets fixed on its own Mapping newMapping = MapperService.mergeMappings(mapperService.documentMapper(), mapping, MapperService.MergeReason.MAPPING_UPDATE); - DocumentMapper newDocMapper = new DocumentMapper(mapperService.documentParser(), newMapping, newMapping.toCompressedXContent()); + DocumentMapper newDocMapper = new DocumentMapper( + mapperService.documentParser(), + newMapping, + newMapping.toCompressedXContent(), + IndexVersion.current() + ); ParsedDocument doc2 = newDocMapper.parse(source(""" { "foo" : { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index d8a75c80177f0..f683cb60c87c3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -227,4 +228,14 @@ public void testSyntheticUpdates() throws Exception { assertFalse(mapper.enabled()); assertFalse(mapper.isSynthetic()); } + + public void testSyntheticSourceInTimeSeries() throws IOException { + XContentBuilder mapping = fieldMapping(b -> { + b.field("type", "keyword"); + b.field("time_series_dimension", true); + }); + DocumentMapper mapper = createTimeSeriesModeDocumentMapper(mapping); + assertTrue(mapper.sourceMapper().isSynthetic()); + assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString()); + } }