Set default index mode for TimeSeries to null (#98808)

* Skip segment for MatchNoDocsQuery filters.

When a query of a filter gets rewritten to MatchNoDocsQuery, segments
will not produce any results. We can therefore skip processing such
segments.

This applies to FilterByFilterAggregator and FiltersAggregator, as well
as to TermsAggregator when it uses StringTermsAggregatorFromFilters
internally; the latter is an adapter aggregator to
FilterByFilterAggregator.

Fixes #94637

* Update docs/changelog/98295.yaml

* Check all filters for `MatchNoDocsQuery`.

* Skip optimization when 'other' bucket is requested.

* Revert "Set default index mode for TimeSeries to `null` (#98586)"

This reverts commit 56abb86.

* Revert "Rollback of #98586 (#98805)"

This reverts commit e370194.

* Skip updating source when missing synthetic mode

* Update docs/changelog/98808.yaml

* Skip matching assert in MapperService too

* Refine the assert

* Extend versions before 8.6, when TS had no synthetic source

* Add source field mapping for non-synthetic TSDB

* Delete 98586.yaml

Duplicate changelog

* Add comment to TSDB_NO_SYNTHETIC mapping

* Spotless fix

* Add yaml test

* Fix version skip in yaml test

docs/changelog/98808.yaml

@@ -0,0 +1,6 @@
+pr: 98808
+summary: Set default index mode for `TimeSeries` to `null`
+area: Aggregations
+type: enhancement
+issues:
+ - 97429

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -415,6 +415,37 @@ nested fields:
                             type: long
                             time_series_metric: gauge
 
+---
+"Synthetic source":
+  - skip:
+      version: " - 8.9.99"
+      reason: "Synthetic source shows up in the mapping in 8.10 and on"
+
+  - do:
+      indices.create:
+        index: tsdb-synthetic
+        body:
+          settings:
+            number_of_replicas: 0
+            mode: time_series
+            routing_path: [field1]
+            time_series:
+              start_time: 2000-01-01T00:00:00Z
+              end_time: 2099-12-31T23:59:59Z
+          mappings:
+            properties:
+              field1:
+                type: keyword
+                time_series_dimension: true
+              field2:
+                type: long
+                time_series_metric: gauge
+
+  - do:
+      indices.get_mapping: {}
+
+  - match: {tsdb-synthetic.mappings._source.mode: synthetic}
+
 ---
 regular source:
   - skip:

server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 
 import java.util.List;
 
@@ -30,18 +31,30 @@ public static DocumentMapper createEmpty(MapperService mapperService) {
         );
         MetadataFieldMapper[] metadata = mapperService.getMetadataMappers().values().toArray(new MetadataFieldMapper[0]);
         Mapping mapping = new Mapping(root, metadata, null);
-        return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent());
+        return new DocumentMapper(mapperService.documentParser(), mapping, mapping.toCompressedXContent(), IndexVersion.current());
     }
 
-    DocumentMapper(DocumentParser documentParser, Mapping mapping, CompressedXContent source) {
+    DocumentMapper(DocumentParser documentParser, Mapping mapping, CompressedXContent source, IndexVersion version) {
         this.documentParser = documentParser;
         this.type = mapping.getRoot().name();
         this.mappingLookup = MappingLookup.fromMapping(mapping);
         this.mappingSource = source;
-        assert mapping.toCompressedXContent().equals(source)
+
+        assert mapping.toCompressedXContent().equals(source) || isSyntheticSourceMalformed(source, version)
             : "provided source [" + source + "] differs from mapping [" + mapping.toCompressedXContent() + "]";
     }
 
+    /**
+     * Indexes built at v.8.7 were missing an explicit entry for synthetic_source.
+     * This got restored in v.8.10 to avoid confusion. The change is only restricted to mapping printout, it has no
+     * functional effect as the synthetic source already applies.
+     */
+    boolean isSyntheticSourceMalformed(CompressedXContent source, IndexVersion version) {
+        return sourceMapper().isSynthetic()
+            && source.string().contains("\"_source\":{\"mode\":\"synthetic\"}") == false
+            && version.onOrBefore(IndexVersion.V_8_10_0);
+    }
+
     public Mapping mapping() {
         return mappingLookup.getMapping();
     }

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -348,13 +348,15 @@ boolean assertNoUpdateRequired(final IndexMetadata newIndexMetadata) {
             Mapping newMapping = parseMapping(mapping.type(), mapping.source());
             final CompressedXContent currentSource = this.mapper.mappingSource();
             final CompressedXContent newSource = newMapping.toCompressedXContent();
-            if (Objects.equals(currentSource, newSource) == false) {
+            if (Objects.equals(currentSource, newSource) == false
+                && mapper.isSyntheticSourceMalformed(currentSource, indexVersionCreated) == false) {
                 throw new IllegalStateException(
                     "expected current mapping [" + currentSource + "] to be the same as new mapping [" + newSource + "]"
                 );
             }
         }
         return true;
+
     }
 
     public void merge(IndexMetadata indexMetadata, MergeReason reason) {
@@ -386,7 +388,7 @@ public DocumentMapper merge(String type, CompressedXContent mappingSource, Merge
     }
 
     private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, CompressedXContent mappingSource) {
-        DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource);
+        DocumentMapper newMapper = new DocumentMapper(documentParser, mapping, mappingSource, indexVersionCreated);
         newMapper.validate(indexSettings, reason != MergeReason.MAPPING_RECOVERY);
         return newMapper;
     }

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.search.lookup.Source;
@@ -61,6 +62,18 @@ private enum Mode {
         IndexMode.TIME_SERIES
     );
 
+    /*
+     * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below
+     * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version.
+     */
+    private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper(
+        null,
+        Explicit.IMPLICIT_TRUE,
+        Strings.EMPTY_ARRAY,
+        Strings.EMPTY_ARRAY,
+        IndexMode.TIME_SERIES
+    );
+
     public static class Defaults {
         public static final String NAME = SourceFieldMapper.NAME;
 
@@ -86,10 +99,15 @@ public static class Builder extends MetadataFieldMapper.Builder {
             .setMergeValidator(
                 (previous, current, conflicts) -> (previous.value() == current.value()) || (previous.value() && current.value() == false)
             );
+
+        /*
+         * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic
+         * source mode.
+         */
         private final Parameter<Mode> mode = new Parameter<>(
             "mode",
             true,
-            () -> getIndexMode() == IndexMode.TIME_SERIES ? Mode.SYNTHETIC : null,
+            () -> null,
             (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)),
             m -> toType(m).enabled.explicit() ? null : toType(m).mode,
             (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)),
@@ -136,10 +154,11 @@ private boolean isDefault() {
 
         @Override
         public SourceFieldMapper build() {
-            if (enabled.getValue().explicit() && mode.get() != null) {
+            if (enabled.getValue().explicit()) {
                 if (indexMode == IndexMode.TIME_SERIES) {
                     throw new MapperParsingException("Time series indices only support synthetic source");
-                } else {
+                }
+                if (mode.get() != null) {
                     throw new MapperParsingException("Cannot set both [mode] and [enabled] parameters");
                 }
             }
@@ -165,7 +184,9 @@ private IndexMode getIndexMode() {
     }
 
     public static final TypeParser PARSER = new ConfigurableTypeParser(
-        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? TSDB_DEFAULT : DEFAULT,
+        c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES
+            ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT
+            : DEFAULT,
         c -> new Builder(c.getIndexSettings().getMode())
     );
 
@@ -217,7 +238,7 @@ private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] include
         this.sourceFilter = buildSourceFilter(includes, excludes);
         this.includes = includes;
         this.excludes = excludes;
-        if (this.sourceFilter != null && mode == Mode.SYNTHETIC) {
+        if (this.sourceFilter != null && (mode == Mode.SYNTHETIC || indexMode == IndexMode.TIME_SERIES)) {
             throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source");
         }
         this.complete = stored() && sourceFilter == null;

server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java

@@ -69,7 +69,7 @@ public void testAddFields() throws Exception {
         assertThat(stage1.mappers().getMapper("obj1.prop1"), nullValue());
         // but merged should
         DocumentParser documentParser = new DocumentParser(null, null, () -> DocumentParsingObserver.EMPTY_INSTANCE);
-        DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent());
+        DocumentMapper mergedMapper = new DocumentMapper(documentParser, merged, merged.toCompressedXContent(), IndexVersion.current());
         assertThat(mergedMapper.mappers().getMapper("age"), notNullValue());
         assertThat(mergedMapper.mappers().getMapper("obj1.prop1"), notNullValue());
     }

server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.CompositeFieldScript;
@@ -2580,7 +2581,12 @@ same name need to be part  of the same mappings (hence the same document). If th
 
         // merge without going through toXContent and reparsing, otherwise the potential leaf path issue gets fixed on its own
         Mapping newMapping = MapperService.mergeMappings(mapperService.documentMapper(), mapping, MapperService.MergeReason.MAPPING_UPDATE);
-        DocumentMapper newDocMapper = new DocumentMapper(mapperService.documentParser(), newMapping, newMapping.toCompressedXContent());
+        DocumentMapper newDocMapper = new DocumentMapper(
+            mapperService.documentParser(),
+            newMapping,
+            newMapping.toCompressedXContent(),
+            IndexVersion.current()
+        );
         ParsedDocument doc2 = newDocMapper.parse(source("""
             {
               "foo" : {

server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
@@ -227,4 +228,14 @@ public void testSyntheticUpdates() throws Exception {
         assertFalse(mapper.enabled());
         assertFalse(mapper.isSynthetic());
     }
+
+    public void testSyntheticSourceInTimeSeries() throws IOException {
+        XContentBuilder mapping = fieldMapping(b -> {
+            b.field("type", "keyword");
+            b.field("time_series_dimension", true);
+        });
+        DocumentMapper mapper = createTimeSeriesModeDocumentMapper(mapping);
+        assertTrue(mapper.sourceMapper().isSynthetic());
+        assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
+    }
 }