From e2281a1158976bd5fac73c8e77e9a67c769dd3b2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:44:03 +0200 Subject: [PATCH] Introduce an `IndexSettingsProvider` to inject logsdb index mode (#113505) Here we introduce a new implementation of `IndexSettingProvider` whose goal is to "inject" the `index.mode` setting with value `logsdb` when a cluster setting `cluster.logsdb.enabled` is `true`. We also make sure that: * the existing `index.mode` is not set * the datastream name matches the `logs-*-*` pattern * `logs@settings` component template is used --- .../LogsIndexModeDisabledRestTestIT.java | 70 +++- .../LogsIndexModeEnabledRestTestIT.java | 87 ++++- .../logsdb/LogsIndexModeRestTestIT.java | 19 +- .../core/src/main/java/module-info.java | 1 + .../cluster/settings/ClusterSettings.java | 19 + .../main/resources/logs@settings-logsdb.json | 26 ++ .../src/main/resources/logs@settings.json | 1 - .../xpack/logsdb/LogsDBPlugin.java | 12 +- .../LogsdbIndexModeSettingsProvider.java | 89 +++++ .../LogsdbIndexModeSettingsProviderTests.java | 326 ++++++++++++++++++ .../stack/LegacyStackTemplateRegistry.java | 7 +- .../xpack/stack/StackPlugin.java | 2 +- .../xpack/stack/StackTemplateRegistry.java | 21 +- 13 files changed, 648 insertions(+), 32 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json create mode 100644 x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java create mode 100644 x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java index c9818a34169de..123ca3b806153 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.hamcrest.Matchers; @@ -23,6 +24,22 @@ public class LogsIndexModeDisabledRestTestIT extends LogsIndexModeRestTestIT { + private static final String MAPPINGS = """ + { + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + @ClassRule() public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) @@ -50,8 +67,59 @@ public void setup() throws Exception { public void testLogsSettingsIndexModeDisabled() throws IOException { assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); } + public void testTogglingLogsdb() throws IOException { + putComponentTemplate(client, "logs@settings", MAPPINGS); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexModeBefore = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeBefore, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "true")); + final String indexModeAfter = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeAfter, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String indexModeLater = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 1), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeLater, equalTo(IndexMode.LOGSDB.getName())); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", "false")); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String indexModeFinal = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 2), + IndexSettings.MODE.getKey() + ); + assertThat(indexModeFinal, Matchers.not(equalTo(IndexMode.LOGSDB.getName()))); + + } + + public void testEnablingLogsdb() throws IOException { + putComponentTemplate(client, "logs@settings", MAPPINGS); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", true)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName())); + assertOK(putClusterSetting(client, "cluster.logsdb.enabled", false)); + } + } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java index d7bdf54007d69..a024a2c0f303c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -10,8 +10,10 @@ package org.elasticsearch.datastreams.logsdb; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.hamcrest.Matchers; @@ -179,7 +181,11 @@ public void setup() throws Exception { public void testCreateDataStream() throws IOException { assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); assertOK(createDataStream(client, "logs-custom-dev")); - final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); assertThat(indexMode, equalTo(IndexMode.LOGSDB.getName())); } @@ -224,4 +230,83 @@ public void testRolloverDataStream() throws IOException { assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); } + + public void testLogsAtSettingWithStandardOverride() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "mode": "standard" + } + } + } + } + """)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.STANDARD.getName())); + } + + public void testLogsAtSettingWithTimeSeriesOverride() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "routing_path": [ "hostname" ], + "mode": "time_series", + "sort.field": [], + "sort.order": [] + } + }, + "mappings": { + "properties": { + "hostname": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + } + """)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting( + client, + getDataStreamBackingIndex(client, "logs-custom-dev", 0), + IndexSettings.MODE.getKey() + ); + assertThat(indexMode, equalTo(IndexMode.TIME_SERIES.getName())); + } + + public void testLogsAtSettingWithTimeSeriesOverrideFailure() { + // NOTE: apm@settings defines sorting on @timestamp and template composition results in index.mode "time_series" + // with a non-allowed index.sort.field '@timestamp'. This fails at template composition stage before the index is even created. + final ResponseException ex = assertThrows(ResponseException.class, () -> putComponentTemplate(client, "logs@custom", """ + { + "template": { + "settings": { + "index": { + "routing_path": [ "hostname" ], + "mode": "time_series" + } + }, + "mappings": { + "properties": { + "hostname": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + } + """)); + assertTrue(ex.getMessage().contains("[index.mode=time_series] is incompatible with [index.sort.field]")); + } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java index 7d65207794598..22ac2b6d7d239 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -33,10 +33,16 @@ protected static void waitForLogs(RestClient client) throws Exception { }); } - protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) + protected static Response putComponentTemplate(final RestClient client, final String componentTemplate, final String contends) throws IOException { - final Request request = new Request("PUT", "/_component_template/" + templateName); - request.setJsonEntity(mappings); + final Request request = new Request("PUT", "/_component_template/" + componentTemplate); + request.setJsonEntity(contends); + return client.performRequest(request); + } + + protected static Response putTemplate(final RestClient client, final String template, final String contents) throws IOException { + final Request request = new Request("PUT", "/_index_template/" + template); + request.setJsonEntity(contents); return client.performRequest(request); } @@ -87,4 +93,11 @@ protected static Response bulkIndex(final RestClient client, final String dataSt bulkRequest.addParameter("refresh", "true"); return client.performRequest(bulkRequest); } + + protected static Response putClusterSetting(final RestClient client, final String settingName, final Object settingValue) + throws IOException { + final Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity("{ \"transient\": { \"" + settingName + "\": " + settingValue + " } }"); + return client.performRequest(request); + } } diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index 72436bb9d5171..47848310fe781 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -228,6 +228,7 @@ exports org.elasticsearch.xpack.core.watcher.trigger; exports org.elasticsearch.xpack.core.watcher.watch; exports org.elasticsearch.xpack.core.watcher; + exports org.elasticsearch.xpack.cluster.settings; provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber with diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java new file mode 100644 index 0000000000000..1127889783f16 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/settings/ClusterSettings.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.cluster.settings; + +import org.elasticsearch.common.settings.Setting; + +public class ClusterSettings { + public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( + "cluster.logsdb.enabled", + false, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json new file mode 100644 index 0000000000000..eabdd6fb9fad2 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json @@ -0,0 +1,26 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs" + }, + "mode": "logsdb", + "codec": "best_compression", + "mapping": { + "ignore_malformed": true, + "total_fields": { + "ignore_dynamic_beyond_limit": true + } + }, + "default_pipeline": "logs@default-pipeline" + } + } + }, + "_meta": { + "description": "default settings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json index e9a9f2611ad7b..ca2659b8d8dea 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json @@ -5,7 +5,6 @@ "lifecycle": { "name": "logs" }, - "mode": "${xpack.stack.template.logsdb.index.mode}", "codec": "best_compression", "mapping": { "ignore_malformed": true, diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index e38f953be96a3..833555a7884ea 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.List; +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING; public class LogsDBPlugin extends Plugin { @@ -24,9 +25,12 @@ public class LogsDBPlugin extends Plugin { private final Settings settings; private final SyntheticSourceLicenseService licenseService; + private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider; + public LogsDBPlugin(Settings settings) { this.settings = settings; this.licenseService = new SyntheticSourceLicenseService(settings); + this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(settings); } @Override @@ -34,6 +38,10 @@ public Collection createComponents(PluginServices services) { licenseService.setLicenseState(XPackPlugin.getSharedLicenseState()); var clusterSettings = services.clusterService().getClusterSettings(); clusterSettings.addSettingsUpdateConsumer(FALLBACK_SETTING, licenseService::setSyntheticSourceFallback); + clusterSettings.addSettingsUpdateConsumer( + CLUSTER_LOGSDB_ENABLED, + logsdbIndexModeSettingsProvider::updateClusterIndexModeLogsdbEnabled + ); // Nothing to share here: return super.createComponents(services); } @@ -43,11 +51,11 @@ public Collection getAdditionalIndexSettingProviders(Index if (DiscoveryNode.isStateless(settings)) { return List.of(); } - return List.of(new SyntheticSourceIndexSettingsProvider(licenseService)); + return List.of(new SyntheticSourceIndexSettingsProvider(licenseService), logsdbIndexModeSettingsProvider); } @Override public List> getSettings() { - return List.of(FALLBACK_SETTING); + return List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED); } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java new file mode 100644 index 0000000000000..3f6bb66dfa438 --- /dev/null +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettings; + +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; + +final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { + private static final String LOGS_PATTERN = "logs-*-*"; + private volatile boolean isLogsdbEnabled; + + LogsdbIndexModeSettingsProvider(final Settings settings) { + this.isLogsdbEnabled = CLUSTER_LOGSDB_ENABLED.get(settings); + } + + void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) { + this.isLogsdbEnabled = isLogsdbEnabled; + } + + @Override + public Settings getAdditionalIndexSettings( + final String indexName, + final String dataStreamName, + boolean isTimeSeries, + final Metadata metadata, + final Instant resolvedAt, + final Settings settings, + final List combinedTemplateMappings + ) { + if (isLogsdbEnabled == false || dataStreamName == null) { + return Settings.EMPTY; + } + + final IndexMode indexMode = resolveIndexMode(settings.get(IndexSettings.MODE.getKey())); + if (indexMode != null) { + return Settings.EMPTY; + } + + if (usesLogsAtSettingsComponentTemplate(metadata, dataStreamName) && matchesLogsPattern(dataStreamName)) { + return Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build(); + } + + return Settings.EMPTY; + } + + private static boolean matchesLogsPattern(final String name) { + return Regex.simpleMatch(LOGS_PATTERN, name); + } + + private IndexMode resolveIndexMode(final String mode) { + return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null; + } + + private boolean usesLogsAtSettingsComponentTemplate(final Metadata metadata, final String name) { + final String template = MetadataIndexTemplateService.findV2Template(metadata, name, false); + if (template == null) { + return false; + } + final ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template); + if (composableIndexTemplate == null) { + return false; + } + for (final String componentTemplate : composableIndexTemplate.composedOf()) { + if ("logs@settings".equals(componentTemplate)) { + return true; + } + } + return false; + } + +} diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java new file mode 100644 index 0000000000000..eeb5389644c02 --- /dev/null +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -0,0 +1,326 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; + +public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { + + public static final String DEFAULT_MAPPING = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "keyword" + }, + "host.name": { + "type": "keyword" + } + } + } + } + """; + + public void testLogsDbDisabled() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", false).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnIndexCreation() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + "logs-apache-production", + null, + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnExplicitStandardIndex() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testOnExplicitTimeSeriesIndex() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testNonLogsDataStream() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testWithoutLogsComponentTemplate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of()), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testWithLogsComponentTemplate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); + } + + public void testWithMultipleComponentTemplates() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); + } + + public void testWithCustomComponentTemplatesOnly() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testNonMatchingTemplateIndexPattern() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testCaseSensitivity() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "LOGS-apache-production", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testMultipleHyphensInDataStreamName() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", true).build() + ); + + final Settings additionalIndexSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production-eu", + false, + Metadata.EMPTY_METADATA, + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(additionalIndexSettings.isEmpty()); + } + + public void testBeforeAndAFterSettingUpdate() throws IOException { + final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + Settings.builder().put("cluster.logsdb.enabled", false).build() + ); + + final Settings beforeSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(beforeSettings.isEmpty()); + + provider.updateClusterIndexModeLogsdbEnabled(true); + + final Settings afterSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertIndexMode(afterSettings, IndexMode.LOGSDB.getName()); + + provider.updateClusterIndexModeLogsdbEnabled(false); + + final Settings laterSettings = provider.getAdditionalIndexSettings( + null, + "logs-apache-production", + false, + buildMetadata(List.of("*"), List.of("logs@settings")), + Instant.now().truncatedTo(ChronoUnit.SECONDS), + Settings.EMPTY, + List.of(new CompressedXContent(DEFAULT_MAPPING)) + ); + + assertTrue(laterSettings.isEmpty()); + } + + private static Metadata buildMetadata(final List indexPatterns, final List componentTemplates) throws IOException { + final Template template = new Template(Settings.EMPTY, new CompressedXContent(DEFAULT_MAPPING), null); + final ComposableIndexTemplate composableTemplate = ComposableIndexTemplate.builder() + .indexPatterns(indexPatterns) + .template(template) + .componentTemplates(componentTemplates) + .priority(1_000L) + .version(1L) + .build(); + return Metadata.builder() + .putCustom(ComposableIndexTemplateMetadata.TYPE, new ComposableIndexTemplateMetadata(Map.of("composable", composableTemplate))) + .build(); + } + + private void assertIndexMode(final Settings settings, final String expectedIndexMode) { + assertEquals(expectedIndexMode, settings.get(IndexSettings.MODE.getKey())); + } + +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java index 62d22c0c0a9cc..b2dc04c1178e4 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java @@ -51,12 +51,7 @@ public class LegacyStackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of( - "xpack.stack.template.deprecated", - "true", - "xpack.stack.template.logsdb.index.mode", - "standard" - ); + private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "true"); // General mappings conventions for any data that ends up in a data stream public static final String DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "data-streams-mappings"; diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java index cc127883652af..71d01798323d3 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -23,7 +23,7 @@ public StackPlugin(Settings settings) { @Override public List> getSettings() { - return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED, StackTemplateRegistry.CLUSTER_LOGSDB_ENABLED); + return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED); } @Override diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 592842f61eee8..b45f17e434388 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.features.FeatureService; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -36,6 +35,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.cluster.settings.ClusterSettings.CLUSTER_LOGSDB_ENABLED; + public class StackTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class); @@ -58,15 +59,6 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { Setting.Property.Dynamic ); - /** - * if index.mode "logsdb" is applied by default in logs@settings for 'logs-*-*' - */ - public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( - "cluster.logsdb.enabled", - false, - Setting.Property.NodeScope - ); - private final ClusterService clusterService; private final FeatureService featureService; private final Map componentTemplateConfigs; @@ -167,15 +159,10 @@ private Map loadComponentTemplateConfigs(boolean logs ), new IndexTemplateConfig( LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, - "/logs@settings.json", + logsDbEnabled ? "/logs@settings-logsdb.json" : "/logs@settings.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, - Map.of( - "xpack.stack.template.deprecated", - "false", - "xpack.stack.template.logsdb.index.mode", - logsDbEnabled ? IndexMode.LOGSDB.getName() : IndexMode.STANDARD.getName() - ) + Map.of("xpack.stack.template.deprecated", "false") ), new IndexTemplateConfig( METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,