Deprecate dot-prefixed indices and composable template index patterns

This commit adds a module emitting a deprecation warning when a dot-prefixed index is manually or automatically created, or when a composable index template with an index pattern that uses a dot-prefix is created. This pattern warns that in the future these indices will not be allowed. In a future breaking change (10.0.0 maybe?) the deprecation can then be changed to an exception.

These deprecations are only displayed when a non-operator user is using the API (one that does not set the `X-elastic-product-origin` header).

modules/dot-prefix-validation/build.gradle

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  description 'Validation for dot-prefixed indices for non-operator users'
+  classname 'org.elasticsearch.validation.DotPrefixValidationPlugin'
+}
+
+restResources {
+  restApi {
+    include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex'
+  }
+}
+
+tasks.named("yamlRestTestV7CompatTest").configure { enabled = false };
+
+tasks.named('yamlRestTest') {
+  usesDefaultDistribution()
+}

modules/dot-prefix-validation/src/main/java/module-info.java

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module org.elasticsearch.validation {
+    requires org.elasticsearch.server;
+    requires org.elasticsearch.base;
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Set;
+
+public class AutoCreateDotValidator extends DotPrefixValidator<CreateIndexRequest> {
+    public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(CreateIndexRequest request) {
+        return Set.of(request.index());
+    }
+
+    @Override
+    public String actionName() {
+        return AutoCreateAction.NAME;
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Set;
+
+public class CreateIndexDotValidator extends DotPrefixValidator<CreateIndexRequest> {
+    public CreateIndexDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(CreateIndexRequest request) {
+        return Set.of(request.index());
+    }
+
+    @Override
+    public String actionName() {
+        return TransportCreateIndexAction.TYPE.name();
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.support.MappedActionFilter;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class DotPrefixValidationPlugin extends Plugin implements ActionPlugin {
+    private final AtomicReference<List<MappedActionFilter>> actionFilters = new AtomicReference<>();
+
+    public DotPrefixValidationPlugin() {}
+
+    @Override
+    public Collection<?> createComponents(PluginServices services) {
+        ThreadContext context = services.threadPool().getThreadContext();
+        ClusterService clusterService = services.clusterService();
+
+        actionFilters.set(
+            List.of(
+                new CreateIndexDotValidator(context, clusterService),
+                new AutoCreateDotValidator(context, clusterService),
+                new IndexTemplateDotValidator(context, clusterService)
+            )
+        );
+
+        return Set.of();
+    }
+
+    @Override
+    public Collection<MappedActionFilter> getMappedActionFilters() {
+        return actionFilters.get();
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.ActionFilterChain;
+import org.elasticsearch.action.support.MappedActionFilter;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationCategory;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.tasks.Task;
+
+import java.util.Optional;
+import java.util.Set;
+
+public abstract class DotPrefixValidator<RequestType> implements MappedActionFilter {
+    public static final Setting<Boolean> VALIDATE_DOT_PREFIXES = Setting.boolSetting(
+        "cluster.indices.validate_dot_prefixes",
+        true,
+        Setting.Property.NodeScope
+    );
+    DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class);
+
+    private final ThreadContext threadContext;
+    private final boolean isEnabled;
+
+    public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService) {
+        this.threadContext = threadContext;
+        this.isEnabled = VALIDATE_DOT_PREFIXES.get(clusterService.getSettings());
+    }
+
+    protected abstract Set<String> getIndicesFromRequest(RequestType request);
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> void apply(
+        Task task,
+        String action,
+        Request request,
+        ActionListener<Response> listener,
+        ActionFilterChain<Request, Response> chain
+    ) {
+        Set<String> indices = getIndicesFromRequest((RequestType) request);
+        if (isEnabled) {
+            validateIndices(indices);
+        }
+        chain.proceed(task, action, request, listener);
+    }
+
+    void validateIndices(@Nullable Set<String> indices) {
+        if (indices != null && isOperator() == false) {
+            for (String index : indices) {
+                if (Strings.hasLength(index)) {
+                    char c = getFirstChar(index);
+                    if (c == '.') {
+                        deprecationLogger.warn(
+                            DeprecationCategory.INDICES,
+                            "dot-prefix",
+                            "Index [{}] name begins with a dot (.), which is deprecated, "
+                                + "and will not be allowed in a future Elasticsearch version.",
+                            index
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    private static char getFirstChar(String index) {
+        char c = index.charAt(0);
+        if (c == '<') {
+            // Date-math is being used for the index, we need to
+            // consider it by stripping the first '<' before we
+            // check for a dot-prefix
+            String strippedLeading = index.substring(1);
+            if (Strings.hasLength(strippedLeading)) {
+                c = strippedLeading.charAt(0);
+            }
+        }
+        return c;
+    }
+
+    private boolean isOperator() {
+        return Optional.ofNullable(threadContext.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER)).map(Strings::hasText).orElse(false);
+    }
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+public class IndexTemplateDotValidator extends DotPrefixValidator<TransportPutComposableIndexTemplateAction.Request> {
+    public IndexTemplateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(TransportPutComposableIndexTemplateAction.Request request) {
+        return new HashSet<>(Arrays.asList(request.indices()));
+    }
+
+    @Override
+    public String actionName() {
+        return TransportPutComposableIndexTemplateAction.TYPE.name();
+    }
+}

modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.validation;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.junit.ClassRule;
+
+import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED;
+
+public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public DotPrefixClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password"));
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build();
+    }
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = createCluster();
+
+    private static ElasticsearchCluster createCluster() {
+        LocalClusterSpecBuilder<ElasticsearchCluster> clusterBuilder = ElasticsearchCluster.local()
+            .distribution(DistributionType.DEFAULT)
+            .feature(FAILURE_STORE_ENABLED)
+            .setting("xpack.security.enabled", "true")
+            .keystore("bootstrap.password", "x-pack-test-password")
+            .user("x_pack_rest_user", "x-pack-test-password");
+        boolean setNodes = Boolean.parseBoolean(System.getProperty("yaml.rest.tests.set_num_nodes", "true"));
+        if (setNodes) {
+            clusterBuilder.nodes(2);
+        }
+        return clusterBuilder.build();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+}