diff --git a/modules/dot-prefix-validation/build.gradle b/modules/dot-prefix-validation/build.gradle new file mode 100644 index 000000000000..422af39b56d0 --- /dev/null +++ b/modules/dot-prefix-validation/build.gradle @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + description 'Validation for dot-prefixed indices for non-operator users' + classname 'org.elasticsearch.validation.DotPrefixValidationPlugin' +} + +restResources { + restApi { + include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex' + } +} + +tasks.named("yamlRestTestV7CompatTest").configure { enabled = false }; + +tasks.named('yamlRestTest') { + usesDefaultDistribution() +} diff --git a/modules/dot-prefix-validation/src/main/java/module-info.java b/modules/dot-prefix-validation/src/main/java/module-info.java new file mode 100644 index 000000000000..918128778742 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/module-info.java @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module org.elasticsearch.validation { + requires org.elasticsearch.server; + requires org.elasticsearch.base; +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java new file mode 100644 index 000000000000..b5dc3769ccc5 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.create.AutoCreateAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Set; + +public class AutoCreateDotValidator extends DotPrefixValidator { + public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(CreateIndexRequest request) { + return Set.of(request.index()); + } + + @Override + public String actionName() { + return AutoCreateAction.NAME; + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java new file mode 100644 index 000000000000..7a1167868901 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/CreateIndexDotValidator.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Set; + +public class CreateIndexDotValidator extends DotPrefixValidator { + public CreateIndexDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(CreateIndexRequest request) { + return Set.of(request.index()); + } + + @Override + public String actionName() { + return TransportCreateIndexAction.TYPE.name(); + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java new file mode 100644 index 000000000000..f511cbe77ee7 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidationPlugin.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.support.MappedActionFilter; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +public class DotPrefixValidationPlugin extends Plugin implements ActionPlugin { + private final AtomicReference> actionFilters = new AtomicReference<>(); + + public DotPrefixValidationPlugin() {} + + @Override + public Collection createComponents(PluginServices services) { + ThreadContext context = services.threadPool().getThreadContext(); + ClusterService clusterService = services.clusterService(); + + actionFilters.set( + List.of( + new CreateIndexDotValidator(context, clusterService), + new AutoCreateDotValidator(context, clusterService), + new IndexTemplateDotValidator(context, clusterService) + ) + ); + + return Set.of(); + } + + @Override + public Collection getMappedActionFilters() { + return actionFilters.get(); + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java new file mode 100644 index 000000000000..5106908e1913 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/DotPrefixValidator.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilterChain; +import org.elasticsearch.action.support.MappedActionFilter; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.tasks.Task; + +import java.util.Optional; +import java.util.Set; + +public abstract class DotPrefixValidator implements MappedActionFilter { + public static final Setting VALIDATE_DOT_PREFIXES = Setting.boolSetting( + "cluster.indices.validate_dot_prefixes", + true, + Setting.Property.NodeScope + ); + DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(DotPrefixValidator.class); + + private final ThreadContext threadContext; + private final boolean isEnabled; + + public DotPrefixValidator(ThreadContext threadContext, ClusterService clusterService) { + this.threadContext = threadContext; + this.isEnabled = VALIDATE_DOT_PREFIXES.get(clusterService.getSettings()); + } + + protected abstract Set getIndicesFromRequest(RequestType request); + + @SuppressWarnings("unchecked") + @Override + public void apply( + Task task, + String action, + Request request, + ActionListener listener, + ActionFilterChain chain + ) { + Set indices = getIndicesFromRequest((RequestType) request); + if (isEnabled) { + validateIndices(indices); + } + chain.proceed(task, action, request, listener); + } + + void validateIndices(@Nullable Set indices) { + if (indices != null && isOperator() == false) { + for (String index : indices) { + if (Strings.hasLength(index)) { + char c = getFirstChar(index); + if (c == '.') { + deprecationLogger.warn( + DeprecationCategory.INDICES, + "dot-prefix", + "Index [{}] name begins with a dot (.), which is deprecated, " + + "and will not be allowed in a future Elasticsearch version.", + index + ); + } + } + } + } + } + + private static char getFirstChar(String index) { + char c = index.charAt(0); + if (c == '<') { + // Date-math is being used for the index, we need to + // consider it by stripping the first '<' before we + // check for a dot-prefix + String strippedLeading = index.substring(1); + if (Strings.hasLength(strippedLeading)) { + c = strippedLeading.charAt(0); + } + } + return c; + } + + private boolean isOperator() { + return Optional.ofNullable(threadContext.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER)).map(Strings::hasText).orElse(false); + } +} diff --git a/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java new file mode 100644 index 000000000000..41edbd89d456 --- /dev/null +++ b/modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/IndexTemplateDotValidator.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.validation; + +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.util.concurrent.ThreadContext; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class IndexTemplateDotValidator extends DotPrefixValidator { + public IndexTemplateDotValidator(ThreadContext threadContext, ClusterService clusterService) { + super(threadContext, clusterService); + } + + @Override + protected Set getIndicesFromRequest(TransportPutComposableIndexTemplateAction.Request request) { + return new HashSet<>(Arrays.asList(request.indices())); + } + + @Override + public String actionName() { + return TransportPutComposableIndexTemplateAction.TYPE.name(); + } +} diff --git a/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java b/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java new file mode 100644 index 000000000000..42d1ce7a7003 --- /dev/null +++ b/modules/dot-prefix-validation/src/yamlRestTest/java/org/elasticsearch/validation/DotPrefixClientYamlTestSuiteIT.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.validation; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterSpecBuilder; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +import static org.elasticsearch.test.cluster.FeatureFlag.FAILURE_STORE_ENABLED; + +public class DotPrefixClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public DotPrefixClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return createParameters(); + } + + private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password")); + + @Override + protected Settings restClientSettings() { + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); + } + + @ClassRule + public static ElasticsearchCluster cluster = createCluster(); + + private static ElasticsearchCluster createCluster() { + LocalClusterSpecBuilder clusterBuilder = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .feature(FAILURE_STORE_ENABLED) + .setting("xpack.security.enabled", "true") + .keystore("bootstrap.password", "x-pack-test-password") + .user("x_pack_rest_user", "x-pack-test-password"); + boolean setNodes = Boolean.parseBoolean(System.getProperty("yaml.rest.tests.set_num_nodes", "true")); + if (setNodes) { + clusterBuilder.nodes(2); + } + return clusterBuilder.build(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + +} diff --git a/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml new file mode 100644 index 000000000000..df257eb8f804 --- /dev/null +++ b/modules/dot-prefix-validation/src/yamlRestTest/resources/rest-api-spec/test/dot_prefix/10_basic.yml @@ -0,0 +1,175 @@ +--- +"Index creation with a dot-prefix is deprecated unless x-elastic-product-origin set": + - requires: + test_runner_features: ["warnings", "warnings_regex", "headers"] + + - do: + warnings: + - "Index [.myindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.myindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + indices.create: + index: .myindex + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + - "index name [.myindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + indices.create: + index: .myindex2 + + - do: + warnings_regex: + - "Index \\[.*\\] name begins with a dot \\(\\.\\), which is deprecated, and will not be allowed in a future Elasticsearch version\\." + - "index name \\[.*\\] starts with a dot '\\.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + indices.create: + index: <.myindex-{now/d}> + + - do: + headers: { X-elastic-product-origin: kibana } + warnings_regex: + - "index name \\[.*\\] starts with a dot '\\.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + indices.create: + index: <.myindex2-{now/d}> + +--- +"Reject auto-creation of dot-prefixed indices": + - requires: + test_runner_features: ["warnings", "headers"] + + - do: + warnings: + - "Index [.myindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.myindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: .myindex + id: "1" + body: {foo: bar} + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + - "index name [.myindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: .myindex2 + id: "1" + body: {foo: bar} + + - do: + warnings: + - "Index [.myindex3] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.myindex3] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + bulk: + body: + - index: + _index: other + - message: foo + - index: + _index: .myindex3 + - message: foo + +--- +"Reject auto-creation of dot-prefixed indices through pipelines": + - requires: + test_runner_features: ["warnings", "headers"] + + - do: + ingest.put_pipeline: + id: mypipeline + body: > + { + "processors": [ + { + "set" : { + "field" : "_index", + "value": "{{redirect_to}}" + } + } + ] + } + - do: + warnings: + - "Index [.other] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + index: + index: myindex + id: "1" + body: {redirect_to: ".other"} + pipeline: mypipeline + + - do: + warnings: + - "Index [.other2] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + bulk: + body: + - index: + _index: other + - message: foo + - index: + _index: myindex + pipeline: mypipeline + - redirect_to: .other2 + + - do: + index: + index: original + id: "1" + body: { "redirect_to": ".other3" } + - do: + indices.refresh: {} + - do: + warnings: + - "Index [.other3] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.other3] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: newindex + pipeline: mypipeline + + - do: + warnings: + - "Index [.reindex] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + - "index name [.reindex] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: .reindex + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + - "index name [.reindex2] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" + reindex: + body: + source: + index: original + dest: + index: .reindex2 + +--- +"Reject index template that has a dot prefix index pattern": + - requires: + test_runner_features: ["warnings", "headers"] + + - do: + warnings: + - "Index [.data-*] name begins with a dot (.), which is deprecated, and will not be allowed in a future Elasticsearch version." + indices.put_index_template: + name: my-template + body: + index_patterns: [regular, .data-*] + data_stream: {} + + - do: + headers: { X-elastic-product-origin: kibana } + warnings: + indices.put_index_template: + name: my-template + body: + index_patterns: [regular, .data2-*] + data_stream: {}