Adding new require_alias option to indexing requests (#58917)

This commit adds the `require_alias` flag to requests that create new documents.

This flag, when `true` prevents the request from automatically creating an index. Instead, the destination of the request MUST be an alias.

When the flag is not set, or `false`, the behavior defaults to the `action.auto_create_index` settings.

This is useful when an alias is required instead of a concrete index.

closes #55267

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/RestNoopBulkAction.java

@@ -65,6 +65,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         String defaultIndex = request.param("index");
         String defaultRouting = request.param("routing");
         String defaultPipeline = request.param("pipeline");
+        Boolean defaultRequireAlias = request.paramAsBoolean("require_alias", null);
 
         String waitForActiveShards = request.param("wait_for_active_shards");
         if (waitForActiveShards != null) {
@@ -73,7 +74,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
         bulkRequest.setRefreshPolicy(request.param("refresh"));
         bulkRequest.add(request.requiredContent(), defaultIndex, defaultRouting,
-            null, defaultPipeline, true, request.getXContentType());
+            null, defaultPipeline, defaultRequireAlias, true, request.getXContentType());
 
         // short circuit the call to the transport layer
         return channel -> {

docs/reference/docs/index_.asciidoc

@@ -25,9 +25,9 @@ and <<update-delete-docs-in-a-backing-index>>.
 
 `POST /<target>/_create/<_id>`
 
-IMPORTANT: You cannot add new documents to a data stream using the 
-`PUT /<target>/_doc/<_id>` request format. To specify a document ID, use the 
-`PUT /<target>/_create/<_id>` format instead. See 
+IMPORTANT: You cannot add new documents to a data stream using the
+`PUT /<target>/_doc/<_id>` request format. To specify a document ID, use the
+`PUT /<target>/_create/<_id>` format instead. See
 <<add-documents-to-a-data-stream>>.
 
 [[docs-index-api-path-params]]
@@ -94,6 +94,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version_type]
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards]
 
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=require-alias]
+
 [[docs-index-api-request-body]]
 ==== {api-request-body-title}
 

docs/reference/docs/update.asciidoc

@@ -53,6 +53,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term]
 `lang`::
 (Optional, string) The script language. Default: `painless`.
 
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=require-alias]
+
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh]
 
 `retry_on_conflict`::

docs/reference/rest-api/common-parms.asciidoc

@@ -570,6 +570,12 @@ such as `1264`.
 A value of `-1` indicates {es} was unable to compute this number.
 end::memory[]
 
+tag::require-alias[]
+`require_alias`::
+(Optional, boolean) When true, this requires the destination to be an alias.
+Defaults to false.
+end::require-alias[]
+
 tag::node-filter[]
 `<node_filter>`::
 (Optional, string)

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/270_set_processor.yml

@@ -53,3 +53,51 @@ teardown:
         index: test
         id: 2
   - match: { _source.foo: "hello" }
+---
+"Test set processor with index change and require_alias":
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "set" : {
+                  "field" : "_index",
+                  "value" : "new_require_alias_index"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+  - do:
+      catch: missing
+      index:
+        index:   test_require_alias
+        pipeline: 1
+        require_alias: true
+        body:    { foo: bar }
+
+  - do:
+      catch: missing
+      indices.get:
+        index: test_require_alias
+  - do:
+      catch: missing
+      indices.get:
+        index: new_require_alias_index
+
+  - do:
+      indices.create:
+        index: backing_index
+        body:
+          mappings: {}
+          aliases:
+            new_require_alias_index: {}
+
+  - do:
+      index:
+        index:   test_require_alias
+        pipeline: 1
+        require_alias: true
+        body:    { foo: bar }

modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexValidator.java

@@ -25,6 +25,7 @@
 import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.AutoCreateIndex;
@@ -35,6 +36,7 @@
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.util.List;
@@ -111,6 +113,14 @@ static void validateAgainstAliases(SearchRequest source, IndexRequest destinatio
             return;
         }
         String target = destination.index();
+        if (destination.isRequireAlias() && (false == clusterState.getMetadata().hasAlias(target))) {
+            throw new IndexNotFoundException("["
+                + DocWriteRequest.REQUIRE_ALIAS
+                + "] request flag is [true] and ["
+                + target
+                + "] is not an alias",
+                target);
+        }
         if (false == autoCreateIndex.shouldAutoCreate(target, clusterState)) {
             /*
              * If we're going to autocreate the index we don't need to resolve

modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestRequest;
@@ -68,6 +69,9 @@ protected ReindexRequest buildRequest(RestRequest request) throws IOException {
         if (request.hasParam("scroll")) {
             internal.setScroll(parseTimeValue(request.param("scroll"), "scroll"));
         }
+        if (request.hasParam(DocWriteRequest.REQUIRE_ALIAS)) {
+            internal.setRequireAlias(request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, false));
+        }
 
         return internal;
     }

rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json

@@ -87,6 +87,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "require_alias": {
+        "type": "boolean",
+        "description": "Sets require_alias for all incoming documents. Defaults to unset (false)"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/index.json

@@ -92,6 +92,10 @@
       "pipeline":{
         "type":"string",
         "description":"The pipeline id to preprocess incoming documents with"
+      },
+      "require_alias": {
+        "type": "boolean",
+        "description": "When true, requires destination to be an alias. Default is false"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/api/update.json

@@ -99,6 +99,10 @@
       "if_primary_term":{
         "type":"number",
         "description":"only perform the update operation if the last operation that has changed the document has the specified primary term"
+      },
+      "require_alias": {
+        "type": "boolean",
+        "description": "When true, requires destination is an alias. Default is false"
       }
     },
     "body":{

rest-api-spec/src/main/resources/rest-api-spec/test/bulk/10_basic.yml

@@ -116,3 +116,88 @@
            {"index": {"_index": "test_index", "_id": "test_id"}}
            {"f1": "v1", "f2": 42}
            {}
+
+---
+"When setting require_alias flag per request":
+  - skip:
+      # TODO adjust after backport
+      version: " - 7.99.99"
+      reason: "require_alias flag was added in version 7.9"
+
+  - do:
+      indices.create:
+        index: backing_index
+        body:
+          mappings: {}
+          aliases:
+            test_require_alias: {}
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: new_index_not_created
+              require_alias: true
+          - f: 1
+          - index:
+              _index: new_index_created
+          - f: 2
+          - index:
+              _index: test_require_alias
+              require_alias: true
+          - f: 3
+          - create:
+              _index: test_require_alias
+          - f: 4
+  - match: { errors: true }
+  - match: { items.0.index.status: 404 }
+  - match: { items.0.index.error.type: index_not_found_exception }
+  - match: { items.0.index.error.reason: "no such index [new_index_not_created] and [require_alias] request flag is [true] and [new_index_not_created] is not an alias" }
+  - match: { items.1.index.result: created }
+  - match: { items.2.index.result: created }
+  - match: { items.3.create.result: created }
+
+  - do:
+      catch: missing
+      indices.get:
+        index: new_index_not_created
+---
+"When setting require_alias flag":
+  - skip:
+      # TODO adjust after backport
+      version: " - 7.99.99"
+      reason: "require_alias flag was added in version 7.9"
+
+  - do:
+      indices.create:
+        index: backing_index
+        body:
+          mappings: {}
+          aliases:
+            test_require_alias: {}
+  - do:
+      bulk:
+        refresh: true
+        require_alias: true
+        body:
+          - index:
+              _index: new_index_not_created
+          - f: 1
+          - index:
+              _index: new_index_created
+              require_alias: false
+          - f: 2
+          - index:
+              _index: test_require_alias
+          - f: 3
+  - match: { errors: true }
+  - match: { items.0.index.status: 404 }
+  - match: { items.0.index.error.type: index_not_found_exception }
+  - match: { items.0.index.error.reason: "no such index [new_index_not_created] and [require_alias] request flag is [true] and [new_index_not_created] is not an alias" }
+  - match: { items.1.index.result: created }
+  - match: { items.2.index.result: created }
+
+  - do:
+      catch: missing
+      indices.get:
+        index: new_index_not_created

rest-api-spec/src/main/resources/rest-api-spec/test/index/70_require_alias.yml

@@ -0,0 +1,30 @@
+---
+"Set require_alias flag":
+  - skip:
+      # TODO adjust after backport
+      version: " - 7.99.99"
+      reason: "require_alias flag added in 7.9+"
+  - do:
+      catch: missing
+      index:
+        index:   test_require_alias
+        require_alias: true
+        body:    { foo: bar }
+  - do:
+      catch: missing
+      indices.get:
+        index: test_require_alias
+
+  - do:
+      indices.create:
+        index: backing_index
+        body:
+          mappings: {}
+          aliases:
+            test_require_alias: {}
+
+  - do:
+      index:
+        index:   test_require_alias
+        require_alias: true
+        body:    { foo: bar }

rest-api-spec/src/main/resources/rest-api-spec/test/update/95_require_alias.yml

@@ -0,0 +1,36 @@
+---
+"Set require_alias flag":
+  - skip:
+      # TODO adjust after backport
+      version: " - 7.99.99"
+      reason: "require_alias flag added in 7.9+"
+  - do:
+      catch: missing
+      update:
+        index:            test_require_alias
+        id:               1
+        require_alias:   true
+        body:
+          doc:            { foo: bar, count: 1 }
+          doc_as_upsert:  true
+  - do:
+      catch: missing
+      indices.get:
+        index: test_require_alias
+
+  - do:
+      indices.create:
+        index: backing_index
+        body:
+          mappings: {}
+          aliases:
+            test_require_alias: {}
+
+  - do:
+      update:
+        index:            test_require_alias
+        id:               1
+        require_alias:   true
+        body:
+          doc:            { foo: bar, count: 1 }
+          doc_as_upsert:  true

server/src/main/java/org/elasticsearch/action/DocWriteRequest.java

@@ -43,6 +43,9 @@
  */
 public interface DocWriteRequest<T> extends IndicesRequest, Accountable {
 
+    // Flag set for disallowing index auto creation for an individual write request.
+    String REQUIRE_ALIAS = "require_alias";
+
     /**
      * Set the index for this request
      * @return the Request
@@ -142,6 +145,11 @@ public interface DocWriteRequest<T> extends IndicesRequest, Accountable {
      */
     OpType opType();
 
+    /**
+     * Should this request override specifically require the destination to be an alias?
+     * @return boolean flag, when true specifically requires an alias
+     */
+    boolean isRequireAlias();
     /**
      * Requested operation type to perform on the document
      */

server/src/main/java/org/elasticsearch/action/bulk/BulkProcessor.java

@@ -393,7 +393,7 @@ public BulkProcessor add(BytesReference data, @Nullable String defaultIndex,
         lock.lock();
         try {
             ensureOpen();
-            bulkRequest.add(data, defaultIndex, null, null, defaultPipeline,
+            bulkRequest.add(data, defaultIndex, null, null, defaultPipeline, null,
                 true, xContentType);
             bulkRequestToExecute = newBulkRequestIfNeeded();
         } finally {