Skip to content

Commit

Permalink
[7.x] Adding new require_alias option to indexing requests (#58917) (
Browse files Browse the repository at this point in the history
…#59769)

* Adding new `require_alias` option to indexing requests (#58917)

This commit adds the `require_alias` flag to requests that create new documents.

This flag, when `true` prevents the request from automatically creating an index. Instead, the destination of the request MUST be an alias.

When the flag is not set, or `false`, the behavior defaults to the `action.auto_create_index` settings.

This is useful when an alias is required instead of a concrete index.

closes #55267
  • Loading branch information
benwtrent authored Jul 17, 2020
1 parent 65f6fb8 commit b7f30fc
Show file tree
Hide file tree
Showing 30 changed files with 447 additions and 36 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
String defaultType = request.param("type");
String defaultRouting = request.param("routing");
String defaultPipeline = request.param("pipeline");
Boolean defaultRequireAlias = request.paramAsBoolean("require_alias", null);

String waitForActiveShards = request.param("wait_for_active_shards");
if (waitForActiveShards != null) {
Expand All @@ -78,7 +79,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
bulkRequest.setRefreshPolicy(request.param("refresh"));
bulkRequest.add(request.requiredContent(), defaultIndex, defaultType, defaultRouting,
null, defaultPipeline, true, request.getXContentType());
null, defaultPipeline, defaultRequireAlias, true, request.getXContentType());

// short circuit the call to the transport layer
return channel -> {
Expand Down
8 changes: 5 additions & 3 deletions docs/reference/docs/index_.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ and <<update-delete-docs-in-a-backing-index>>.

`POST /<target>/_create/<_id>`

IMPORTANT: You cannot add new documents to a data stream using the
`PUT /<target>/_doc/<_id>` request format. To specify a document ID, use the
`PUT /<target>/_create/<_id>` format instead. See
IMPORTANT: You cannot add new documents to a data stream using the
`PUT /<target>/_doc/<_id>` request format. To specify a document ID, use the
`PUT /<target>/_create/<_id>` format instead. See
<<add-documents-to-a-data-stream>>.

[[docs-index-api-path-params]]
Expand Down Expand Up @@ -94,6 +94,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version_type]

include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards]

include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=require-alias]

[[docs-index-api-request-body]]
==== {api-request-body-title}

Expand Down
2 changes: 2 additions & 0 deletions docs/reference/docs/update.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term]
`lang`::
(Optional, string) The script language. Default: `painless`.

include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=require-alias]

include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh]

`retry_on_conflict`::
Expand Down
6 changes: 6 additions & 0 deletions docs/reference/rest-api/common-parms.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,12 @@ such as `1264`.
A value of `-1` indicates {es} was unable to compute this number.
end::memory[]

tag::require-alias[]
`require_alias`::
(Optional, boolean) When true, this requires the destination to be an alias.
Defaults to false.
end::require-alias[]

tag::node-filter[]
`<node_filter>`::
(Optional, string)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,51 @@ teardown:
index: test
id: 2
- match: { _source.foo: "hello" }
---
"Test set processor with index change and require_alias":
- do:
ingest.put_pipeline:
id: "1"
body: >
{
"processors": [
{
"set" : {
"field" : "_index",
"value" : "new_require_alias_index"
}
}
]
}
- match: { acknowledged: true }
- do:
catch: missing
index:
index: test_require_alias
pipeline: 1
require_alias: true
body: { foo: bar }

- do:
catch: missing
indices.get:
index: test_require_alias
- do:
catch: missing
indices.get:
index: new_require_alias_index

- do:
indices.create:
index: backing_index
body:
mappings: {}
aliases:
new_require_alias_index: {}

- do:
index:
index: test_require_alias
pipeline: 1
require_alias: true
body: { foo: bar }
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.lucene.util.automaton.MinimizationOperations;
import org.apache.lucene.util.automaton.Operations;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.AutoCreateIndex;
Expand All @@ -37,6 +38,7 @@
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.search.builder.SearchSourceBuilder;

import java.util.List;
Expand Down Expand Up @@ -114,6 +116,14 @@ static void validateAgainstAliases(SearchRequest source, IndexRequest destinatio
return;
}
String target = destination.index();
if (destination.isRequireAlias() && (false == clusterState.getMetadata().hasAlias(target))) {
throw new IndexNotFoundException("["
+ DocWriteRequest.REQUIRE_ALIAS
+ "] request flag is [true] and ["
+ target
+ "] is not an alias",
target);
}
if (false == autoCreateIndex.shouldAutoCreate(target, clusterState)) {
/*
* If we're going to autocreate the index we don't need to resolve
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.index.reindex;

import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.rest.RestRequest;
Expand Down Expand Up @@ -69,6 +70,10 @@ protected ReindexRequest buildRequest(RestRequest request) throws IOException {
if (request.hasParam("scroll")) {
internal.setScroll(parseTimeValue(request.param("scroll"), "scroll"));
}
if (request.hasParam(DocWriteRequest.REQUIRE_ALIAS)) {
internal.setRequireAlias(request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, false));
}

return internal;
}
}
4 changes: 4 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@
"pipeline":{
"type":"string",
"description":"The pipeline id to preprocess incoming documents with"
},
"require_alias": {
"type": "boolean",
"description": "Sets require_alias for all incoming documents. Defaults to unset (false)"
}
},
"body":{
Expand Down
4 changes: 4 additions & 0 deletions rest-api-spec/src/main/resources/rest-api-spec/api/index.json
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,10 @@
"pipeline":{
"type":"string",
"description":"The pipeline id to preprocess incoming documents with"
},
"require_alias": {
"type": "boolean",
"description": "When true, requires destination to be an alias. Default is false"
}
},
"body":{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,10 @@
"if_primary_term":{
"type":"number",
"description":"only perform the update operation if the last operation that has changed the document has the specified primary term"
},
"require_alias": {
"type": "boolean",
"description": "When true, requires destination is an alias. Default is false"
}
},
"body":{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,3 +122,86 @@
{"index": {"_index": "test_index", "_id": "test_id"}}
{"f1": "v1", "f2": 42}
{}
---
"When setting require_alias flag per request":
- skip:
version: " - 7.9.99"
reason: "require_alias flag was added in version 7.10"

- do:
indices.create:
index: backing_index
body:
mappings: {}
aliases:
test_require_alias: {}
- do:
bulk:
refresh: true
body:
- index:
_index: new_index_not_created
require_alias: true
- f: 1
- index:
_index: new_index_created
- f: 2
- index:
_index: test_require_alias
require_alias: true
- f: 3
- create:
_index: test_require_alias
- f: 4
- match: { errors: true }
- match: { items.0.index.status: 404 }
- match: { items.0.index.error.type: index_not_found_exception }
- match: { items.0.index.error.reason: "no such index [new_index_not_created] and [require_alias] request flag is [true] and [new_index_not_created] is not an alias" }
- match: { items.1.index.result: created }
- match: { items.2.index.result: created }
- match: { items.3.create.result: created }

- do:
catch: missing
indices.get:
index: new_index_not_created
---
"When setting require_alias flag":
- skip:
version: " - 7.9.99"
reason: "require_alias flag was added in version 7.10"

- do:
indices.create:
index: backing_index
body:
mappings: {}
aliases:
test_require_alias: {}
- do:
bulk:
refresh: true
require_alias: true
body:
- index:
_index: new_index_not_created
- f: 1
- index:
_index: new_index_created
require_alias: false
- f: 2
- index:
_index: test_require_alias
- f: 3
- match: { errors: true }
- match: { items.0.index.status: 404 }
- match: { items.0.index.error.type: index_not_found_exception }
- match: { items.0.index.error.reason: "no such index [new_index_not_created] and [require_alias] request flag is [true] and [new_index_not_created] is not an alias" }
- match: { items.1.index.result: created }
- match: { items.2.index.result: created }

- do:
catch: missing
indices.get:
index: new_index_not_created
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
"Set require_alias flag":
- skip:
version: " - 7.9.99"
reason: "require_alias flag added in 7.10"
- do:
catch: missing
index:
index: test_require_alias
require_alias: true
body: { foo: bar }
- do:
catch: missing
indices.get:
index: test_require_alias

- do:
indices.create:
index: backing_index
body:
mappings: {}
aliases:
test_require_alias: {}

- do:
index:
index: test_require_alias
require_alias: true
body: { foo: bar }
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
---
"Set require_alias flag":
- skip:
version: " - 7.9.99"
reason: "require_alias flag added in 7.10"
- do:
catch: missing
update:
index: test_require_alias
id: 1
require_alias: true
body:
doc: { foo: bar, count: 1 }
doc_as_upsert: true
- do:
catch: missing
indices.get:
index: test_require_alias

- do:
indices.create:
index: backing_index
body:
mappings: {}
aliases:
test_require_alias: {}

- do:
update:
index: test_require_alias
id: 1
require_alias: true
body:
doc: { foo: bar, count: 1 }
doc_as_upsert: true
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@
*/
public interface DocWriteRequest<T> extends IndicesRequest, Accountable {

// Flag set for disallowing index auto creation for an individual write request.
String REQUIRE_ALIAS = "require_alias";

/**
* Set the index for this request
* @return the Request
Expand Down Expand Up @@ -163,6 +166,11 @@ public interface DocWriteRequest<T> extends IndicesRequest, Accountable {
*/
OpType opType();

/**
* Should this request override specifically require the destination to be an alias?
* @return boolean flag, when true specifically requires an alias
*/
boolean isRequireAlias();
/**
* Requested operation type to perform on the document
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ public BulkProcessor add(BytesReference data, @Nullable String defaultIndex, @Nu
lock.lock();
try {
ensureOpen();
bulkRequest.add(data, defaultIndex, defaultType, null, null, defaultPipeline,
bulkRequest.add(data, defaultIndex, defaultType, null, null, defaultPipeline, null,
true, xContentType);
bulkRequestToExecute = newBulkRequestIfNeeded();
} finally {
Expand Down
Loading

0 comments on commit b7f30fc

Please sign in to comment.