From 7bc8bc960568735a1116a34c34cae944e714ed81 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Mon, 4 Feb 2019 16:36:04 +0100 Subject: [PATCH 01/24] ensureGreen (#38324) --- .../cluster/coordination/ElasticsearchNodeCommandIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java index 8ff8cde653d16..4ea9cc87dd89a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java @@ -391,7 +391,6 @@ public void testNoInitialBootstrapAfterDetach() throws Exception { internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38267") public void testCanRunUnsafeBootstrapAfterErroneousDetachWithoutLoosingMetaData() throws Exception { internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startMasterOnlyNode(); @@ -410,7 +409,7 @@ public void testCanRunUnsafeBootstrapAfterErroneousDetachWithoutLoosingMetaData( unsafeBootstrap(environment); internalCluster().startMasterOnlyNode(); - ensureStableCluster(1); + ensureGreen(); state = internalCluster().client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.metaData().settings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()), From 1899658a38f587d7c7d1a95b9a1fb781439f4401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 4 Feb 2019 17:13:54 +0100 Subject: [PATCH 02/24] Mute ClusterClientIT#testClusterHealthYellowSpecificIndex (#38343) --- .../src/test/java/org/elasticsearch/client/ClusterClientIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java index 2044a5ac56c92..0e1834cd3ac3b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java @@ -215,6 +215,8 @@ private static void assertYellowShards(ClusterHealthResponse response) { assertThat(response.getActiveShardsPercent(), equalTo(50d)); } + + @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/35450") public void testClusterHealthYellowSpecificIndex() throws IOException { createIndex("index", Settings.EMPTY); createIndex("index2", Settings.EMPTY); From f872c721ac2c9c4b6a168366b62f62c1258ab574 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 4 Feb 2019 09:43:28 -0700 Subject: [PATCH 03/24] Run Node deprecation checks locally (#38065) (#38250) At times, we need to check for usage of deprecated settings in settings which should not be returned by the NodeInfo API. This commit changes the deprecation info API to run all node checks locally so that these settings can be checked without exposing them via any externally accessible API. --- .../deprecation/DeprecationInfoAction.java | 44 ++++--- .../NodesDeprecationCheckAction.java | 121 ++++++++++++++++++ .../NodesDeprecationCheckRequest.java | 50 ++++++++ .../NodesDeprecationCheckResponse.java | 53 ++++++++ .../DeprecationInfoActionResponseTests.java | 32 ++--- .../NodesDeprecationCheckRequestTests.java | 33 +++++ .../NodesDeprecationCheckResponseTests.java | 84 ++++++++++++ .../xpack/deprecation/Deprecation.java | 7 +- .../xpack/deprecation/DeprecationChecks.java | 6 +- .../TransportDeprecationInfoAction.java | 71 +++++----- .../TransportNodeDeprecationCheckAction.java | 73 +++++++++++ 11 files changed, 500 insertions(+), 74 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java index 1241b136c7a6e..b917dbf260c9c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; import org.elasticsearch.action.support.master.MasterNodeReadRequest; @@ -35,7 +34,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -62,6 +60,23 @@ public static List filterChecks(List checks, Function mergeNodeIssues(NodesDeprecationCheckResponse response) { + Map> issueListMap = new HashMap<>(); + for (NodesDeprecationCheckAction.NodeResponse resp : response.getNodes()) { + for (DeprecationIssue issue : resp.getDeprecationIssues()) { + issueListMap.computeIfAbsent(issue, (key) -> new ArrayList<>()).add(resp.getNode().getName()); + } + } + + return issueListMap.entrySet().stream() + .map(entry -> { + DeprecationIssue issue = entry.getKey(); + String details = issue.getDetails() != null ? issue.getDetails() + " " : ""; + return new DeprecationIssue(issue.getLevel(), issue.getMessage(), issue.getUrl(), + details + "(nodes impacted: " + entry.getValue() + ")"); + }).collect(Collectors.toList()); + } + @Override public Response newResponse() { return new Response(); @@ -159,32 +174,29 @@ public int hashCode() { * this function will run through all the checks and build out the final list of issues that exist in the * cluster. * - * @param nodesInfo The list of {@link NodeInfo} metadata objects for retrieving node-level information - * @param nodesStats The list of {@link NodeStats} metadata objects for retrieving node-level information * @param state The cluster state * @param indexNameExpressionResolver Used to resolve indices into their concrete names * @param indices The list of index expressions to evaluate using `indexNameExpressionResolver` * @param indicesOptions The options to use when resolving and filtering which indices to check * @param datafeeds The ml datafeed configurations - * @param clusterSettingsChecks The list of cluster-level checks - * @param nodeSettingsChecks The list of node-level checks + * @param nodeDeprecationResponse The response containing the deprecation issues found on each node * @param indexSettingsChecks The list of index-level checks that will be run across all specified * concrete indices + * @param clusterSettingsChecks The list of cluster-level checks * @param mlSettingsCheck The list of ml checks * @return The list of deprecation issues found in the cluster */ - public static DeprecationInfoAction.Response from(List nodesInfo, List nodesStats, ClusterState state, - IndexNameExpressionResolver indexNameExpressionResolver, - String[] indices, IndicesOptions indicesOptions, - List datafeeds, - List>clusterSettingsChecks, - List, List, DeprecationIssue>> nodeSettingsChecks, - List> indexSettingsChecks, - List> mlSettingsCheck) { + public static DeprecationInfoAction.Response from(ClusterState state, + IndexNameExpressionResolver indexNameExpressionResolver, + String[] indices, IndicesOptions indicesOptions, + List datafeeds, + NodesDeprecationCheckResponse nodeDeprecationResponse, + List> indexSettingsChecks, + List> clusterSettingsChecks, + List> mlSettingsCheck) { List clusterSettingsIssues = filterChecks(clusterSettingsChecks, (c) -> c.apply(state)); - List nodeSettingsIssues = filterChecks(nodeSettingsChecks, - (c) -> c.apply(nodesInfo, nodesStats)); + List nodeSettingsIssues = mergeNodeIssues(nodeDeprecationResponse); List mlSettingsIssues = new ArrayList<>(); for (DatafeedConfig config : datafeeds) { mlSettingsIssues.addAll(filterChecks(mlSettingsCheck, (c) -> c.apply(config))); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java new file mode 100644 index 0000000000000..db0b0a0603de5 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckAction.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.support.nodes.BaseNodeRequest; +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Runs deprecation checks on each node. Deprecation checks are performed locally so that filtered settings + * can be accessed in the deprecation checks. + */ +public class NodesDeprecationCheckAction extends Action { + public static final NodesDeprecationCheckAction INSTANCE = new NodesDeprecationCheckAction(); + public static final String NAME = "cluster:admin/xpack/deprecation/nodes/info"; + + private NodesDeprecationCheckAction() { + super(NAME); + } + + @Override + public NodesDeprecationCheckResponse newResponse() { + return new NodesDeprecationCheckResponse(); + } + + public static class NodeRequest extends BaseNodeRequest { + + NodesDeprecationCheckRequest request; + + public NodeRequest() {} + public NodeRequest(String nodeId, NodesDeprecationCheckRequest request) { + super(nodeId); + this.request = request; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + request = new NodesDeprecationCheckRequest(); + request.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + request.writeTo(out); + } + } + + public static class NodeResponse extends BaseNodeResponse { + private List deprecationIssues; + + public NodeResponse() { + super(); + } + + public NodeResponse(DiscoveryNode node, List deprecationIssues) { + super(node); + this.deprecationIssues = deprecationIssues; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + deprecationIssues = in.readList(DeprecationIssue::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeList(this.deprecationIssues); + } + + public static NodeResponse readNodeResponse(StreamInput in) throws IOException { + NodeResponse nodeResponse = new NodeResponse(); + nodeResponse.readFrom(in); + return nodeResponse; + } + + public List getDeprecationIssues() { + return deprecationIssues; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeResponse that = (NodeResponse) o; + return Objects.equals(getDeprecationIssues(), that.getDeprecationIssues()) + && Objects.equals(getNode(), that.getNode()); + } + + @Override + public int hashCode() { + return Objects.hash(getNode(), getDeprecationIssues()); + } + } + + public static class RequestBuilder extends NodesOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, + Action action, + NodesDeprecationCheckRequest request) { + super(client, action, request); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java new file mode 100644 index 0000000000000..af7b2da6f55eb --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequest.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +public class NodesDeprecationCheckRequest extends BaseNodesRequest { + public NodesDeprecationCheckRequest() {} + + public NodesDeprecationCheckRequest(String... nodesIds) { + super(nodesIds); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash((Object[]) this.nodesIds()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + NodesDeprecationCheckRequest that = (NodesDeprecationCheckRequest) obj; + return Arrays.equals(this.nodesIds(), that.nodesIds()); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java new file mode 100644 index 0000000000000..db7dbc6a381e2 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponse.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class NodesDeprecationCheckResponse extends BaseNodesResponse { + + public NodesDeprecationCheckResponse() {} + + public NodesDeprecationCheckResponse(ClusterName clusterName, + List nodes, + List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(NodesDeprecationCheckAction.NodeResponse::readNodeResponse); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeStreamableList(nodes); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodesDeprecationCheckResponse that = (NodesDeprecationCheckResponse) o; + return Objects.equals(getClusterName(), that.getClusterName()) + && Objects.equals(getNodes(), that.getNodes()) + && Objects.equals(failures(), that.failures()); + } + + @Override + public int hashCode() { + return Objects.hash(getClusterName(), getNodes(), failures()); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java index b878f1c5d404d..59ed1dcd17bbc 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/DeprecationInfoActionResponseTests.java @@ -5,10 +5,7 @@ */ package org.elasticsearch.xpack.core.deprecation; -import org.elasticsearch.Build; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -31,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -78,12 +74,6 @@ public void testFrom() throws IOException { DiscoveryNode discoveryNode = DiscoveryNode.createLocal(Settings.EMPTY, new TransportAddress(TransportAddress.META_ADDRESS, 9300), "test"); ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metaData(metadata).build(); - List nodeInfos = Collections.singletonList(new NodeInfo(Version.CURRENT, Build.CURRENT, - discoveryNode, null, null, null, null, - null, null, null, null, null, null)); - List nodeStats = Collections.singletonList(new NodeStats(discoveryNode, 0L, null, - null, null, null, null, null, null, null, null, - null, null, null, null)); List datafeeds = Collections.singletonList(DatafeedConfigTests.createRandomizedDatafeedConfig("foo")); IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(); IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, @@ -97,11 +87,6 @@ public void testFrom() throws IOException { Collections.unmodifiableList(Arrays.asList( (s) -> clusterIssueFound ? foundIssue : null )); - List, List, DeprecationIssue>> nodeSettingsChecks = - Collections.unmodifiableList(Arrays.asList( - (ln, ls) -> nodeIssueFound ? foundIssue : null - )); - List> indexSettingsChecks = Collections.unmodifiableList(Arrays.asList( (idx) -> indexIssueFound ? foundIssue : null @@ -111,9 +96,17 @@ public void testFrom() throws IOException { (idx) -> mlIssueFound ? foundIssue : null )); - DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(nodeInfos, nodeStats, state, + NodesDeprecationCheckResponse nodeDeprecationIssues = new NodesDeprecationCheckResponse( + new ClusterName(randomAlphaOfLength(5)), + nodeIssueFound + ? Collections.singletonList( + new NodesDeprecationCheckAction.NodeResponse(discoveryNode, Collections.singletonList(foundIssue))) + : Collections.emptyList(), + Collections.emptyList()); + + DeprecationInfoAction.Response response = DeprecationInfoAction.Response.from(state, resolver, Strings.EMPTY_ARRAY, indicesOptions, datafeeds, - clusterSettingsChecks, nodeSettingsChecks, indexSettingsChecks, mlSettingsChecks); + nodeDeprecationIssues, indexSettingsChecks, clusterSettingsChecks, mlSettingsChecks); if (clusterIssueFound) { assertThat(response.getClusterSettingsIssues(), equalTo(Collections.singletonList(foundIssue))); @@ -122,7 +115,10 @@ public void testFrom() throws IOException { } if (nodeIssueFound) { - assertThat(response.getNodeSettingsIssues(), equalTo(Collections.singletonList(foundIssue))); + String details = foundIssue.getDetails() != null ? foundIssue.getDetails() + " " : ""; + DeprecationIssue mergedFoundIssue = new DeprecationIssue(foundIssue.getLevel(), foundIssue.getMessage(), foundIssue.getUrl(), + details + "(nodes impacted: [" + discoveryNode.getName() + "])"); + assertThat(response.getNodeSettingsIssues(), equalTo(Collections.singletonList(mergedFoundIssue))); } else { assertTrue(response.getNodeSettingsIssues().isEmpty()); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java new file mode 100644 index 0000000000000..8dd7255a7f15a --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckRequestTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.test.AbstractStreamableTestCase; + +import java.io.IOException; + +public class NodesDeprecationCheckRequestTests + extends AbstractStreamableTestCase { + + @Override + protected NodesDeprecationCheckRequest createBlankInstance() { + return new NodesDeprecationCheckRequest(); + } + + @Override + protected NodesDeprecationCheckRequest mutateInstance(NodesDeprecationCheckRequest instance) throws IOException { + int newSize = randomValueOtherThan(instance.nodesIds().length, () -> randomIntBetween(0,10)); + String[] newNodeIds = randomArray(newSize, newSize, String[]::new, () -> randomAlphaOfLengthBetween(5, 10)); + return new NodesDeprecationCheckRequest(newNodeIds); + } + + @Override + protected NodesDeprecationCheckRequest createTestInstance() { + return new NodesDeprecationCheckRequest(randomArray(0, 10, String[]::new, + ()-> randomAlphaOfLengthBetween(5,10))); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java new file mode 100644 index 0000000000000..143c0e2f5ad50 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/deprecation/NodesDeprecationCheckResponseTests.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.deprecation; + +import org.elasticsearch.Version; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.AbstractStreamableTestCase; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class NodesDeprecationCheckResponseTests + extends AbstractStreamableTestCase { + + @Override + protected NodesDeprecationCheckResponse createBlankInstance() { + return new NodesDeprecationCheckResponse(); + } + + @Override + protected NodesDeprecationCheckResponse createTestInstance() { + + List responses = + Arrays.asList(randomArray(1, 10, NodesDeprecationCheckAction.NodeResponse[]::new, + NodesDeprecationCheckResponseTests::randomNodeResponse)); + return new NodesDeprecationCheckResponse(new ClusterName(randomAlphaOfLength(10)), + responses, + Collections.emptyList()); + } + + @Override + protected NodesDeprecationCheckResponse mutateInstance(NodesDeprecationCheckResponse instance) throws IOException { + int mutate = randomIntBetween(1,3); + switch (mutate) { + case 1: + List responses = new ArrayList<>(instance.getNodes()); + responses.add(randomNodeResponse()); + return new NodesDeprecationCheckResponse(instance.getClusterName(), responses, instance.failures()); + case 2: + ArrayList failures = new ArrayList<>(instance.failures()); + failures.add(new FailedNodeException("test node", "test failure", new RuntimeException(randomAlphaOfLength(10)))); + return new NodesDeprecationCheckResponse(instance.getClusterName(), instance.getNodes(), failures); + case 3: + String clusterName = randomValueOtherThan(instance.getClusterName().value(), () -> randomAlphaOfLengthBetween(5,15)); + return new NodesDeprecationCheckResponse(new ClusterName(clusterName), instance.getNodes(), instance.failures()); + default: + fail("invalid mutation"); + } + + return super.mutateInstance(instance); + } + + private static DiscoveryNode randomDiscoveryNode() throws Exception { + InetAddress inetAddress = InetAddress.getByAddress(randomAlphaOfLength(5), + new byte[] { (byte) 192, (byte) 168, (byte) 0, (byte) 1}); + TransportAddress transportAddress = new TransportAddress(inetAddress, randomIntBetween(0, 65535)); + + return new DiscoveryNode(randomAlphaOfLength(5), randomAlphaOfLength(5), transportAddress, + Collections.emptyMap(), Collections.emptySet(), Version.CURRENT); + } + + private static NodesDeprecationCheckAction.NodeResponse randomNodeResponse() { + DiscoveryNode node; + try { + node = randomDiscoveryNode(); + } catch (Exception e) { + throw new RuntimeException(e); + } + List issuesList = Arrays.asList(randomArray(0,10, DeprecationIssue[]::new, + DeprecationIssueTests::createTestInstance)); + return new NodesDeprecationCheckAction.NodeResponse(node, issuesList); + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java index cede3eb309151..9bfbe352f839a 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java @@ -19,7 +19,9 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.function.Supplier; @@ -30,7 +32,10 @@ public class Deprecation extends Plugin implements ActionPlugin { @Override public List> getActions() { - return Collections.singletonList(new ActionHandler<>(DeprecationInfoAction.INSTANCE, TransportDeprecationInfoAction.class)); + return Collections.unmodifiableList(Arrays.asList( + new ActionHandler<>(DeprecationInfoAction.INSTANCE, TransportDeprecationInfoAction.class), + new ActionHandler<>(NodesDeprecationCheckAction.INSTANCE, TransportNodeDeprecationCheckAction.class) + )); } @Override diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index c6c3d5fd840c0..97e273b213b79 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -5,10 +5,10 @@ */ package org.elasticsearch.xpack.deprecation; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; @@ -33,7 +33,7 @@ private DeprecationChecks() { static List> CLUSTER_SETTINGS_CHECKS = Collections.emptyList(); - static List, List, DeprecationIssue>> NODE_SETTINGS_CHECKS = + static List> NODE_SETTINGS_CHECKS = Collections.unmodifiableList(Arrays.asList( // STUB )); diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 6ae416248e9b7..bac290d41a5eb 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -5,11 +5,10 @@ */ package org.elasticsearch.xpack.deprecation; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; +import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.client.node.NodeClient; @@ -20,7 +19,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.threadpool.ThreadPool; @@ -29,14 +27,22 @@ import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckRequest; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS; +import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS; +import static org.elasticsearch.xpack.deprecation.DeprecationChecks.ML_SETTINGS_CHECKS; public class TransportDeprecationInfoAction extends TransportMasterNodeReadAction { + private static final Logger logger = LogManager.getLogger(TransportDeprecationInfoAction.class); private final XPackLicenseState licenseState; private final NodeClient client; @@ -76,39 +82,32 @@ protected ClusterBlockException checkBlock(DeprecationInfoAction.Request request protected final void masterOperation(final DeprecationInfoAction.Request request, ClusterState state, final ActionListener listener) { if (licenseState.isDeprecationAllowed()) { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest("_local").settings(true).plugins(true); - NodesStatsRequest nodesStatsRequest = new NodesStatsRequest("_local").fs(true); - final ThreadContext threadContext = client.threadPool().getThreadContext(); - ClientHelper.executeAsyncWithOrigin(threadContext, ClientHelper.DEPRECATION_ORIGIN, nodesInfoRequest, - ActionListener.wrap( - nodesInfoResponse -> { - if (nodesInfoResponse.hasFailures()) { - throw nodesInfoResponse.failures().get(0); - } - ClientHelper.executeAsyncWithOrigin(threadContext, ClientHelper.DEPRECATION_ORIGIN, nodesStatsRequest, - ActionListener.wrap( - nodesStatsResponse -> { - if (nodesStatsResponse.hasFailures()) { - throw nodesStatsResponse.failures().get(0); - } + NodesDeprecationCheckRequest nodeDepReq = new NodesDeprecationCheckRequest("_all"); + ClientHelper.executeAsyncWithOrigin(client, ClientHelper.DEPRECATION_ORIGIN, + NodesDeprecationCheckAction.INSTANCE, nodeDepReq, + ActionListener.wrap(response -> { + if (response.hasFailures()) { + List failedNodeIds = response.failures().stream() + .map(failure -> failure.nodeId() + ": " + failure.getMessage()) + .collect(Collectors.toList()); + logger.warn("nodes failed to run deprecation checks: {}", failedNodeIds); + for (FailedNodeException failure : response.failures()) { + logger.debug("node {} failed to run deprecation checks: {}", failure.nodeId(), failure); + } + } + getDatafeedConfigs(ActionListener.wrap( + datafeeds -> { + listener.onResponse( + DeprecationInfoAction.Response.from(state, indexNameExpressionResolver, + request.indices(), request.indicesOptions(), datafeeds, + response, INDEX_SETTINGS_CHECKS, CLUSTER_SETTINGS_CHECKS, + ML_SETTINGS_CHECKS)); + }, + listener::onFailure + )); - getDatafeedConfigs(ActionListener.wrap( - datafeeds -> { - listener.onResponse( - DeprecationInfoAction.Response.from(nodesInfoResponse.getNodes(), - nodesStatsResponse.getNodes(), state, indexNameExpressionResolver, - request.indices(), request.indicesOptions(), datafeeds, - DeprecationChecks.CLUSTER_SETTINGS_CHECKS, - DeprecationChecks.NODE_SETTINGS_CHECKS, - DeprecationChecks.INDEX_SETTINGS_CHECKS, - DeprecationChecks.ML_SETTINGS_CHECKS)); - }, - listener::onFailure - )); - }, listener::onFailure), - client.admin().cluster()::nodesStats); - }, listener::onFailure), client.admin().cluster()::nodesInfo); + }, listener::onFailure)); } else { listener.onFailure(LicenseUtils.newComplianceException(XPackField.DEPRECATION)); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java new file mode 100644 index 0000000000000..a315559a2f9a7 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckRequest; +import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckResponse; + +import java.util.List; + +public class TransportNodeDeprecationCheckAction extends TransportNodesAction { + + private final Settings settings; + private final PluginsService pluginsService; + + @Inject + public TransportNodeDeprecationCheckAction(Settings settings, ThreadPool threadPool, + ClusterService clusterService, TransportService transportService, + PluginsService pluginsService, ActionFilters actionFilters) { + super(NodesDeprecationCheckAction.NAME, threadPool, clusterService, transportService, actionFilters, + NodesDeprecationCheckRequest::new, + NodesDeprecationCheckAction.NodeRequest::new, + ThreadPool.Names.GENERIC, + NodesDeprecationCheckAction.NodeResponse.class); + this.settings = settings; + this.pluginsService = pluginsService; + } + + @Override + protected NodesDeprecationCheckResponse newResponse(NodesDeprecationCheckRequest request, + List nodeResponses, + List failures) { + return new NodesDeprecationCheckResponse(clusterService.getClusterName(), nodeResponses, failures); + } + + @Override + protected NodesDeprecationCheckAction.NodeRequest newNodeRequest(String nodeId, NodesDeprecationCheckRequest request) { + return new NodesDeprecationCheckAction.NodeRequest(nodeId, request); + } + + @Override + protected NodesDeprecationCheckAction.NodeResponse newNodeResponse() { + return new NodesDeprecationCheckAction.NodeResponse(); + } + + @Override + protected NodesDeprecationCheckAction.NodeResponse nodeOperation(NodesDeprecationCheckAction.NodeRequest request) { + List issues = DeprecationInfoAction.filterChecks(DeprecationChecks.NODE_SETTINGS_CHECKS, + (c) -> c.apply(settings, pluginsService.info())); + + return new NodesDeprecationCheckAction.NodeResponse(transportService.getLocalNode(), issues); + } + + +} From ac07386eabe9ff535c03b77d91ac76784f835478 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 4 Feb 2019 17:48:08 +0100 Subject: [PATCH 04/24] Ignore the `>test-mute` label when generating release notes. (#38307) Muted tests are irrelevant for release notes. --- dev-tools/es_release_notes.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl index e911b5a5a4a4c..16a00d4eff2ae 100755 --- a/dev-tools/es_release_notes.pl +++ b/dev-tools/es_release_notes.pl @@ -32,7 +32,7 @@ ">enhancement", ">bug", ">regression", ">upgrade" ); my %Ignore = map { $_ => 1 } - ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Infra/Build", "backport" ); + ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ">test-mute", ":Core/Infra/Build", "backport" ); my %Group_Labels = ( '>breaking' => 'Breaking changes', From 2c1eab2b8a65663c17bc86df32e209e39757ff24 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 4 Feb 2019 17:44:00 +0000 Subject: [PATCH 05/24] Clarify slow cluster-state log messages (#38302) The message `... took [31s] above the warn threshold of 30s` suggests incorrectly that the task took 61 seconds. This commit adds the clarifying words `which is`. --- .../cluster/service/ClusterApplierService.java | 2 +- .../org/elasticsearch/cluster/service/ClusterService.java | 4 ---- .../org/elasticsearch/cluster/service/MasterService.java | 2 +- .../cluster/service/ClusterApplierServiceTests.java | 6 +++--- .../elasticsearch/cluster/service/MasterServiceTests.java | 8 ++++---- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java index 313ff4c660866..e254196caa47b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java @@ -517,7 +517,7 @@ public void onSuccess(String source) { protected void warnAboutSlowTaskIfNeeded(TimeValue executionTime, String source) { if (executionTime.getMillis() > slowTaskLoggingThreshold.getMillis()) { - logger.warn("cluster state applier task [{}] took [{}] above the warn threshold of {}", source, executionTime, + logger.warn("cluster state applier task [{}] took [{}] which is above the warn threshold of {}", source, executionTime, slowTaskLoggingThreshold); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java index f66ca0738954b..f83f2606b14b6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java @@ -19,8 +19,6 @@ package org.elasticsearch.cluster.service; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateApplier; @@ -45,8 +43,6 @@ import java.util.Map; public class ClusterService extends AbstractLifecycleComponent { - private static final Logger logger = LogManager.getLogger(ClusterService.class); - private final MasterService masterService; private final ClusterApplierService clusterApplierService; diff --git a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java index beb42fa1c6814..c355592890855 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java @@ -569,7 +569,7 @@ public TimeValue ackTimeout() { protected void warnAboutSlowTaskIfNeeded(TimeValue executionTime, String source) { if (executionTime.getMillis() > slowTaskLoggingThreshold.getMillis()) { - logger.warn("cluster state update task [{}] took [{}] above the warn threshold of {}", source, executionTime, + logger.warn("cluster state update task [{}] took [{}] which is above the warn threshold of {}", source, executionTime, slowTaskLoggingThreshold); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java index 770ae68e1285f..0d0ed96bf12aa 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java @@ -195,19 +195,19 @@ public void testLongClusterStateUpdateLogging() throws Exception { "test1 shouldn't see because setting is too low", ClusterApplierService.class.getCanonicalName(), Level.WARN, - "*cluster state applier task [test1] took [*] above the warn threshold of *")); + "*cluster state applier task [test1] took [*] which is above the warn threshold of *")); mockAppender.addExpectation( new MockLogAppender.SeenEventExpectation( "test2", ClusterApplierService.class.getCanonicalName(), Level.WARN, - "*cluster state applier task [test2] took [32s] above the warn threshold of *")); + "*cluster state applier task [test2] took [32s] which is above the warn threshold of *")); mockAppender.addExpectation( new MockLogAppender.SeenEventExpectation( "test4", ClusterApplierService.class.getCanonicalName(), Level.WARN, - "*cluster state applier task [test3] took [34s] above the warn threshold of *")); + "*cluster state applier task [test3] took [34s] which is above the warn threshold of *")); Logger clusterLogger = LogManager.getLogger(ClusterApplierService.class); Loggers.addAppender(clusterLogger, mockAppender); diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java index 7ed3f45e505f9..1136ab857ca4e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java @@ -652,25 +652,25 @@ public void testLongClusterStateUpdateLogging() throws Exception { "test1 shouldn't see because setting is too low", MasterService.class.getCanonicalName(), Level.WARN, - "*cluster state update task [test1] took [*] above the warn threshold of *")); + "*cluster state update task [test1] took [*] which is above the warn threshold of *")); mockAppender.addExpectation( new MockLogAppender.SeenEventExpectation( "test2", MasterService.class.getCanonicalName(), Level.WARN, - "*cluster state update task [test2] took [32s] above the warn threshold of *")); + "*cluster state update task [test2] took [32s] which is above the warn threshold of *")); mockAppender.addExpectation( new MockLogAppender.SeenEventExpectation( "test3", MasterService.class.getCanonicalName(), Level.WARN, - "*cluster state update task [test3] took [33s] above the warn threshold of *")); + "*cluster state update task [test3] took [33s] which is above the warn threshold of *")); mockAppender.addExpectation( new MockLogAppender.SeenEventExpectation( "test4", MasterService.class.getCanonicalName(), Level.WARN, - "*cluster state update task [test4] took [34s] above the warn threshold of *")); + "*cluster state update task [test4] took [34s] which is above the warn threshold of *")); Logger clusterLogger = LogManager.getLogger(MasterService.class); Loggers.addAppender(clusterLogger, mockAppender); From ab1150378befec03b1e48722bc2dc416f2f3b329 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Mon, 4 Feb 2019 13:47:04 -0500 Subject: [PATCH 06/24] Add Composite to AggregationBuilders (#38207) --- .../org/elasticsearch/client/SearchIT.java | 32 +++++++++++++++++++ .../aggregations/AggregationBuilders.java | 10 ++++++ 2 files changed, 42 insertions(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index bf16ce93c147d..54826e963cb83 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -59,7 +59,11 @@ import org.elasticsearch.script.mustache.SearchTemplateRequest; import org.elasticsearch.script.mustache.SearchTemplateResponse; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -275,6 +279,34 @@ public void testSearchWithTermsAgg() throws IOException { assertEquals(0, type2.getAggregations().asList().size()); } + public void testSearchWithCompositeAgg() throws IOException { + SearchRequest searchRequest = new SearchRequest(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + List> sources + = Collections.singletonList(new TermsValuesSourceBuilder("terms").field("type.keyword").missingBucket(true).order("asc")); + searchSourceBuilder.aggregation(AggregationBuilders.composite("composite", sources)); + searchSourceBuilder.size(0); + searchRequest.source(searchSourceBuilder); + searchRequest.indices("index"); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); + assertSearchHeader(searchResponse); + assertNull(searchResponse.getSuggest()); + assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); + assertEquals(0, searchResponse.getHits().getHits().length); + assertEquals(Float.NaN, searchResponse.getHits().getMaxScore(), 0f); + CompositeAggregation compositeAgg = searchResponse.getAggregations().get("composite"); + assertEquals("composite", compositeAgg.getName()); + assertEquals(2, compositeAgg.getBuckets().size()); + CompositeAggregation.Bucket bucket1 = compositeAgg.getBuckets().get(0); + assertEquals(3, bucket1.getDocCount()); + assertEquals("{terms=type1}", bucket1.getKeyAsString()); + assertEquals(0, bucket1.getAggregations().asList().size()); + CompositeAggregation.Bucket bucket2 = compositeAgg.getBuckets().get(1); + assertEquals(2, bucket2.getDocCount()); + assertEquals("{terms=type2}", bucket2.getKeyAsString()); + assertEquals(0, bucket2.getAggregations().asList().size()); + } + public void testSearchWithRangeAgg() throws IOException { { SearchRequest searchRequest = new SearchRequest(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java index d78e42ba89603..72ac99d94b951 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java @@ -23,6 +23,8 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrix; import org.elasticsearch.search.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.filter.Filters; @@ -88,6 +90,7 @@ import org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviation; +import java.util.List; import java.util.Map; /** @@ -368,4 +371,11 @@ public static GeoCentroidAggregationBuilder geoCentroid(String name) { public static ScriptedMetricAggregationBuilder scriptedMetric(String name) { return new ScriptedMetricAggregationBuilder(name); } + + /** + * Create a new {@link CompositeAggregationBuilder} aggregation with the given name. + */ + public static CompositeAggregationBuilder composite(String name, List> sources) { + return new CompositeAggregationBuilder(name, sources); + } } From be1bb0ec7d72a63232c99015836c9034af4e016e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 4 Feb 2019 10:58:03 -0800 Subject: [PATCH 07/24] Remove types from Monitoring plugin "backend" code (#37745) This PR removes the use of document types from the monitoring exporters and template + watches setup code. It does not remove the notion of types from the monitoring bulk API endpoint "front end" code as that code will eventually just go away in 8.0 and be replaced with Beats as collectors/shippers directly to the monitoring cluster. --- .../monitoring/exporter/http/HttpExportBulk.java | 6 ++++-- .../xpack/monitoring/exporter/local/LocalBulk.java | 6 +++--- .../watches/elasticsearch_cluster_status.json | 1 - .../monitoring/watches/elasticsearch_nodes.json | 3 +-- .../watches/elasticsearch_version_mismatch.json | 1 - .../monitoring/watches/kibana_version_mismatch.json | 1 - .../watches/logstash_version_mismatch.json | 1 - .../watches/xpack_license_expiration.json | 1 - .../action/MonitoringBulkRequestTests.java | 1 - .../exporter/local/LocalExporterIntegTests.java | 1 - .../local/LocalExporterResourceIntegTests.java | 1 + .../xpack/monitoring/integration/MonitoringIT.java | 13 +++++-------- 12 files changed, 14 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java index 7d17705decfbe..cd307322cb547 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java @@ -143,7 +143,6 @@ private byte[] toBulkBytes(final MonitoringDoc doc) throws IOException { builder.startObject("index"); { builder.field("_index", index); - builder.field("_type", "doc"); if (id != null) { builder.field("_id", id); } @@ -163,7 +162,10 @@ private byte[] toBulkBytes(final MonitoringDoc doc) throws IOException { // Adds final bulk separator out.write(xContent.streamSeparator()); - logger.trace("added index request [index={}, type={}, id={}]", index, doc.getType(), id); + logger.trace( + "http exporter [{}] - added index request [index={}, id={}, monitoring data type={}]", + name, index, id, doc.getType() + ); return BytesReference.toBytes(out.bytes()); } catch (Exception e) { diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java index b8337bea17d92..3eb92fd68a3cf 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java @@ -66,7 +66,7 @@ public void doAdd(Collection docs) throws ExportException { try { final String index = MonitoringTemplateUtils.indexName(formatter, doc.getSystem(), doc.getTimestamp()); - final IndexRequest request = new IndexRequest(index, "doc"); + final IndexRequest request = new IndexRequest(index); if (Strings.hasText(doc.getId())) { request.id(doc.getId()); } @@ -82,8 +82,8 @@ public void doAdd(Collection docs) throws ExportException { requestBuilder.add(request); if (logger.isTraceEnabled()) { - logger.trace("local exporter [{}] - added index request [index={}, type={}, id={}, pipeline={}]", - name, request.index(), request.type(), request.id(), request.getPipeline()); + logger.trace("local exporter [{}] - added index request [index={}, id={}, pipeline={}, monitoring data type={}]", + name, request.index(), request.id(), request.getPipeline(), doc.getType()); } } catch (Exception e) { if (exception == null) { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json index 1a177a26e7cf2..fca6171b62329 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json @@ -152,7 +152,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json index 4b45cb38f07d4..0566d03f21f5f 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json @@ -157,8 +157,7 @@ "actions": { "add_to_alerts_index": { "index": { - "index": ".monitoring-alerts-6", - "doc_type": "doc" + "index": ".monitoring-alerts-6" } }, "send_email_to_admin": { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json index 7cb494ce8e712..61d77d2b602fb 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json @@ -148,7 +148,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json index 7ec176c57897a..95de1a1a6383e 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json @@ -168,7 +168,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json index 69fc05f7ccee8..7448000fa8516 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json @@ -168,7 +168,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json index e7c05fbcde11b..d61bb3cd952cc 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json @@ -141,7 +141,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java index a8ab2960194f2..fc3bf633a3964 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java @@ -232,7 +232,6 @@ public void testAddRequestContentWithUnrecognizedIndexName() throws IOException assertThat(e.getMessage(), containsString("unrecognized index name [" + indexName + "]")); //This test's JSON contains outdated references to types assertWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE); - } public void testSerialization() throws IOException { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java index cf82f3e55cd78..171eeedf88cab 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java @@ -275,7 +275,6 @@ private void checkMonitoringDocs() { assertTrue("document is missing cluster_uuid field", Strings.hasText((String) source.get("cluster_uuid"))); assertTrue("document is missing timestamp field", Strings.hasText(timestamp)); assertTrue("document is missing type field", Strings.hasText(type)); - assertEquals("document _type is 'doc'", "doc", hit.getType()); @SuppressWarnings("unchecked") Map docSource = (Map) source.get("doc"); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java index 5448521594dfe..16ea27488d8a4 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java @@ -81,6 +81,7 @@ private static BytesReference generateTemplateSource(final String name, final In .field("index.number_of_replicas", 0) .endObject() .startObject("mappings") + // Still need use type, RestPutIndexTemplateAction#prepareRequestSource has logic that adds type if missing .startObject("doc") .startObject("_meta") .field("test", true) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index 1685de2667a76..f5ed570acb51f 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.license.License; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.rest.action.document.RestBulkAction; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.collapse.CollapseBuilder; @@ -105,11 +104,11 @@ protected Collection> getPlugins() { } private String createBulkEntity() { - return "{\"index\":{\"_type\":\"test\"}}\n" + + return "{\"index\":{}}\n" + "{\"foo\":{\"bar\":0}}\n" + - "{\"index\":{\"_type\":\"test\"}}\n" + + "{\"index\":{}}\n" + "{\"foo\":{\"bar\":1}}\n" + - "{\"index\":{\"_type\":\"test\"}}\n" + + "{\"index\":{}}\n" + "{\"foo\":{\"bar\":2}}\n" + "\n"; } @@ -128,7 +127,7 @@ public void testMonitoringBulk() throws Exception { final MonitoringBulkResponse bulkResponse = new MonitoringBulkRequestBuilder(client()) - .add(system, null, new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON, + .add(system, "monitoring_data_type", new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON, System.currentTimeMillis(), interval.millis()) .get(); @@ -179,10 +178,9 @@ public void testMonitoringBulk() throws Exception { equalTo(1L)); for (final SearchHit hit : hits.getHits()) { - assertMonitoringDoc(toMap(hit), system, "test", interval); + assertMonitoringDoc(toMap(hit), system, "monitoring_data_type", interval); } }); - assertWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE); } /** @@ -265,7 +263,6 @@ private void assertMonitoringDoc(final Map document, final String index = (String) document.get("_index"); assertThat(index, containsString(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-")); - assertThat(document.get("_type"), equalTo("doc")); assertThat((String) document.get("_id"), not(isEmptyOrNullString())); final Map source = (Map) document.get("_source"); From 66530dbde01fa30ae0be0defe708e87c462cb49e Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Mon, 4 Feb 2019 13:23:00 -0600 Subject: [PATCH 08/24] Deprecate HLRC security methods (#37883) This commit deprecates the few methods that had their parameters reordered to facilitate the move from EmptyResponse to boolean. This commit also readds the boolean based methods with the proper signatures. Relates #37540 Relates #36938 --- .../elasticsearch/client/SecurityClient.java | 86 +++++++++---------- .../client/security/EmptyResponse.java | 39 --------- .../SecurityDocumentationIT.java | 12 +-- .../client/security/EmptyResponseTests.java | 51 ----------- 4 files changed, 45 insertions(+), 143 deletions(-) delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/EmptyResponse.java delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/EmptyResponseTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java index 4d8d1d5db43aa..de0032c6c2a8f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java @@ -38,7 +38,6 @@ import org.elasticsearch.client.security.DeleteUserRequest; import org.elasticsearch.client.security.DeleteUserResponse; import org.elasticsearch.client.security.DisableUserRequest; -import org.elasticsearch.client.security.EmptyResponse; import org.elasticsearch.client.security.EnableUserRequest; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetPrivilegesResponse; @@ -235,14 +234,12 @@ public void getRoleMappingsAsync(final GetRoleMappingsRequest request, final Req * * @param request the request with the user to enable * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response from the enable user call + * @return {@code true} if the request succeeded (the user is enabled) * @throws IOException in case there is a problem sending the request or parsing back the response - * @deprecated use {@link #enableUser(RequestOptions, EnableUserRequest)} instead */ - @Deprecated - public EmptyResponse enableUser(EnableUserRequest request, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::enableUser, options, - EmptyResponse::fromXContent, emptySet()); + public boolean enableUser(EnableUserRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequest(request, SecurityRequestConverters::enableUser, options, + RestHighLevelClient::convertExistsResponse, emptySet()); } /** @@ -254,10 +251,11 @@ public EmptyResponse enableUser(EnableUserRequest request, RequestOptions option * @param request the request with the user to enable * @return {@code true} if the request succeeded (the user is enabled) * @throws IOException in case there is a problem sending the request or parsing back the response + * @deprecated use {@link #enableUser(EnableUserRequest, RequestOptions)} instead */ + @Deprecated public boolean enableUser(RequestOptions options, EnableUserRequest request) throws IOException { - return restHighLevelClient.performRequest(request, SecurityRequestConverters::enableUser, options, - RestHighLevelClient::convertExistsResponse, emptySet()); + return enableUser(request, options); } /** @@ -268,13 +266,11 @@ public boolean enableUser(RequestOptions options, EnableUserRequest request) thr * @param request the request with the user to enable * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion - * @deprecated use {@link #enableUserAsync(RequestOptions, EnableUserRequest, ActionListener)} instead */ - @Deprecated public void enableUserAsync(EnableUserRequest request, RequestOptions options, - ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::enableUser, options, - EmptyResponse::fromXContent, listener, emptySet()); + ActionListener listener) { + restHighLevelClient.performRequestAsync(request, SecurityRequestConverters::enableUser, options, + RestHighLevelClient::convertExistsResponse, listener, emptySet()); } /** @@ -285,11 +281,12 @@ public void enableUserAsync(EnableUserRequest request, RequestOptions options, * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param request the request with the user to enable * @param listener the listener to be notified upon request completion + * @deprecated use {@link #enableUserAsync(EnableUserRequest, RequestOptions, ActionListener)} instead */ + @Deprecated public void enableUserAsync(RequestOptions options, EnableUserRequest request, ActionListener listener) { - restHighLevelClient.performRequestAsync(request, SecurityRequestConverters::enableUser, options, - RestHighLevelClient::convertExistsResponse, listener, emptySet()); + enableUserAsync(request, options, listener); } /** @@ -299,14 +296,12 @@ public void enableUserAsync(RequestOptions options, EnableUserRequest request, * * @param request the request with the user to disable * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response from the enable user call + * @return {@code true} if the request succeeded (the user is disabled) * @throws IOException in case there is a problem sending the request or parsing back the response - * @deprecated use {@link #disableUser(RequestOptions, DisableUserRequest)} instead */ - @Deprecated - public EmptyResponse disableUser(DisableUserRequest request, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::disableUser, options, - EmptyResponse::fromXContent, emptySet()); + public boolean disableUser(DisableUserRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequest(request, SecurityRequestConverters::disableUser, options, + RestHighLevelClient::convertExistsResponse, emptySet()); } /** @@ -318,10 +313,11 @@ public EmptyResponse disableUser(DisableUserRequest request, RequestOptions opti * @param request the request with the user to disable * @return {@code true} if the request succeeded (the user is disabled) * @throws IOException in case there is a problem sending the request or parsing back the response + * @deprecated use {@link #disableUser(DisableUserRequest, RequestOptions)} instead */ + @Deprecated public boolean disableUser(RequestOptions options, DisableUserRequest request) throws IOException { - return restHighLevelClient.performRequest(request, SecurityRequestConverters::disableUser, options, - RestHighLevelClient::convertExistsResponse, emptySet()); + return disableUser(request, options); } /** @@ -332,13 +328,11 @@ public boolean disableUser(RequestOptions options, DisableUserRequest request) t * @param request the request with the user to disable * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion - * @deprecated use {@link #disableUserAsync(RequestOptions, DisableUserRequest, ActionListener)} instead */ - @Deprecated public void disableUserAsync(DisableUserRequest request, RequestOptions options, - ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::disableUser, options, - EmptyResponse::fromXContent, listener, emptySet()); + ActionListener listener) { + restHighLevelClient.performRequestAsync(request, SecurityRequestConverters::disableUser, options, + RestHighLevelClient::convertExistsResponse, listener, emptySet()); } /** @@ -349,11 +343,12 @@ public void disableUserAsync(DisableUserRequest request, RequestOptions options, * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param request the request with the user to disable * @param listener the listener to be notified upon request completion + * @deprecated use {@link #disableUserAsync(DisableUserRequest, RequestOptions, ActionListener)} instead */ + @Deprecated public void disableUserAsync(RequestOptions options, DisableUserRequest request, ActionListener listener) { - restHighLevelClient.performRequestAsync(request, SecurityRequestConverters::disableUser, options, - RestHighLevelClient::convertExistsResponse, listener, emptySet()); + disableUserAsync(request, options, listener); } /** @@ -523,14 +518,12 @@ public void getSslCertificatesAsync(RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::changePassword, options, - EmptyResponse::fromXContent, listener, emptySet()); + ActionListener listener) { + restHighLevelClient.performRequestAsync(request, SecurityRequestConverters::changePassword, options, + RestHighLevelClient::convertExistsResponse, listener, emptySet()); } /** @@ -573,14 +565,14 @@ public void changePasswordAsync(ChangePasswordRequest request, RequestOptions op * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param request the request with the user's new password * @param listener the listener to be notified upon request completion + * @deprecated use {@link #changePasswordAsync(ChangePasswordRequest, RequestOptions, ActionListener)} instead */ + @Deprecated public void changePasswordAsync(RequestOptions options, ChangePasswordRequest request, ActionListener listener) { - restHighLevelClient.performRequestAsync(request, SecurityRequestConverters::changePassword, options, - RestHighLevelClient::convertExistsResponse, listener, emptySet()); + changePasswordAsync(request, options, listener); } - /** * Delete a role mapping. * See diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/EmptyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/EmptyResponse.java deleted file mode 100644 index 961a9cb3cdfb4..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/EmptyResponse.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.security; - -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; - -/** - * Response for a request which simply returns an empty object. - @deprecated Use a boolean instead of this class - */ -@Deprecated -public final class EmptyResponse { - - private static final ObjectParser PARSER = new ObjectParser<>("empty_response", false, EmptyResponse::new); - - public static EmptyResponse fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java index fa10de4fe4ce9..0edd862eb6371 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java @@ -517,7 +517,7 @@ public void testEnableUser() throws Exception { { //tag::enable-user-execute EnableUserRequest request = new EnableUserRequest("enable_user", RefreshPolicy.NONE); - boolean response = client.security().enableUser(RequestOptions.DEFAULT, request); + boolean response = client.security().enableUser(request, RequestOptions.DEFAULT); //end::enable-user-execute assertTrue(response); @@ -544,7 +544,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::enable-user-execute-async - client.security().enableUserAsync(RequestOptions.DEFAULT, request, listener); // <1> + client.security().enableUserAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::enable-user-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -561,7 +561,7 @@ public void testDisableUser() throws Exception { { //tag::disable-user-execute DisableUserRequest request = new DisableUserRequest("disable_user", RefreshPolicy.NONE); - boolean response = client.security().disableUser(RequestOptions.DEFAULT, request); + boolean response = client.security().disableUser(request, RequestOptions.DEFAULT); //end::disable-user-execute assertTrue(response); @@ -588,7 +588,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::disable-user-execute-async - client.security().disableUserAsync(RequestOptions.DEFAULT, request, listener); // <1> + client.security().disableUserAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::disable-user-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -1038,7 +1038,7 @@ public void testChangePassword() throws Exception { { //tag::change-password-execute ChangePasswordRequest request = new ChangePasswordRequest("change_password_user", newPassword, RefreshPolicy.NONE); - boolean response = client.security().changePassword(RequestOptions.DEFAULT, request); + boolean response = client.security().changePassword(request, RequestOptions.DEFAULT); //end::change-password-execute assertTrue(response); @@ -1064,7 +1064,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); //tag::change-password-execute-async - client.security().changePasswordAsync(RequestOptions.DEFAULT, request, listener); // <1> + client.security().changePasswordAsync(request, RequestOptions.DEFAULT, listener); // <1> //end::change-password-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/EmptyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/EmptyResponseTests.java deleted file mode 100644 index 37e2e6bb51565..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/EmptyResponseTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.security; - -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentParseException; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; - -import static org.hamcrest.Matchers.containsString; - -public class EmptyResponseTests extends ESTestCase { - - public void testParseFromXContent() throws IOException { - try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, "{}")) { - - EmptyResponse response = EmptyResponse.fromXContent(parser); - assertNotNull(response); - } - - try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, "{\"foo\": \"bar\"}")) { - - XContentParseException exception = - expectThrows(XContentParseException.class, () -> EmptyResponse.fromXContent(parser)); - assertThat(exception.getMessage(), containsString("field [foo]")); - } - } -} From 5d949dddfb76e35a3c1cdcf85f86d9b7bd6cd594 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 4 Feb 2019 14:57:38 -0500 Subject: [PATCH 09/24] Docs: Drop inline callout from scroll example (#38340) Coalesces two calls into one in a scroll example so all callouts are at the end of the line. This is the only sort of callouts that are supported by asciidoctor and we'd like to start building our docs with asciidoctor. At present we don't have any mechanism to stop folks adding more inline callouts but we ought to be able to have one in a few weeks. For now, though, removing these inline callouts is a step in the right direction. Relates to #38335 --- docs/reference/search/request/scroll.asciidoc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc index f46a4a91e7f3c..ebc2f0aca6cb0 100644 --- a/docs/reference/search/request/scroll.asciidoc +++ b/docs/reference/search/request/scroll.asciidoc @@ -57,21 +57,20 @@ results. [source,js] -------------------------------------------------- -POST <1> /_search/scroll <2> +POST /_search/scroll <1> { - "scroll" : "1m", <3> - "scroll_id" : "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ==" <4> + "scroll" : "1m", <2> + "scroll_id" : "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ==" <3> } -------------------------------------------------- // CONSOLE // TEST[continued s/DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ==/$body._scroll_id/] -<1> `GET` or `POST` can be used. -<2> The URL should not include the `index` name -- this - is specified in the original `search` request instead. -<3> The `scroll` parameter tells Elasticsearch to keep the search context open +<1> `GET` or `POST` can be used and the URL should not include the `index` + name -- this is specified in the original `search` request instead. +<2> The `scroll` parameter tells Elasticsearch to keep the search context open for another `1m`. -<4> The `scroll_id` parameter +<3> The `scroll_id` parameter The `size` parameter allows you to configure the maximum number of hits to be returned with each batch of results. Each call to the `scroll` API returns the From fb1e350c815c6f5a1a75a80ddf34eb9dc7e1bb80 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 4 Feb 2019 15:04:46 -0500 Subject: [PATCH 10/24] Mute testFollowIndexAndCloseNode (#38360) Tracked at #33337 --- .../java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java index 440a5fbc37e1e..3dd20c4385fee 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java @@ -121,6 +121,7 @@ public void testFailOverOnFollower() throws Exception { pauseFollow("follower-index"); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33337") public void testFollowIndexAndCloseNode() throws Exception { getFollowerCluster().ensureAtLeastNumDataNodes(3); String leaderIndexSettings = getIndexSettings(3, 1, singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); From 24db0534650ebcf3ca3e3002109511de19b2a4ca Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Mon, 4 Feb 2019 14:05:00 -0600 Subject: [PATCH 11/24] Fix ILM explain response to allow unknown fields (#38054) IndexLifecycleExplainResponse did not allow unknown fields. This commit fixes the test and ConstructingObjectParser such that it allows unknown fields. --- .../IndexLifecycleExplainResponse.java | 2 +- .../IndexLifecycleExplainResponseTests.java | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java index 8bdc3b195acd0..772dfbc0c5c13 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java @@ -54,7 +54,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject { private static final ParseField PHASE_EXECUTION_INFO = new ParseField("phase_execution"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "index_lifecycle_explain_response", + "index_lifecycle_explain_response", true, a -> new IndexLifecycleExplainResponse( (String) a[0], (boolean) a[1], diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java index 29f7a8db89f57..89e580dfd33dd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; @@ -99,7 +100,16 @@ protected IndexLifecycleExplainResponse doParseInstance(XContentParser parser) t @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return (field) -> + // actions are plucked from the named registry, and it fails if the action is not in the named registry + field.endsWith("phase_definition.actions") + // This is a bytes reference, so any new fields are tested for equality in this bytes reference. + || field.contains("step_info"); } private static class RandomStepInfo implements ToXContentObject { From 578fd14257b5881bf28149e90382a4ce8b5d2306 Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 4 Feb 2019 20:09:07 +0000 Subject: [PATCH 12/24] Types removal - fix FullClusterRestartIT warning expectations (#38310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relax test warning message checking to pre-empt PR 38022 landing in 6.7 with new warning messages. The relaxed test now just assumes any warning message starting with “[types removal]” is tolerated rather than the precise phrasing used in the 6.7 branch. --- .../upgrades/FullClusterRestartIT.java | 6 ++--- .../test/rest/ESRestTestCase.java | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 93b3354afdc82..a5d939af5e405 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -36,8 +36,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; -import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.rest.action.document.RestBulkAction; import org.elasticsearch.rest.action.document.RestGetAction; import org.elasticsearch.rest.action.document.RestUpdateAction; @@ -925,8 +923,8 @@ public void testSnapshotRestore() throws IOException { // We therefore use the deprecated typed APIs when running against the current version. if (isRunningAgainstOldCluster() == false) { createTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); - createTemplateRequest.setOptions(expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); } + createTemplateRequest.setOptions(allowTypeRemovalWarnings()); client().performRequest(createTemplateRequest); @@ -1135,8 +1133,8 @@ && getOldClusterVersion().onOrAfter(Version.V_6_1_0) && getOldClusterVersion().b // We therefore use the deprecated typed APIs when running against the current version. if (isRunningAgainstOldCluster() == false) { getTemplateRequest.addParameter(INCLUDE_TYPE_NAME_PARAMETER, "true"); - getTemplateRequest.setOptions(expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); } + getTemplateRequest.setOptions(allowTypeRemovalWarnings()); Map getTemplateResponse = entityAsMap(client().performRequest(getTemplateRequest)); Map expectedTemplate = new HashMap<>(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index c363b7f4f6c92..56f8881a5f529 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -257,6 +257,28 @@ public static RequestOptions expectVersionSpecificWarnings(Consumer consumer.current(warnings)); } + + /** + * Creates RequestOptions designed to ignore [types removal] warnings but nothing else + * @deprecated this method is only required while we deprecate types and can be removed in 8.0 + */ + @Deprecated + public static RequestOptions allowTypeRemovalWarnings() { + Builder builder = RequestOptions.DEFAULT.toBuilder(); + builder.setWarningsHandler(new WarningsHandler() { + @Override + public boolean warningsShouldFailRequest(List warnings) { + for (String warning : warnings) { + if(warning.startsWith("[types removal]") == false) { + //Something other than a types removal message - return true + return true; + } + } + return false; + } + }); + return builder.build(); + } /** * Construct an HttpHost from the given host and port From 641704464d7b59ef60c215efda2222bec0bb345d Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 4 Feb 2019 16:07:45 -0500 Subject: [PATCH 13/24] Deprecate types in rollover index API (#38039) Relates to #35190 --- .../elasticsearch/client/IndicesClient.java | 44 +++++- .../client/IndicesRequestConverters.java | 23 ++- .../elasticsearch/client/TimedRequest.java | 8 + .../client/indices/CreateIndexRequest.java | 8 +- .../indices/rollover/RolloverRequest.java | 148 ++++++++++++++++++ .../indices/rollover/RolloverResponse.java | 129 +++++++++++++++ .../elasticsearch/client/IndicesClientIT.java | 32 +++- .../client/IndicesRequestConvertersTests.java | 50 +++++- .../IndicesClientDocumentationIT.java | 15 +- .../indices/RandomCreateIndexGenerator.java | 13 ++ .../rollover/RolloverRequestTests.java | 65 ++++++++ .../rollover/RolloverResponseTests.java | 100 ++++++++++++ .../high-level/indices/rollover.asciidoc | 15 +- .../rest-api-spec/api/indices.rollover.json | 4 + .../test/indices.rollover/40_mapping.yml | 43 +++++ .../41_mapping_with_types.yml | 47 ++++++ .../admin/indices/rollover/Condition.java | 4 + .../indices/rollover/RolloverRequest.java | 22 ++- .../indices/rollover/RolloverResponse.java | 7 + .../indices/RestRolloverIndexAction.java | 12 +- .../rollover/RolloverRequestTests.java | 9 +- .../rollover/RolloverResponseTests.java | 3 +- .../index/RandomCreateIndexGenerator.java | 2 +- 23 files changed, 758 insertions(+), 45 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 2f5bd65fba189..8cae8630cd21d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -41,8 +41,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -64,6 +62,8 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -853,6 +853,46 @@ public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions option RolloverResponse::fromXContent, listener, emptySet()); } + + /** + * Rolls over an index using the Rollover Index API. + * See + * Rollover Index API on elastic.co + * @param rolloverRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rollover(RolloverRequest, RequestOptions)} should be used instead, which accepts a new request object. + */ + @Deprecated + public org.elasticsearch.action.admin.indices.rollover.RolloverResponse rollover( + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously rolls over an index using the Rollover Index API. + * See + * Rollover Index API on elastic.co + * @param rolloverRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} should be used instead, which + * accepts a new request object. + */ + @Deprecated + public void rolloverAsync(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, listener, emptySet()); + } + /** * Gets one or more aliases using the Get Index Aliases API. * See Indices Aliases API on diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index 13bc2b8c149db..9c2ba8b30bb23 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -37,7 +37,6 @@ import org.elasticsearch.client.indices.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -52,6 +51,7 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; import org.elasticsearch.common.Strings; import java.io.IOException; @@ -339,7 +339,7 @@ private static Request resize(ResizeRequest resizeRequest) throws IOException { static Request rollover(RolloverRequest rolloverRequest) throws IOException { String endpoint = new RequestConverters.EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") - .addPathPart(rolloverRequest.getNewIndexName()).build(); + .addPathPart(rolloverRequest.getNewIndexName()).build(); Request request = new Request(HttpPost.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(request); @@ -354,6 +354,25 @@ static Request rollover(RolloverRequest rolloverRequest) throws IOException { return request; } + @Deprecated + static Request rollover(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest) throws IOException { + String endpoint = new RequestConverters.EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") + .addPathPart(rolloverRequest.getNewIndexName()).build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + params.withTimeout(rolloverRequest.timeout()); + params.withMasterTimeout(rolloverRequest.masterNodeTimeout()); + params.withWaitForActiveShards(rolloverRequest.getCreateIndexRequest().waitForActiveShards()); + if (rolloverRequest.isDryRun()) { + params.putParam("dry_run", Boolean.TRUE.toString()); + } + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); + + return request; + } + static Request getSettings(GetSettingsRequest getSettingsRequest) { String[] indices = getSettingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.indices(); String[] names = getSettingsRequest.names() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.names(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java index be2b2f5ed5c6b..0e6e7077118b6 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java @@ -36,10 +36,18 @@ public abstract class TimedRequest implements Validatable { private TimeValue timeout = DEFAULT_ACK_TIMEOUT; private TimeValue masterTimeout = DEFAULT_MASTER_NODE_TIMEOUT; + /** + * Sets the timeout to wait for the all the nodes to acknowledge + * @param timeout timeout as a {@link TimeValue} + */ public void setTimeout(TimeValue timeout) { this.timeout = timeout; } + /** + * Sets the timeout to connect to the master node + * @param masterTimeout timeout as a {@link TimeValue} + */ public void setMasterTimeout(TimeValue masterTimeout) { this.masterTimeout = masterTimeout; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java index f0bff6e6f4307..1a018591dc770 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java @@ -338,10 +338,14 @@ public CreateIndexRequest waitForActiveShards(ActiveShardCount waitForActiveShar return this; } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + innerToXContent(builder, params); + builder.endObject(); + return builder; + } + public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(SETTINGS.getPreferredName()); settings.toXContent(builder, params); builder.endObject(); @@ -356,8 +360,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws for (Alias alias : aliases) { alias.toXContent(builder, params); } - builder.endObject(); - builder.endObject(); return builder; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java new file mode 100644 index 0000000000000..ef78fb7353067 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.client.TimedRequest; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Request class to swap index under an alias upon satisfying conditions + */ +public class RolloverRequest extends TimedRequest implements ToXContentObject { + + private final String alias; + private final String newIndexName; + private boolean dryRun; + private final Map> conditions = new HashMap<>(2); + //the index name "_na_" is never read back, what matters are settings, mappings and aliases + private final CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); + + public RolloverRequest(String alias, String newIndexName) { + if (alias == null) { + throw new IllegalArgumentException("The index alias cannot be null!"); + } + this.alias = alias; + this.newIndexName = newIndexName; + } + + /** + * Returns the alias of the rollover operation + */ + public String getAlias() { + return alias; + } + + /** + * Returns the new index name for the rollover + */ + public String getNewIndexName() { + return newIndexName; + } + + + /** + * Sets if the rollover should not be executed when conditions are met + */ + public RolloverRequest dryRun(boolean dryRun) { + this.dryRun = dryRun; + return this; + } + /** + * Returns if the rollover should not be executed when conditions are met + */ + public boolean isDryRun() { + return dryRun; + } + + /** + * Adds condition to check if the index is at least age old + */ + public RolloverRequest addMaxIndexAgeCondition(TimeValue age) { + MaxAgeCondition maxAgeCondition = new MaxAgeCondition(age); + if (this.conditions.containsKey(maxAgeCondition.name())) { + throw new IllegalArgumentException(maxAgeCondition.name() + " condition is already set"); + } + this.conditions.put(maxAgeCondition.name(), maxAgeCondition); + return this; + } + + /** + * Adds condition to check if the index has at least numDocs + */ + public RolloverRequest addMaxIndexDocsCondition(long numDocs) { + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(numDocs); + if (this.conditions.containsKey(maxDocsCondition.name())) { + throw new IllegalArgumentException(maxDocsCondition.name() + " condition is already set"); + } + this.conditions.put(maxDocsCondition.name(), maxDocsCondition); + return this; + } + /** + * Adds a size-based condition to check if the index size is at least size. + */ + public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) { + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(size); + if (this.conditions.containsKey(maxSizeCondition.name())) { + throw new IllegalArgumentException(maxSizeCondition + " condition is already set"); + } + this.conditions.put(maxSizeCondition.name(), maxSizeCondition); + return this; + } + /** + * Returns all set conditions + */ + public Map> getConditions() { + return conditions; + } + + /** + * Returns the inner {@link CreateIndexRequest}. Allows to configure mappings, settings and aliases for the new index. + */ + public CreateIndexRequest getCreateIndexRequest() { + return createIndexRequest; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + createIndexRequest.innerToXContent(builder, params); + + builder.startObject("conditions"); + for (Condition condition : conditions.values()) { + condition.toXContent(builder, params); + } + builder.endObject(); + + builder.endObject(); + return builder; + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java new file mode 100644 index 0000000000000..2bcd683d7b1f6 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * Response object for {@link RolloverRequest} API + */ +public final class RolloverResponse extends ShardsAcknowledgedResponse { + + private static final ParseField NEW_INDEX = new ParseField("new_index"); + private static final ParseField OLD_INDEX = new ParseField("old_index"); + private static final ParseField DRY_RUN = new ParseField("dry_run"); + private static final ParseField ROLLED_OVER = new ParseField("rolled_over"); + private static final ParseField CONDITIONS = new ParseField("conditions"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rollover", + true, args -> new RolloverResponse((String) args[0], (String) args[1], (Map) args[2], + (Boolean)args[3], (Boolean)args[4], (Boolean) args[5], (Boolean) args[6])); + + static { + PARSER.declareString(constructorArg(), OLD_INDEX); + PARSER.declareString(constructorArg(), NEW_INDEX); + PARSER.declareObject(constructorArg(), (parser, context) -> parser.map(), CONDITIONS); + PARSER.declareBoolean(constructorArg(), DRY_RUN); + PARSER.declareBoolean(constructorArg(), ROLLED_OVER); + declareAcknowledgedAndShardsAcknowledgedFields(PARSER); + } + + private final String oldIndex; + private final String newIndex; + private final Map conditionStatus; + private final boolean dryRun; + private final boolean rolledOver; + + public RolloverResponse(String oldIndex, String newIndex, Map conditionResults, + boolean dryRun, boolean rolledOver, boolean acknowledged, boolean shardsAcknowledged) { + super(acknowledged, shardsAcknowledged); + this.oldIndex = oldIndex; + this.newIndex = newIndex; + this.dryRun = dryRun; + this.rolledOver = rolledOver; + this.conditionStatus = conditionResults; + } + + /** + * Returns the name of the index that the request alias was pointing to + */ + public String getOldIndex() { + return oldIndex; + } + + /** + * Returns the name of the index that the request alias currently points to + */ + public String getNewIndex() { + return newIndex; + } + + /** + * Returns the statuses of all the request conditions + */ + public Map getConditionStatus() { + return conditionStatus; + } + + /** + * Returns if the rollover execution was skipped even when conditions were met + */ + public boolean isDryRun() { + return dryRun; + } + + /** + * Returns true if the rollover was not simulated and the conditions were met + */ + public boolean isRolledOver() { + return rolledOver; + } + + public static RolloverResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (super.equals(o)) { + RolloverResponse that = (RolloverResponse) o; + return dryRun == that.dryRun && + rolledOver == that.rolledOver && + Objects.equals(oldIndex, that.oldIndex) && + Objects.equals(newIndex, that.newIndex) && + Objects.equals(conditionStatus, that.conditionStatus); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), oldIndex, newIndex, conditionStatus, dryRun, rolledOver); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 306929d78a67a..ee57c32b23796 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -45,8 +45,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -76,6 +74,8 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ValidationException; @@ -100,6 +100,7 @@ import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import java.io.IOException; import java.util.Arrays; @@ -1102,6 +1103,8 @@ public void testRollover() throws IOException { assertEquals("test_new", rolloverResponse.getNewIndex()); } { + String mappings = "{\"properties\":{\"field2\":{\"type\":\"keyword\"}}}"; + rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON); rolloverRequest.dryRun(false); rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, @@ -1118,6 +1121,31 @@ public void testRollover() throws IOException { } } + public void testRolloverWithTypes() throws IOException { + highLevelClient().indices().create(new CreateIndexRequest("test").alias(new Alias("alias")), RequestOptions.DEFAULT); + highLevelClient().index(new IndexRequest("test").id("1").source("field", "value"), RequestOptions.DEFAULT); + highLevelClient().index(new IndexRequest("test").id("2").source("field", "value") + .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL), RequestOptions.DEFAULT); + + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest = + new org.elasticsearch.action.admin.indices.rollover.RolloverRequest("alias", "test_new"); + rolloverRequest.addMaxIndexDocsCondition(1); + rolloverRequest.getCreateIndexRequest().mapping("_doc", "field2", "type=keyword"); + + org.elasticsearch.action.admin.indices.rollover.RolloverResponse rolloverResponse = execute( + rolloverRequest, + highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, + expectWarnings(RestRolloverIndexAction.TYPES_DEPRECATION_MESSAGE) + ); + assertTrue(rolloverResponse.isRolledOver()); + assertFalse(rolloverResponse.isDryRun()); + Map conditionStatus = rolloverResponse.getConditionStatus(); + assertTrue(conditionStatus.get("[max_docs: 1]")); + assertEquals("test", rolloverResponse.getOldIndex()); + assertEquals("test_new", rolloverResponse.getNewIndex()); + } + public void testGetAlias() throws IOException { { createIndex("index1", Settings.EMPTY); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index 6d873ec2b944c..0c94cb61a7923 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -55,6 +54,7 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.RandomCreateIndexGenerator; +import org.elasticsearch.client.indices.rollover.RolloverRequest; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -75,7 +75,8 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; -import static org.elasticsearch.index.RandomCreateIndexGenerator.randomAliases; +import static org.elasticsearch.client.indices.RandomCreateIndexGenerator.randomAliases; +import static org.elasticsearch.client.indices.RandomCreateIndexGenerator.randomMapping; import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings; import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomAliasAction; import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; @@ -808,7 +809,7 @@ private void resizeTest(ResizeType resizeType, CheckedFunction expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomTimeout(rolloverRequest, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); + RequestConvertersTests.setRandomMasterTimeout(rolloverRequest, expectedParams); + if (ESTestCase.randomBoolean()) { + rolloverRequest.dryRun(ESTestCase.randomBoolean()); + if (rolloverRequest.isDryRun()) { + expectedParams.put("dry_run", "true"); + } + } + if (ESTestCase.randomBoolean()) { + rolloverRequest.addMaxIndexAgeCondition(new TimeValue(ESTestCase.randomNonNegativeLong())); + } + if (ESTestCase.randomBoolean()) { + rolloverRequest.getCreateIndexRequest().mapping(randomMapping()); + } + if (ESTestCase.randomBoolean()) { + randomAliases(rolloverRequest.getCreateIndexRequest()); + } + if (ESTestCase.randomBoolean()) { + rolloverRequest.getCreateIndexRequest().settings( + org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings()); + } + RequestConvertersTests.setRandomWaitForActiveShards(rolloverRequest.getCreateIndexRequest()::waitForActiveShards, expectedParams); + + Request request = IndicesRequestConverters.rollover(rolloverRequest); + if (rolloverRequest.getNewIndexName() == null) { + Assert.assertEquals("/" + rolloverRequest.getAlias() + "/_rollover", request.getEndpoint()); + } else { + Assert.assertEquals("/" + rolloverRequest.getAlias() + "/_rollover/" + rolloverRequest.getNewIndexName(), + request.getEndpoint()); + } + Assert.assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + RequestConvertersTests.assertToXContentBody(rolloverRequest, request.getEntity()); + Assert.assertEquals(expectedParams, request.getParameters()); + } + + public void testRolloverWithTypes() throws IOException { + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest = + new org.elasticsearch.action.admin.indices.rollover.RolloverRequest(ESTestCase.randomAlphaOfLengthBetween(3, 10), + ESTestCase.randomBoolean() ? null : ESTestCase.randomAlphaOfLengthBetween(3, 10)); + Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomTimeout(rolloverRequest::timeout, rolloverRequest.timeout(), expectedParams); RequestConvertersTests.setRandomMasterTimeout(rolloverRequest, expectedParams); if (ESTestCase.randomBoolean()) { @@ -835,6 +876,7 @@ public void testRollover() throws IOException { expectedParams.put("dry_run", "true"); } } + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); if (ESTestCase.randomBoolean()) { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(ESTestCase.randomNonNegativeLong())); } @@ -844,7 +886,7 @@ public void testRollover() throws IOException { org.elasticsearch.index.RandomCreateIndexGenerator.randomMapping(type)); } if (ESTestCase.randomBoolean()) { - randomAliases(rolloverRequest.getCreateIndexRequest()); + org.elasticsearch.index.RandomCreateIndexGenerator.randomAliases(rolloverRequest.getCreateIndexRequest()); } if (ESTestCase.randomBoolean()) { rolloverRequest.getCreateIndexRequest().settings( diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index d358655f2355a..64741da12249a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -46,8 +46,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -80,6 +78,8 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -1832,18 +1832,16 @@ public void testRolloverIndex() throws Exception { // end::rollover-index-request // tag::rollover-index-request-timeout - request.timeout(TimeValue.timeValueMinutes(2)); // <1> - request.timeout("2m"); // <2> + request.setTimeout(TimeValue.timeValueMinutes(2)); // <1> // end::rollover-index-request-timeout // tag::rollover-index-request-masterTimeout - request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> - request.masterNodeTimeout("1m"); // <2> + request.setMasterTimeout(TimeValue.timeValueMinutes(1)); // <1> // end::rollover-index-request-masterTimeout // tag::rollover-index-request-dryRun request.dryRun(true); // <1> // end::rollover-index-request-dryRun // tag::rollover-index-request-waitForActiveShards - request.getCreateIndexRequest().waitForActiveShards(2); // <1> + request.getCreateIndexRequest().waitForActiveShards(ActiveShardCount.from(2)); // <1> request.getCreateIndexRequest().waitForActiveShards(ActiveShardCount.DEFAULT); // <2> // end::rollover-index-request-waitForActiveShards // tag::rollover-index-request-settings @@ -1851,7 +1849,8 @@ public void testRolloverIndex() throws Exception { .put("index.number_of_shards", 4)); // <1> // end::rollover-index-request-settings // tag::rollover-index-request-mapping - request.getCreateIndexRequest().mapping("type", "field", "type=keyword"); // <1> + String mappings = "{\"properties\":{\"field-1\":{\"type\":\"keyword\"}}}"; + request.getCreateIndexRequest().mapping(mappings, XContentType.JSON); // <1> // end::rollover-index-request-mapping // tag::rollover-index-request-alias request.getCreateIndexRequest().alias(new Alias("another_alias")); // <1> diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java index 179b7e728b620..610cc54678ae0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java @@ -24,6 +24,9 @@ import java.io.IOException; +import static org.elasticsearch.index.RandomCreateIndexGenerator.randomAlias; +import static org.elasticsearch.test.ESTestCase.randomIntBetween; + public class RandomCreateIndexGenerator { /** @@ -58,4 +61,14 @@ public static XContentBuilder randomMapping() throws IOException { builder.endObject(); return builder; } + + /** + * Sets random aliases to the provided {@link CreateIndexRequest} + */ + public static void randomAliases(CreateIndexRequest request) { + int aliasesNo = randomIntBetween(0, 2); + for (int i = 0; i < aliasesNo; i++) { + request.alias(randomAlias()); + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java new file mode 100644 index 0000000000000..57798c393db8f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.containsInAnyOrder; + + +public class RolloverRequestTests extends ESTestCase { + public void testConstructorAndFieldAssignments() { + // test constructor + String alias = randomAlphaOfLength(5); + String newIndexName = null; + if (randomBoolean()) { + newIndexName = randomAlphaOfLength(8); + } + RolloverRequest rolloverRequest = new RolloverRequest(alias, newIndexName); + assertEquals(alias, rolloverRequest.getAlias()); + assertEquals(newIndexName, rolloverRequest.getNewIndexName()); + + // test assignment of conditions + MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(10)); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000)); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(10000L); + Condition[] expectedConditions = new Condition[] {maxAgeCondition, maxSizeCondition, maxDocsCondition}; + rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value()); + rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); + rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value()); + List> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values()); + assertThat(requestConditions, containsInAnyOrder(expectedConditions)); + } + + public void testValidation() { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> + new RolloverRequest(null, null)); + assertEquals("The index alias cannot be null!", exception.getMessage()); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java new file mode 100644 index 0000000000000..53fe3bb279e3f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.common.xcontent.ToXContent.Params; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.Collections; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class RolloverResponseTests extends ESTestCase { + + private static final List>> conditionSuppliers = new ArrayList<>(); + static { + conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + } + + public void testFromXContent() throws IOException { + xContentTester( + this::createParser, + RolloverResponseTests::createTestInstance, + RolloverResponseTests::toXContent, + RolloverResponse::fromXContent) + .supportsUnknownFields(true) + .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter()) + .test(); + } + + private static RolloverResponse createTestInstance() { + final String oldIndex = randomAlphaOfLength(8); + final String newIndex = randomAlphaOfLength(8); + final boolean dryRun = randomBoolean(); + final boolean rolledOver = randomBoolean(); + final boolean acknowledged = randomBoolean(); + final boolean shardsAcknowledged = acknowledged && randomBoolean(); + + Map results = new HashMap<>(); + int numResults = randomIntBetween(0, 3); + List>> conditions = randomSubsetOf(numResults, conditionSuppliers); + conditions.forEach(condition -> results.put(condition.get().name(), randomBoolean())); + + return new RolloverResponse(oldIndex, newIndex, results, dryRun, rolledOver, acknowledged, shardsAcknowledged); + } + + private Predicate getRandomFieldsExcludeFilter() { + return field -> field.startsWith("conditions"); + } + + private static void toXContent(RolloverResponse response, XContentBuilder builder) throws IOException { + Params params = new ToXContent.MapParams( + Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "false")); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse serverResponse = + new org.elasticsearch.action.admin.indices.rollover.RolloverResponse( + response.getOldIndex(), + response.getNewIndex(), + response.getConditionStatus(), + response.isDryRun(), + response.isRolledOver(), + response.isAcknowledged(), + response.isShardsAcknowledged() + ); + serverResponse.toXContent(builder, params); + } +} diff --git a/docs/java-rest/high-level/indices/rollover.asciidoc b/docs/java-rest/high-level/indices/rollover.asciidoc index c6134cd5579df..6b7a82a11ae2b 100644 --- a/docs/java-rest/high-level/indices/rollover.asciidoc +++ b/docs/java-rest/high-level/indices/rollover.asciidoc @@ -19,7 +19,8 @@ one or more conditions that determine when the index has to be rolled over: include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> The alias (first argument) that points to the index to rollover, and -optionally the name of the new index in case the rollover operation is performed +the name of the new index in case the rollover operation is performed. +The new index argument is optional, and can be set to null <2> Condition on the age of the index <3> Condition on the number of documents in the index <4> Condition on the size of the index @@ -39,24 +40,20 @@ include-tagged::{doc-tests-file}[{api}-request-timeout] -------------------------------------------------- <1> Timeout to wait for the all the nodes to acknowledge the index is opened as a `TimeValue` -<2> Timeout to wait for the all the nodes to acknowledge the index is opened -as a `String` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-masterTimeout] -------------------------------------------------- <1> Timeout to connect to the master node as a `TimeValue` -<2> Timeout to connect to the master node as a `String` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-waitForActiveShards] -------------------------------------------------- -<1> The number of active shard copies to wait for before the rollover index API -returns a response, as an `int` -<2> The number of active shard copies to wait for before the rollover index API -returns a response, as an `ActiveShardCount` +<1> Sets the number of active shard copies to wait for before the rollover +index API returns a response +<2> Resets the number of active shard copies to wait for to the default value ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -98,5 +95,3 @@ each shard in the index before timing out <5> Whether the index has been rolled over <6> Whether the operation was performed or it was a dry run <7> The different conditions and whether they were matched or not - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json index 5e5ba1367ad3e..7bf1513969fb3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json @@ -18,6 +18,10 @@ } }, "params": { + "include_type_name": { + "type" : "boolean", + "description" : "Whether a type should be included in the body of the mappings." + }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml new file mode 100644 index 0000000000000..59e027fb98457 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml @@ -0,0 +1,43 @@ +--- +"Typeless mapping": + - skip: + version: " - 6.99.99" + reason: include_type_name defaults to true before 7.0.0 + + - do: + indices.create: + index: logs-1 + body: + aliases: + logs_search: {} + + # index first document and wait for refresh + - do: + index: + index: logs-1 + id: "1" + body: { "foo": "hello world" } + refresh: true + + # index second document and wait for refresh + - do: + index: + index: logs-1 + id: "2" + body: { "foo": "hello world" } + refresh: true + + # perform alias rollover with new typeless mapping + - do: + indices.rollover: + alias: "logs_search" + body: + conditions: + max_docs: 2 + mappings: + properties: + foo2: + type: keyword + + - match: { conditions: { "[max_docs: 2]": true } } + - match: { rolled_over: true } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml new file mode 100644 index 0000000000000..36389f3ce8bba --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml @@ -0,0 +1,47 @@ +--- +"Typeless mapping": + - skip: + version: " - 6.99.99" + reason: include_type_name defaults to true before 7.0.0 + + - do: + indices.create: + index: logs-1 + body: + aliases: + logs_search: {} + + # index first document and wait for refresh + - do: + index: + index: logs-1 + type: test + id: "1" + body: { "foo": "hello world" } + refresh: true + + # index second document and wait for refresh + - do: + index: + index: logs-1 + type: test + id: "2" + body: { "foo": "hello world" } + refresh: true + + # perform alias rollover with new typeless mapping + - do: + indices.rollover: + include_type_name: true + alias: "logs_search" + body: + conditions: + max_docs: 2 + mappings: + _doc: + properties: + foo2: + type: keyword + + - match: { conditions: { "[max_docs: 2]": true } } + - match: { rolled_over: true } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java index 4a65427f34e17..7e9ccda8f90b7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java @@ -75,6 +75,10 @@ public T value() { return value; } + public String name() { + return name; + } + /** * Holder for index stats used to evaluate conditions */ diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 1475e4bd42088..3bd3153d83180 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -41,10 +42,13 @@ /** * Request class to swap index under an alias upon satisfying conditions + * + * Note: there is a new class with the same name for the Java HLRC that uses a typeless format. + * Any changes done to this class should also go to that client class. */ public class RolloverRequest extends AcknowledgedRequest implements IndicesRequest, ToXContentObject { - private static final ObjectParser PARSER = new ObjectParser<>("rollover"); + private static final ObjectParser PARSER = new ObjectParser<>("rollover"); private static final ObjectParser>, Void> CONDITION_PARSER = new ObjectParser<>("conditions"); private static final ParseField CONDITIONS = new ParseField("conditions"); @@ -66,9 +70,14 @@ public class RolloverRequest extends AcknowledgedRequest implem CONDITIONS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.settings(parser.map()), CreateIndexRequest.SETTINGS, ObjectParser.ValueType.OBJECT); - PARSER.declareField((parser, request, context) -> { - for (Map.Entry mappingsEntry : parser.map().entrySet()) { - request.createIndexRequest.mapping(mappingsEntry.getKey(), (Map) mappingsEntry.getValue()); + PARSER.declareField((parser, request, isTypeIncluded) -> { + if (isTypeIncluded) { + for (Map.Entry mappingsEntry : parser.map().entrySet()) { + request.createIndexRequest.mapping(mappingsEntry.getKey(), (Map) mappingsEntry.getValue()); + } + } else { + // a type is not included, add a dummy _doc type + request.createIndexRequest.mapping(MapperService.SINGLE_MAPPING_NAME, parser.map()); } }, CreateIndexRequest.MAPPINGS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), @@ -230,7 +239,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public void fromXContent(XContentParser parser) throws IOException { - PARSER.parse(parser, this, null); + // param isTypeIncluded decides how mappings should be parsed from XContent + public void fromXContent(boolean isTypeIncluded, XContentParser parser) throws IOException { + PARSER.parse(parser, this, isTypeIncluded); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java index 4fb5b6a19f117..52c5470618671 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java @@ -37,6 +37,13 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * Response object for {@link RolloverRequest} API + * + * Note: there is a new class with the same name for the Java HLRC that uses a typeless format. + * Any changes done to this class should also go to that client class. + */ public final class RolloverResponse extends ShardsAcknowledgedResponse implements ToXContentObject { private static final ParseField NEW_INDEX = new ParseField("new_index"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index 489001bf2a14f..f79d3247e647d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -19,9 +19,11 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -31,6 +33,10 @@ import java.io.IOException; public class RestRolloverIndexAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(RestRolloverIndexAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in rollover " + + "index requests is deprecated. The parameter will be removed in the next major version."; public RestRolloverIndexAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this); @@ -44,8 +50,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.deprecatedAndMaybeLog("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); + } RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index")); - request.applyContentParser(rolloverIndexRequest::fromXContent); + request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser)); rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false)); rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout())); rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 6443c0e5ce961..b3e89e5054ff3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -52,7 +52,6 @@ import static org.hamcrest.Matchers.equalTo; public class RolloverRequestTests extends ESTestCase { - private NamedWriteableRegistry writeableRegistry; @Override @@ -72,7 +71,7 @@ public void testConditionsParsing() throws Exception { .field("max_size", "45gb") .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); Map> conditions = request.getConditions(); assertThat(conditions.size(), equalTo(3)); MaxAgeCondition maxAgeCondition = (MaxAgeCondition)conditions.get(MaxAgeCondition.NAME); @@ -108,7 +107,7 @@ public void testParsingWithIndexSettings() throws Exception { .startObject("alias1").endObject() .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(true, createParser(builder)); Map> conditions = request.getConditions(); assertThat(conditions.size(), equalTo(2)); assertThat(request.getCreateIndexRequest().mappings().size(), equalTo(1)); @@ -147,7 +146,7 @@ public void testToAndFromXContent() throws IOException { BytesReference originalBytes = toShuffledXContent(rolloverRequest, xContentType, EMPTY_PARAMS, humanReadable); RolloverRequest parsedRolloverRequest = new RolloverRequest(); - parsedRolloverRequest.fromXContent(createParser(xContentType.xContent(), originalBytes)); + parsedRolloverRequest.fromXContent(true, createParser(xContentType.xContent(), originalBytes)); CreateIndexRequest createIndexRequest = rolloverRequest.getCreateIndexRequest(); CreateIndexRequest parsedCreateIndexRequest = parsedRolloverRequest.getCreateIndexRequest(); @@ -172,7 +171,7 @@ public void testUnknownFields() throws IOException { } builder.endObject(); BytesReference mutated = XContentTestUtils.insertRandomFields(xContentType, BytesReference.bytes(builder), null, random()); - expectThrows(XContentParseException.class, () -> request.fromXContent(createParser(xContentType.xContent(), mutated))); + expectThrows(XContentParseException.class, () -> request.fromXContent(false, createParser(xContentType.xContent(), mutated))); } public void testSameConditionCanOnlyBeAddedOnce() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java index 37c9dc3dab328..0cc3f455e83df 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractStreamableXContentTestCase; @@ -58,7 +59,7 @@ private static Map randomResults(boolean allowNoItems) { private static final List>> conditionSuppliers = new ArrayList<>(); static { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java index 345ef1f58bcac..9732504cac6d4 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java @@ -126,7 +126,7 @@ public static void randomAliases(CreateIndexRequest request) { } } - private static Alias randomAlias() { + public static Alias randomAlias() { Alias alias = new Alias(randomAlphaOfLength(5)); if (randomBoolean()) { From 0ced7753891489e3b26cf3552b1e05bbb2149f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 4 Feb 2019 22:30:34 +0100 Subject: [PATCH 14/24] Mute RareClusterStateIT.testDelayedMappingPropagationOnReplica (#38357) --- .../elasticsearch/cluster/coordination/RareClusterStateIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java index 49b4086372d21..f072fd4fb9c63 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java @@ -283,6 +283,7 @@ public void testDelayedMappingPropagationOnPrimary() throws Exception { }); } + @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/36813") public void testDelayedMappingPropagationOnReplica() throws Exception { // This is essentially the same thing as testDelayedMappingPropagationOnPrimary // but for replicas From c3cdf84c04e6137da256561d773df0a9a066c8ec Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Mon, 4 Feb 2019 14:34:37 -0700 Subject: [PATCH 15/24] Fix SSLContext pinning to TLSV1.2 in reload tests (#38341) This commit fixes the pinning of SSLContexts to TLSv1.2 in the SSLConfigurationReloaderTests. The pinning was added for the initial creation of clients and webservers but the updated contexts would default to TLSv1.3, which is known to cause hangs with the MockWebServer that we use. Relates #38103 Closes #38247 --- .../xpack/core/ssl/SSLConfigurationReloaderTests.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java index a9227649159ad..674e14ca0e196 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java @@ -91,6 +91,7 @@ public void testReloadingKeyStore() throws Exception { final Settings settings = Settings.builder() .put("path.home", createTempDir()) .put("xpack.security.transport.ssl.keystore.path", keystorePath) + .put("xpack.security.transport.ssl.supported_protocols", "TLSv1.2") .setSecureSettings(secureSettings) .build(); final Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings); @@ -149,6 +150,7 @@ public void testPEMKeyConfigReloading() throws Exception { .put("xpack.security.transport.ssl.key", keyPath) .put("xpack.security.transport.ssl.certificate", certPath) .putList("xpack.security.transport.ssl.certificate_authorities", certPath.toString()) + .put("xpack.security.transport.ssl.supported_protocols", "TLSv1.2") .setSecureSettings(secureSettings) .build(); final Environment env = randomBoolean() ? null : @@ -193,7 +195,6 @@ public void testPEMKeyConfigReloading() throws Exception { * Tests the reloading of SSLContext when the trust store is modified. The same store is used as a TrustStore (for the * reloadable SSLContext used in the HTTPClient) and as a KeyStore for the MockWebServer */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38247") public void testReloadingTrustStore() throws Exception { assumeFalse("Can't run in a FIPS JVM", inFipsJvm()); Path tempDir = createTempDir(); @@ -206,6 +207,7 @@ public void testReloadingTrustStore() throws Exception { secureSettings.setString("xpack.security.transport.ssl.truststore.secure_password", "testnode"); Settings settings = Settings.builder() .put("xpack.security.transport.ssl.truststore.path", trustStorePath) + .put("xpack.security.transport.ssl.supported_protocols", "TLSv1.2") .put("path.home", createTempDir()) .setSecureSettings(secureSettings) .build(); @@ -241,10 +243,10 @@ public void testReloadingTrustStore() throws Exception { validateSSLConfigurationIsReloaded(settings, env, trustMaterialPreChecks, modifier, trustMaterialPostChecks); } } + /** * Test the reloading of SSLContext whose trust config is backed by PEM certificate files. */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38247") public void testReloadingPEMTrustConfig() throws Exception { Path tempDir = createTempDir(); Path serverCertPath = tempDir.resolve("testnode.crt"); @@ -257,6 +259,7 @@ public void testReloadingPEMTrustConfig() throws Exception { Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_updated.crt"), updatedCert); Settings settings = Settings.builder() .putList("xpack.security.transport.ssl.certificate_authorities", serverCertPath.toString()) + .put("xpack.security.transport.ssl.supported_protocols", "TLSv1.2") .put("path.home", createTempDir()) .build(); Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings); @@ -305,6 +308,7 @@ public void testReloadingKeyStoreException() throws Exception { secureSettings.setString("xpack.security.transport.ssl.keystore.secure_password", "testnode"); Settings settings = Settings.builder() .put("xpack.security.transport.ssl.keystore.path", keystorePath) + .put("xpack.security.transport.ssl.supported_protocols", "TLSv1.2") .setSecureSettings(secureSettings) .put("path.home", createTempDir()) .build(); @@ -346,6 +350,7 @@ public void testReloadingPEMKeyConfigException() throws Exception { .put("xpack.security.transport.ssl.key", keyPath) .put("xpack.security.transport.ssl.certificate", certPath) .putList("xpack.security.transport.ssl.certificate_authorities", certPath.toString(), clientCertPath.toString()) + .put("xpack.security.transport.ssl.supported_protocols", "TLSv1.2") .put("path.home", createTempDir()) .setSecureSettings(secureSettings) .build(); @@ -381,6 +386,7 @@ public void testTrustStoreReloadException() throws Exception { secureSettings.setString("xpack.security.transport.ssl.truststore.secure_password", "testnode"); Settings settings = Settings.builder() .put("xpack.security.transport.ssl.truststore.path", trustStorePath) + .put("xpack.security.transport.ssl.supported_protocols", "TLSv1.2") .put("path.home", createTempDir()) .setSecureSettings(secureSettings) .build(); @@ -414,6 +420,7 @@ public void testPEMTrustReloadException() throws Exception { Files.copy(getDataPath("/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt"), clientCertPath); Settings settings = Settings.builder() .putList("xpack.security.transport.ssl.certificate_authorities", clientCertPath.toString()) + .put("xpack.security.transport.ssl.supported_protocols", "TLSv1.2") .put("path.home", createTempDir()) .build(); Environment env = randomBoolean() ? null : TestEnvironment.newEnvironment(settings); From 5a33816c86e4fdf0459be428ebe2328e3e4ede20 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 4 Feb 2019 16:37:42 -0600 Subject: [PATCH 16/24] Add test for `PutFollowAction` on a closed index (#38236) This is related to #35975. Currently when an index falls behind a leader it encounters a fatal exception. This commit adds a test for that scenario. Additionally, it tests that the user can stop following, close the follower index, and put follow again. After the indexing is re-bootstrapped, it will recover the documents it lost in normal following operations. --- .../xpack/ccr/IndexFollowingIT.java | 109 +++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 74c44704e2e1c..eee28b5875bcc 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -8,6 +8,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; @@ -16,10 +19,13 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -53,6 +59,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.snapshots.SnapshotRestoreException; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.xpack.CcrIntegTestCase; @@ -75,6 +82,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -86,6 +94,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -947,6 +956,98 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception { assertThat(hasFollowIndexBeenClosedChecker.getAsBoolean(), is(true)); } + public void testMustCloseIndexAndPauseToRestartWithPutFollowing() throws Exception { + final int numberOfPrimaryShards = randomIntBetween(1, 3); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); + assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); + ensureLeaderYellow("index1"); + + final PutFollowAction.Request followRequest = putFollow("index1", "index2"); + PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); + assertTrue(response.isFollowIndexCreated()); + assertTrue(response.isFollowIndexShardsAcked()); + assertTrue(response.isIndexFollowingStarted()); + + final PutFollowAction.Request followRequest2 = putFollow("index1", "index2"); + expectThrows(SnapshotRestoreException.class, + () -> followerClient().execute(PutFollowAction.INSTANCE, followRequest2).actionGet()); + + followerClient().admin().indices().prepareClose("index2").get(); + expectThrows(ResourceAlreadyExistsException.class, + () -> followerClient().execute(PutFollowAction.INSTANCE, followRequest2).actionGet()); + } + + public void testIndexFallBehind() throws Exception { + final int numberOfPrimaryShards = randomIntBetween(1, 3); + final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1), + singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); + assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); + ensureLeaderYellow("index1"); + + final int numDocs = randomIntBetween(2, 64); + logger.info("Indexing [{}] docs as first batch", numDocs); + for (int i = 0; i < numDocs; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + + final PutFollowAction.Request followRequest = putFollow("index1", "index2"); + PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); + assertTrue(response.isFollowIndexCreated()); + assertTrue(response.isFollowIndexShardsAcked()); + assertTrue(response.isIndexFollowingStarted()); + + assertIndexFullyReplicatedToFollower("index1", "index2"); + for (int i = 0; i < numDocs; i++) { + assertBusy(assertExpectedDocumentRunnable(i)); + } + + pauseFollow("index2"); + + for (int i = 0; i < numDocs; i++) { + final String source = String.format(Locale.ROOT, "{\"f\":%d}", i * 2); + leaderClient().prepareIndex("index1", "doc", Integer.toString(i)).setSource(source, XContentType.JSON).get(); + } + leaderClient().prepareDelete("index1", "doc", "1").get(); + leaderClient().admin().indices().refresh(new RefreshRequest("index1")).actionGet(); + leaderClient().admin().indices().flush(new FlushRequest("index1").force(true)).actionGet(); + ForceMergeRequest forceMergeRequest = new ForceMergeRequest("index1"); + forceMergeRequest.maxNumSegments(1); + leaderClient().admin().indices().forceMerge(forceMergeRequest).actionGet(); + + followerClient().execute(ResumeFollowAction.INSTANCE, followRequest.getFollowRequest()).get(); + + assertBusy(() -> { + List statuses = getFollowTaskStatuses("index2"); + Set exceptions = statuses.stream() + .map(ShardFollowNodeTaskStatus::getFatalException) + .filter(Objects::nonNull) + .map(ExceptionsHelper::unwrapCause) + .filter(e -> e instanceof ResourceNotFoundException) + .map(e -> (ResourceNotFoundException) e) + .filter(e -> e.getMetadataKeys().contains("es.requested_operations_missing")) + .collect(Collectors.toSet()); + assertThat(exceptions.size(), greaterThan(0)); + }); + + followerClient().admin().indices().prepareClose("index2").get(); + pauseFollow("index2"); + + + final PutFollowAction.Request followRequest2 = putFollow("index1", "index2"); + PutFollowAction.Response response2 = followerClient().execute(PutFollowAction.INSTANCE, followRequest2).get(); + assertTrue(response2.isFollowIndexCreated()); + assertTrue(response2.isFollowIndexShardsAcked()); + assertTrue(response2.isIndexFollowingStarted()); + + ensureFollowerGreen("index2"); + assertIndexFullyReplicatedToFollower("index1", "index2"); + for (int i = 2; i < numDocs; i++) { + assertBusy(assertExpectedDocumentRunnable(i, i * 2)); + } + } + private long getFollowTaskSettingsVersion(String followerIndex) { long settingsVersion = -1L; for (ShardFollowNodeTaskStatus status : getFollowTaskStatuses(followerIndex)) { @@ -1032,9 +1133,13 @@ private CheckedRunnable assertTask(final int numberOfPrimaryShards, f } private CheckedRunnable assertExpectedDocumentRunnable(final int value) { + return assertExpectedDocumentRunnable(value, value); + } + + private CheckedRunnable assertExpectedDocumentRunnable(final int key, final int value) { return () -> { - final GetResponse getResponse = followerClient().prepareGet("index2", "doc", Integer.toString(value)).get(); - assertTrue("Doc with id [" + value + "] is missing", getResponse.isExists()); + final GetResponse getResponse = followerClient().prepareGet("index2", "doc", Integer.toString(key)).get(); + assertTrue("Doc with id [" + key + "] is missing", getResponse.isExists()); assertTrue((getResponse.getSource().containsKey("f"))); assertThat(getResponse.getSource().get("f"), equalTo(value)); }; From cecfa5bd6def76e2799f2d7a1b96a80f0220dac2 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 4 Feb 2019 17:53:41 -0500 Subject: [PATCH 17/24] Tighten mapping syncing in ccr remote restore (#38071) There are two issues regarding the way that we sync mapping from leader to follower when a ccr restore is completed: 1. The returned mapping from a cluster service might not be up to date as the mapping of the restored index commit. 2. We should not compare the mapping version of the follower and the leader. They are not related to one another. Moreover, I think we should only ensure that once the restore is done, the mapping on the follower should be at least the mapping of the copied index commit. We don't have to sync the mapping which is updated after we have opened a session. Relates #36879 Closes #37887 --- .../xpack/ccr/action/CcrRequests.java | 38 +++++++++++++++++ .../ccr/action/ShardFollowTasksExecutor.java | 39 ++---------------- .../PutCcrRestoreSessionAction.java | 14 ++++++- .../xpack/ccr/repository/CcrRepository.java | 41 +++++++++++-------- .../xpack/ccr/CcrRepositoryIT.java | 17 +------- 5 files changed, 79 insertions(+), 70 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java index 87d913c337642..f039810ed940c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java @@ -6,11 +6,15 @@ package org.elasticsearch.xpack.ccr.action; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.MappingRequestValidator; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.rest.RestStatus; @@ -18,6 +22,7 @@ import java.util.Arrays; import java.util.List; +import java.util.function.Supplier; import java.util.stream.Collectors; public final class CcrRequests { @@ -40,6 +45,39 @@ public static PutMappingRequest putMappingRequest(String followerIndex, MappingM return putMappingRequest; } + /** + * Gets an {@link IndexMetaData} of the given index. The mapping version and metadata version of the returned {@link IndexMetaData} + * must be at least the provided {@code mappingVersion} and {@code metadataVersion} respectively. + */ + public static void getIndexMetadata(Client client, Index index, long mappingVersion, long metadataVersion, + Supplier timeoutSupplier, ActionListener listener) { + final ClusterStateRequest request = CcrRequests.metaDataRequest(index.getName()); + if (metadataVersion > 0) { + request.waitForMetaDataVersion(metadataVersion).waitForTimeout(timeoutSupplier.get()); + } + client.admin().cluster().state(request, ActionListener.wrap( + response -> { + if (response.getState() == null) { + assert metadataVersion > 0 : metadataVersion; + throw new IllegalStateException("timeout to get cluster state with" + + " metadata version [" + metadataVersion + "], mapping version [" + mappingVersion + "]"); + } + final MetaData metaData = response.getState().metaData(); + final IndexMetaData indexMetaData = metaData.getIndexSafe(index); + if (indexMetaData.getMappingVersion() >= mappingVersion) { + listener.onResponse(indexMetaData); + return; + } + if (timeoutSupplier.get().nanos() < 0) { + throw new IllegalStateException("timeout to get cluster state with mapping version [" + mappingVersion + "]"); + } + // ask for the next version. + getIndexMetadata(client, index, mappingVersion, metaData.version() + 1, timeoutSupplier, listener); + }, + listener::onFailure + )); + } + public static final MappingRequestValidator CCR_PUT_MAPPING_REQUEST_VALIDATOR = (request, state, indices) -> { if (request.origin() == null) { return null; // a put-mapping-request on old versions does not have origin. diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index 56538d395feda..c0e2d7f54b318 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -24,7 +24,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedConsumer; @@ -59,6 +58,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.LongConsumer; +import java.util.function.Supplier; import static org.elasticsearch.xpack.ccr.CcrLicenseChecker.wrapClient; import static org.elasticsearch.xpack.ccr.action.TransportResumeFollowAction.extractLeaderShardHistoryUUIDs; @@ -111,7 +111,9 @@ protected AllocatedPersistentTask createTask(long id, String type, String action @Override protected void innerUpdateMapping(long minRequiredMappingVersion, LongConsumer handler, Consumer errorHandler) { final Index followerIndex = params.getFollowShardId().getIndex(); - getIndexMetadata(minRequiredMappingVersion, 0L, params, ActionListener.wrap( + final Index leaderIndex = params.getLeaderShardId().getIndex(); + final Supplier timeout = () -> isStopped() ? TimeValue.MINUS_ONE : waitForMetadataTimeOut; + CcrRequests.getIndexMetadata(remoteClient(params), leaderIndex, minRequiredMappingVersion, 0L, timeout, ActionListener.wrap( indexMetaData -> { if (indexMetaData.getMappings().isEmpty()) { assert indexMetaData.getMappingVersion() == 1; @@ -246,39 +248,6 @@ private Client remoteClient(ShardFollowTask params) { return wrapClient(client.getRemoteClusterClient(params.getRemoteCluster()), params.getHeaders()); } - private void getIndexMetadata(long minRequiredMappingVersion, long minRequiredMetadataVersion, - ShardFollowTask params, ActionListener listener) { - final Index leaderIndex = params.getLeaderShardId().getIndex(); - final ClusterStateRequest clusterStateRequest = CcrRequests.metaDataRequest(leaderIndex.getName()); - if (minRequiredMetadataVersion > 0) { - clusterStateRequest.waitForMetaDataVersion(minRequiredMetadataVersion).waitForTimeout(waitForMetadataTimeOut); - } - try { - remoteClient(params).admin().cluster().state(clusterStateRequest, ActionListener.wrap( - r -> { - // if wait_for_metadata_version timeout, the response is empty - if (r.getState() == null) { - assert minRequiredMetadataVersion > 0; - getIndexMetadata(minRequiredMappingVersion, minRequiredMetadataVersion, params, listener); - return; - } - final MetaData metaData = r.getState().metaData(); - final IndexMetaData indexMetaData = metaData.getIndexSafe(leaderIndex); - if (indexMetaData.getMappingVersion() < minRequiredMappingVersion) { - // ask for the next version. - getIndexMetadata(minRequiredMappingVersion, metaData.version() + 1, params, listener); - } else { - assert metaData.version() >= minRequiredMetadataVersion : metaData.version() + " < " + minRequiredMetadataVersion; - listener.onResponse(indexMetaData); - } - }, - listener::onFailure - )); - } catch (Exception e) { - listener.onFailure(e); - } - } - interface FollowerStatsInfoHandler { void accept(String followerHistoryUUID, long globalCheckpoint, long maxSeqNo); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java index 07ee076135a1b..eceacc1d926d8 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/PutCcrRestoreSessionAction.java @@ -72,7 +72,8 @@ protected PutCcrRestoreSessionResponse shardOperation(PutCcrRestoreSessionReques throw new ShardNotFoundException(shardId); } Store.MetadataSnapshot storeFileMetaData = ccrRestoreService.openSession(request.getSessionUUID(), indexShard); - return new PutCcrRestoreSessionResponse(clusterService.localNode(), storeFileMetaData); + long mappingVersion = indexShard.indexSettings().getIndexMetaData().getMappingVersion(); + return new PutCcrRestoreSessionResponse(clusterService.localNode(), storeFileMetaData, mappingVersion); } @Override @@ -97,19 +98,22 @@ public static class PutCcrRestoreSessionResponse extends ActionResponse { private DiscoveryNode node; private Store.MetadataSnapshot storeFileMetaData; + private long mappingVersion; PutCcrRestoreSessionResponse() { } - PutCcrRestoreSessionResponse(DiscoveryNode node, Store.MetadataSnapshot storeFileMetaData) { + PutCcrRestoreSessionResponse(DiscoveryNode node, Store.MetadataSnapshot storeFileMetaData, long mappingVersion) { this.node = node; this.storeFileMetaData = storeFileMetaData; + this.mappingVersion = mappingVersion; } PutCcrRestoreSessionResponse(StreamInput in) throws IOException { super(in); node = new DiscoveryNode(in); storeFileMetaData = new Store.MetadataSnapshot(in); + mappingVersion = in.readVLong(); } @Override @@ -117,6 +121,7 @@ public void readFrom(StreamInput in) throws IOException { super.readFrom(in); node = new DiscoveryNode(in); storeFileMetaData = new Store.MetadataSnapshot(in); + mappingVersion = in.readVLong(); } @Override @@ -124,6 +129,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); node.writeTo(out); storeFileMetaData.writeTo(out); + out.writeVLong(mappingVersion); } public DiscoveryNode getNode() { @@ -133,5 +139,9 @@ public DiscoveryNode getNode() { public Store.MetadataSnapshot getStoreFileMetaData() { return storeFileMetaData; } + + public long getMappingVersion() { + return mappingVersion; + } } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index bcf0e5f6dc6e9..baad95d5a94df 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -29,9 +29,9 @@ import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.CombinedRateLimiter; import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardRecoveryException; @@ -72,6 +72,8 @@ import java.util.Map; import java.util.Set; import java.util.function.LongConsumer; +import java.util.function.Supplier; + /** * This repository relies on a remote cluster for Ccr restores. It is read-only so it can only be used to @@ -288,11 +290,10 @@ public void restoreShard(IndexShard indexShard, SnapshotId snapshotId, Version v String name = metadata.name(); try (RestoreSession restoreSession = openSession(name, remoteClient, leaderShardId, indexShard, recoveryState)) { restoreSession.restoreFiles(); + updateMappings(remoteClient, leaderIndex, restoreSession.mappingVersion, client, indexShard.routingEntry().index()); } catch (Exception e) { throw new IndexShardRestoreFailedException(indexShard.shardId(), "failed to restore snapshot [" + snapshotId + "]", e); } - - maybeUpdateMappings(client, remoteClient, leaderIndex, indexShard.indexSettings()); } @Override @@ -300,18 +301,20 @@ public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Ve throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } - private void maybeUpdateMappings(Client localClient, Client remoteClient, Index leaderIndex, IndexSettings followerIndexSettings) { - ClusterStateRequest clusterStateRequest = CcrRequests.metaDataRequest(leaderIndex.getName()); - ClusterStateResponse clusterState = remoteClient.admin().cluster().state(clusterStateRequest) - .actionGet(ccrSettings.getRecoveryActionTimeout()); - IndexMetaData leaderIndexMetadata = clusterState.getState().metaData().getIndexSafe(leaderIndex); - long leaderMappingVersion = leaderIndexMetadata.getMappingVersion(); - - if (leaderMappingVersion > followerIndexSettings.getIndexMetaData().getMappingVersion()) { - Index followerIndex = followerIndexSettings.getIndex(); - MappingMetaData mappingMetaData = leaderIndexMetadata.mapping(); - PutMappingRequest putMappingRequest = CcrRequests.putMappingRequest(followerIndex.getName(), mappingMetaData); - localClient.admin().indices().putMapping(putMappingRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); + private void updateMappings(Client leaderClient, Index leaderIndex, long leaderMappingVersion, + Client followerClient, Index followerIndex) { + final PlainActionFuture indexMetadataFuture = new PlainActionFuture<>(); + final long startTimeInNanos = System.nanoTime(); + final Supplier timeout = () -> { + final long elapsedInNanos = System.nanoTime() - startTimeInNanos; + return TimeValue.timeValueNanos(ccrSettings.getRecoveryActionTimeout().nanos() - elapsedInNanos); + }; + CcrRequests.getIndexMetadata(leaderClient, leaderIndex, leaderMappingVersion, 0L, timeout, indexMetadataFuture); + final IndexMetaData leaderIndexMetadata = indexMetadataFuture.actionGet(ccrSettings.getRecoveryActionTimeout()); + final MappingMetaData mappingMetaData = leaderIndexMetadata.mapping(); + if (mappingMetaData != null) { + final PutMappingRequest putMappingRequest = CcrRequests.putMappingRequest(followerIndex.getName(), mappingMetaData); + followerClient.admin().indices().putMapping(putMappingRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); } } @@ -321,7 +324,7 @@ private RestoreSession openSession(String repositoryName, Client remoteClient, S PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse response = remoteClient.execute(PutCcrRestoreSessionAction.INSTANCE, new PutCcrRestoreSessionRequest(sessionUUID, leaderShardId)).actionGet(ccrSettings.getRecoveryActionTimeout()); return new RestoreSession(repositoryName, remoteClient, sessionUUID, response.getNode(), indexShard, recoveryState, - response.getStoreFileMetaData(), ccrSettings, throttledTime::inc); + response.getStoreFileMetaData(), response.getMappingVersion(), ccrSettings, throttledTime::inc); } private static class RestoreSession extends FileRestoreContext implements Closeable { @@ -332,17 +335,19 @@ private static class RestoreSession extends FileRestoreContext implements Closea private final String sessionUUID; private final DiscoveryNode node; private final Store.MetadataSnapshot sourceMetaData; + private final long mappingVersion; private final CcrSettings ccrSettings; private final LongConsumer throttleListener; RestoreSession(String repositoryName, Client remoteClient, String sessionUUID, DiscoveryNode node, IndexShard indexShard, - RecoveryState recoveryState, Store.MetadataSnapshot sourceMetaData, CcrSettings ccrSettings, - LongConsumer throttleListener) { + RecoveryState recoveryState, Store.MetadataSnapshot sourceMetaData, long mappingVersion, + CcrSettings ccrSettings, LongConsumer throttleListener) { super(repositoryName, indexShard, SNAPSHOT_ID, recoveryState, BUFFER_SIZE); this.remoteClient = remoteClient; this.sessionUUID = sessionUUID; this.node = node; this.sourceMetaData = sourceMetaData; + this.mappingVersion = mappingVersion; this.ccrSettings = ccrSettings; this.throttleListener = throttleListener; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index f34f73ef70592..d4d6d13f7a292 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -390,7 +390,6 @@ public void testIndividualActionsTimeout() throws Exception { assertAcked(followerClient().admin().cluster().updateSettings(settingsRequest).actionGet()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37887") public void testFollowerMappingIsUpdated() throws IOException { String leaderClusterRepoName = CcrRepository.NAME_PREFIX + "leader_cluster"; String leaderIndex = "index1"; @@ -413,16 +412,8 @@ public void testFollowerMappingIsUpdated() throws IOException { .renameReplacement(followerIndex).masterNodeTimeout(new TimeValue(1L, TimeUnit.HOURS)) .indexSettings(settingsBuilder); - // TODO: Eventually when the file recovery work is complete, we should test updated mappings by - // indexing to the leader while the recovery is happening. However, into order to that test mappings - // are updated prior to that work, we index documents in the clear session callback. This will - // ensure a mapping change prior to the final mapping check on the follower side. - for (CcrRestoreSourceService restoreSourceService : getLeaderCluster().getDataNodeInstances(CcrRestoreSourceService.class)) { - restoreSourceService.addCloseSessionListener(s -> { - final String source = String.format(Locale.ROOT, "{\"k\":%d}", 1); - leaderClient().prepareIndex("index1", "doc", Long.toString(1)).setSource(source, XContentType.JSON).get(); - }); - } + final String source = String.format(Locale.ROOT, "{\"k\":%d}", 1); + leaderClient().prepareIndex("index1", "doc", Long.toString(1)).setSource(source, XContentType.JSON).get(); PlainActionFuture future = PlainActionFuture.newFuture(); restoreService.restoreSnapshot(restoreRequest, waitForRestore(clusterService, future)); @@ -435,10 +426,6 @@ public void testFollowerMappingIsUpdated() throws IOException { clusterStateRequest.clear(); clusterStateRequest.metaData(true); clusterStateRequest.indices(followerIndex); - ClusterStateResponse clusterState = followerClient().admin().cluster().state(clusterStateRequest).actionGet(); - IndexMetaData followerIndexMetadata = clusterState.getState().metaData().index(followerIndex); - assertEquals(2, followerIndexMetadata.getMappingVersion()); - MappingMetaData mappingMetaData = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings() .get("index2").get("doc"); assertThat(XContentMapValues.extractValue("properties.k.type", mappingMetaData.sourceAsMap()), equalTo("long")); From b866417650e450e6b9e478e6711b22fec59bfffd Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 4 Feb 2019 16:06:19 -0700 Subject: [PATCH 18/24] Mute testCannotShrinkLeaderIndex (#38374) This test should not pass until CCR finishes integrating shard history retention leases. It currently sometimes passes (which is a bug in the test), but cannot pass reliably until the linked issue is resolved. --- .../xpack/indexlifecycle/CCRIndexLifecycleIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java index cd8aac7bf1007..01f0eb4c7d0d4 100644 --- a/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java +++ b/x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java @@ -326,6 +326,9 @@ public void testUnfollowInjectedBeforeShrink() throws Exception { } } + // Specifically, this is waiting for this bullet to be complete: + // - integrate shard history retention leases with cross-cluster replication + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37165") public void testCannotShrinkLeaderIndex() throws Exception { String indexName = "shrink-leader-test"; String shrunkenIndexName = "shrink-" + indexName; From 292e0f6fb76a1e280580cb4d5798a7cc876eb8ae Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 4 Feb 2019 16:11:44 -0700 Subject: [PATCH 19/24] Deprecate `_type` in simulate pipeline requests (#37949) As mapping types are being removed throughout Elasticsearch, the use of `_type` in pipeline simulation requests is deprecated. Additionally, the default `_type` used if one is not supplied has been changed to `_doc` for consistency with the rest of Elasticsearch. --- .../elasticsearch/client/IngestClientIT.java | 1 - .../IngestClientDocumentationIT.java | 8 +-- .../ingest/apis/simulate-pipeline.asciidoc | 4 -- .../processors/date-index-name.asciidoc | 2 +- .../reference/ingest/processors/grok.asciidoc | 4 +- .../ingest/SimulatePipelineRequest.java | 12 +++- .../SimulatePipelineRequestParsingTests.java | 63 ++++++++++++++----- 7 files changed, 67 insertions(+), 27 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java index 84bf43ab019d5..1c10f65fb3677 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java @@ -130,7 +130,6 @@ private void testSimulatePipeline(boolean isVerbose, { builder.startObject() .field("_index", "index") - .field("_type", "doc") .field("_id", "doc_" + 1) .startObject("_source").field("foo", "rab_" + 1).field("rank", rankValue).endObject() .endObject(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java index 00bee27807f5f..df27b1f1c1a41 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java @@ -296,8 +296,8 @@ public void testSimulatePipeline() throws IOException { "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" + "}," + "\"docs\":[" + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + "]" + "}"; SimulatePipelineRequest request = new SimulatePipelineRequest( @@ -353,8 +353,8 @@ public void testSimulatePipelineAsync() throws Exception { "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" + "}," + "\"docs\":[" + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + "]" + "}"; SimulatePipelineRequest request = new SimulatePipelineRequest( diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index d4f043e802159..deb464eac7a53 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -65,7 +65,6 @@ POST _ingest/pipeline/_simulate "docs": [ { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "bar" @@ -73,7 +72,6 @@ POST _ingest/pipeline/_simulate }, { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "rab" @@ -158,7 +156,6 @@ POST _ingest/pipeline/_simulate?verbose "docs": [ { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "bar" @@ -166,7 +163,6 @@ POST _ingest/pipeline/_simulate?verbose }, { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "rab" diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index fcece261bd440..6dd54dab056e8 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -112,7 +112,7 @@ and the result: "doc" : { "_id" : "_id", "_index" : "", - "_type" : "_type", + "_type" : "_doc", "_source" : { "date1" : "2016-04-25T12:02:01.789Z" }, diff --git a/docs/reference/ingest/processors/grok.asciidoc b/docs/reference/ingest/processors/grok.asciidoc index b266879e40b16..f6f5fb3c92881 100644 --- a/docs/reference/ingest/processors/grok.asciidoc +++ b/docs/reference/ingest/processors/grok.asciidoc @@ -193,7 +193,7 @@ response: "docs": [ { "doc": { - "_type": "_type", + "_type": "_doc", "_index": "_index", "_id": "_id", "_source": { @@ -254,7 +254,7 @@ POST _ingest/pipeline/_simulate "docs": [ { "doc": { - "_type": "_type", + "_type": "_doc", "_index": "_index", "_id": "_id", "_source": { diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 4c2736e3d86de..eb15b56db31cc 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.ingest; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -43,6 +46,9 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContentObject { + private static final Logger logger = LogManager.getLogger(SimulatePipelineRequest.class); + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger); + private String id; private boolean verbose; private BytesReference source; @@ -178,8 +184,12 @@ private static List parseDocs(Map config) { dataMap, Fields.SOURCE); String index = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.INDEX.getFieldName(), "_index"); + if (dataMap.containsKey(MetaData.TYPE.getFieldName())) { + deprecationLogger.deprecatedAndMaybeLog("simulate_pipeline_with_types", + "[types removal] specifying _type in pipeline simulation requests is deprecated"); + } String type = ConfigurationUtils.readStringOrIntProperty(null, null, - dataMap, MetaData.TYPE.getFieldName(), "_type"); + dataMap, MetaData.TYPE.getFieldName(), "_doc"); String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.ID.getFieldName(), "_id"); String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 1711d16891083..8e313e7cdbb1a 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -19,15 +19,6 @@ package org.elasticsearch.action.ingest; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import org.elasticsearch.index.VersionType; import org.elasticsearch.ingest.CompoundProcessor; import org.elasticsearch.ingest.IngestDocument; @@ -38,6 +29,15 @@ import org.elasticsearch.test.ESTestCase; import org.junit.Before; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + import static org.elasticsearch.action.ingest.SimulatePipelineRequest.Fields; import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID; import static org.elasticsearch.ingest.IngestDocument.MetaData.ID; @@ -67,7 +67,15 @@ public void init() throws IOException { when(ingestService.getProcessorFactories()).thenReturn(registry); } - public void testParseUsingPipelineStore() throws Exception { + public void testParseUsingPipelineStoreNoType() throws Exception { + innerTestParseUsingPipelineStore(false); + } + + public void testParseUsingPipelineStoreWithType() throws Exception { + innerTestParseUsingPipelineStore(true); + } + + private void innerTestParseUsingPipelineStore(boolean useExplicitType) throws Exception { int numDocs = randomIntBetween(1, 10); Map requestContent = new HashMap<>(); @@ -80,7 +88,9 @@ public void testParseUsingPipelineStore() throws Exception { String type = randomAlphaOfLengthBetween(1, 10); String id = randomAlphaOfLengthBetween(1, 10); doc.put(INDEX.getFieldName(), index); - doc.put(TYPE.getFieldName(), type); + if (useExplicitType) { + doc.put(TYPE.getFieldName(), type); + } doc.put(ID.getFieldName(), id); String fieldName = randomAlphaOfLengthBetween(1, 10); String fieldValue = randomAlphaOfLengthBetween(1, 10); @@ -88,7 +98,11 @@ public void testParseUsingPipelineStore() throws Exception { docs.add(doc); Map expectedDoc = new HashMap<>(); expectedDoc.put(INDEX.getFieldName(), index); - expectedDoc.put(TYPE.getFieldName(), type); + if (useExplicitType) { + expectedDoc.put(TYPE.getFieldName(), type); + } else { + expectedDoc.put(TYPE.getFieldName(), "_doc"); + } expectedDoc.put(ID.getFieldName(), id); expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue)); expectedDocs.add(expectedDoc); @@ -111,9 +125,20 @@ public void testParseUsingPipelineStore() throws Exception { assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID)); assertThat(actualRequest.getPipeline().getDescription(), nullValue()); assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(1)); + if (useExplicitType) { + assertWarnings("[types removal] specifying _type in pipeline simulation requests is deprecated"); + } + } + + public void testParseWithProvidedPipelineNoType() throws Exception { + innerTestParseWithProvidedPipeline(false); } - public void testParseWithProvidedPipeline() throws Exception { + public void testParseWithProvidedPipelineWithType() throws Exception { + innerTestParseWithProvidedPipeline(true); + } + + private void innerTestParseWithProvidedPipeline(boolean useExplicitType) throws Exception { int numDocs = randomIntBetween(1, 10); Map requestContent = new HashMap<>(); @@ -135,6 +160,14 @@ public void testParseWithProvidedPipeline() throws Exception { ); doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); + } else if (field == TYPE) { + if (useExplicitType) { + String value = randomAlphaOfLengthBetween(1, 10); + doc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), value); + } else { + expectedDoc.put(field.getFieldName(), "_doc"); + } } else { if (randomBoolean()) { String value = randomAlphaOfLengthBetween(1, 10); @@ -191,7 +224,6 @@ public void testParseWithProvidedPipeline() throws Exception { Map expectedDocument = expectedDocsIterator.next(); Map metadataMap = ingestDocument.extractMetadata(); assertThat(metadataMap.get(INDEX), equalTo(expectedDocument.get(INDEX.getFieldName()))); - assertThat(metadataMap.get(TYPE), equalTo(expectedDocument.get(TYPE.getFieldName()))); assertThat(metadataMap.get(ID), equalTo(expectedDocument.get(ID.getFieldName()))); assertThat(metadataMap.get(ROUTING), equalTo(expectedDocument.get(ROUTING.getFieldName()))); assertThat(metadataMap.get(VERSION), equalTo(expectedDocument.get(VERSION.getFieldName()))); @@ -202,6 +234,9 @@ public void testParseWithProvidedPipeline() throws Exception { assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID)); assertThat(actualRequest.getPipeline().getDescription(), nullValue()); assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(numProcessors)); + if (useExplicitType) { + assertWarnings("[types removal] specifying _type in pipeline simulation requests is deprecated"); + } } public void testNullPipelineId() { From b5b319ec9a21e85ff0978fce74c36db14b054e7a Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> Date: Tue, 5 Feb 2019 11:01:13 +1100 Subject: [PATCH 20/24] Skip unsupported languages for tests (#38328) Skip the languages in tests for which SimpleKdcServer does not handle generalized time correctly. Closes#38320 --- .../xpack/security/authc/kerberos/KerberosTestCase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java index 5011aa1d307f1..ecaf67205ac80 100644 --- a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java +++ b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java @@ -82,6 +82,8 @@ public abstract class KerberosTestCase extends ESTestCase { unsupportedLocaleLanguages.add("ps"); unsupportedLocaleLanguages.add("ur"); unsupportedLocaleLanguages.add("pa"); + unsupportedLocaleLanguages.add("ig"); + unsupportedLocaleLanguages.add("sd"); } @BeforeClass From 48f09471f8c091b96c6a36ed7d476ebad69e331e Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 4 Feb 2019 17:00:40 -0800 Subject: [PATCH 21/24] add docs saying mixed-cluster ILM is not supported (#37954) Closes #37085. --- docs/reference/ilm/index.asciidoc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/reference/ilm/index.asciidoc b/docs/reference/ilm/index.asciidoc index aa27ab1386b80..b906f9ade4447 100644 --- a/docs/reference/ilm/index.asciidoc +++ b/docs/reference/ilm/index.asciidoc @@ -46,6 +46,16 @@ to a single shard. . After 7 days, move the index into the cold stage and move it to less expensive hardware. . Delete the index once the required 30 day retention period is reached. + +[IMPORTANT] +=========================== +{ilm} does not support mixed-version cluster usage. Although it +may be possible to create such new policies against +newer-versioned nodes, there is no guarantee they will +work as intended. New policies using new actions that +do not exist in the oldest versioned node will cause errors. +=========================== + -- include::getting-started-ilm.asciidoc[] From 9d3f0578943063f5f6858cced3fb5289505d8b9c Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> Date: Tue, 5 Feb 2019 12:02:36 +1100 Subject: [PATCH 22/24] Limit token expiry to 1 hour maximum (#38244) We mention in our documentation for the token expiration configuration maximum value is 1 hour but do not enforce it. This commit adds max limit to the TOKEN_EXPIRATION setting. --- .../xpack/security/authc/TokenService.java | 2 +- .../security/authc/TokenServiceTests.java | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index daa20aeb9e19e..0ea689f3ac5f4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -156,7 +156,7 @@ public final class TokenService { public static final String THREAD_POOL_NAME = XPackField.SECURITY + "-token-key"; public static final Setting TOKEN_EXPIRATION = Setting.timeSetting("xpack.security.authc.token.timeout", - TimeValue.timeValueMinutes(20L), TimeValue.timeValueSeconds(1L), Property.NodeScope); + TimeValue.timeValueMinutes(20L), TimeValue.timeValueSeconds(1L), TimeValue.timeValueHours(1L), Property.NodeScope); public static final Setting DELETE_INTERVAL = Setting.timeSetting("xpack.security.authc.token.delete.interval", TimeValue.timeValueMinutes(30L), Property.NodeScope); public static final Setting DELETE_TIMEOUT = Setting.timeSetting("xpack.security.authc.token.delete.timeout", diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 47770288b1b66..6744bd8e099c0 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -65,6 +65,7 @@ import static java.time.Clock.systemUTC; import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; @@ -408,6 +409,29 @@ public void testComputeSecretKeyIsConsistent() throws Exception { assertArrayEquals(key.getEncoded(), key2.getEncoded()); } + public void testTokenExpiryConfig() { + TimeValue expiration = TokenService.TOKEN_EXPIRATION.get(tokenServiceEnabledSettings); + assertThat(expiration, equalTo(TimeValue.timeValueMinutes(20L))); + // Configure Minimum expiration + tokenServiceEnabledSettings = Settings.builder().put(TokenService.TOKEN_EXPIRATION.getKey(), "1s").build(); + expiration = TokenService.TOKEN_EXPIRATION.get(tokenServiceEnabledSettings); + assertThat(expiration, equalTo(TimeValue.timeValueSeconds(1L))); + // Configure Maximum expiration + tokenServiceEnabledSettings = Settings.builder().put(TokenService.TOKEN_EXPIRATION.getKey(), "60m").build(); + expiration = TokenService.TOKEN_EXPIRATION.get(tokenServiceEnabledSettings); + assertThat(expiration, equalTo(TimeValue.timeValueHours(1L))); + // Outside range should fail + tokenServiceEnabledSettings = Settings.builder().put(TokenService.TOKEN_EXPIRATION.getKey(), "1ms").build(); + IllegalArgumentException ile = expectThrows(IllegalArgumentException.class, + () -> TokenService.TOKEN_EXPIRATION.get(tokenServiceEnabledSettings)); + assertThat(ile.getMessage(), + containsString("failed to parse value [1ms] for setting [xpack.security.authc.token.timeout], must be >= [1s]")); + tokenServiceEnabledSettings = Settings.builder().put(TokenService.TOKEN_EXPIRATION.getKey(), "120m").build(); + ile = expectThrows(IllegalArgumentException.class, () -> TokenService.TOKEN_EXPIRATION.get(tokenServiceEnabledSettings)); + assertThat(ile.getMessage(), + containsString("failed to parse value [120m] for setting [xpack.security.authc.token.timeout], must be <= [1h]")); + } + public void testTokenExpiry() throws Exception { ClockMock clock = ClockMock.frozen(); TokenService tokenService = new TokenService(tokenServiceEnabledSettings, clock, client, securityIndex, clusterService); From d25530358452b4abbf223017003692ea92e76f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 5 Feb 2019 03:41:05 +0100 Subject: [PATCH 23/24] Add typless client side GetIndexRequest calls and response class (#37778) The HLRC client currently uses `org.elasticsearch.action.admin.indices.get.GetIndexRequest` and `org.elasticsearch.action.admin.indices.get.GetIndexResponse` in its get index calls. Both request and response are designed for the typed APIs, including some return types e.g. for `getMappings()` which in the maps it returns still use a level including the type name. In order to change this without breaking existing users of the HLRC API, this PR introduces two new request and response objects in the `org.elasticsearch.client.indices` client package. These are used by the IndicesClient#get and IndicesClient#exists calls now by default and support the type-less API. The old request and response objects are still kept for use in similarly named, but deprecated methods. The newly introduced client side classes are simplified versions of the server side request/response classes since they don't need to support wire serialization, and only the response needs fromXContent parsing (but no xContent-serialization, since this is the responsibility of the server-side class). Also changing the return type of `GetIndexResponse#getMapping` to `Map getMappings()`, while it previously was returning another map keyed by the type-name. Similar getters return simple Maps instead of the ImmutableOpenMaps that the server side response objects return. --- .../elasticsearch/client/IndicesClient.java | 122 ++++++++-- .../client/IndicesRequestConverters.java | 62 ++++- .../client/indices/GetIndexRequest.java | 132 +++++++++++ .../client/indices/GetIndexResponse.java | 222 ++++++++++++++++++ .../client/ClusterRequestConvertersTests.java | 2 +- .../java/org/elasticsearch/client/CrudIT.java | 6 +- .../elasticsearch/client/IndicesClientIT.java | 137 +++++++---- .../client/IndicesRequestConvertersTests.java | 99 ++++++-- .../client/RequestConvertersTests.java | 14 +- .../SnapshotRequestConvertersTests.java | 2 +- .../IndicesClientDocumentationIT.java | 29 +-- .../client/indices/GetIndexRequestTests.java | 68 ++++++ .../client/indices/GetIndexResponseTests.java | 195 +++++++++++++++ .../admin/indices/get/GetIndexResponse.java | 16 +- .../admin/indices/RestGetIndicesAction.java | 4 +- .../indices/get/GetIndexResponseTests.java | 24 +- 16 files changed, 990 insertions(+), 144 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexResponseTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 8cae8630cd21d..cbb1d95feae1b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -33,10 +33,6 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.client.indices.GetFieldMappingsRequest; -import org.elasticsearch.client.indices.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -54,10 +50,14 @@ import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.FreezeIndexRequest; +import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetFieldMappingsResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; -import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; @@ -649,6 +649,41 @@ public void getAsync(GetIndexRequest getIndexRequest, RequestOptions options, GetIndexResponse::fromXContent, listener, emptySet()); } + /** + * Retrieve information about one or more indexes + * See + * Indices Get Index API on elastic.co + * @param getIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #get(GetIndexRequest, RequestOptions)} should be used instead, which accepts a new request object. + */ + @Deprecated + public org.elasticsearch.action.admin.indices.get.GetIndexResponse get( + org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getIndexRequest, IndicesRequestConverters::getIndex, options, + org.elasticsearch.action.admin.indices.get.GetIndexResponse::fromXContent, emptySet()); + } + + /** + * Retrieve information about one or more indexes + * See + * Indices Get Index API on elastic.co + * @param getIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #getAsync(GetIndexRequest, RequestOptions, ActionListener)} should be used instead, which accepts a new request object. + */ + @Deprecated + public void getAsync(org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getIndexRequest, IndicesRequestConverters::getIndex, options, + org.elasticsearch.action.admin.indices.get.GetIndexResponse::fromXContent, listener, emptySet()); + } + /** * Force merge one or more indices using the Force Merge API. * See @@ -772,6 +807,51 @@ public void existsAsync(GetIndexRequest request, RequestOptions options, ActionL ); } + /** + * Checks if the index (indices) exists or not. + * See + * Indices Exists API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #exists(GetIndexRequest, RequestOptions)} should be used instead, which accepts a new request object. + */ + @Deprecated + public boolean exists(org.elasticsearch.action.admin.indices.get.GetIndexRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequest( + request, + IndicesRequestConverters::indicesExist, + options, + RestHighLevelClient::convertExistsResponse, + Collections.emptySet() + ); + } + + /** + * Asynchronously checks if the index (indices) exists or not. + * See + * Indices Exists API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #existsAsync(GetIndexRequest, RequestOptions, ActionListener)} should be used instead, which accepts a new request object. + */ + @Deprecated + public void existsAsync(org.elasticsearch.action.admin.indices.get.GetIndexRequest request, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsync( + request, + IndicesRequestConverters::indicesExist, + options, + RestHighLevelClient::convertExistsResponse, + listener, + Collections.emptySet() + ); + } + /** * Shrinks an index using the Shrink Index API. * See @@ -948,7 +1028,7 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques AcknowledgedResponse::fromXContent, listener, emptySet()); } - + /** * Puts an index template using the Index Templates API. * See Index Templates API @@ -957,7 +1037,7 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response - * @deprecated This old form of request allows types in mappings. Use {@link #putTemplate(PutIndexTemplateRequest, RequestOptions)} + * @deprecated This old form of request allows types in mappings. Use {@link #putTemplate(PutIndexTemplateRequest, RequestOptions)} * instead which introduces a new request object without types. */ @Deprecated @@ -975,18 +1055,18 @@ public AcknowledgedResponse putTemplate( * @param putIndexTemplateRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion - * @deprecated This old form of request allows types in mappings. - * Use {@link #putTemplateAsync(PutIndexTemplateRequest, RequestOptions, ActionListener)} + * @deprecated This old form of request allows types in mappings. + * Use {@link #putTemplateAsync(PutIndexTemplateRequest, RequestOptions, ActionListener)} * instead which introduces a new request object without types. */ @Deprecated - public void putTemplateAsync(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, + public void putTemplateAsync(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } - - + + /** * Puts an index template using the Index Templates API. * See Index Templates API @@ -1011,7 +1091,7 @@ public AcknowledgedResponse putTemplate( * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion */ - public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, + public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, AcknowledgedResponse::fromXContent, listener, emptySet()); @@ -1056,7 +1136,7 @@ public void validateQueryAsync(ValidateQueryRequest validateQueryRequest, Reques * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response - * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use + * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use * {@link #getIndexTemplate(GetIndexTemplatesRequest, RequestOptions)} instead which returns a new response object */ @Deprecated @@ -1066,7 +1146,7 @@ public org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResp IndicesRequestConverters::getTemplatesWithDocumentTypes, options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, emptySet()); } - + /** * Gets index templates using the Index Templates API * See Index Templates API @@ -1081,17 +1161,17 @@ public GetIndexTemplatesResponse getIndexTemplate(GetIndexTemplatesRequest getIn return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates, options, GetIndexTemplatesResponse::fromXContent, emptySet()); - } + } /** - * Asynchronously gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, + * Asynchronously gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, * where the mapping definition is nested under the type name. * See Index Templates API * on elastic.co * @param getIndexTemplatesRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion - * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use + * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use * {@link #getIndexTemplateAsync(GetIndexTemplatesRequest, RequestOptions, ActionListener)} instead which returns a new response object */ @Deprecated @@ -1101,7 +1181,7 @@ public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, IndicesRequestConverters::getTemplatesWithDocumentTypes, options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, listener, emptySet()); } - + /** * Asynchronously gets index templates using the Index Templates API * See Index Templates API @@ -1110,12 +1190,12 @@ public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion */ - public void getIndexTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, + public void getIndexTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates, options, GetIndexTemplatesResponse::fromXContent, listener, emptySet()); - } + } /** * Uses the Index Templates API to determine if index templates exist diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index 9c2ba8b30bb23..cc5adffd33483 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -33,8 +33,6 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.client.indices.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; @@ -45,6 +43,8 @@ import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.FreezeIndexRequest; +import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; @@ -148,6 +148,10 @@ static Request putMapping(PutMappingRequest putMappingRequest) throws IOExceptio return request; } + /** + * converter for the legacy server-side {@link org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest} that still supports + * types + */ @Deprecated static Request putMapping(org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest putMappingRequest) throws IOException { // The concreteIndex is an internal concept, not applicable to requests made over the REST API. @@ -389,6 +393,28 @@ static Request getSettings(GetSettingsRequest getSettingsRequest) { return request; } + /** + * converter for the legacy server-side {@link org.elasticsearch.action.admin.indices.get.GetIndexRequest} that + * still supports types + */ + @Deprecated + static Request getIndex(org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest) { + String[] indices = getIndexRequest.indices() == null ? Strings.EMPTY_ARRAY : getIndexRequest.indices(); + + String endpoint = RequestConverters.endpoint(indices); + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + params.withIndicesOptions(getIndexRequest.indicesOptions()); + params.withLocal(getIndexRequest.local()); + params.withIncludeDefaults(getIndexRequest.includeDefaults()); + params.withHuman(getIndexRequest.humanReadable()); + params.withMasterTimeout(getIndexRequest.masterNodeTimeout()); + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + + return request; + } + static Request getIndex(GetIndexRequest getIndexRequest) { String[] indices = getIndexRequest.indices() == null ? Strings.EMPTY_ARRAY : getIndexRequest.indices(); @@ -405,6 +431,28 @@ static Request getIndex(GetIndexRequest getIndexRequest) { return request; } + /** + * converter for the legacy server-side {@link org.elasticsearch.action.admin.indices.get.GetIndexRequest} that + * still supports types + */ + @Deprecated + static Request indicesExist(org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest) { + // this can be called with no indices as argument by transport client, not via REST though + if (getIndexRequest.indices() == null || getIndexRequest.indices().length == 0) { + throw new IllegalArgumentException("indices are mandatory"); + } + String endpoint = RequestConverters.endpoint(getIndexRequest.indices(), ""); + Request request = new Request(HttpHead.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + params.withLocal(getIndexRequest.local()); + params.withHuman(getIndexRequest.humanReadable()); + params.withIndicesOptions(getIndexRequest.indicesOptions()); + params.withIncludeDefaults(getIndexRequest.includeDefaults()); + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + return request; + } + static Request indicesExist(GetIndexRequest getIndexRequest) { // this can be called with no indices as argument by transport client, not via REST though if (getIndexRequest.indices() == null || getIndexRequest.indices().length == 0) { @@ -436,11 +484,11 @@ static Request indexPutSettings(UpdateSettingsRequest updateSettingsRequest) thr } /** - * @deprecated This uses the old form of PutIndexTemplateRequest which uses types. + * @deprecated This uses the old form of PutIndexTemplateRequest which uses types. * Use (@link {@link #putTemplate(PutIndexTemplateRequest)} instead */ @Deprecated - static Request putTemplate(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest) + static Request putTemplate(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest) throws IOException { String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template") .addPathPart(putIndexTemplateRequest.name()).build(); @@ -503,11 +551,11 @@ static Request getAlias(GetAliasesRequest getAliasesRequest) { static Request getTemplatesWithDocumentTypes(GetIndexTemplatesRequest getIndexTemplatesRequest) { return getTemplates(getIndexTemplatesRequest, true); } - + static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) { return getTemplates(getIndexTemplatesRequest, false); } - + private static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest, boolean includeTypeName) { final String endpoint = new RequestConverters.EndpointBuilder() .addPathPartAsIs("_template") @@ -521,7 +569,7 @@ private static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRe params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); } return request; - } + } static Request templatesExist(IndexTemplatesExistRequest indexTemplatesExistRequest) { final String endpoint = new RequestConverters.EndpointBuilder() diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexRequest.java new file mode 100644 index 0000000000000..227b1b4d36abc --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexRequest.java @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.TimedRequest; +import org.elasticsearch.common.util.ArrayUtils; + +/** + * A request to retrieve information about an index. + */ +public class GetIndexRequest extends TimedRequest { + + public enum Feature { + ALIASES, + MAPPINGS, + SETTINGS; + } + + static final Feature[] DEFAULT_FEATURES = new Feature[] { Feature.ALIASES, Feature.MAPPINGS, Feature.SETTINGS }; + private Feature[] features = DEFAULT_FEATURES; + private boolean humanReadable = false; + private transient boolean includeDefaults = false; + + private final String[] indices; + private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true); + private boolean local = false; + + public GetIndexRequest(String... indices) { + this.indices = indices; + } + + /** + * The indices into which the mappings will be put. + */ + public String[] indices() { + return indices; + } + + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public GetIndexRequest indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + public final GetIndexRequest local(boolean local) { + this.local = local; + return this; + } + + /** + * Return local information, do not retrieve the state from master node (default: false). + * @return true if local information is to be returned; + * false if information is to be retrieved from master node (default). + */ + public final boolean local() { + return local; + } + + public GetIndexRequest features(Feature... features) { + if (features == null) { + throw new IllegalArgumentException("features cannot be null"); + } else { + this.features = features; + } + return this; + } + + public GetIndexRequest addFeatures(Feature... features) { + if (this.features == DEFAULT_FEATURES) { + return features(features); + } else { + return features(ArrayUtils.concat(features(), features, Feature.class)); + } + } + + public Feature[] features() { + return features; + } + + public GetIndexRequest humanReadable(boolean humanReadable) { + this.humanReadable = humanReadable; + return this; + } + + public boolean humanReadable() { + return humanReadable; + } + + /** + * Sets the value of "include_defaults". + * + * @param includeDefaults value of "include_defaults" to be set. + * @return this request + */ + public GetIndexRequest includeDefaults(boolean includeDefaults) { + this.includeDefaults = includeDefaults; + return this; + } + + /** + * Whether to return all default settings for each of the indices. + * + * @return true if defaults settings for each of the indices need to returned; + * false otherwise. + */ + public boolean includeDefaults() { + return includeDefaults; + } + + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexResponse.java new file mode 100644 index 0000000000000..3d98f93df47d9 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexResponse.java @@ -0,0 +1,222 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * A client side response for a get index action. + */ +public class GetIndexResponse { + + private Map mappings; + private Map> aliases; + private Map settings; + private Map defaultSettings; + private String[] indices; + + GetIndexResponse(String[] indices, + Map mappings, + Map> aliases, + Map settings, + Map defaultSettings) { + this.indices = indices; + // to have deterministic order + Arrays.sort(indices); + if (mappings != null) { + this.mappings = mappings; + } + if (aliases != null) { + this.aliases = aliases; + } + if (settings != null) { + this.settings = settings; + } + if (defaultSettings != null) { + this.defaultSettings = defaultSettings; + } + } + + public String[] getIndices() { + return indices; + } + + public Map getMappings() { + return mappings; + } + + public Map> getAliases() { + return aliases; + } + + /** + * If the originating {@link GetIndexRequest} object was configured to include + * defaults, this will contain a mapping of index name to {@link Settings} objects. + * The returned {@link Settings} objects will contain only those settings taking + * effect as defaults. Any settings explicitly set on the index will be available + * via {@link #getSettings()}. + * See also {@link GetIndexRequest#includeDefaults(boolean)} + */ + public Map getDefaultSettings() { + return defaultSettings; + } + + public Map getSettings() { + return settings; + } + + /** + * Returns the string value for the specified index and setting. If the includeDefaults flag was not set or set to + * false on the {@link GetIndexRequest}, this method will only return a value where the setting was explicitly set + * on the index. If the includeDefaults flag was set to true on the {@link GetIndexRequest}, this method will fall + * back to return the default value if the setting was not explicitly set. + */ + public String getSetting(String index, String setting) { + Settings indexSettings = settings.get(index); + if (setting != null) { + if (indexSettings != null && indexSettings.hasValue(setting)) { + return indexSettings.get(setting); + } else { + Settings defaultIndexSettings = defaultSettings.get(index); + if (defaultIndexSettings != null) { + return defaultIndexSettings.get(setting); + } else { + return null; + } + } + } else { + return null; + } + } + + private static List parseAliases(XContentParser parser) throws IOException { + List indexAliases = new ArrayList<>(); + // We start at START_OBJECT since parseIndexEntry ensures that + while (parser.nextToken() != Token.END_OBJECT) { + ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + indexAliases.add(AliasMetaData.Builder.fromXContent(parser)); + } + return indexAliases; + } + + private static MappingMetaData parseMappings(XContentParser parser) throws IOException { + return new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, parser.map()); + } + + private static IndexEntry parseIndexEntry(XContentParser parser) throws IOException { + List indexAliases = null; + MappingMetaData indexMappings = null; + Settings indexSettings = null; + Settings indexDefaultSettings = null; + // We start at START_OBJECT since fromXContent ensures that + while (parser.nextToken() != Token.END_OBJECT) { + ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + if (parser.currentToken() == Token.START_OBJECT) { + switch (parser.currentName()) { + case "aliases": + indexAliases = parseAliases(parser); + break; + case "mappings": + indexMappings = parseMappings(parser); + break; + case "settings": + indexSettings = Settings.fromXContent(parser); + break; + case "defaults": + indexDefaultSettings = Settings.fromXContent(parser); + break; + default: + parser.skipChildren(); + } + } else if (parser.currentToken() == Token.START_ARRAY) { + parser.skipChildren(); + } + } + return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings); + } + + // This is just an internal container to make stuff easier for returning + private static class IndexEntry { + List indexAliases = new ArrayList<>(); + MappingMetaData indexMappings; + Settings indexSettings = Settings.EMPTY; + Settings indexDefaultSettings = Settings.EMPTY; + IndexEntry(List indexAliases, MappingMetaData indexMappings, Settings indexSettings, Settings indexDefaultSettings) { + if (indexAliases != null) this.indexAliases = indexAliases; + if (indexMappings != null) this.indexMappings = indexMappings; + if (indexSettings != null) this.indexSettings = indexSettings; + if (indexDefaultSettings != null) this.indexDefaultSettings = indexDefaultSettings; + } + } + + public static GetIndexResponse fromXContent(XContentParser parser) throws IOException { + Map> aliases = new HashMap<>(); + Map mappings = new HashMap<>(); + Map settings = new HashMap<>(); + Map defaultSettings = new HashMap<>(); + List indices = new ArrayList<>(); + + if (parser.currentToken() == null) { + parser.nextToken(); + } + ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + + while (!parser.isClosed()) { + if (parser.currentToken() == Token.START_OBJECT) { + // we assume this is an index entry + String indexName = parser.currentName(); + indices.add(indexName); + IndexEntry indexEntry = parseIndexEntry(parser); + // make the order deterministic + CollectionUtil.timSort(indexEntry.indexAliases, Comparator.comparing(AliasMetaData::alias)); + aliases.put(indexName, Collections.unmodifiableList(indexEntry.indexAliases)); + mappings.put(indexName, indexEntry.indexMappings); + settings.put(indexName, indexEntry.indexSettings); + if (indexEntry.indexDefaultSettings.isEmpty() == false) { + defaultSettings.put(indexName, indexEntry.indexDefaultSettings); + } + } else if (parser.currentToken() == Token.START_ARRAY) { + parser.skipChildren(); + } else { + parser.nextToken(); + } + } + return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java index 9a7596957d02a..9b7b5b0d284dc 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java @@ -72,7 +72,7 @@ public void testClusterGetSettings() throws IOException { public void testClusterHealth() { ClusterHealthRequest healthRequest = new ClusterHealthRequest(); Map expectedParams = new HashMap<>(); - RequestConvertersTests.setRandomLocal(healthRequest, expectedParams); + RequestConvertersTests.setRandomLocal(healthRequest::local, expectedParams); String timeoutType = ESTestCase.randomFrom("timeout", "masterTimeout", "both", "none"); String timeout = ESTestCase.randomTimeValue(); String masterTimeout = ESTestCase.randomTimeValue(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 1bf1f2487cd29..3bd3c79072dc9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkRequest; @@ -48,6 +47,7 @@ import org.elasticsearch.client.core.MultiTermVectorsResponse; import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.client.core.TermVectorsResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -1105,7 +1105,7 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) }; try (BulkProcessor processor = BulkProcessor.builder( - (request, bulkListener) -> highLevelClient().bulkAsync(request, + (request, bulkListener) -> highLevelClient().bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener) .setConcurrentRequests(0) .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.GB)) @@ -1231,7 +1231,7 @@ public void testUrlEncode() throws IOException { assertEquals(docId, getResponse.getId()); } - assertTrue(highLevelClient().indices().exists(new GetIndexRequest().indices(indexPattern, "index"), RequestOptions.DEFAULT)); + assertTrue(highLevelClient().indices().exists(new GetIndexRequest(indexPattern, "index"), RequestOptions.DEFAULT)); } public void testParamsEncode() throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index ee57c32b23796..a7aa517709391 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -39,8 +39,6 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -65,10 +63,12 @@ import org.elasticsearch.client.indices.FreezeIndexRequest; import org.elasticsearch.client.indices.GetFieldMappingsRequest; import org.elasticsearch.client.indices.GetFieldMappingsResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; -import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.IndexTemplateMetaData; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutIndexTemplateRequest; @@ -78,6 +78,7 @@ import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Setting; @@ -96,10 +97,11 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; -import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; -import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction; +import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import java.io.IOException; @@ -137,8 +139,7 @@ public void testIndicesExists() throws IOException { String indexName = "test_index_exists_index_present"; createIndex(indexName, Settings.EMPTY); - GetIndexRequest request = new GetIndexRequest(); - request.indices(indexName); + GetIndexRequest request = new GetIndexRequest(indexName); boolean response = execute( request, @@ -152,8 +153,7 @@ public void testIndicesExists() throws IOException { { String indexName = "non_existent_index"; - GetIndexRequest request = new GetIndexRequest(); - request.indices(indexName); + GetIndexRequest request = new GetIndexRequest(indexName); boolean response = execute( request, @@ -170,8 +170,7 @@ public void testIndicesExists() throws IOException { String nonExistentIndex = "oranges"; - GetIndexRequest request = new GetIndexRequest(); - request.indices(existingIndex, nonExistentIndex); + GetIndexRequest request = new GetIndexRequest(existingIndex, nonExistentIndex); boolean response = execute( request, @@ -180,7 +179,20 @@ public void testIndicesExists() throws IOException { ); assertFalse(response); } + } + + public void testIndicesExistsWithTypes() throws IOException { + // Index present + String indexName = "test_index_exists_index_present"; + createIndex(indexName, Settings.EMPTY); + + org.elasticsearch.action.admin.indices.get.GetIndexRequest request + = new org.elasticsearch.action.admin.indices.get.GetIndexRequest(); + request.indices(indexName); + boolean response = execute(request, highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync, + expectWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE)); + assertTrue(response); } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -416,8 +428,7 @@ public void testGetIndex() throws IOException { String mappings = "\"properties\":{\"field-1\":{\"type\":\"integer\"}}"; createIndex(indexName, basicSettings, mappings); - GetIndexRequest getIndexRequest = new GetIndexRequest() - .indices(indexName).includeDefaults(false); + GetIndexRequest getIndexRequest = new GetIndexRequest(indexName).includeDefaults(false); GetIndexResponse getIndexResponse = execute(getIndexRequest, highLevelClient().indices()::get, highLevelClient().indices()::getAsync); @@ -426,8 +437,12 @@ public void testGetIndex() throws IOException { assertEquals("1", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_SHARDS)); assertEquals("0", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_REPLICAS)); assertNotNull(getIndexResponse.getMappings().get(indexName)); - assertNotNull(getIndexResponse.getMappings().get(indexName).get("_doc")); - Object o = getIndexResponse.getMappings().get(indexName).get("_doc").getSourceAsMap().get("properties"); + assertNotNull(getIndexResponse.getMappings().get(indexName)); + MappingMetaData mappingMetaData = getIndexResponse.getMappings().get(indexName); + assertNotNull(mappingMetaData); + assertEquals("_doc", mappingMetaData.type()); + assertEquals("{\"properties\":{\"field-1\":{\"type\":\"integer\"}}}", mappingMetaData.source().string()); + Object o = mappingMetaData.getSourceAsMap().get("properties"); assertThat(o, instanceOf(Map.class)); //noinspection unchecked assertThat(((Map) o).get("field-1"), instanceOf(Map.class)); @@ -436,6 +451,33 @@ public void testGetIndex() throws IOException { assertEquals("integer", fieldMapping.get("type")); } + @SuppressWarnings("unchecked") + public void testGetIndexWithTypes() throws IOException { + String indexName = "get_index_test"; + Settings basicSettings = Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + String mappings = "\"properties\":{\"field-1\":{\"type\":\"integer\"}}"; + createIndex(indexName, basicSettings, mappings); + + org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest = + new org.elasticsearch.action.admin.indices.get.GetIndexRequest().indices(indexName).includeDefaults(false); + org.elasticsearch.action.admin.indices.get.GetIndexResponse getIndexResponse = execute(getIndexRequest, + highLevelClient().indices()::get, highLevelClient().indices()::getAsync, + expectWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE)); + + // default settings should be null + assertNull(getIndexResponse.getSetting(indexName, "index.refresh_interval")); + assertEquals("1", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_SHARDS)); + assertEquals("0", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_REPLICAS)); + assertNotNull(getIndexResponse.getMappings().get(indexName)); + MappingMetaData mappingMetaData = getIndexResponse.getMappings().get(indexName).get("_doc"); + assertNotNull(mappingMetaData); + assertEquals("_doc", mappingMetaData.type()); + assertEquals("{\"properties\":{\"field-1\":{\"type\":\"integer\"}}}", mappingMetaData.source().string()); + } + @SuppressWarnings("unchecked") public void testGetIndexWithDefaults() throws IOException { String indexName = "get_index_test"; @@ -446,19 +488,18 @@ public void testGetIndexWithDefaults() throws IOException { String mappings = "\"properties\":{\"field-1\":{\"type\":\"integer\"}}"; createIndex(indexName, basicSettings, mappings); - GetIndexRequest getIndexRequest = new GetIndexRequest() - .indices(indexName).includeDefaults(true); + GetIndexRequest getIndexRequest = new GetIndexRequest(indexName).includeDefaults(true); GetIndexResponse getIndexResponse = execute(getIndexRequest, highLevelClient().indices()::get, highLevelClient().indices()::getAsync); assertNotNull(getIndexResponse.getSetting(indexName, "index.refresh_interval")); assertEquals(IndexSettings.DEFAULT_REFRESH_INTERVAL, - getIndexResponse.defaultSettings().get(indexName).getAsTime("index.refresh_interval", null)); + getIndexResponse.getDefaultSettings().get(indexName).getAsTime("index.refresh_interval", null)); assertEquals("1", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_SHARDS)); assertEquals("0", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_REPLICAS)); assertNotNull(getIndexResponse.getMappings().get(indexName)); - assertNotNull(getIndexResponse.getMappings().get(indexName).get("_doc")); - Object o = getIndexResponse.getMappings().get(indexName).get("_doc").getSourceAsMap().get("properties"); + assertNotNull(getIndexResponse.getMappings().get(indexName)); + Object o = getIndexResponse.getMappings().get(indexName).getSourceAsMap().get("properties"); assertThat(o, instanceOf(Map.class)); assertThat(((Map) o).get("field-1"), instanceOf(Map.class)); Map fieldMapping = (Map) ((Map) o).get("field-1"); @@ -469,7 +510,7 @@ public void testGetIndexNonExistentIndex() throws IOException { String nonExistentIndex = "index_that_doesnt_exist"; assertFalse(indexExists(nonExistentIndex)); - GetIndexRequest getIndexRequest = new GetIndexRequest().indices(nonExistentIndex); + GetIndexRequest getIndexRequest = new GetIndexRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(getIndexRequest, highLevelClient().indices()::get, highLevelClient().indices()::getAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); @@ -1432,7 +1473,7 @@ public void testIndexPutSettingNonExistent() throws IOException { @SuppressWarnings("unchecked") public void testPutTemplateWithTypes() throws Exception { - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() .name("my-template") .patterns(Arrays.asList("pattern-1", "name-*")) @@ -1459,7 +1500,7 @@ public void testPutTemplateWithTypes() throws Exception { assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); } - + @SuppressWarnings("unchecked") public void testPutTemplate() throws Exception { PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest("my-template") @@ -1487,7 +1528,7 @@ public void testPutTemplate() throws Exception { assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); } - + public void testPutTemplateWithTypesUsingUntypedAPI() throws Exception { PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest("my-template") .patterns(Arrays.asList("pattern-1", "name-*")) @@ -1503,17 +1544,17 @@ public void testPutTemplateWithTypesUsingUntypedAPI() throws Exception { + "}", XContentType.JSON) .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); - + ElasticsearchStatusException badMappingError = expectThrows(ElasticsearchStatusException.class, () -> execute(putTemplateRequest, highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync)); - assertThat(badMappingError.getDetailedMessage(), + assertThat(badMappingError.getDetailedMessage(), containsString("Root mapping definition has unsupported parameters: [my_doc_type")); - } - + } + @SuppressWarnings("unchecked") public void testPutTemplateWithNoTypesUsingTypedApi() throws Exception { - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() .name("my-template") .patterns(Arrays.asList("pattern-1", "name-*")) @@ -1521,7 +1562,7 @@ public void testPutTemplateWithNoTypesUsingTypedApi() throws Exception { .create(randomBoolean()) .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) .mapping("my_doc_type", - // Note that the declared type is missing from the mapping + // Note that the declared type is missing from the mapping "{ " + "\"properties\":{" + "\"host_name\": {\"type\":\"keyword\"}," @@ -1546,7 +1587,7 @@ public void testPutTemplateWithNoTypesUsingTypedApi() throws Exception { assertThat(extractValue("my-template.mappings.properties.description.type", templates), equalTo("text")); assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); - } + } public void testPutTemplateBadRequests() throws Exception { RestHighLevelClient client = highLevelClient(); @@ -1615,35 +1656,35 @@ public void testInvalidValidateQuery() throws IOException{ public void testCRUDIndexTemplateWithTypes() throws Exception { RestHighLevelClient client = highLevelClient(); - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate1 = + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate1 = new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest().name("template-1") .patterns(Arrays.asList("pattern-1", "name-1")).alias(new Alias("alias-1")); assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync , expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)) .isAcknowledged(), equalTo(true)); - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate2 = + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate2 = new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest().name("template-2") .patterns(Arrays.asList("pattern-2", "name-2")) .mapping("custom_doc_type", "name", "type=text") .settings(Settings.builder().put("number_of_shards", "2").put("number_of_replicas", "0")); - assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync, + assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync, expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)) .isAcknowledged(), equalTo(true)); org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate1 = execute( new GetIndexTemplatesRequest("template-1"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, + client.indices()::getTemplate, client.indices()::getTemplateAsync, expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); assertThat(getTemplate1.getIndexTemplates(), hasSize(1)); org.elasticsearch.cluster.metadata.IndexTemplateMetaData template1 = getTemplate1.getIndexTemplates().get(0); assertThat(template1.name(), equalTo("template-1")); assertThat(template1.patterns(), contains("pattern-1", "name-1")); assertTrue(template1.aliases().containsKey("alias-1")); - - //Check the typed version of the call - org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate2 = + + //Check the typed version of the call + org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate2 = execute(new GetIndexTemplatesRequest("template-2"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, + client.indices()::getTemplate, client.indices()::getTemplateAsync, expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); assertThat(getTemplate2.getIndexTemplates(), hasSize(1)); org.elasticsearch.cluster.metadata.IndexTemplateMetaData template2 = getTemplate2.getIndexTemplates().get(0); @@ -1651,7 +1692,7 @@ public void testCRUDIndexTemplateWithTypes() throws Exception { assertThat(template2.patterns(), contains("pattern-2", "name-2")); assertTrue(template2.aliases().isEmpty()); assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); - assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); + assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); // Ugly deprecated form of API requires use of doc type to get at mapping object which is CompressedXContent assertTrue(template2.mappings().containsKey("custom_doc_type")); @@ -1683,21 +1724,21 @@ public void testCRUDIndexTemplateWithTypes() throws Exception { client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, + client.indices()::getTemplate, client.indices()::getTemplateAsync, expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)).getIndexTemplates(), hasSize(1)); assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, + client.indices()::getTemplate, client.indices()::getTemplateAsync, expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)).getIndexTemplates() .get(0).name(), equalTo("template-2")); assertTrue(execute(new DeleteIndexTemplateRequest("template-*"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, + client.indices()::getTemplate, client.indices()::getTemplateAsync, expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE))).status(), equalTo(RestStatus.NOT_FOUND)); } - + public void testCRUDIndexTemplate() throws Exception { RestHighLevelClient client = highLevelClient(); @@ -1720,7 +1761,7 @@ public void testCRUDIndexTemplate() throws Exception { assertThat(template1.name(), equalTo("template-1")); assertThat(template1.patterns(), contains("pattern-1", "name-1")); assertTrue(template1.aliases().containsKey("alias-1")); - + GetIndexTemplatesResponse getTemplate2 = execute(new GetIndexTemplatesRequest("template-2"), client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); assertThat(getTemplate2.getIndexTemplates(), hasSize(1)); @@ -1729,14 +1770,14 @@ public void testCRUDIndexTemplate() throws Exception { assertThat(template2.patterns(), contains("pattern-2", "name-2")); assertTrue(template2.aliases().isEmpty()); assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); - assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); + assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); // New API returns a MappingMetaData class rather than CompressedXContent for the mapping assertTrue(template2.mappings().sourceAsMap().containsKey("properties")); @SuppressWarnings("unchecked") Map props = (Map) template2.mappings().sourceAsMap().get("properties"); assertTrue(props.containsKey("name")); - - + + List names = randomBoolean() ? Arrays.asList("*-1", "template-2") diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index 0c94cb61a7923..f7d5ac51a73ac 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; @@ -48,6 +47,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; @@ -104,13 +104,13 @@ public void testAnalyzeRequest() throws Exception { public void testIndicesExist() { String[] indices = RequestConvertersTests.randomIndicesNames(1, 10); - GetIndexRequest getIndexRequest = new GetIndexRequest().indices(indices); + GetIndexRequest getIndexRequest = new GetIndexRequest(indices); Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomIndicesOptions(getIndexRequest::indicesOptions, getIndexRequest::indicesOptions, expectedParams); - RequestConvertersTests.setRandomLocal(getIndexRequest, expectedParams); - RequestConvertersTests.setRandomHumanReadable(getIndexRequest, expectedParams); - RequestConvertersTests.setRandomIncludeDefaults(getIndexRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getIndexRequest::local, expectedParams); + RequestConvertersTests.setRandomHumanReadable(getIndexRequest::humanReadable, expectedParams); + RequestConvertersTests.setRandomIncludeDefaults(getIndexRequest::includeDefaults, expectedParams); final Request request = IndicesRequestConverters.indicesExist(getIndexRequest); @@ -124,7 +124,35 @@ public void testIndicesExistEmptyIndices() { LuceneTestCase.expectThrows(IllegalArgumentException.class, () -> IndicesRequestConverters.indicesExist(new GetIndexRequest())); LuceneTestCase.expectThrows(IllegalArgumentException.class, () - -> IndicesRequestConverters.indicesExist(new GetIndexRequest().indices((String[]) null))); + -> IndicesRequestConverters.indicesExist(new GetIndexRequest((String[]) null))); + } + + public void testIndicesExistEmptyIndicesWithTypes() { + LuceneTestCase.expectThrows(IllegalArgumentException.class, + () -> IndicesRequestConverters.indicesExist(new org.elasticsearch.action.admin.indices.get.GetIndexRequest())); + LuceneTestCase.expectThrows(IllegalArgumentException.class, () -> IndicesRequestConverters + .indicesExist(new org.elasticsearch.action.admin.indices.get.GetIndexRequest().indices((String[]) null))); + } + + public void testIndicesExistWithTypes() { + String[] indices = RequestConvertersTests.randomIndicesNames(1, 10); + + org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest = + new org.elasticsearch.action.admin.indices.get.GetIndexRequest().indices(indices); + + Map expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomIndicesOptions(getIndexRequest::indicesOptions, getIndexRequest::indicesOptions, expectedParams); + RequestConvertersTests.setRandomLocal(getIndexRequest::local, expectedParams); + RequestConvertersTests.setRandomHumanReadable(getIndexRequest::humanReadable, expectedParams); + RequestConvertersTests.setRandomIncludeDefaults(getIndexRequest::includeDefaults, expectedParams); + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); + + final Request request = IndicesRequestConverters.indicesExist(getIndexRequest); + + Assert.assertEquals(HttpHead.METHOD_NAME, request.getMethod()); + Assert.assertEquals("/" + String.join(",", indices), request.getEndpoint()); + Assert.assertThat(expectedParams, equalTo(request.getParameters())); + Assert.assertNull(request.getEntity()); } public void testCreateIndex() throws IOException { @@ -288,7 +316,7 @@ public void testGetMappingWithTypes() { RequestConvertersTests.setRandomIndicesOptions(getMappingRequest::indicesOptions, getMappingRequest::indicesOptions, expectedParams); RequestConvertersTests.setRandomMasterTimeout(getMappingRequest, expectedParams); - RequestConvertersTests.setRandomLocal(getMappingRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getMappingRequest::local, expectedParams); expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); Request request = IndicesRequestConverters.getMappings(getMappingRequest); @@ -436,7 +464,7 @@ public void testGetSettings() throws IOException { RequestConvertersTests.setRandomIndicesOptions(getSettingsRequest::indicesOptions, getSettingsRequest::indicesOptions, expectedParams); - RequestConvertersTests.setRandomLocal(getSettingsRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getSettingsRequest::local, expectedParams); if (ESTestCase.randomBoolean()) { // the request object will not have include_defaults present unless it is set to @@ -477,13 +505,48 @@ public void testGetSettings() throws IOException { public void testGetIndex() throws IOException { String[] indicesUnderTest = ESTestCase.randomBoolean() ? null : RequestConvertersTests.randomIndicesNames(0, 5); - GetIndexRequest getIndexRequest = new GetIndexRequest().indices(indicesUnderTest); + GetIndexRequest getIndexRequest = new GetIndexRequest(indicesUnderTest); + + Map expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomMasterTimeout(getIndexRequest, expectedParams); + RequestConvertersTests.setRandomIndicesOptions(getIndexRequest::indicesOptions, getIndexRequest::indicesOptions, expectedParams); + RequestConvertersTests.setRandomLocal(getIndexRequest::local, expectedParams); + RequestConvertersTests.setRandomHumanReadable(getIndexRequest::humanReadable, expectedParams); + + if (ESTestCase.randomBoolean()) { + // the request object will not have include_defaults present unless it is set to + // true + getIndexRequest.includeDefaults(ESTestCase.randomBoolean()); + if (getIndexRequest.includeDefaults()) { + expectedParams.put("include_defaults", Boolean.toString(true)); + } + } + + StringJoiner endpoint = new StringJoiner("/", "/", ""); + if (indicesUnderTest != null && indicesUnderTest.length > 0) { + endpoint.add(String.join(",", indicesUnderTest)); + } + + Request request = IndicesRequestConverters.getIndex(getIndexRequest); + + Assert.assertThat(endpoint.toString(), equalTo(request.getEndpoint())); + Assert.assertThat(request.getParameters(), equalTo(expectedParams)); + Assert.assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME)); + Assert.assertThat(request.getEntity(), nullValue()); + } + + public void testGetIndexWithTypes() throws IOException { + String[] indicesUnderTest = ESTestCase.randomBoolean() ? null : RequestConvertersTests.randomIndicesNames(0, 5); + + org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest = + new org.elasticsearch.action.admin.indices.get.GetIndexRequest().indices(indicesUnderTest); Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomMasterTimeout(getIndexRequest, expectedParams); RequestConvertersTests.setRandomIndicesOptions(getIndexRequest::indicesOptions, getIndexRequest::indicesOptions, expectedParams); - RequestConvertersTests.setRandomLocal(getIndexRequest, expectedParams); - RequestConvertersTests.setRandomHumanReadable(getIndexRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getIndexRequest::local, expectedParams); + RequestConvertersTests.setRandomHumanReadable(getIndexRequest::humanReadable, expectedParams); + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); if (ESTestCase.randomBoolean()) { // the request object will not have include_defaults present unless it is set to @@ -734,7 +797,7 @@ public void testExistsAlias() { } getAliasesRequest.aliases(aliases); Map expectedParams = new HashMap<>(); - RequestConvertersTests.setRandomLocal(getAliasesRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getAliasesRequest::local, expectedParams); RequestConvertersTests.setRandomIndicesOptions(getAliasesRequest::indicesOptions, getAliasesRequest::indicesOptions, expectedParams); @@ -910,7 +973,7 @@ public void testGetAlias() { GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); Map expectedParams = new HashMap<>(); - RequestConvertersTests.setRandomLocal(getAliasesRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getAliasesRequest::local, expectedParams); RequestConvertersTests.setRandomIndicesOptions(getAliasesRequest::indicesOptions, getAliasesRequest::indicesOptions, expectedParams); @@ -971,7 +1034,7 @@ public void testPutTemplateRequestWithTypes() throws Exception { names.put("-#template", "-%23template"); names.put("foo^bar", "foo%5Ebar"); - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = + org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() .name(ESTestCase.randomFrom(names.keySet())) .patterns(Arrays.asList(ESTestCase.generateRandomStringArray(20, 100, false, false))); @@ -1001,7 +1064,7 @@ public void testPutTemplateRequestWithTypes() throws Exception { String cause = ESTestCase.randomUnicodeOfCodepointLengthBetween(1, 50); putTemplateRequest.cause(cause); expectedParams.put("cause", cause); - } + } RequestConvertersTests.setRandomMasterTimeout(putTemplateRequest, expectedParams); Request request = IndicesRequestConverters.putTemplate(putTemplateRequest); @@ -1017,7 +1080,7 @@ public void testPutTemplateRequest() throws Exception { names.put("-#template", "-%23template"); names.put("foo^bar", "foo%5Ebar"); - PutIndexTemplateRequest putTemplateRequest = + PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest(ESTestCase.randomFrom(names.keySet())) .patterns(Arrays.asList(ESTestCase.generateRandomStringArray(20, 100, false, false))); if (ESTestCase.randomBoolean()) { @@ -1031,7 +1094,7 @@ public void testPutTemplateRequest() throws Exception { } Map expectedParams = new HashMap<>(); if (ESTestCase.randomBoolean()) { - putTemplateRequest.mapping("{ \"properties\": { \"field-" + ESTestCase.randomInt() + + putTemplateRequest.mapping("{ \"properties\": { \"field-" + ESTestCase.randomInt() + "\" : { \"type\" : \"" + ESTestCase.randomFrom("text", "keyword") + "\" }}}", XContentType.JSON); } if (ESTestCase.randomBoolean()) { @@ -1045,7 +1108,7 @@ public void testPutTemplateRequest() throws Exception { String cause = ESTestCase.randomUnicodeOfCodepointLengthBetween(1, 50); putTemplateRequest.cause(cause); expectedParams.put("cause", cause); - } + } RequestConvertersTests.setRandomMasterTimeout(putTemplateRequest, expectedParams); Request request = IndicesRequestConverters.putTemplate(putTemplateRequest); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 95971ad40ced0..b58e5ae8852d3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.delete.DeleteRequest; @@ -50,7 +49,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; @@ -1905,20 +1903,20 @@ static IndicesOptions setRandomIndicesOptions(IndicesOptions indicesOptions, Map return indicesOptions; } - static void setRandomIncludeDefaults(GetIndexRequest request, Map expectedParams) { + static void setRandomIncludeDefaults(Consumer setter, Map expectedParams) { if (randomBoolean()) { boolean includeDefaults = randomBoolean(); - request.includeDefaults(includeDefaults); + setter.accept(includeDefaults); if (includeDefaults) { expectedParams.put("include_defaults", String.valueOf(includeDefaults)); } } } - static void setRandomHumanReadable(GetIndexRequest request, Map expectedParams) { + static void setRandomHumanReadable(Consumer setter, Map expectedParams) { if (randomBoolean()) { boolean humanReadable = randomBoolean(); - request.humanReadable(humanReadable); + setter.accept(humanReadable); if (humanReadable) { expectedParams.put("human", String.valueOf(humanReadable)); } @@ -1935,10 +1933,6 @@ static void setRandomLocal(Consumer setter, Map expecte } } - static void setRandomLocal(MasterNodeReadRequest request, Map expectedParams) { - setRandomLocal(request::local, expectedParams); - } - static void setRandomTimeout(TimedRequest request, TimeValue defaultTimeout, Map expectedParams) { setRandomTimeout(s -> request.setTimeout(TimeValue.parseTimeValue(s, request.getClass().getName() + ".timeout")), diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java index ca86a9120422b..66720b70ee3a6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java @@ -58,7 +58,7 @@ public void testGetRepositories() { GetRepositoriesRequest getRepositoriesRequest = new GetRepositoriesRequest(); RequestConvertersTests.setRandomMasterTimeout(getRepositoriesRequest, expectedParams); - RequestConvertersTests.setRandomLocal(getRepositoriesRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getRepositoriesRequest::local, expectedParams); if (randomBoolean()) { String[] entries = new String[]{"a", "b", "c"}; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 64741da12249a..02b7d597ce24e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -38,10 +38,6 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.client.indices.GetFieldMappingsRequest; -import org.elasticsearch.client.indices.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -69,10 +65,14 @@ import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.FreezeIndexRequest; +import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetFieldMappingsResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; +import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; -import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.IndexTemplateMetaData; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutIndexTemplateRequest; @@ -82,7 +82,6 @@ import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -139,8 +138,7 @@ public void testIndicesExist() throws IOException { { // tag::indices-exists-request - GetIndexRequest request = new GetIndexRequest(); - request.indices("twitter"); // <1> + GetIndexRequest request = new GetIndexRequest("twitter"); // <1> // end::indices-exists-request IndicesOptions indicesOptions = IndicesOptions.strictExpand(); @@ -167,8 +165,7 @@ public void testIndicesExistAsync() throws Exception { } { - GetIndexRequest request = new GetIndexRequest(); - request.indices("twitter"); + GetIndexRequest request = new GetIndexRequest("twitter"); // tag::indices-exists-execute-listener ActionListener listener = new ActionListener() { @@ -1230,7 +1227,7 @@ public void testGetIndex() throws Exception { } // tag::get-index-request - GetIndexRequest request = new GetIndexRequest().indices("index"); // <1> + GetIndexRequest request = new GetIndexRequest("index"); // <1> // end::get-index-request // tag::get-index-request-indicesOptions @@ -1246,13 +1243,13 @@ public void testGetIndex() throws Exception { // end::get-index-execute // tag::get-index-response - ImmutableOpenMap indexMappings = getIndexResponse.getMappings().get("index"); // <1> - Map indexTypeMappings = indexMappings.get("_doc").getSourceAsMap(); // <2> + MappingMetaData indexMappings = getIndexResponse.getMappings().get("index"); // <1> + Map indexTypeMappings = indexMappings.getSourceAsMap(); // <2> List indexAliases = getIndexResponse.getAliases().get("index"); // <3> String numberOfShardsString = getIndexResponse.getSetting("index", "index.number_of_shards"); // <4> Settings indexSettings = getIndexResponse.getSettings().get("index"); // <5> Integer numberOfShards = indexSettings.getAsInt("index.number_of_shards", null); // <6> - TimeValue time = getIndexResponse.defaultSettings().get("index") + TimeValue time = getIndexResponse.getDefaultSettings().get("index") .getAsTime("index.refresh_interval", null); // <7> // end::get-index-response @@ -2100,7 +2097,7 @@ public void testPutTemplate() throws Exception { " \"type\": \"text\"\n" + " }\n" + " }\n" + - "}", + "}", XContentType.JSON); // end::put-template-request-mappings-json assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); @@ -2115,7 +2112,7 @@ public void testPutTemplate() throws Exception { message.put("type", "text"); properties.put("message", message); } - jsonMap.put("properties", properties); + jsonMap.put("properties", properties); } request.mapping(jsonMap); // <1> //end::put-template-request-mappings-map diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexRequestTests.java new file mode 100644 index 0000000000000..46b64aab6d406 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexRequestTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.indices.GetIndexRequest.Feature; +import org.elasticsearch.test.ESTestCase; + +public class GetIndexRequestTests extends ESTestCase { + + public void testIndices() { + String[] indices = generateRandomStringArray(5, 5, false, true); + GetIndexRequest request = new GetIndexRequest(indices); + assertArrayEquals(indices, request.indices()); + } + + public void testFeatures() { + int numFeature = randomIntBetween(0, 3); + Feature[] features = new Feature[numFeature]; + for (int i = 0; i < numFeature; i++) { + features[i] = randomFrom(GetIndexRequest.DEFAULT_FEATURES); + } + GetIndexRequest request = new GetIndexRequest().addFeatures(features); + assertArrayEquals(features, request.features()); + } + + public void testLocal() { + boolean local = randomBoolean(); + GetIndexRequest request = new GetIndexRequest().local(local); + assertEquals(local, request.local()); + } + + public void testHumanReadable() { + boolean humanReadable = randomBoolean(); + GetIndexRequest request = new GetIndexRequest().humanReadable(humanReadable); + assertEquals(humanReadable, request.humanReadable()); + } + + public void testIncludeDefaults() { + boolean includeDefaults = randomBoolean(); + GetIndexRequest request = new GetIndexRequest().includeDefaults(includeDefaults); + assertEquals(includeDefaults, request.includeDefaults()); + } + + public void testIndicesOptions() { + IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()); + GetIndexRequest request = new GetIndexRequest().indicesOptions(indicesOptions); + assertEquals(indicesOptions, request.indicesOptions()); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexResponseTests.java new file mode 100644 index 0000000000000..19c25fd11f6ed --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexResponseTests.java @@ -0,0 +1,195 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.client.GetAliasesResponseTests; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.RandomCreateIndexGenerator; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class GetIndexResponseTests extends ESTestCase { + + // Because the client-side class does not have a toXContent method, we test xContent serialization by creating + // a random client object, converting it to a server object then serializing it to xContent, and finally + // parsing it back as a client object. We check equality between the original client object, and the parsed one. + public void testFromXContent() throws IOException { + xContentTester( + this::createParser, + GetIndexResponseTests::createTestInstance, + GetIndexResponseTests::toXContent, + GetIndexResponse::fromXContent) + .supportsUnknownFields(false) + .assertToXContentEquivalence(false) + .assertEqualsConsumer(GetIndexResponseTests::assertEqualInstances) + .test(); + } + + private static void assertEqualInstances(GetIndexResponse expected, GetIndexResponse actual) { + assertArrayEquals(expected.getIndices(), actual.getIndices()); + assertEquals(expected.getMappings(), actual.getMappings()); + assertEquals(expected.getSettings(), actual.getSettings()); + assertEquals(expected.getDefaultSettings(), actual.getDefaultSettings()); + assertEquals(expected.getAliases(), actual.getAliases()); + } + + private static GetIndexResponse createTestInstance() { + String[] indices = generateRandomStringArray(5, 5, false, false); + Map mappings = new HashMap<>(); + Map> aliases = new HashMap<>(); + Map settings = new HashMap<>(); + Map defaultSettings = new HashMap<>(); + IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS; + boolean includeDefaults = randomBoolean(); + for (String index: indices) { + mappings.put(index, createMappingsForIndex()); + + List aliasMetaDataList = new ArrayList<>(); + int aliasesNum = randomIntBetween(0, 3); + for (int i=0; i mappings = new HashMap<>(); + mappings.put("field-" + i, randomFieldMapping()); + if (randomBoolean()) { + mappings.put("field2-" + i, randomFieldMapping()); + } + + try { + String typeName = MapperService.SINGLE_MAPPING_NAME; + mmd = new MappingMetaData(typeName, mappings); + } catch (IOException e) { + fail("shouldn't have failed " + e); + } + } + } + return mmd; + } + + // Not meant to be exhaustive + private static Map randomFieldMapping() { + Map mappings = new HashMap<>(); + if (randomBoolean()) { + mappings.put("type", randomBoolean() ? "text" : "keyword"); + mappings.put("index", "analyzed"); + mappings.put("analyzer", "english"); + } else if (randomBoolean()) { + mappings.put("type", randomFrom("integer", "float", "long", "double")); + mappings.put("index", Objects.toString(randomBoolean())); + } else if (randomBoolean()) { + mappings.put("type", "object"); + mappings.put("dynamic", "strict"); + Map properties = new HashMap<>(); + Map props1 = new HashMap<>(); + props1.put("type", randomFrom("text", "keyword")); + props1.put("analyzer", "keyword"); + properties.put("subtext", props1); + Map props2 = new HashMap<>(); + props2.put("type", "object"); + Map prop2properties = new HashMap<>(); + Map props3 = new HashMap<>(); + props3.put("type", "integer"); + props3.put("index", "false"); + prop2properties.put("subsubfield", props3); + props2.put("properties", prop2properties); + mappings.put("properties", properties); + } else { + mappings.put("type", "keyword"); + } + return mappings; + } + + private static void toXContent(GetIndexResponse response, XContentBuilder builder) throws IOException { + // first we need to repackage from GetIndexResponse to org.elasticsearch.action.admin.indices.get.GetIndexResponse + ImmutableOpenMap.Builder> allMappings = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder> aliases = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder settings = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder defaultSettings = ImmutableOpenMap.builder(); + + Map indexMappings = response.getMappings(); + for (String index : response.getIndices()) { + MappingMetaData mmd = indexMappings.get(index); + ImmutableOpenMap.Builder typedMappings = ImmutableOpenMap.builder(); + if (mmd != null) { + typedMappings.put(MapperService.SINGLE_MAPPING_NAME, mmd); + } + allMappings.put(index, typedMappings.build()); + aliases.put(index, response.getAliases().get(index)); + settings.put(index, response.getSettings().get(index)); + defaultSettings.put(index, response.getDefaultSettings().get(index)); + } + + org.elasticsearch.action.admin.indices.get.GetIndexResponse serverResponse + = new org.elasticsearch.action.admin.indices.get.GetIndexResponse( + response.getIndices(), + allMappings.build(), + aliases.build(), + settings.build(), + defaultSettings.build()); + + // then we can call its toXContent method, forcing no output of types + Params params = new ToXContent.MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "false")); + serverResponse.toXContent(builder, params); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index 9482a42a56e45..235df6d4c33b1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -43,7 +43,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Objects; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -61,7 +60,7 @@ public class GetIndexResponse extends ActionResponse implements ToXContentObject private ImmutableOpenMap defaultSettings = ImmutableOpenMap.of(); private String[] indices; - GetIndexResponse(String[] indices, + public GetIndexResponse(String[] indices, ImmutableOpenMap> mappings, ImmutableOpenMap> aliases, ImmutableOpenMap settings, @@ -315,9 +314,16 @@ private static List parseAliases(XContentParser parser) throws IO private static ImmutableOpenMap parseMappings(XContentParser parser) throws IOException { ImmutableOpenMap.Builder indexMappings = ImmutableOpenMap.builder(); - Map map = parser.map(); - if (map.isEmpty() == false) { - indexMappings.put(MapperService.SINGLE_MAPPING_NAME, new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, map)); + // We start at START_OBJECT since parseIndexEntry ensures that + while (parser.nextToken() != Token.END_OBJECT) { + ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + if (parser.currentToken() == Token.START_OBJECT) { + String mappingType = parser.currentName(); + indexMappings.put(mappingType, new MappingMetaData(mappingType, parser.map())); + } else if (parser.currentToken() == Token.START_ARRAY) { + parser.skipChildren(); + } } return indexMappings.build(); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java index f38df9326949f..842741872fdb2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java @@ -47,8 +47,8 @@ public class RestGetIndicesAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetIndicesAction.class)); - static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using `include_type_name` in get indices requests is deprecated. " - + "The parameter will be removed in the next major version."; + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using `include_type_name` in get indices requests" + + " is deprecated. The parameter will be removed in the next major version."; private static final Set allowedResponseParameters = Collections .unmodifiableSet(Stream.concat(Collections.singleton(INCLUDE_TYPE_NAME_PARAMETER).stream(), Settings.FORMAT_PARAMS.stream()) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java index af3ab33e915db..86e1973ed0afa 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java @@ -31,9 +31,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.RandomCreateIndexGenerator; +import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.test.AbstractStreamableXContentTestCase; import org.junit.Assert; @@ -73,10 +74,6 @@ protected GetIndexResponse createBlankInstance() { @Override protected GetIndexResponse createTestInstance() { - return createTestInstance(randomBoolean()); - } - - private GetIndexResponse createTestInstance(boolean randomTypeName) { String[] indices = generateRandomStringArray(5, 5, false, false); ImmutableOpenMap.Builder> mappings = ImmutableOpenMap.builder(); ImmutableOpenMap.Builder> aliases = ImmutableOpenMap.builder(); @@ -87,7 +84,7 @@ private GetIndexResponse createTestInstance(boolean randomTypeName) { for (String index: indices) { // rarely have no types int typeCount = rarely() ? 0 : 1; - mappings.put(index, GetMappingsResponseTests.createMappingsForIndex(typeCount, randomTypeName)); + mappings.put(index, GetMappingsResponseTests.createMappingsForIndex(typeCount, true)); List aliasMetaDataList = new ArrayList<>(); int aliasesNum = randomIntBetween(0, 3); @@ -110,12 +107,6 @@ private GetIndexResponse createTestInstance(boolean randomTypeName) { ); } - @Override - protected GetIndexResponse createXContextTestInstance(XContentType xContentType) { - // don't use random type names for XContent roundtrip tests because we cannot parse them back anymore - return createTestInstance(false); - } - @Override protected Predicate getRandomFieldsExcludeFilter() { //we do not want to add new fields at the root (index-level), or inside the blocks @@ -203,4 +194,13 @@ public void testCanOutput622Response() throws IOException { Assert.assertEquals(TEST_6_3_0_RESPONSE_BYTES, base64OfResponse); } + + /** + * For xContent roundtrip testing we force the xContent output to still contain types because the parser still expects them. + * The new typeless parsing is implemented in the client side GetIndexResponse. + */ + @Override + protected ToXContent.Params getToXContentParams() { + return new ToXContent.MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "true")); + } } From fe36861ada3e75259ff601a9f6b0e1cd5fbb8afe Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> Date: Tue, 5 Feb 2019 14:21:57 +1100 Subject: [PATCH 24/24] Add support for API keys to access Elasticsearch (#38291) X-Pack security supports built-in authentication service `token-service` that allows access tokens to be used to access Elasticsearch without using Basic authentication. The tokens are generated by `token-service` based on OAuth2 spec. The access token is a short-lived token (defaults to 20m) and refresh token with a lifetime of 24 hours, making them unsuitable for long-lived or recurring tasks where the system might go offline thereby failing refresh of tokens. This commit introduces a built-in authentication service `api-key-service` that adds support for long-lived tokens aka API keys to access Elasticsearch. The `api-key-service` is consulted after `token-service` in the authentication chain. By default, if TLS is enabled then `api-key-service` is also enabled. The service can be disabled using the configuration setting. The API keys:- - by default do not have an expiration but expiration can be configured where the API keys need to be expired after a certain amount of time. - when generated will keep authentication information of the user that generated them. - can be defined with a role describing the privileges for accessing Elasticsearch and will be limited by the role of the user that generated them - can be invalidated via invalidation API - information can be retrieved via a get API - that have been expired or invalidated will be retained for 1 week before being deleted. The expired API keys remover task handles this. Following are the API key management APIs:- 1. Create API Key - `PUT/POST /_security/api_key` 2. Get API key(s) - `GET /_security/api_key` 3. Invalidate API Key(s) `DELETE /_security/api_key` The API keys can be used to access Elasticsearch using `Authorization` header, where the auth scheme is `ApiKey` and the credentials, is the base64 encoding of API key Id and API key separated by a colon. Example:- ``` curl -H "Authorization: ApiKey YXBpLWtleS1pZDphcGkta2V5" http://localhost:9200/_cluster/health ``` Closes #34383 --- client/rest-high-level/build.gradle | 1 + .../elasticsearch/client/SecurityClient.java | 97 ++ .../client/SecurityRequestConverters.java | 35 + .../client/security/CreateApiKeyRequest.java | 128 +++ .../client/security/CreateApiKeyResponse.java | 105 +++ .../client/security/GetApiKeyRequest.java | 133 +++ .../client/security/GetApiKeyResponse.java | 91 ++ .../security/InvalidateApiKeyRequest.java | 145 +++ .../security/InvalidateApiKeyResponse.java | 121 +++ .../client/security/support/ApiKey.java | 152 ++++ .../SecurityRequestConvertersTests.java | 54 +- .../SecurityDocumentationIT.java | 400 ++++++++- .../security/CreateApiKeyRequestTests.java | 105 +++ .../security/CreateApiKeyResponseTests.java | 101 +++ .../security/GetApiKeyRequestTests.java | 72 ++ .../security/GetApiKeyResponseTests.java | 100 +++ .../InvalidateApiKeyRequestTests.java | 73 ++ .../InvalidateApiKeyResponseTests.java | 111 +++ .../security/create-api-key.asciidoc | 40 + .../high-level/security/get-api-key.asciidoc | 67 ++ .../security/invalidate-api-key.asciidoc | 75 ++ .../high-level/supported-apis.asciidoc | 6 + .../rest-api-spec/test/README.asciidoc | 25 + .../common/RandomBasedUUIDGenerator.java | 34 +- .../java/org/elasticsearch/common/UUIDs.java | 7 + .../common/io/stream/StreamInput.java | 17 + .../common/io/stream/StreamOutput.java | 21 + .../elasticsearch/common/util/set/Sets.java | 15 + .../common/io/stream/StreamTests.java | 32 + .../common/util/set/SetsTests.java | 12 + .../test/rest/yaml/Features.java | 3 +- .../rest/yaml/section/ExecutableSection.java | 1 + .../yaml/section/TransformAndSetSection.java | 106 +++ .../section/TransformAndSetSectionTests.java | 96 ++ x-pack/docs/build.gradle | 1 + x-pack/docs/en/rest-api/security.asciidoc | 14 + .../security/create-api-keys.asciidoc | 99 ++ .../rest-api/security/get-api-keys.asciidoc | 118 +++ .../security/invalidate-api-keys.asciidoc | 140 +++ x-pack/plugin/build.gradle | 1 + .../xpack/ccr/CcrLicenseChecker.java | 9 +- .../xpack/core/XPackClientPlugin.java | 6 + .../xpack/core/XPackSettings.java | 7 +- .../xpack/core/security/SecurityContext.java | 11 +- .../xpack/core/security/action/ApiKey.java | 165 ++++ .../security/action/CreateApiKeyAction.java | 33 + .../security/action/CreateApiKeyRequest.java | 132 +++ .../action/CreateApiKeyRequestBuilder.java | 84 ++ .../security/action/CreateApiKeyResponse.java | 168 ++++ .../core/security/action/GetApiKeyAction.java | 33 + .../security/action/GetApiKeyRequest.java | 146 +++ .../security/action/GetApiKeyResponse.java | 88 ++ .../action/InvalidateApiKeyAction.java | 33 + .../action/InvalidateApiKeyRequest.java | 146 +++ .../action/InvalidateApiKeyResponse.java | 141 +++ .../action/role/GetRolesResponse.java | 4 +- .../action/user/HasPrivilegesResponse.java | 61 +- .../core/security/authc/Authentication.java | 62 +- .../DefaultAuthenticationFailureHandler.java | 6 +- .../core/security/authz/RoleDescriptor.java | 71 +- .../accesscontrol/IndicesAccessControl.java | 72 +- .../SecurityIndexSearcherWrapper.java | 179 +--- .../permission/ApplicationPermission.java | 36 + .../authz/permission/ClusterPermission.java | 10 + .../authz/permission/DocumentPermissions.java | 262 ++++++ .../authz/permission/FieldPermissions.java | 39 +- .../authz/permission/IndicesPermission.java | 48 +- .../authz/permission/LimitedRole.java | 152 ++++ .../authz/permission/ResourcePrivileges.java | 93 ++ .../permission/ResourcePrivilegesMap.java | 121 +++ .../core/security/authz/permission/Role.java | 81 +- .../SecurityQueryTemplateEvaluator.java | 92 ++ .../core/security/client/SecurityClient.java | 31 + .../core/security/support/Automatons.java | 6 + .../resources/security-index-template.json | 34 + .../CreateApiKeyRequestBuilderTests.java | 62 ++ .../action/CreateApiKeyRequestTests.java | 113 +++ .../action/CreateApiKeyResponseTests.java | 81 ++ .../action/GetApiKeyRequestTests.java | 103 +++ .../action/GetApiKeyResponseTests.java | 64 ++ .../action/InvalidateApiKeyRequestTests.java | 104 +++ .../action/InvalidateApiKeyResponseTests.java | 88 ++ .../user/HasPrivilegesResponseTests.java | 36 +- ...aultAuthenticationFailureHandlerTests.java | 5 +- ...yIndexSearcherWrapperIntegrationTests.java | 131 ++- ...SecurityIndexSearcherWrapperUnitTests.java | 152 +--- .../permission/DocumentPermissionsTests.java | 123 +++ .../permission/FieldPermissionsTests.java | 81 ++ .../authz/permission/LimitedRoleTests.java | 403 +++++++++ .../ResourcePrivilegesMapTests.java | 91 ++ .../permission/ResourcePrivilegesTests.java | 70 ++ .../SecurityQueryTemplateEvaluatorTests.java | 94 ++ .../ml/action/TransportPutDatafeedAction.java | 5 +- .../security/ApiKeySSLBootstrapCheck.java | 37 + .../xpack/security/Security.java | 57 +- .../action/TransportCreateApiKeyAction.java | 48 + .../action/TransportGetApiKeyAction.java | 46 + .../TransportInvalidateApiKeyAction.java | 44 + .../BulkShardRequestInterceptor.java | 2 +- ...cumentLevelSecurityRequestInterceptor.java | 2 +- .../IndicesAliasesRequestInterceptor.java | 2 +- .../interceptor/ResizeRequestInterceptor.java | 2 +- .../TransportGetUserPrivilegesAction.java | 5 +- .../user/TransportHasPrivilegesAction.java | 124 +-- .../xpack/security/authc/ApiKeyService.java | 844 ++++++++++++++++++ .../security/authc/AuthenticationService.java | 40 +- .../security/authc/ExpiredApiKeysRemover.java | 116 +++ .../xpack/security/authc/TokenService.java | 11 +- .../security/authz/AuthorizationService.java | 59 +- .../security/authz/AuthorizationUtils.java | 5 +- .../security/authz/AuthorizedIndices.java | 2 +- .../authz/store/CompositeRolesStore.java | 128 ++- .../rest/action/RestCreateApiKeyAction.java | 56 ++ .../rest/action/RestGetApiKeyAction.java | 63 ++ .../action/RestInvalidateApiKeyAction.java | 70 ++ .../ApiKeySSLBootstrapCheckTests.java | 32 + .../xpack/security/SecurityContextTests.java | 7 + .../xpack/security/SecurityTests.java | 7 +- .../security/TokenSSLBootsrapCheckTests.java | 9 +- .../filter/SecurityActionFilterTests.java | 16 +- ...IndicesAliasesRequestInterceptorTests.java | 4 +- .../ResizeRequestInterceptorTests.java | 4 +- .../TransportHasPrivilegesActionTests.java | 142 ++- .../security/authc/ApiKeyIntegTests.java | 494 ++++++++++ .../security/authc/ApiKeyServiceTests.java | 295 ++++++ .../authc/AuthenticationServiceTests.java | 153 +++- .../security/authc/TokenServiceTests.java | 41 +- .../authz/AuthorizationServiceTests.java | 238 +++-- .../authz/IndicesAndAliasesResolverTests.java | 13 +- .../security/authz/RoleDescriptorTests.java | 8 +- .../IndicesAccessControlTests.java | 101 +++ .../accesscontrol/IndicesPermissionTests.java | 53 +- .../accesscontrol/OptOutQueryCacheTests.java | 28 +- .../authz/store/CompositeRolesStoreTests.java | 74 +- .../action/RestCreateApiKeyActionTests.java | 111 +++ .../rest/action/RestGetApiKeyActionTests.java | 138 +++ .../RestInvalidateApiKeyActionTests.java | 121 +++ .../transport/ServerTransportFilterTests.java | 12 +- .../api/security.create_api_key.json | 22 + .../api/security.get_api_key.json | 30 + .../api/security.invalidate_api_key.json | 15 + .../rest-api-spec/test/api_key/10_basic.yml | 290 ++++++ 142 files changed, 10697 insertions(+), 946 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java create mode 100644 docs/java-rest/high-level/security/create-api-key.asciidoc create mode 100644 docs/java-rest/high-level/security/get-api-key.asciidoc create mode 100644 docs/java-rest/high-level/security/invalidate-api-key.asciidoc create mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java create mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java create mode 100644 x-pack/docs/en/rest-api/security/create-api-keys.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/get-api-keys.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissionsTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMapTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ApiKeySSLBootstrapCheck.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGetApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportInvalidateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredApiKeysRemover.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestCreateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestGetApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestInvalidateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ApiKeySSLBootstrapCheckTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestCreateApiKeyActionTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestGetApiKeyActionTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestInvalidateApiKeyActionTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/security.create_api_key.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_api_key.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_api_key.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/api_key/10_basic.yml diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 22e6252892a7d..44262f09346de 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -104,6 +104,7 @@ integTestCluster { setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' + setting 'xpack.security.authc.api_key.enabled', 'true' // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt' setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks' diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java index de0032c6c2a8f..fefb5771dc801 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java @@ -27,6 +27,8 @@ import org.elasticsearch.client.security.ClearRealmCacheResponse; import org.elasticsearch.client.security.ClearRolesCacheRequest; import org.elasticsearch.client.security.ClearRolesCacheResponse; +import org.elasticsearch.client.security.CreateApiKeyRequest; +import org.elasticsearch.client.security.CreateApiKeyResponse; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.CreateTokenResponse; import org.elasticsearch.client.security.DeletePrivilegesRequest; @@ -39,6 +41,8 @@ import org.elasticsearch.client.security.DeleteUserResponse; import org.elasticsearch.client.security.DisableUserRequest; import org.elasticsearch.client.security.EnableUserRequest; +import org.elasticsearch.client.security.GetApiKeyRequest; +import org.elasticsearch.client.security.GetApiKeyResponse; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetPrivilegesResponse; import org.elasticsearch.client.security.GetRoleMappingsRequest; @@ -53,6 +57,8 @@ import org.elasticsearch.client.security.GetUsersResponse; import org.elasticsearch.client.security.HasPrivilegesRequest; import org.elasticsearch.client.security.HasPrivilegesResponse; +import org.elasticsearch.client.security.InvalidateApiKeyRequest; +import org.elasticsearch.client.security.InvalidateApiKeyResponse; import org.elasticsearch.client.security.InvalidateTokenRequest; import org.elasticsearch.client.security.InvalidateTokenResponse; import org.elasticsearch.client.security.PutPrivilegesRequest; @@ -842,4 +848,95 @@ public void deletePrivilegesAsync(DeletePrivilegesRequest request, RequestOption restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::deletePrivileges, options, DeletePrivilegesResponse::fromXContent, listener, singleton(404)); } + + /** + * Create an API Key.
+ * See + * the docs for more. + * + * @param request the request to create a API key + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response from the create API key call + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public CreateApiKeyResponse createApiKey(final CreateApiKeyRequest request, final RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::createApiKey, options, + CreateApiKeyResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously creates an API key.
+ * See + * the docs for more. + * + * @param request the request to create a API key + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void createApiKeyAsync(final CreateApiKeyRequest request, final RequestOptions options, + final ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::createApiKey, options, + CreateApiKeyResponse::fromXContent, listener, emptySet()); + } + + /** + * Retrieve API Key(s) information.
+ * See + * the docs for more. + * + * @param request the request to retrieve API key(s) + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response from the create API key call + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetApiKeyResponse getApiKey(final GetApiKeyRequest request, final RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::getApiKey, options, + GetApiKeyResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously retrieve API Key(s) information.
+ * See + * the docs for more. + * + * @param request the request to retrieve API key(s) + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void getApiKeyAsync(final GetApiKeyRequest request, final RequestOptions options, + final ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::getApiKey, options, + GetApiKeyResponse::fromXContent, listener, emptySet()); + } + + /** + * Invalidate API Key(s).
+ * See + * the docs for more. + * + * @param request the request to invalidate API key(s) + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response from the invalidate API key call + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public InvalidateApiKeyResponse invalidateApiKey(final InvalidateApiKeyRequest request, final RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::invalidateApiKey, options, + InvalidateApiKeyResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously invalidates API key(s).
+ * See + * the docs for more. + * + * @param request the request to invalidate API key(s) + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void invalidateApiKeyAsync(final InvalidateApiKeyRequest request, final RequestOptions options, + final ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::invalidateApiKey, options, + InvalidateApiKeyResponse::fromXContent, listener, emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java index 9e9698ded1cd8..f686167e211bb 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java @@ -26,6 +26,7 @@ import org.elasticsearch.client.security.ChangePasswordRequest; import org.elasticsearch.client.security.ClearRealmCacheRequest; import org.elasticsearch.client.security.ClearRolesCacheRequest; +import org.elasticsearch.client.security.CreateApiKeyRequest; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.DeletePrivilegesRequest; import org.elasticsearch.client.security.DeleteRoleMappingRequest; @@ -33,11 +34,13 @@ import org.elasticsearch.client.security.DeleteUserRequest; import org.elasticsearch.client.security.DisableUserRequest; import org.elasticsearch.client.security.EnableUserRequest; +import org.elasticsearch.client.security.GetApiKeyRequest; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetRoleMappingsRequest; import org.elasticsearch.client.security.GetRolesRequest; import org.elasticsearch.client.security.GetUsersRequest; import org.elasticsearch.client.security.HasPrivilegesRequest; +import org.elasticsearch.client.security.InvalidateApiKeyRequest; import org.elasticsearch.client.security.InvalidateTokenRequest; import org.elasticsearch.client.security.PutPrivilegesRequest; import org.elasticsearch.client.security.PutRoleMappingRequest; @@ -256,4 +259,36 @@ static Request putRole(final PutRoleRequest putRoleRequest) throws IOException { params.withRefreshPolicy(putRoleRequest.getRefreshPolicy()); return request; } + + static Request createApiKey(final CreateApiKeyRequest createApiKeyRequest) throws IOException { + final Request request = new Request(HttpPost.METHOD_NAME, "/_security/api_key"); + request.setEntity(createEntity(createApiKeyRequest, REQUEST_BODY_CONTENT_TYPE)); + final RequestConverters.Params params = new RequestConverters.Params(request); + params.withRefreshPolicy(createApiKeyRequest.getRefreshPolicy()); + return request; + } + + static Request getApiKey(final GetApiKeyRequest getApiKeyRequest) throws IOException { + final Request request = new Request(HttpGet.METHOD_NAME, "/_security/api_key"); + if (Strings.hasText(getApiKeyRequest.getId())) { + request.addParameter("id", getApiKeyRequest.getId()); + } + if (Strings.hasText(getApiKeyRequest.getName())) { + request.addParameter("name", getApiKeyRequest.getName()); + } + if (Strings.hasText(getApiKeyRequest.getUserName())) { + request.addParameter("username", getApiKeyRequest.getUserName()); + } + if (Strings.hasText(getApiKeyRequest.getRealmName())) { + request.addParameter("realm_name", getApiKeyRequest.getRealmName()); + } + return request; + } + + static Request invalidateApiKey(final InvalidateApiKeyRequest invalidateApiKeyRequest) throws IOException { + final Request request = new Request(HttpDelete.METHOD_NAME, "/_security/api_key"); + request.setEntity(createEntity(invalidateApiKeyRequest, REQUEST_BODY_CONTENT_TYPE)); + final RequestConverters.Params params = new RequestConverters.Params(request); + return request; + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java new file mode 100644 index 0000000000000..ad5f0a9ba2cf6 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.client.security.user.privileges.Role; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Request to create API key + */ +public final class CreateApiKeyRequest implements Validatable, ToXContentObject { + + private final String name; + private final TimeValue expiration; + private final List roles; + private final RefreshPolicy refreshPolicy; + + /** + * Create API Key request constructor + * @param name name for the API key + * @param roles list of {@link Role}s + * @param expiration to specify expiration for the API key + */ + public CreateApiKeyRequest(String name, List roles, @Nullable TimeValue expiration, @Nullable final RefreshPolicy refreshPolicy) { + if (Strings.hasText(name)) { + this.name = name; + } else { + throw new IllegalArgumentException("name must not be null or empty"); + } + this.roles = Objects.requireNonNull(roles, "roles may not be null"); + this.expiration = expiration; + this.refreshPolicy = (refreshPolicy == null) ? RefreshPolicy.getDefault() : refreshPolicy; + } + + public String getName() { + return name; + } + + public TimeValue getExpiration() { + return expiration; + } + + public List getRoles() { + return roles; + } + + public RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + @Override + public int hashCode() { + return Objects.hash(name, refreshPolicy, roles, expiration); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final CreateApiKeyRequest that = (CreateApiKeyRequest) o; + return Objects.equals(name, that.name) && Objects.equals(refreshPolicy, that.refreshPolicy) && Objects.equals(roles, that.roles) + && Objects.equals(expiration, that.expiration); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject().field("name", name); + if (expiration != null) { + builder.field("expiration", expiration.getStringRep()); + } + builder.startObject("role_descriptors"); + for (Role role : roles) { + builder.startObject(role.getName()); + if (role.getApplicationPrivileges() != null) { + builder.field(Role.APPLICATIONS.getPreferredName(), role.getApplicationPrivileges()); + } + if (role.getClusterPrivileges() != null) { + builder.field(Role.CLUSTER.getPreferredName(), role.getClusterPrivileges()); + } + if (role.getGlobalPrivileges() != null) { + builder.field(Role.GLOBAL.getPreferredName(), role.getGlobalPrivileges()); + } + if (role.getIndicesPrivileges() != null) { + builder.field(Role.INDICES.getPreferredName(), role.getIndicesPrivileges()); + } + if (role.getMetadata() != null) { + builder.field(Role.METADATA.getPreferredName(), role.getMetadata()); + } + if (role.getRunAsPrivilege() != null) { + builder.field(Role.RUN_AS.getPreferredName(), role.getRunAsPrivilege()); + } + builder.endObject(); + } + builder.endObject(); + return builder.endObject(); + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java new file mode 100644 index 0000000000000..9c5037237407b --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for create API key + */ +public final class CreateApiKeyResponse { + + private final String name; + private final String id; + private final SecureString key; + private final Instant expiration; + + public CreateApiKeyResponse(String name, String id, SecureString key, Instant expiration) { + this.name = name; + this.id = id; + this.key = key; + // As we do not yet support the nanosecond precision when we serialize to JSON, + // here creating the 'Instant' of milliseconds precision. + // This Instant can then be used for date comparison. + this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null; + } + + public String getName() { + return name; + } + + public String getId() { + return id; + } + + public SecureString getKey() { + return key; + } + + @Nullable + public Instant getExpiration() { + return expiration; + } + + @Override + public int hashCode() { + return Objects.hash(id, name, key, expiration); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final CreateApiKeyResponse other = (CreateApiKeyResponse) obj; + return Objects.equals(id, other.id) + && Objects.equals(key, other.key) + && Objects.equals(name, other.name) + && Objects.equals(expiration, other.expiration); + } + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("create_api_key_response", + args -> new CreateApiKeyResponse((String) args[0], (String) args[1], new SecureString((String) args[2]), + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]))); + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareString(constructorArg(), new ParseField("api_key")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); + } + + public static CreateApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java new file mode 100644 index 0000000000000..6fa98ec549b07 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Request for get API key + */ +public final class GetApiKeyRequest implements Validatable, ToXContentObject { + + private final String realmName; + private final String userName; + private final String id; + private final String name; + + // pkg scope for testing + GetApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId, + @Nullable String apiKeyName) { + if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false + && Strings.hasText(apiKeyName) == false) { + throwValidationError("One of [api key id, api key name, username, realm name] must be specified"); + } + if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) { + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + throwValidationError( + "username or realm name must not be specified when the api key id or api key name is specified"); + } + } + if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) { + throwValidationError("only one of [api key id, api key name] can be specified"); + } + this.realmName = realmName; + this.userName = userName; + this.id = apiKeyId; + this.name = apiKeyName; + } + + private void throwValidationError(String message) { + throw new IllegalArgumentException(message); + } + + public String getRealmName() { + return realmName; + } + + public String getUserName() { + return userName; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + /** + * Creates get API key request for given realm name + * @param realmName realm name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingRealmName(String realmName) { + return new GetApiKeyRequest(realmName, null, null, null); + } + + /** + * Creates get API key request for given user name + * @param userName user name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingUserName(String userName) { + return new GetApiKeyRequest(null, userName, null, null); + } + + /** + * Creates get API key request for given realm and user name + * @param realmName realm name + * @param userName user name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingRealmAndUserName(String realmName, String userName) { + return new GetApiKeyRequest(realmName, userName, null, null); + } + + /** + * Creates get API key request for given api key id + * @param apiKeyId api key id + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingApiKeyId(String apiKeyId) { + return new GetApiKeyRequest(null, null, apiKeyId, null); + } + + /** + * Creates get API key request for given api key name + * @param apiKeyName api key name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingApiKeyName(String apiKeyName) { + return new GetApiKeyRequest(null, null, null, apiKeyName); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder; + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java new file mode 100644 index 0000000000000..58e3e8effbb09 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.security.support.ApiKey; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for get API keys.
+ * The result contains information about the API keys that were found. + */ +public final class GetApiKeyResponse { + + private final List foundApiKeysInfo; + + public GetApiKeyResponse(List foundApiKeysInfo) { + Objects.requireNonNull(foundApiKeysInfo, "found_api_keys_info must be provided"); + this.foundApiKeysInfo = Collections.unmodifiableList(foundApiKeysInfo); + } + + public static GetApiKeyResponse emptyResponse() { + return new GetApiKeyResponse(Collections.emptyList()); + } + + public List getApiKeyInfos() { + return foundApiKeysInfo; + } + + @Override + public int hashCode() { + return Objects.hash(foundApiKeysInfo); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final GetApiKeyResponse other = (GetApiKeyResponse) obj; + return Objects.equals(foundApiKeysInfo, other.foundApiKeysInfo); + } + + @SuppressWarnings("unchecked") + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("get_api_key_response", args -> { + return (args[0] == null) ? GetApiKeyResponse.emptyResponse() : new GetApiKeyResponse((List) args[0]); + }); + static { + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ApiKey.fromXContent(p), new ParseField("api_keys")); + } + + public static GetApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "GetApiKeyResponse [foundApiKeysInfo=" + foundApiKeysInfo + "]"; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java new file mode 100644 index 0000000000000..d3203354b7ab1 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java @@ -0,0 +1,145 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Request for invalidating API key(s) so that it can no longer be used + */ +public final class InvalidateApiKeyRequest implements Validatable, ToXContentObject { + + private final String realmName; + private final String userName; + private final String id; + private final String name; + + // pkg scope for testing + InvalidateApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId, + @Nullable String apiKeyName) { + if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false + && Strings.hasText(apiKeyName) == false) { + throwValidationError("One of [api key id, api key name, username, realm name] must be specified"); + } + if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) { + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + throwValidationError( + "username or realm name must not be specified when the api key id or api key name is specified"); + } + } + if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) { + throwValidationError("only one of [api key id, api key name] can be specified"); + } + this.realmName = realmName; + this.userName = userName; + this.id = apiKeyId; + this.name = apiKeyName; + } + + private void throwValidationError(String message) { + throw new IllegalArgumentException(message); + } + + public String getRealmName() { + return realmName; + } + + public String getUserName() { + return userName; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + /** + * Creates invalidate API key request for given realm name + * @param realmName realm name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingRealmName(String realmName) { + return new InvalidateApiKeyRequest(realmName, null, null, null); + } + + /** + * Creates invalidate API key request for given user name + * @param userName user name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingUserName(String userName) { + return new InvalidateApiKeyRequest(null, userName, null, null); + } + + /** + * Creates invalidate API key request for given realm and user name + * @param realmName realm name + * @param userName user name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingRealmAndUserName(String realmName, String userName) { + return new InvalidateApiKeyRequest(realmName, userName, null, null); + } + + /** + * Creates invalidate API key request for given api key id + * @param apiKeyId api key id + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingApiKeyId(String apiKeyId) { + return new InvalidateApiKeyRequest(null, null, apiKeyId, null); + } + + /** + * Creates invalidate API key request for given api key name + * @param apiKeyName api key name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingApiKeyName(String apiKeyName) { + return new InvalidateApiKeyRequest(null, null, null, apiKeyName); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (realmName != null) { + builder.field("realm_name", realmName); + } + if (userName != null) { + builder.field("username", userName); + } + if (id != null) { + builder.field("id", id); + } + if (name != null) { + builder.field("name", name); + } + return builder.endObject(); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java new file mode 100644 index 0000000000000..48df9d0f7f12b --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java @@ -0,0 +1,121 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public final class InvalidateApiKeyResponse { + + private final List invalidatedApiKeys; + private final List previouslyInvalidatedApiKeys; + private final List errors; + + /** + * Constructor for API keys invalidation response + * @param invalidatedApiKeys list of invalidated API key ids + * @param previouslyInvalidatedApiKeys list of previously invalidated API key ids + * @param errors list of encountered errors while invalidating API keys + */ + public InvalidateApiKeyResponse(List invalidatedApiKeys, List previouslyInvalidatedApiKeys, + @Nullable List errors) { + this.invalidatedApiKeys = Objects.requireNonNull(invalidatedApiKeys, "invalidated_api_keys must be provided"); + this.previouslyInvalidatedApiKeys = Objects.requireNonNull(previouslyInvalidatedApiKeys, + "previously_invalidated_api_keys must be provided"); + if (null != errors) { + this.errors = errors; + } else { + this.errors = Collections.emptyList(); + } + } + + public static InvalidateApiKeyResponse emptyResponse() { + return new InvalidateApiKeyResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + } + + public List getInvalidatedApiKeys() { + return invalidatedApiKeys; + } + + public List getPreviouslyInvalidatedApiKeys() { + return previouslyInvalidatedApiKeys; + } + + public List getErrors() { + return errors; + } + + @SuppressWarnings("unchecked") + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("invalidate_api_key_response", + args -> { + return new InvalidateApiKeyResponse((List) args[0], (List) args[1], (List) args[3]); + }); + static { + PARSER.declareStringArray(constructorArg(), new ParseField("invalidated_api_keys")); + PARSER.declareStringArray(constructorArg(), new ParseField("previously_invalidated_api_keys")); + // error count is parsed but ignored as we have list of errors + PARSER.declareInt(constructorArg(), new ParseField("error_count")); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ElasticsearchException.fromXContent(p), + new ParseField("error_details")); + } + + public static InvalidateApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public int hashCode() { + return Objects.hash(invalidatedApiKeys, previouslyInvalidatedApiKeys, errors); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + InvalidateApiKeyResponse other = (InvalidateApiKeyResponse) obj; + return Objects.equals(invalidatedApiKeys, other.invalidatedApiKeys) + && Objects.equals(previouslyInvalidatedApiKeys, other.previouslyInvalidatedApiKeys) + && Objects.equals(errors, other.errors); + } + + @Override + public String toString() { + return "ApiKeysInvalidationResult [invalidatedApiKeys=" + invalidatedApiKeys + ", previouslyInvalidatedApiKeys=" + + previouslyInvalidatedApiKeys + ", errors=" + errors + "]"; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java new file mode 100644 index 0000000000000..d021628f750cb --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java @@ -0,0 +1,152 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security.support; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * API key information + */ +public final class ApiKey { + + private final String name; + private final String id; + private final Instant creation; + private final Instant expiration; + private final boolean invalidated; + private final String username; + private final String realm; + + public ApiKey(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm) { + this.name = name; + this.id = id; + // As we do not yet support the nanosecond precision when we serialize to JSON, + // here creating the 'Instant' of milliseconds precision. + // This Instant can then be used for date comparison. + this.creation = Instant.ofEpochMilli(creation.toEpochMilli()); + this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null; + this.invalidated = invalidated; + this.username = username; + this.realm = realm; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + /** + * @return a instance of {@link Instant} when this API key was created. + */ + public Instant getCreation() { + return creation; + } + + /** + * @return a instance of {@link Instant} when this API key will expire. In case the API key does not expire then will return + * {@code null} + */ + public Instant getExpiration() { + return expiration; + } + + /** + * @return {@code true} if this API key has been invalidated else returns {@code false} + */ + public boolean isInvalidated() { + return invalidated; + } + + /** + * @return the username for which this API key was created. + */ + public String getUsername() { + return username; + } + + /** + * @return the realm name of the user for which this API key was created. + */ + public String getRealm() { + return realm; + } + + @Override + public int hashCode() { + return Objects.hash(name, id, creation, expiration, invalidated, username, realm); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ApiKey other = (ApiKey) obj; + return Objects.equals(name, other.name) + && Objects.equals(id, other.id) + && Objects.equals(creation, other.creation) + && Objects.equals(expiration, other.expiration) + && Objects.equals(invalidated, other.invalidated) + && Objects.equals(username, other.username) + && Objects.equals(realm, other.realm); + } + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("api_key", args -> { + return new ApiKey((String) args[0], (String) args[1], Instant.ofEpochMilli((Long) args[2]), + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]), (Boolean) args[4], (String) args[5], (String) args[6]); + }); + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareLong(constructorArg(), new ParseField("creation")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); + PARSER.declareBoolean(constructorArg(), new ParseField("invalidated")); + PARSER.declareString(constructorArg(), new ParseField("username")); + PARSER.declareString(constructorArg(), new ParseField("realm")); + } + + public static ApiKey fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "ApiKey [name=" + name + ", id=" + id + ", creation=" + creation + ", expiration=" + expiration + ", invalidated=" + + invalidated + ", username=" + username + ", realm=" + realm + "]"; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java index d9bd606167370..b2c2028d0fbbd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java @@ -24,6 +24,7 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.security.ChangePasswordRequest; +import org.elasticsearch.client.security.CreateApiKeyRequest; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.DeletePrivilegesRequest; import org.elasticsearch.client.security.DeleteRoleMappingRequest; @@ -31,10 +32,12 @@ import org.elasticsearch.client.security.DeleteUserRequest; import org.elasticsearch.client.security.DisableUserRequest; import org.elasticsearch.client.security.EnableUserRequest; +import org.elasticsearch.client.security.GetApiKeyRequest; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetRoleMappingsRequest; import org.elasticsearch.client.security.GetRolesRequest; import org.elasticsearch.client.security.GetUsersRequest; +import org.elasticsearch.client.security.InvalidateApiKeyRequest; import org.elasticsearch.client.security.PutPrivilegesRequest; import org.elasticsearch.client.security.PutRoleMappingRequest; import org.elasticsearch.client.security.PutRoleRequest; @@ -44,11 +47,14 @@ import org.elasticsearch.client.security.support.expressiondsl.expressions.AnyRoleMapperExpression; import org.elasticsearch.client.security.support.expressiondsl.fields.FieldRoleMapperExpression; import org.elasticsearch.client.security.user.User; -import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges; import org.elasticsearch.client.security.user.privileges.ApplicationPrivilege; +import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges; import org.elasticsearch.client.security.user.privileges.IndicesPrivileges; import org.elasticsearch.client.security.user.privileges.Role; +import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName; +import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.test.ESTestCase; @@ -61,6 +67,7 @@ import java.util.Map; import static org.elasticsearch.client.RequestConvertersTests.assertToXContentBody; +import static org.hamcrest.Matchers.equalTo; public class SecurityRequestConvertersTests extends ESTestCase { @@ -411,4 +418,47 @@ public void testPutRole() throws IOException { assertEquals(expectedParams, request.getParameters()); assertToXContentBody(putRoleRequest, request.getEntity()); } -} + + public void testCreateApiKey() throws IOException { + final String name = randomAlphaOfLengthBetween(4, 7); + final List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = randomBoolean() ? null : TimeValue.timeValueHours(24); + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + final Map expectedParams; + if (refreshPolicy != RefreshPolicy.NONE) { + expectedParams = Collections.singletonMap("refresh", refreshPolicy.getValue()); + } else { + expectedParams = Collections.emptyMap(); + } + final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + final Request request = SecurityRequestConverters.createApiKey(createApiKeyRequest); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + assertEquals("/_security/api_key", request.getEndpoint()); + assertEquals(expectedParams, request.getParameters()); + assertToXContentBody(createApiKeyRequest, request.getEntity()); + } + + public void testGetApiKey() throws IOException { + String realmName = randomAlphaOfLength(5); + String userName = randomAlphaOfLength(7); + final GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName(realmName, userName); + final Request request = SecurityRequestConverters.getApiKey(getApiKeyRequest); + assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals("/_security/api_key", request.getEndpoint()); + Map mapOfParameters = new HashMap<>(); + mapOfParameters.put("realm_name", realmName); + mapOfParameters.put("username", userName); + assertThat(request.getParameters(), equalTo(mapOfParameters)); + } + + public void testInvalidateApiKey() throws IOException { + String realmName = randomAlphaOfLength(5); + String userName = randomAlphaOfLength(7); + final InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName(realmName, userName); + final Request request = SecurityRequestConverters.invalidateApiKey(invalidateApiKeyRequest); + assertEquals(HttpDelete.METHOD_NAME, request.getMethod()); + assertEquals("/_security/api_key", request.getEndpoint()); + assertToXContentBody(invalidateApiKeyRequest, request.getEntity()); + } + } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java index 0edd862eb6371..ea070868c6821 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java @@ -33,6 +33,8 @@ import org.elasticsearch.client.security.ClearRealmCacheResponse; import org.elasticsearch.client.security.ClearRolesCacheRequest; import org.elasticsearch.client.security.ClearRolesCacheResponse; +import org.elasticsearch.client.security.CreateApiKeyRequest; +import org.elasticsearch.client.security.CreateApiKeyResponse; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.CreateTokenResponse; import org.elasticsearch.client.security.DeletePrivilegesRequest; @@ -46,6 +48,8 @@ import org.elasticsearch.client.security.DisableUserRequest; import org.elasticsearch.client.security.EnableUserRequest; import org.elasticsearch.client.security.ExpressionRoleMapping; +import org.elasticsearch.client.security.GetApiKeyRequest; +import org.elasticsearch.client.security.GetApiKeyResponse; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetPrivilegesResponse; import org.elasticsearch.client.security.GetRoleMappingsRequest; @@ -58,6 +62,8 @@ import org.elasticsearch.client.security.GetUsersResponse; import org.elasticsearch.client.security.HasPrivilegesRequest; import org.elasticsearch.client.security.HasPrivilegesResponse; +import org.elasticsearch.client.security.InvalidateApiKeyRequest; +import org.elasticsearch.client.security.InvalidateApiKeyResponse; import org.elasticsearch.client.security.InvalidateTokenRequest; import org.elasticsearch.client.security.InvalidateTokenResponse; import org.elasticsearch.client.security.PutPrivilegesRequest; @@ -69,6 +75,7 @@ import org.elasticsearch.client.security.PutUserRequest; import org.elasticsearch.client.security.PutUserResponse; import org.elasticsearch.client.security.RefreshPolicy; +import org.elasticsearch.client.security.support.ApiKey; import org.elasticsearch.client.security.support.CertificateInfo; import org.elasticsearch.client.security.support.expressiondsl.RoleMapperExpression; import org.elasticsearch.client.security.support.expressiondsl.expressions.AnyRoleMapperExpression; @@ -78,13 +85,17 @@ import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges; import org.elasticsearch.client.security.user.privileges.IndicesPrivileges; import org.elasticsearch.client.security.user.privileges.Role; +import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName; +import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName; import org.elasticsearch.client.security.user.privileges.UserIndicesPrivileges; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.hamcrest.Matchers; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.PBEKeySpec; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -97,15 +108,20 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; + import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isIn; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase { @@ -336,7 +352,7 @@ public void onFailure(Exception e) { private void addUser(RestHighLevelClient client, String userName, String password) throws IOException { User user = new User(userName, Collections.singletonList(userName)); - PutUserRequest request = new PutUserRequest(user, password.toCharArray(), true, RefreshPolicy.NONE); + PutUserRequest request = PutUserRequest.withPassword(user, password.toCharArray(), true, RefreshPolicy.NONE); PutUserResponse response = client.security().putUser(request, RequestOptions.DEFAULT); assertTrue(response.isCreated()); } @@ -510,7 +526,7 @@ public void testEnableUser() throws Exception { RestHighLevelClient client = highLevelClient(); char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'}; User enable_user = new User("enable_user", Collections.singletonList("superuser")); - PutUserRequest putUserRequest = new PutUserRequest(enable_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(enable_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); @@ -555,7 +571,7 @@ public void testDisableUser() throws Exception { RestHighLevelClient client = highLevelClient(); char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'}; User disable_user = new User("disable_user", Collections.singletonList("superuser")); - PutUserRequest putUserRequest = new PutUserRequest(disable_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(disable_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); { @@ -1032,7 +1048,7 @@ public void testChangePassword() throws Exception { char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'}; char[] newPassword = new char[]{'n', 'e', 'w', 'p', 'a', 's', 's', 'w', 'o', 'r', 'd'}; User user = new User("change_password_user", Collections.singletonList("superuser"), Collections.emptyMap(), null, null); - PutUserRequest putUserRequest = new PutUserRequest(user, password, true, RefreshPolicy.NONE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(user, password, true, RefreshPolicy.NONE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); { @@ -1249,7 +1265,8 @@ public void testCreateToken() throws Exception { { // Setup user User token_user = new User("token_user", Collections.singletonList("kibana_user")); - PutUserRequest putUserRequest = new PutUserRequest(token_user, "password".toCharArray(), true, RefreshPolicy.IMMEDIATE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(token_user, "password".toCharArray(), true, + RefreshPolicy.IMMEDIATE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); } @@ -1327,27 +1344,27 @@ public void testInvalidateToken() throws Exception { // Setup users final char[] password = "password".toCharArray(); User user = new User("user", Collections.singletonList("kibana_user")); - PutUserRequest putUserRequest = new PutUserRequest(user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); User this_user = new User("this_user", Collections.singletonList("kibana_user")); - PutUserRequest putThisUserRequest = new PutUserRequest(this_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putThisUserRequest = PutUserRequest.withPassword(this_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putThisUserResponse = client.security().putUser(putThisUserRequest, RequestOptions.DEFAULT); assertTrue(putThisUserResponse.isCreated()); User that_user = new User("that_user", Collections.singletonList("kibana_user")); - PutUserRequest putThatUserRequest = new PutUserRequest(that_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putThatUserRequest = PutUserRequest.withPassword(that_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putThatUserResponse = client.security().putUser(putThatUserRequest, RequestOptions.DEFAULT); assertTrue(putThatUserResponse.isCreated()); User other_user = new User("other_user", Collections.singletonList("kibana_user")); - PutUserRequest putOtherUserRequest = new PutUserRequest(other_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putOtherUserRequest = PutUserRequest.withPassword(other_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putOtherUserResponse = client.security().putUser(putOtherUserRequest, RequestOptions.DEFAULT); assertTrue(putOtherUserResponse.isCreated()); User extra_user = new User("extra_user", Collections.singletonList("kibana_user")); - PutUserRequest putExtraUserRequest = new PutUserRequest(extra_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putExtraUserRequest = PutUserRequest.withPassword(extra_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putExtraUserResponse = client.security().putUser(putExtraUserRequest, RequestOptions.DEFAULT); assertTrue(putExtraUserResponse.isCreated()); @@ -1747,4 +1764,363 @@ public void onFailure(Exception e) { } } + public void testCreateApiKey() throws Exception { + RestHighLevelClient client = highLevelClient(); + + List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = TimeValue.timeValueHours(24); + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + { + final String name = randomAlphaOfLength(5); + // tag::create-api-key-request + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + // end::create-api-key-request + + // tag::create-api-key-execute + CreateApiKeyResponse createApiKeyResponse = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + // end::create-api-key-execute + + // tag::create-api-key-response + SecureString apiKey = createApiKeyResponse.getKey(); // <1> + Instant apiKeyExpiration = createApiKeyResponse.getExpiration(); // <2> + // end::create-api-key-response + assertThat(createApiKeyResponse.getName(), equalTo(name)); + assertNotNull(apiKey); + assertNotNull(apiKeyExpiration); + } + + { + final String name = randomAlphaOfLength(5); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + + ActionListener listener; + // tag::create-api-key-execute-listener + listener = new ActionListener() { + @Override + public void onResponse(CreateApiKeyResponse createApiKeyResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::create-api-key-execute-listener + + // Avoid unused variable warning + assertNotNull(listener); + + // Replace the empty listener by a blocking listener in test + final PlainActionFuture future = new PlainActionFuture<>(); + listener = future; + + // tag::create-api-key-execute-async + client.security().createApiKeyAsync(createApiKeyRequest, RequestOptions.DEFAULT, listener); // <1> + // end::create-api-key-execute-async + + assertNotNull(future.get(30, TimeUnit.SECONDS)); + assertThat(future.get().getName(), equalTo(name)); + assertNotNull(future.get().getKey()); + assertNotNull(future.get().getExpiration()); + } + } + + public void testGetApiKey() throws Exception { + RestHighLevelClient client = highLevelClient(); + + List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = TimeValue.timeValueHours(24); + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + // Create API Keys + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse1 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse1.getName(), equalTo("k1")); + assertNotNull(createApiKeyResponse1.getKey()); + + final ApiKey expectedApiKeyInfo = new ApiKey(createApiKeyResponse1.getName(), createApiKeyResponse1.getId(), Instant.now(), + Instant.now().plusMillis(expiration.getMillis()), false, "test_user", "default_file"); + { + // tag::get-api-key-id-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId()); + // end::get-api-key-id-request + + // tag::get-api-key-execute + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + // end::get-api-key-execute + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + // tag::get-api-key-name-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyName(createApiKeyResponse1.getName()); + // end::get-api-key-name-request + + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + // tag::get-realm-api-keys-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmName("default_file"); + // end::get-realm-api-keys-request + + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + // tag::get-user-api-keys-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingUserName("test_user"); + // end::get-user-api-keys-request + + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + // tag::get-user-realm-api-keys-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName("default_file", "test_user"); + // end::get-user-realm-api-keys-request + + // tag::get-api-key-response + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + // end::get-api-key-response + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId()); + + ActionListener listener; + // tag::get-api-key-execute-listener + listener = new ActionListener() { + @Override + public void onResponse(GetApiKeyResponse getApiKeyResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::get-api-key-execute-listener + + // Avoid unused variable warning + assertNotNull(listener); + + // Replace the empty listener by a blocking listener in test + final PlainActionFuture future = new PlainActionFuture<>(); + listener = future; + + // tag::get-api-key-execute-async + client.security().getApiKeyAsync(getApiKeyRequest, RequestOptions.DEFAULT, listener); // <1> + // end::get-api-key-execute-async + + final GetApiKeyResponse response = future.get(30, TimeUnit.SECONDS); + assertNotNull(response); + + assertThat(response.getApiKeyInfos(), is(notNullValue())); + assertThat(response.getApiKeyInfos().size(), is(1)); + verifyApiKey(response.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + } + + private void verifyApiKey(final ApiKey actual, final ApiKey expected) { + assertThat(actual.getId(), is(expected.getId())); + assertThat(actual.getName(), is(expected.getName())); + assertThat(actual.getUsername(), is(expected.getUsername())); + assertThat(actual.getRealm(), is(expected.getRealm())); + assertThat(actual.isInvalidated(), is(expected.isInvalidated())); + assertThat(actual.getExpiration(), is(greaterThan(Instant.now()))); + } + + public void testInvalidateApiKey() throws Exception { + RestHighLevelClient client = highLevelClient(); + + List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = TimeValue.timeValueHours(24); + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + // Create API Keys + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse1 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse1.getName(), equalTo("k1")); + assertNotNull(createApiKeyResponse1.getKey()); + + { + // tag::invalidate-api-key-id-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId()); + // end::invalidate-api-key-id-request + + // tag::invalidate-api-key-execute + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + // end::invalidate-api-key-execute + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse1.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k2", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse2 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse2.getName(), equalTo("k2")); + assertNotNull(createApiKeyResponse2.getKey()); + + // tag::invalidate-api-key-name-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyName(createApiKeyResponse2.getName()); + // end::invalidate-api-key-name-request + + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse2.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k3", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse3 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse3.getName(), equalTo("k3")); + assertNotNull(createApiKeyResponse3.getKey()); + + // tag::invalidate-realm-api-keys-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmName("default_file"); + // end::invalidate-realm-api-keys-request + + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse3.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k4", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse4 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse4.getName(), equalTo("k4")); + assertNotNull(createApiKeyResponse4.getKey()); + + // tag::invalidate-user-api-keys-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingUserName("test_user"); + // end::invalidate-user-api-keys-request + + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse4.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k5", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse5 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse5.getName(), equalTo("k5")); + assertNotNull(createApiKeyResponse5.getKey()); + + // tag::invalidate-user-realm-api-keys-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName("default_file", "test_user"); + // end::invalidate-user-realm-api-keys-request + + // tag::invalidate-api-key-response + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + // end::invalidate-api-key-response + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse5.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k6", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse6 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse6.getName(), equalTo("k6")); + assertNotNull(createApiKeyResponse6.getKey()); + + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(createApiKeyResponse6.getId()); + + ActionListener listener; + // tag::invalidate-api-key-execute-listener + listener = new ActionListener() { + @Override + public void onResponse(InvalidateApiKeyResponse invalidateApiKeyResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::invalidate-api-key-execute-listener + + // Avoid unused variable warning + assertNotNull(listener); + + // Replace the empty listener by a blocking listener in test + final PlainActionFuture future = new PlainActionFuture<>(); + listener = future; + + // tag::invalidate-api-key-execute-async + client.security().invalidateApiKeyAsync(invalidateApiKeyRequest, RequestOptions.DEFAULT, listener); // <1> + // end::invalidate-api-key-execute-async + + final InvalidateApiKeyResponse response = future.get(30, TimeUnit.SECONDS); + assertNotNull(response); + final List invalidatedApiKeyIds = response.getInvalidatedApiKeys(); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse6.getId()); + assertTrue(response.getErrors().isEmpty()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(response.getPreviouslyInvalidatedApiKeys().size(), equalTo(0)); + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java new file mode 100644 index 0000000000000..188493deeb78a --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.security.user.privileges.IndicesPrivileges; +import org.elasticsearch.client.security.user.privileges.Role; +import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName; +import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; + +public class CreateApiKeyRequestTests extends ESTestCase { + + public void test() throws IOException { + List roles = new ArrayList<>(); + roles.add(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + roles.add(Role.builder().name("r2").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-y").privileges(IndexPrivilegeName.ALL).build()).build()); + + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("api-key", roles, null, null); + final XContentBuilder builder = XContentFactory.jsonBuilder(); + createApiKeyRequest.toXContent(builder, ToXContent.EMPTY_PARAMS); + final String output = Strings.toString(builder); + assertThat(output, equalTo( + "{\"name\":\"api-key\",\"role_descriptors\":{\"r1\":{\"applications\":[],\"cluster\":[\"all\"],\"indices\":[{\"names\":" + + "[\"ind-x\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":false}],\"metadata\":{},\"run_as\":[]}," + + "\"r2\":{\"applications\":[],\"cluster\":" + + "[\"all\"],\"indices\":[{\"names\":[\"ind-y\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":false}]," + + "\"metadata\":{},\"run_as\":[]}}}")); + } + + public void testEqualsHashCode() { + final String name = randomAlphaOfLength(5); + List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = null; + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyRequest, (original) -> { + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyRequest, (original) -> { + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy()); + }, CreateApiKeyRequestTests::mutateTestItem); + } + + private static CreateApiKeyRequest mutateTestItem(CreateApiKeyRequest original) { + switch (randomIntBetween(0, 3)) { + case 0: + return new CreateApiKeyRequest(randomAlphaOfLength(5), original.getRoles(), original.getExpiration(), + original.getRefreshPolicy()); + case 1: + return new CreateApiKeyRequest(original.getName(), + Collections.singletonList(Role.builder().name(randomAlphaOfLength(6)).clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges( + IndicesPrivileges.builder().indices(randomAlphaOfLength(4)).privileges(IndexPrivilegeName.ALL).build()) + .build()), + original.getExpiration(), original.getRefreshPolicy()); + case 2: + return new CreateApiKeyRequest(original.getName(), original.getRoles(), TimeValue.timeValueSeconds(10000), + original.getRefreshPolicy()); + case 3: + List values = Arrays.stream(RefreshPolicy.values()).filter(rp -> rp != original.getRefreshPolicy()) + .collect(Collectors.toList()); + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), randomFrom(values)); + default: + return new CreateApiKeyRequest(randomAlphaOfLength(5), original.getRoles(), original.getExpiration(), + original.getRefreshPolicy()); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java new file mode 100644 index 0000000000000..4481d70c80b37 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java @@ -0,0 +1,101 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.common.CharArrays; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; + +import static org.hamcrest.Matchers.equalTo; + +public class CreateApiKeyResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + final String id = randomAlphaOfLengthBetween(4, 8); + final String name = randomAlphaOfLength(5); + final SecureString apiKey = UUIDs.randomBase64UUIDSecureString(); + final Instant expiration = randomBoolean() ? null : Instant.ofEpochMilli(10000); + + final XContentType xContentType = randomFrom(XContentType.values()); + final XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + builder.startObject().field("id", id).field("name", name); + if (expiration != null) { + builder.field("expiration", expiration.toEpochMilli()); + } + byte[] charBytes = CharArrays.toUtf8Bytes(apiKey.getChars()); + try { + builder.field("api_key").utf8Value(charBytes, 0, charBytes.length); + } finally { + Arrays.fill(charBytes, (byte) 0); + } + builder.endObject(); + BytesReference xContent = BytesReference.bytes(builder); + + final CreateApiKeyResponse response = CreateApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent)); + assertThat(response.getId(), equalTo(id)); + assertThat(response.getName(), equalTo(name)); + assertThat(response.getKey(), equalTo(apiKey)); + if (expiration != null) { + assertThat(response.getExpiration(), equalTo(expiration)); + } + } + + public void testEqualsHashCode() { + final String id = randomAlphaOfLengthBetween(4, 8); + final String name = randomAlphaOfLength(5); + final SecureString apiKey = UUIDs.randomBase64UUIDSecureString(); + final Instant expiration = Instant.ofEpochMilli(10000); + CreateApiKeyResponse createApiKeyResponse = new CreateApiKeyResponse(name, id, apiKey, expiration); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> { + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> { + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration()); + }, CreateApiKeyResponseTests::mutateTestItem); + } + + private static CreateApiKeyResponse mutateTestItem(CreateApiKeyResponse original) { + switch (randomIntBetween(0, 3)) { + case 0: + return new CreateApiKeyResponse(randomAlphaOfLength(7), original.getId(), original.getKey(), original.getExpiration()); + case 1: + return new CreateApiKeyResponse(original.getName(), randomAlphaOfLengthBetween(4, 8), original.getKey(), + original.getExpiration()); + case 2: + return new CreateApiKeyResponse(original.getName(), original.getId(), UUIDs.randomBase64UUIDSecureString(), + original.getExpiration()); + case 3: + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), Instant.ofEpochMilli(150000)); + default: + return new CreateApiKeyResponse(randomAlphaOfLength(7), original.getId(), original.getKey(), original.getExpiration()); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java new file mode 100644 index 0000000000000..79551e1e73e92 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.ValidationException; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Optional; + +import static org.hamcrest.Matchers.equalTo; + +public class GetApiKeyRequestTests extends ESTestCase { + + public void testRequestValidation() { + GetApiKeyRequest request = GetApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5)); + Optional ve = request.validate(); + assertFalse(ve.isPresent()); + request = GetApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5)); + ve = request.validate(); + assertFalse(ve.isPresent()); + request = GetApiKeyRequest.usingRealmName(randomAlphaOfLength(5)); + ve = request.validate(); + assertFalse(ve.isPresent()); + request = GetApiKeyRequest.usingUserName(randomAlphaOfLength(5)); + ve = request.validate(); + assertFalse(ve.isPresent()); + request = GetApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7)); + ve = request.validate(); + assertFalse(ve.isPresent()); + } + + public void testRequestValidationFailureScenarios() throws IOException { + String[][] inputs = new String[][] { + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), + randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" }, + { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" }, + { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } }; + String[] expectedErrorMessages = new String[] { "One of [api key id, api key name, username, realm name] must be specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }; + + for (int i = 0; i < inputs.length; i++) { + final int caseNo = i; + IllegalArgumentException ve = expectThrows(IllegalArgumentException.class, + () -> new GetApiKeyRequest(inputs[caseNo][0], inputs[caseNo][1], inputs[caseNo][2], inputs[caseNo][3])); + assertNotNull(ve); + assertThat(ve.getMessage(), equalTo(expectedErrorMessages[caseNo])); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java new file mode 100644 index 0000000000000..7aa92e4f212a4 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.security.support.ApiKey; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; + +import static org.hamcrest.Matchers.equalTo; + +public class GetApiKeyResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false, + "user-a", "realm-x"); + ApiKey apiKeyInfo2 = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true, + "user-b", "realm-y"); + GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1, apiKeyInfo2)); + final XContentType xContentType = randomFrom(XContentType.values()); + final XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + toXContent(response, builder); + BytesReference xContent = BytesReference.bytes(builder); + GetApiKeyResponse responseParsed = GetApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent)); + assertThat(responseParsed, equalTo(response)); + } + + private void toXContent(GetApiKeyResponse response, final XContentBuilder builder) throws IOException { + builder.startObject(); + builder.startArray("api_keys"); + for (ApiKey apiKey : response.getApiKeyInfos()) { + builder.startObject() + .field("id", apiKey.getId()) + .field("name", apiKey.getName()) + .field("creation", apiKey.getCreation().toEpochMilli()); + if (apiKey.getExpiration() != null) { + builder.field("expiration", apiKey.getExpiration().toEpochMilli()); + } + builder.field("invalidated", apiKey.isInvalidated()) + .field("username", apiKey.getUsername()) + .field("realm", apiKey.getRealm()); + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + } + + public void testEqualsHashCode() { + ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false, + "user-a", "realm-x"); + GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1)); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(response, (original) -> { + return new GetApiKeyResponse(original.getApiKeyInfos()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(response, (original) -> { + return new GetApiKeyResponse(original.getApiKeyInfos()); + }, GetApiKeyResponseTests::mutateTestItem); + } + + private static GetApiKeyResponse mutateTestItem(GetApiKeyResponse original) { + ApiKey apiKeyInfo = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true, + "user-b", "realm-y"); + switch (randomIntBetween(0, 2)) { + case 0: + return new GetApiKeyResponse(Arrays.asList(apiKeyInfo)); + default: + return new GetApiKeyResponse(Arrays.asList(apiKeyInfo)); + } + } + + private static ApiKey createApiKeyInfo(String name, String id, Instant creation, Instant expiration, boolean invalidated, + String username, String realm) { + return new ApiKey(name, id, creation, expiration, invalidated, username, realm); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java new file mode 100644 index 0000000000000..25ee4bb05bcc4 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.ValidationException; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Optional; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class InvalidateApiKeyRequestTests extends ESTestCase { + + public void testRequestValidation() { + InvalidateApiKeyRequest request = InvalidateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5)); + Optional ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + request = InvalidateApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5)); + ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + request = InvalidateApiKeyRequest.usingRealmName(randomAlphaOfLength(5)); + ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + request = InvalidateApiKeyRequest.usingUserName(randomAlphaOfLength(5)); + ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + request = InvalidateApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7)); + ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + } + + public void testRequestValidationFailureScenarios() throws IOException { + String[][] inputs = new String[][] { + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), + randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" }, + { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" }, + { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } }; + String[] expectedErrorMessages = new String[] { "One of [api key id, api key name, username, realm name] must be specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }; + + for (int i = 0; i < inputs.length; i++) { + final int caseNo = i; + IllegalArgumentException ve = expectThrows(IllegalArgumentException.class, + () -> new InvalidateApiKeyRequest(inputs[caseNo][0], inputs[caseNo][1], inputs[caseNo][2], inputs[caseNo][3])); + assertNotNull(ve); + assertThat(ve.getMessage(), equalTo(expectedErrorMessages[caseNo])); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java new file mode 100644 index 0000000000000..f5cd403536fc2 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class InvalidateApiKeyResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + List invalidatedApiKeys = Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))); + List previouslyInvalidatedApiKeys = Arrays.asList(randomArray(2, 3, String[]::new, () -> randomAlphaOfLength(5))); + List errors = Arrays.asList(randomArray(2, 5, ElasticsearchException[]::new, + () -> new ElasticsearchException(randomAlphaOfLength(5), new IllegalArgumentException(randomAlphaOfLength(4))))); + + final XContentType xContentType = randomFrom(XContentType.values()); + final XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + builder.startObject().array("invalidated_api_keys", invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)) + .array("previously_invalidated_api_keys", previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)) + .field("error_count", errors.size()); + if (errors.isEmpty() == false) { + builder.field("error_details"); + builder.startArray(); + for (ElasticsearchException e : errors) { + builder.startObject(); + ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, e); + builder.endObject(); + } + builder.endArray(); + } + builder.endObject(); + BytesReference xContent = BytesReference.bytes(builder); + + final InvalidateApiKeyResponse response = InvalidateApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent)); + assertThat(response.getInvalidatedApiKeys(), containsInAnyOrder(invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY))); + assertThat(response.getPreviouslyInvalidatedApiKeys(), + containsInAnyOrder(previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY))); + assertThat(response.getErrors(), is(notNullValue())); + assertThat(response.getErrors().size(), is(errors.size())); + assertThat(response.getErrors().get(0).toString(), containsString("type=illegal_argument_exception")); + assertThat(response.getErrors().get(1).toString(), containsString("type=illegal_argument_exception")); + } + + public void testEqualsHashCode() { + List invalidatedApiKeys = Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))); + List previouslyInvalidatedApiKeys = Arrays.asList(randomArray(2, 3, String[]::new, () -> randomAlphaOfLength(5))); + List errors = Arrays.asList(randomArray(2, 5, ElasticsearchException[]::new, + () -> new ElasticsearchException(randomAlphaOfLength(5), new IllegalArgumentException(randomAlphaOfLength(4))))); + InvalidateApiKeyResponse invalidateApiKeyResponse = new InvalidateApiKeyResponse(invalidatedApiKeys, previouslyInvalidatedApiKeys, + errors); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(invalidateApiKeyResponse, (original) -> { + return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(), + original.getErrors()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(invalidateApiKeyResponse, (original) -> { + return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(), + original.getErrors()); + }, InvalidateApiKeyResponseTests::mutateTestItem); + } + + private static InvalidateApiKeyResponse mutateTestItem(InvalidateApiKeyResponse original) { + switch (randomIntBetween(0, 2)) { + case 0: + return new InvalidateApiKeyResponse(Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))), + original.getPreviouslyInvalidatedApiKeys(), original.getErrors()); + case 1: + return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), Collections.emptyList(), original.getErrors()); + case 2: + return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(), + Collections.emptyList()); + default: + return new InvalidateApiKeyResponse(Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))), + original.getPreviouslyInvalidatedApiKeys(), original.getErrors()); + } + } +} diff --git a/docs/java-rest/high-level/security/create-api-key.asciidoc b/docs/java-rest/high-level/security/create-api-key.asciidoc new file mode 100644 index 0000000000000..93c3fa16de1da --- /dev/null +++ b/docs/java-rest/high-level/security/create-api-key.asciidoc @@ -0,0 +1,40 @@ +-- +:api: create-api-key +:request: CreateApiKeyRequest +:response: CreateApiKeyResponse +-- + +[id="{upid}-{api}"] +=== Create API Key API + +API Key can be created using this API. + +[id="{upid}-{api}-request"] +==== Create API Key Request + +A +{request}+ contains name for the API key, +list of role descriptors to define permissions and +optional expiration for the generated API key. +If expiration is not provided then by default the API +keys do not expire. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Create API Key Response + +The returned +{response}+ contains an id, +API key, name for the API key and optional +expiration. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> the API key that can be used to authenticate to Elasticsearch. +<2> expiration if the API keys expire \ No newline at end of file diff --git a/docs/java-rest/high-level/security/get-api-key.asciidoc b/docs/java-rest/high-level/security/get-api-key.asciidoc new file mode 100644 index 0000000000000..bb98b527d22ba --- /dev/null +++ b/docs/java-rest/high-level/security/get-api-key.asciidoc @@ -0,0 +1,67 @@ +-- +:api: get-api-key +:request: GetApiKeyRequest +:response: GetApiKeyResponse +-- + +[id="{upid}-{api}"] +=== Get API Key information API + +API Key(s) information can be retrieved using this API. + +[id="{upid}-{api}-request"] +==== Get API Key Request +The +{request}+ supports retrieving API key information for + +. A specific API key + +. All API keys for a specific realm + +. All API keys for a specific user + +. All API keys for a specific user in a specific realm + +===== Retrieve a specific API key by its id +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-api-key-id-request] +-------------------------------------------------- + +===== Retrieve a specific API key by its name +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-api-key-name-request] +-------------------------------------------------- + +===== Retrieve all API keys for given realm +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-realm-api-keys-request] +-------------------------------------------------- + +===== Retrieve all API keys for a given user +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-user-api-keys-request] +-------------------------------------------------- + +===== Retrieve all API keys for given user in a realm +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-user-realm-api-keys-request] +-------------------------------------------------- + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Get API Key information API Response + +The returned +{response}+ contains the information regarding the API keys that were +requested. + +`api_keys`:: Available using `getApiKeyInfos`, contains list of API keys that were retrieved for this request. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- diff --git a/docs/java-rest/high-level/security/invalidate-api-key.asciidoc b/docs/java-rest/high-level/security/invalidate-api-key.asciidoc new file mode 100644 index 0000000000000..7f9c43b3165a8 --- /dev/null +++ b/docs/java-rest/high-level/security/invalidate-api-key.asciidoc @@ -0,0 +1,75 @@ +-- +:api: invalidate-api-key +:request: InvalidateApiKeyRequest +:response: InvalidateApiKeyResponse +-- + +[id="{upid}-{api}"] +=== Invalidate API Key API + +API Key(s) can be invalidated using this API. + +[id="{upid}-{api}-request"] +==== Invalidate API Key Request +The +{request}+ supports invalidating + +. A specific API key + +. All API keys for a specific realm + +. All API keys for a specific user + +. All API keys for a specific user in a specific realm + +===== Specific API key by API key id +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-api-key-id-request] +-------------------------------------------------- + +===== Specific API key by API key name +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-api-key-name-request] +-------------------------------------------------- + +===== All API keys for realm +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-realm-api-keys-request] +-------------------------------------------------- + +===== All API keys for user +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-user-api-keys-request] +-------------------------------------------------- + +===== All API key for user in realm +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-user-realm-api-keys-request] +-------------------------------------------------- + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Invalidate API Key Response + +The returned +{response}+ contains the information regarding the API keys that the request +invalidated. + +`invalidatedApiKeys`:: Available using `getInvalidatedApiKeys` lists the API keys + that this request invalidated. + +`previouslyInvalidatedApiKeys`:: Available using `getPreviouslyInvalidatedApiKeys` lists the API keys + that this request attempted to invalidate + but were already invalid. + +`errors`:: Available using `getErrors` contains possible errors that were encountered while + attempting to invalidate API keys. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 70f06e457e9b5..1df10985e7e3b 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -411,6 +411,9 @@ The Java High Level REST Client supports the following Security APIs: * <<{upid}-get-privileges>> * <<{upid}-put-privileges>> * <<{upid}-delete-privileges>> +* <<{upid}-create-api-key>> +* <<{upid}-get-api-key>> +* <<{upid}-invalidate-api-key>> include::security/put-user.asciidoc[] include::security/get-users.asciidoc[] @@ -435,6 +438,9 @@ include::security/delete-role-mapping.asciidoc[] include::security/create-token.asciidoc[] include::security/invalidate-token.asciidoc[] include::security/put-privileges.asciidoc[] +include::security/create-api-key.asciidoc[] +include::security/get-api-key.asciidoc[] +include::security/invalidate-api-key.asciidoc[] == Watcher APIs diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc index a9b6639359e24..c83edb69b3e62 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc @@ -280,6 +280,31 @@ example above), but the same goes for actual values: The stash should be reset at the beginning of each test file. +=== `transform_and_set` + +For some tests, it is necessary to extract a value and transform it from the previous `response`, in +order to reuse it in a subsequent `do` and other tests. +Currently, it only has support for `base64EncodeCredentials`, for unknown transformations it will not +do anything and stash the value as is. +For instance, when testing you may want to base64 encode username and password for +`Basic` authorization header: + +.... + - do: + index: + index: test + type: test + - transform_and_set: { login_creds: "#base64EncodeCredentials(user,password)" } # stash the base64 encoded credentials of `response.user` and `response.password` as `login_creds` + - do: + headers: + Authorization: Basic ${login_creds} # replace `$login_creds` with the stashed value + get: + index: test + type: test +.... + +Stashed values can be used as described in the `set` section + === `is_true` The specified key exists and has a true value (ie not `0`, `false`, `undefined`, `null` diff --git a/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java b/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java index 59e5960b99d09..b5b35b477efbd 100644 --- a/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java +++ b/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java @@ -20,6 +20,9 @@ package org.elasticsearch.common; +import org.elasticsearch.common.settings.SecureString; + +import java.util.Arrays; import java.util.Base64; import java.util.Random; @@ -34,12 +37,37 @@ public String getBase64UUID() { return getBase64UUID(SecureRandomHolder.INSTANCE); } + /** + * Returns a Base64 encoded {@link SecureString} of a Version 4.0 compatible UUID + * as defined here: http://www.ietf.org/rfc/rfc4122.txt + */ + public SecureString getBase64UUIDSecureString() { + byte[] uuidBytes = null; + byte[] encodedBytes = null; + try { + uuidBytes = getUUIDBytes(SecureRandomHolder.INSTANCE); + encodedBytes = Base64.getUrlEncoder().withoutPadding().encode(uuidBytes); + return new SecureString(CharArrays.utf8BytesToChars(encodedBytes)); + } finally { + if (uuidBytes != null) { + Arrays.fill(uuidBytes, (byte) 0); + } + if (encodedBytes != null) { + Arrays.fill(encodedBytes, (byte) 0); + } + } + } + /** * Returns a Base64 encoded version of a Version 4.0 compatible UUID * randomly initialized by the given {@link java.util.Random} instance * as defined here: http://www.ietf.org/rfc/rfc4122.txt */ public String getBase64UUID(Random random) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(getUUIDBytes(random)); + } + + private byte[] getUUIDBytes(Random random) { final byte[] randomBytes = new byte[16]; random.nextBytes(randomBytes); /* Set the version to version 4 (see http://www.ietf.org/rfc/rfc4122.txt) @@ -48,12 +76,12 @@ public String getBase64UUID(Random random) { * stamp (bits 4 through 7 of the time_hi_and_version field).*/ randomBytes[6] &= 0x0f; /* clear the 4 most significant bits for the version */ randomBytes[6] |= 0x40; /* set the version to 0100 / 0x40 */ - - /* Set the variant: + + /* Set the variant: * The high field of th clock sequence multiplexed with the variant. * We set only the MSB of the variant*/ randomBytes[8] &= 0x3f; /* clear the 2 most significant bits */ randomBytes[8] |= 0x80; /* set the variant (MSB is set)*/ - return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes); + return randomBytes; } } diff --git a/server/src/main/java/org/elasticsearch/common/UUIDs.java b/server/src/main/java/org/elasticsearch/common/UUIDs.java index 63fcaedde0f5c..a6a314c2cccb0 100644 --- a/server/src/main/java/org/elasticsearch/common/UUIDs.java +++ b/server/src/main/java/org/elasticsearch/common/UUIDs.java @@ -19,6 +19,8 @@ package org.elasticsearch.common; +import org.elasticsearch.common.settings.SecureString; + import java.util.Random; public class UUIDs { @@ -50,4 +52,9 @@ public static String randomBase64UUID() { return RANDOM_UUID_GENERATOR.getBase64UUID(); } + /** Returns a Base64 encoded {@link SecureString} of a Version 4.0 compatible UUID as defined here: http://www.ietf.org/rfc/rfc4122.txt, + * using a private {@code SecureRandom} instance */ + public static SecureString randomBase64UUIDSecureString() { + return RANDOM_UUID_GENERATOR.getBase64UUIDSecureString(); + } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 2de583b460f84..f361225b48f2d 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -588,6 +588,23 @@ public Object readGenericValue() throws IOException { } } + /** + * Read an {@link Instant} from the stream with nanosecond resolution + */ + public final Instant readInstant() throws IOException { + return Instant.ofEpochSecond(readLong(), readInt()); + } + + /** + * Read an optional {@link Instant} from the stream. Returns null when + * no instant is present. + */ + @Nullable + public final Instant readOptionalInstant() throws IOException { + final boolean present = readBoolean(); + return present ? readInstant() : null; + } + @SuppressWarnings("unchecked") private List readArrayList() throws IOException { int size = readArraySize(); diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 175f800a7d8cf..1c9dfd7ea4433 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -56,6 +56,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.NotDirectoryException; import java.time.ZoneId; +import java.time.Instant; import java.time.ZonedDateTime; import java.util.Collection; import java.util.Collections; @@ -573,6 +574,26 @@ public final void writeMap(final Map map, final Writer keyWriter } } + /** + * Writes an {@link Instant} to the stream with nanosecond resolution + */ + public final void writeInstant(Instant instant) throws IOException { + writeLong(instant.getEpochSecond()); + writeInt(instant.getNano()); + } + + /** + * Writes an {@link Instant} to the stream, which could possibly be null + */ + public final void writeOptionalInstant(@Nullable Instant instant) throws IOException { + if (instant == null) { + writeBoolean(false); + } else { + writeBoolean(true); + writeInstant(instant); + } + } + private static final Map, Writer> WRITERS; static { diff --git a/server/src/main/java/org/elasticsearch/common/util/set/Sets.java b/server/src/main/java/org/elasticsearch/common/util/set/Sets.java index 0f1fe22c02010..02d534552100c 100644 --- a/server/src/main/java/org/elasticsearch/common/util/set/Sets.java +++ b/server/src/main/java/org/elasticsearch/common/util/set/Sets.java @@ -144,4 +144,19 @@ public static Set union(Set left, Set right) { union.addAll(right); return union; } + + public static Set intersection(Set set1, Set set2) { + Objects.requireNonNull(set1); + Objects.requireNonNull(set2); + final Set left; + final Set right; + if (set1.size() < set2.size()) { + left = set1; + right = set2; + } else { + left = set2; + right = set1; + } + return left.stream().filter(o -> right.contains(o)).collect(Collectors.toSet()); + } } diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java index e2cdaf3c7d5b8..837c0202faf92 100644 --- a/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java +++ b/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java @@ -30,6 +30,7 @@ import java.io.ByteArrayInputStream; import java.io.EOFException; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -336,6 +337,37 @@ public void testSetOfLongs() throws IOException { assertThat(targetSet, equalTo(sourceSet)); } + public void testInstantSerialization() throws IOException { + final Instant instant = Instant.now(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeInstant(instant); + try (StreamInput in = out.bytes().streamInput()) { + final Instant serialized = in.readInstant(); + assertEquals(instant, serialized); + } + } + } + + public void testOptionalInstantSerialization() throws IOException { + final Instant instant = Instant.now(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeOptionalInstant(instant); + try (StreamInput in = out.bytes().streamInput()) { + final Instant serialized = in.readOptionalInstant(); + assertEquals(instant, serialized); + } + } + + final Instant missing = null; + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeOptionalInstant(missing); + try (StreamInput in = out.bytes().streamInput()) { + final Instant serialized = in.readOptionalInstant(); + assertEquals(missing, serialized); + } + } + } + static final class WriteableString implements Writeable { final String string; diff --git a/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java b/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java index 0c1869a6b4086..f4337daf4346c 100644 --- a/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -56,6 +57,17 @@ public void testSortedDifference() { } } + public void testIntersection() { + final int endExclusive = randomIntBetween(0, 256); + final Tuple, Set> sets = randomSets(endExclusive); + final Set intersection = Sets.intersection(sets.v1(), sets.v2()); + final Set expectedIntersection = IntStream.range(0, endExclusive) + .boxed() + .filter(i -> (sets.v1().contains(i) && sets.v2().contains(i))) + .collect(Collectors.toSet()); + assertThat(intersection, containsInAnyOrder(expectedIntersection.toArray(new Integer[0]))); + } + /** * Assert the difference between two sets is as expected. * diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java index bb5354e4fedd3..fea1c3997530c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java @@ -46,7 +46,8 @@ public final class Features { "stash_path_replace", "warnings", "yaml", - "contains" + "contains", + "transform_and_set" )); private Features() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java index ff02d6d16aa4a..135a60cca3431 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java @@ -40,6 +40,7 @@ public interface ExecutableSection { List DEFAULT_EXECUTABLE_CONTEXTS = unmodifiableList(Arrays.asList( new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("do"), DoSection::parse), new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("set"), SetSection::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("transform_and_set"), TransformAndSetSection::parse), new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("match"), MatchAssertion::parse), new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_true"), IsTrueAssertion::parse), new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_false"), IsFalseAssertion::parse), diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java new file mode 100644 index 0000000000000..7b0b915dd97df --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java @@ -0,0 +1,106 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest.yaml.section; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +/** + * Represents a transform_and_set section: + *

+ * + * In the following example,
+ * - transform_and_set: { login_creds: "#base64EncodeCredentials(user,password)" }
+ * user and password are from the response which are joined by ':' and Base64 encoded and then stashed as 'login_creds' + * + */ +public class TransformAndSetSection implements ExecutableSection { + public static TransformAndSetSection parse(XContentParser parser) throws IOException { + String currentFieldName = null; + XContentParser.Token token; + + TransformAndSetSection transformAndStashSection = new TransformAndSetSection(parser.getTokenLocation()); + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + transformAndStashSection.addSet(currentFieldName, parser.text()); + } + } + + parser.nextToken(); + + if (transformAndStashSection.getStash().isEmpty()) { + throw new ParsingException(transformAndStashSection.location, "transform_and_set section must set at least a value"); + } + + return transformAndStashSection; + } + + private final Map transformStash = new HashMap<>(); + private final XContentLocation location; + + public TransformAndSetSection(XContentLocation location) { + this.location = location; + } + + public void addSet(String stashedField, String transformThis) { + transformStash.put(stashedField, transformThis); + } + + public Map getStash() { + return transformStash; + } + + @Override + public XContentLocation getLocation() { + return location; + } + + @Override + public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { + for (Map.Entry entry : transformStash.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (value.startsWith("#base64EncodeCredentials(") && value.endsWith(")")) { + value = entry.getValue().substring("#base64EncodeCredentials(".length(), entry.getValue().lastIndexOf(")")); + String[] idAndPassword = value.split(","); + if (idAndPassword.length == 2) { + String credentials = executionContext.response(idAndPassword[0].trim()) + ":" + + executionContext.response(idAndPassword[1].trim()); + value = Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + } else { + throw new IllegalArgumentException("base64EncodeCredentials requires a username/id and a password parameters"); + } + } + executionContext.stash().stashValue(key, value); + } + } + +} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java new file mode 100644 index 0000000000000..a61f91de287e7 --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest.yaml.section; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.Stash; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class TransformAndSetSectionTests extends AbstractClientYamlTestFragmentParserTestCase { + + public void testParseSingleValue() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "{ key: value }" + ); + + TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser); + assertThat(transformAndSet, notNullValue()); + assertThat(transformAndSet.getStash(), notNullValue()); + assertThat(transformAndSet.getStash().size(), equalTo(1)); + assertThat(transformAndSet.getStash().get("key"), equalTo("value")); + } + + public void testParseMultipleValues() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "{ key1: value1, key2: value2 }" + ); + + TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser); + assertThat(transformAndSet, notNullValue()); + assertThat(transformAndSet.getStash(), notNullValue()); + assertThat(transformAndSet.getStash().size(), equalTo(2)); + assertThat(transformAndSet.getStash().get("key1"), equalTo("value1")); + assertThat(transformAndSet.getStash().get("key2"), equalTo("value2")); + } + + public void testTransformation() throws Exception { + parser = createParser(YamlXContent.yamlXContent, "{ login_creds: \"#base64EncodeCredentials(id,api_key)\" }"); + + TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser); + assertThat(transformAndSet, notNullValue()); + assertThat(transformAndSet.getStash(), notNullValue()); + assertThat(transformAndSet.getStash().size(), equalTo(1)); + assertThat(transformAndSet.getStash().get("login_creds"), equalTo("#base64EncodeCredentials(id,api_key)")); + + ClientYamlTestExecutionContext executionContext = mock(ClientYamlTestExecutionContext.class); + when(executionContext.response("id")).thenReturn("user"); + when(executionContext.response("api_key")).thenReturn("password"); + Stash stash = new Stash(); + when(executionContext.stash()).thenReturn(stash); + transformAndSet.execute(executionContext); + verify(executionContext).response("id"); + verify(executionContext).response("api_key"); + verify(executionContext).stash(); + assertThat(stash.getValue("$login_creds"), + equalTo(Base64.getEncoder().encodeToString("user:password".getBytes(StandardCharsets.UTF_8)))); + verifyNoMoreInteractions(executionContext); + } + + public void testParseSetSectionNoValues() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "{ }" + ); + + Exception e = expectThrows(ParsingException.class, () -> TransformAndSetSection.parse(parser)); + assertThat(e.getMessage(), is("transform_and_set section must set at least a value")); + } +} diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 27f815b1637f1..ecfd30bb7469b 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -73,6 +73,7 @@ project.copyRestSpec.from(xpackResources) { } integTestCluster { setting 'xpack.security.enabled', 'true' + setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' // Disable monitoring exporters for the docs tests setting 'xpack.monitoring.exporters._local.type', 'local' diff --git a/x-pack/docs/en/rest-api/security.asciidoc b/x-pack/docs/en/rest-api/security.asciidoc index 851bd2ba327b2..c59c44312ae60 100644 --- a/x-pack/docs/en/rest-api/security.asciidoc +++ b/x-pack/docs/en/rest-api/security.asciidoc @@ -51,6 +51,17 @@ without requiring basic authentication: * <> * <> +[float] +[[security-api-keys]] +=== API Keys + +You can use the following APIs to create, retrieve and invalidate API keys for access +without requiring basic authentication: + +* <> +* <> +* <> + [float] [[security-user-apis]] === Users @@ -88,3 +99,6 @@ include::security/get-users.asciidoc[] include::security/has-privileges.asciidoc[] include::security/invalidate-tokens.asciidoc[] include::security/ssl.asciidoc[] +include::security/create-api-keys.asciidoc[] +include::security/invalidate-api-keys.asciidoc[] +include::security/get-api-keys.asciidoc[] diff --git a/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc new file mode 100644 index 0000000000000..e4fa1be71d40e --- /dev/null +++ b/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc @@ -0,0 +1,99 @@ +[role="xpack"] +[[security-api-create-api-key]] +=== Create API Key API + +Creates an API key for access without requiring basic authentication. + +==== Request + +`POST /_security/api_key` +`PUT /_security/api_key` + +==== Description + +The API keys are created by the {es} API key service, which is automatically enabled +when you configure TLS on the HTTP interface. See <>. Alternatively, +you can explicitly enable the `xpack.security.authc.api_key.enabled` setting. When +you are running in production mode, a bootstrap check prevents you from enabling +the API key service unless you also enable TLS on the HTTP interface. + +A successful create API key API call returns a JSON structure that contains +the unique id, the name to identify API key, the API key and the expiration if +applicable for the API key in milliseconds. + +NOTE: By default API keys never expire. You can specify expiration at the time of +creation for the API keys. + +==== Request Body + +The following parameters can be specified in the body of a POST or PUT request: + +`name`:: +(string) Specifies the name for this API key. + +`role_descriptors`:: +(array-of-role-descriptor) Optional array of role descriptor for this API key. The role descriptor +must be a subset of permissions of the authenticated user. The structure of role +descriptor is same as the request for create role API. For more details on role +see <>. +If the role descriptors are not provided then permissions of the authenticated user are applied. + +`expiration`:: +(string) Optional expiration time for the API key. By default API keys never expire. + +==== Examples + +The following example creates an API key: + +[source, js] +------------------------------------------------------------ +POST /_security/api_key +{ + "name": "my-api-key", + "expiration": "1d", <1> + "role_descriptors": { <2> + "role-a": { + "cluster": ["all"], + "index": [ + { + "names": ["index-a*"], + "privileges": ["read"] + } + ] + }, + "role-b": { + "cluster": ["all"], + "index": [ + { + "names": ["index-b*"], + "privileges": ["all"] + } + ] + } + } +} +------------------------------------------------------------ +// CONSOLE +<1> optional expiration for the API key being generated. If expiration is not + provided then the API keys do not expire. +<2> optional role descriptors for this API key, if not provided then permissions + of authenticated user are applied. + +A successful call returns a JSON structure that provides +API key information. + +[source,js] +-------------------------------------------------- +{ + "id":"VuaCfGcBCdbkQm-e5aOx", <1> + "name":"my-api-key", + "expiration":1544068612110, <2> + "api_key":"ui2lp2axTNmsyakw9tvNnw" <3> +} +-------------------------------------------------- +// TESTRESPONSE[s/VuaCfGcBCdbkQm-e5aOx/$body.id/] +// TESTRESPONSE[s/1544068612110/$body.expiration/] +// TESTRESPONSE[s/ui2lp2axTNmsyakw9tvNnw/$body.api_key/] +<1> unique id for this API key +<2> optional expiration in milliseconds for this API key +<3> generated API key diff --git a/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc new file mode 100644 index 0000000000000..ab2ef770cb124 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc @@ -0,0 +1,118 @@ +[role="xpack"] +[[security-api-get-api-key]] +=== Get API Key information API +++++ +Get API key information +++++ + +Retrieves information for one or more API keys. + +==== Request + +`GET /_security/api_key` + +==== Description + +The information for the API keys created by <> can be retrieved +using this API. + +==== Request Body + +The following parameters can be specified in the query parameters of a GET request and +pertain to retrieving api keys: + +`id` (optional):: +(string) An API key id. This parameter cannot be used with any of `name`, `realm_name` or + `username` are used. + +`name` (optional):: +(string) An API key name. This parameter cannot be used with any of `id`, `realm_name` or + `username` are used. + +`realm_name` (optional):: +(string) The name of an authentication realm. This parameter cannot be used with either `id` or `name`. + +`username` (optional):: +(string) The username of a user. This parameter cannot be used with either `id` or `name`. + +NOTE: While all parameters are optional, at least one of them is required. + +==== Examples + +The following example to retrieve the API key identified by specified `id`: + +[source,js] +-------------------------------------------------- +GET /_security/api_key?id=dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ== +-------------------------------------------------- +// NOTCONSOLE + +whereas the following example to retrieve the API key identified by specified `name`: + +[source,js] +-------------------------------------------------- +GET /_security/api_key?name=hadoop_myuser_key +-------------------------------------------------- +// NOTCONSOLE + +The following example retrieves all API keys for the `native1` realm: + +[source,js] +-------------------------------------------------- +GET /_xpack/api_key?realm_name=native1 +-------------------------------------------------- +// NOTCONSOLE + +The following example retrieves all API keys for the user `myuser` in all realms: + +[source,js] +-------------------------------------------------- +GET /_xpack/api_key?username=myuser +-------------------------------------------------- +// NOTCONSOLE + +Finally, the following example retrieves all API keys for the user `myuser` in + the `native1` realm immediately: + +[source,js] +-------------------------------------------------- +GET /_xpack/api_key?username=myuser&realm_name=native1 +-------------------------------------------------- +// NOTCONSOLE + +A successful call returns a JSON structure that contains the information of one or more API keys that were retrieved. + +[source,js] +-------------------------------------------------- +{ + "api_keys": [ <1> + { + "id": "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==", <2> + "name": "hadoop_myuser_key", <3> + "creation": 1548550550158, <4> + "expiration": 1548551550158, <5> + "invalidated": false, <6> + "username": "myuser", <7> + "realm": "native1" <8> + }, + { + "id": "api-key-id-2", + "name": "api-key-name-2", + "creation": 1548550550158, + "invalidated": false, + "username": "user-y", + "realm": "realm-2" + } + ] +} +-------------------------------------------------- +// NOTCONSOLE + +<1> The list of API keys that were retrieved for this request. +<2> Id for the API key +<3> Name of the API key +<4> Creation time for the API key in milliseconds +<5> optional expiration time for the API key in milliseconds +<6> invalidation status for the API key, `true` if the key has been invalidated else `false` +<7> principal for which this API key was created +<8> realm name of the principal for which this API key was created diff --git a/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc new file mode 100644 index 0000000000000..4809e267ebd80 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc @@ -0,0 +1,140 @@ +[role="xpack"] +[[security-api-invalidate-api-key]] +=== Invalidate API Key API +++++ +Invalidate API key +++++ + +Invalidates one or more API keys. + +==== Request + +`DELETE /_security/api_key` + +==== Description + +The API keys created by <> can be invalidated +using this API. + +==== Request Body + +The following parameters can be specified in the body of a DELETE request and +pertain to invalidating api keys: + +`id` (optional):: +(string) An API key id. This parameter cannot be used with any of `name`, `realm_name` or + `username` are used. + +`name` (optional):: +(string) An API key name. This parameter cannot be used with any of `id`, `realm_name` or + `username` are used. + +`realm_name` (optional):: +(string) The name of an authentication realm. This parameter cannot be used with either `api_key_id` or `api_key_name`. + +`username` (optional):: +(string) The username of a user. This parameter cannot be used with either `api_key_id` or `api_key_name`. + +NOTE: While all parameters are optional, at least one of them is required. + +==== Examples + +The following example invalidates the API key identified by specified `id` immediately: + +[source,js] +-------------------------------------------------- +DELETE /_security/api_key +{ + "id" : "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==" +} +-------------------------------------------------- +// NOTCONSOLE + +whereas the following example invalidates the API key identified by specified `name` immediately: + +[source,js] +-------------------------------------------------- +DELETE /_security/api_key +{ + "name" : "hadoop_myuser_key" +} +-------------------------------------------------- +// NOTCONSOLE + +The following example invalidates all API keys for the `native1` realm immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/api_key +{ + "realm_name" : "native1" +} +-------------------------------------------------- +// NOTCONSOLE + +The following example invalidates all API keys for the user `myuser` in all realms immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/api_key +{ + "username" : "myuser" +} +-------------------------------------------------- +// NOTCONSOLE + +Finally, the following example invalidates all API keys for the user `myuser` in + the `native1` realm immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/api_key +{ + "username" : "myuser", + "realm_name" : "native1" +} +-------------------------------------------------- +// NOTCONSOLE + +A successful call returns a JSON structure that contains the ids of the API keys that were invalidated, the ids +of the API keys that had already been invalidated, and potentially a list of errors encountered while invalidating +specific api keys. + +[source,js] +-------------------------------------------------- +{ + "invalidated_api_keys": [ <1> + "api-key-id-1" + ], + "previously_invalidated_api_keys": [ <2> + "api-key-id-2", + "api-key-id-3" + ], + "error_count": 2, <3> + "error_details": [ <4> + { + "type": "exception", + "reason": "error occurred while invalidating api keys", + "caused_by": { + "type": "illegal_argument_exception", + "reason": "invalid api key id" + } + }, + { + "type": "exception", + "reason": "error occurred while invalidating api keys", + "caused_by": { + "type": "illegal_argument_exception", + "reason": "invalid api key id" + } + } + ] +} +-------------------------------------------------- +// NOTCONSOLE + +<1> The ids of the API keys that were invalidated as part of this request. +<2> The ids of the API keys that were already invalidated. +<3> The number of errors that were encountered when invalidating the API keys. +<4> Details about these errors. This field is not present in the response when + `error_count` is 0. diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index c16a80ff37d8b..6ce71982f5b1d 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,6 +133,7 @@ integTestCluster { setting 'xpack.monitoring.exporters._local.type', 'local' setting 'xpack.monitoring.exporters._local.enabled', 'false' setting 'xpack.security.authc.token.enabled', 'true' + setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.transport.ssl.key', nodeKey.name setting 'xpack.security.transport.ssl.certificate', nodeCert.name diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java index d6005b6d8308b..a0a81b1a51677 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java @@ -13,21 +13,21 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.client.FilterClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.Engine; @@ -35,14 +35,15 @@ import org.elasticsearch.license.RemoteClusterLicenseChecker; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.ccr.action.ShardFollowTask; import org.elasticsearch.xpack.ccr.action.ShardChangesAction; +import org.elasticsearch.xpack.ccr.action.ShardFollowTask; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges; import org.elasticsearch.xpack.core.security.support.Exceptions; import java.util.Arrays; @@ -328,7 +329,7 @@ public void hasPrivilegesToFollowIndices(final Client remoteClient, final String message.append(indices.length == 1 ? " index " : " indices "); message.append(Arrays.toString(indices)); - HasPrivilegesResponse.ResourcePrivileges resourcePrivileges = response.getIndexPrivileges().iterator().next(); + ResourcePrivileges resourcePrivileges = response.getIndexPrivileges().iterator().next(); for (Map.Entry entry : resourcePrivileges.getPrivileges().entrySet()) { if (entry.getValue() == false) { message.append(", privilege for action ["); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 4e3feb3c8a2fa..6b1fcb67950e9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -137,6 +137,9 @@ import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage; import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.core.security.SecuritySettings; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.GetApiKeyAction; +import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction; import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction; import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction; @@ -314,6 +317,9 @@ public List> getClientActions() { InvalidateTokenAction.INSTANCE, GetCertificateInfoAction.INSTANCE, RefreshTokenAction.INSTANCE, + CreateApiKeyAction.INSTANCE, + InvalidateApiKeyAction.INSTANCE, + GetApiKeyAction.INSTANCE, // upgrade IndexUpgradeInfoAction.INSTANCE, IndexUpgradeAction.INSTANCE, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index dd18e3b319468..dd8b1d5bb4681 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -99,10 +99,14 @@ private XPackSettings() { public static final Setting RESERVED_REALM_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.reserved_realm.enabled", true, Setting.Property.NodeScope); - /** Setting for enabling or disabling the token service. Defaults to true */ + /** Setting for enabling or disabling the token service. Defaults to the value of https being enabled */ public static final Setting TOKEN_SERVICE_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.token.enabled", XPackSettings.HTTP_SSL_ENABLED::getRaw, Setting.Property.NodeScope); + /** Setting for enabling or disabling the api key service. Defaults to the value of https being enabled */ + public static final Setting API_KEY_SERVICE_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.api_key.enabled", + XPackSettings.HTTP_SSL_ENABLED::getRaw, Setting.Property.NodeScope); + /** Setting for enabling or disabling FIPS mode. Defaults to false */ public static final Setting FIPS_MODE_ENABLED = Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope); @@ -199,6 +203,7 @@ public static List> getAllSettings() { settings.add(HTTP_SSL_ENABLED); settings.add(RESERVED_REALM_ENABLED_SETTING); settings.add(TOKEN_SERVICE_ENABLED_SETTING); + settings.add(API_KEY_SERVICE_ENABLED_SETTING); settings.add(SQL_ENABLED); settings.add(USER_SETTING); settings.add(ROLLUP_ENABLED); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java index c737ab75d81aa..0da07a52996ad 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java @@ -13,9 +13,11 @@ import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; import org.elasticsearch.node.Node; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType; import org.elasticsearch.xpack.core.security.user.User; import java.io.IOException; +import java.util.Collections; import java.util.Objects; import java.util.function.Consumer; @@ -71,7 +73,8 @@ public void setUser(User user, Version version) { } else { lookedUpBy = null; } - setAuthentication(new Authentication(user, authenticatedBy, lookedUpBy, version)); + setAuthentication( + new Authentication(user, authenticatedBy, lookedUpBy, version, AuthenticationType.INTERNAL, Collections.emptyMap())); } /** Writes the authentication to the thread context */ @@ -89,7 +92,7 @@ private void setAuthentication(Authentication authentication) { */ public void executeAsUser(User user, Consumer consumer, Version version) { final StoredContext original = threadContext.newStoredContext(true); - try (ThreadContext.StoredContext ctx = threadContext.stashContext()) { + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { setUser(user, version); consumer.accept(original); } @@ -102,9 +105,9 @@ public void executeAsUser(User user, Consumer consumer, Version v public void executeAfterRewritingAuthentication(Consumer consumer, Version version) { final StoredContext original = threadContext.newStoredContext(true); final Authentication authentication = Objects.requireNonNull(userSettings.getAuthentication()); - try (ThreadContext.StoredContext ctx = threadContext.stashContext()) { + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { setAuthentication(new Authentication(authentication.getUser(), authentication.getAuthenticatedBy(), - authentication.getLookedUpBy(), version)); + authentication.getLookedUpBy(), version, authentication.getAuthenticationType(), authentication.getMetadata())); consumer.accept(original); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java new file mode 100644 index 0000000000000..bfe9f523062a0 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * API key information + */ +public final class ApiKey implements ToXContentObject, Writeable { + + private final String name; + private final String id; + private final Instant creation; + private final Instant expiration; + private final boolean invalidated; + private final String username; + private final String realm; + + public ApiKey(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm) { + this.name = name; + this.id = id; + // As we do not yet support the nanosecond precision when we serialize to JSON, + // here creating the 'Instant' of milliseconds precision. + // This Instant can then be used for date comparison. + this.creation = Instant.ofEpochMilli(creation.toEpochMilli()); + this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null; + this.invalidated = invalidated; + this.username = username; + this.realm = realm; + } + + public ApiKey(StreamInput in) throws IOException { + this.name = in.readString(); + this.id = in.readString(); + this.creation = in.readInstant(); + this.expiration = in.readOptionalInstant(); + this.invalidated = in.readBoolean(); + this.username = in.readString(); + this.realm = in.readString(); + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public Instant getCreation() { + return creation; + } + + public Instant getExpiration() { + return expiration; + } + + public boolean isInvalidated() { + return invalidated; + } + + public String getUsername() { + return username; + } + + public String getRealm() { + return realm; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field("id", id) + .field("name", name) + .field("creation", creation.toEpochMilli()); + if (expiration != null) { + builder.field("expiration", expiration.toEpochMilli()); + } + builder.field("invalidated", invalidated) + .field("username", username) + .field("realm", realm); + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeString(id); + out.writeInstant(creation); + out.writeOptionalInstant(expiration); + out.writeBoolean(invalidated); + out.writeString(username); + out.writeString(realm); + } + + @Override + public int hashCode() { + return Objects.hash(name, id, creation, expiration, invalidated, username, realm); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ApiKey other = (ApiKey) obj; + return Objects.equals(name, other.name) + && Objects.equals(id, other.id) + && Objects.equals(creation, other.creation) + && Objects.equals(expiration, other.expiration) + && Objects.equals(invalidated, other.invalidated) + && Objects.equals(username, other.username) + && Objects.equals(realm, other.realm); + } + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("api_key", args -> { + return new ApiKey((String) args[0], (String) args[1], Instant.ofEpochMilli((Long) args[2]), + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]), (Boolean) args[4], (String) args[5], (String) args[6]); + }); + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareLong(constructorArg(), new ParseField("creation")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); + PARSER.declareBoolean(constructorArg(), new ParseField("invalidated")); + PARSER.declareString(constructorArg(), new ParseField("username")); + PARSER.declareString(constructorArg(), new ParseField("realm")); + } + + public static ApiKey fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "ApiKey [name=" + name + ", id=" + id + ", creation=" + creation + ", expiration=" + expiration + ", invalidated=" + + invalidated + ", username=" + username + ", realm=" + realm + "]"; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java new file mode 100644 index 0000000000000..5d211ea70b522 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.common.io.stream.Writeable; + +/** + * Action for the creation of an API key + */ +public final class CreateApiKeyAction extends Action { + + public static final String NAME = "cluster:admin/xpack/security/api_key/create"; + public static final CreateApiKeyAction INSTANCE = new CreateApiKeyAction(); + + private CreateApiKeyAction() { + super(NAME); + } + + @Override + public CreateApiKeyResponse newResponse() { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return CreateApiKeyResponse::new; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java new file mode 100644 index 0000000000000..28a872c2222dd --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Request class used for the creation of an API key. The request requires a name to be provided + * and optionally an expiration time and permission limitation can be provided. + */ +public final class CreateApiKeyRequest extends ActionRequest { + public static final WriteRequest.RefreshPolicy DEFAULT_REFRESH_POLICY = WriteRequest.RefreshPolicy.WAIT_UNTIL; + + private String name; + private TimeValue expiration; + private List roleDescriptors = Collections.emptyList(); + private WriteRequest.RefreshPolicy refreshPolicy = DEFAULT_REFRESH_POLICY; + + public CreateApiKeyRequest() {} + + /** + * Create API Key request constructor + * @param name name for the API key + * @param roleDescriptors list of {@link RoleDescriptor}s + * @param expiration to specify expiration for the API key + */ + public CreateApiKeyRequest(String name, List roleDescriptors, @Nullable TimeValue expiration) { + if (Strings.hasText(name)) { + this.name = name; + } else { + throw new IllegalArgumentException("name must not be null or empty"); + } + this.roleDescriptors = Objects.requireNonNull(roleDescriptors, "role descriptors may not be null"); + this.expiration = expiration; + } + + public CreateApiKeyRequest(StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.expiration = in.readOptionalTimeValue(); + this.roleDescriptors = Collections.unmodifiableList(in.readList(RoleDescriptor::new)); + this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in); + } + + public String getName() { + return name; + } + + public void setName(String name) { + if (Strings.hasText(name)) { + this.name = name; + } else { + throw new IllegalArgumentException("name must not be null or empty"); + } + } + + public TimeValue getExpiration() { + return expiration; + } + + public void setExpiration(TimeValue expiration) { + this.expiration = expiration; + } + + public List getRoleDescriptors() { + return roleDescriptors; + } + + public void setRoleDescriptors(List roleDescriptors) { + this.roleDescriptors = Collections.unmodifiableList(Objects.requireNonNull(roleDescriptors, "role descriptors may not be null")); + } + + public WriteRequest.RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { + this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null"); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isNullOrEmpty(name)) { + validationException = addValidationError("name is required", validationException); + } else { + if (name.length() > 256) { + validationException = addValidationError("name may not be more than 256 characters long", validationException); + } + if (name.equals(name.trim()) == false) { + validationException = addValidationError("name may not begin or end with whitespace", validationException); + } + if (name.startsWith("_")) { + validationException = addValidationError("name may not begin with an underscore", validationException); + } + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeOptionalTimeValue(expiration); + out.writeList(roleDescriptors); + refreshPolicy.writeTo(out); + } + + @Override + public void readFrom(StreamInput in) { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java new file mode 100644 index 0000000000000..1a711aa7d9a26 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Request builder for populating a {@link CreateApiKeyRequest} + */ +public final class CreateApiKeyRequestBuilder extends ActionRequestBuilder { + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "api_key_request", false, (args, v) -> { + return new CreateApiKeyRequest((String) args[0], (List) args[1], + TimeValue.parseTimeValue((String) args[2], null, "expiration")); + }); + + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareNamedObjects(constructorArg(), (p, c, n) -> { + p.nextToken(); + return RoleDescriptor.parse(n, p, false); + }, new ParseField("role_descriptors")); + PARSER.declareString(optionalConstructorArg(), new ParseField("expiration")); + } + + public CreateApiKeyRequestBuilder(ElasticsearchClient client) { + super(client, CreateApiKeyAction.INSTANCE, new CreateApiKeyRequest()); + } + + public CreateApiKeyRequestBuilder setName(String name) { + request.setName(name); + return this; + } + + public CreateApiKeyRequestBuilder setExpiration(TimeValue expiration) { + request.setExpiration(expiration); + return this; + } + + public CreateApiKeyRequestBuilder setRoleDescriptors(List roleDescriptors) { + request.setRoleDescriptors(roleDescriptors); + return this; + } + + public CreateApiKeyRequestBuilder setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { + request.setRefreshPolicy(refreshPolicy); + return this; + } + + public CreateApiKeyRequestBuilder source(BytesReference source, XContentType xContentType) throws IOException { + final NamedXContentRegistry registry = NamedXContentRegistry.EMPTY; + try (InputStream stream = source.streamInput(); + XContentParser parser = xContentType.xContent().createParser(registry, LoggingDeprecationHandler.INSTANCE, stream)) { + CreateApiKeyRequest createApiKeyRequest = PARSER.parse(parser, null); + setName(createApiKeyRequest.getName()); + setRoleDescriptors(createApiKeyRequest.getRoleDescriptors()); + setExpiration(createApiKeyRequest.getExpiration()); + } + return this; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java new file mode 100644 index 0000000000000..a774413c3c4a2 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.CharArrays; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for the successful creation of an api key + */ +public final class CreateApiKeyResponse extends ActionResponse implements ToXContentObject { + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("create_api_key_response", + args -> new CreateApiKeyResponse((String) args[0], (String) args[1], new SecureString((String) args[2]), + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]))); + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareString(constructorArg(), new ParseField("api_key")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); + } + + private final String name; + private final String id; + private final SecureString key; + private final Instant expiration; + + public CreateApiKeyResponse(String name, String id, SecureString key, Instant expiration) { + this.name = name; + this.id = id; + this.key = key; + // As we do not yet support the nanosecond precision when we serialize to JSON, + // here creating the 'Instant' of milliseconds precision. + // This Instant can then be used for date comparison. + this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null; + } + + public CreateApiKeyResponse(StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.id = in.readString(); + byte[] bytes = null; + try { + bytes = in.readByteArray(); + this.key = new SecureString(CharArrays.utf8BytesToChars(bytes)); + } finally { + if (bytes != null) { + Arrays.fill(bytes, (byte) 0); + } + } + this.expiration = in.readOptionalInstant(); + } + + public String getName() { + return name; + } + + public String getId() { + return id; + } + + public SecureString getKey() { + return key; + } + + @Nullable + public Instant getExpiration() { + return expiration; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((expiration == null) ? 0 : expiration.hashCode()); + result = prime * result + Objects.hash(id, name, key); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final CreateApiKeyResponse other = (CreateApiKeyResponse) obj; + if (expiration == null) { + if (other.expiration != null) + return false; + } else if (!Objects.equals(expiration, other.expiration)) + return false; + return Objects.equals(id, other.id) + && Objects.equals(key, other.key) + && Objects.equals(name, other.name); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeString(id); + byte[] bytes = null; + try { + bytes = CharArrays.toUtf8Bytes(key.getChars()); + out.writeByteArray(bytes); + } finally { + if (bytes != null) { + Arrays.fill(bytes, (byte) 0); + } + } + out.writeOptionalInstant(expiration); + } + + @Override + public void readFrom(StreamInput in) { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + public static CreateApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field("id", id) + .field("name", name); + if (expiration != null) { + builder.field("expiration", expiration.toEpochMilli()); + } + byte[] charBytes = CharArrays.toUtf8Bytes(key.getChars()); + try { + builder.field("api_key").utf8Value(charBytes, 0, charBytes.length); + } finally { + Arrays.fill(charBytes, (byte) 0); + } + return builder.endObject(); + } + + @Override + public String toString() { + return "CreateApiKeyResponse [name=" + name + ", id=" + id + ", expiration=" + expiration + "]"; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java new file mode 100644 index 0000000000000..2af331909a3af --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.common.io.stream.Writeable; + +/** + * Action for retrieving API key(s) + */ +public final class GetApiKeyAction extends Action { + + public static final String NAME = "cluster:admin/xpack/security/api_key/get"; + public static final GetApiKeyAction INSTANCE = new GetApiKeyAction(); + + private GetApiKeyAction() { + super(NAME); + } + + @Override + public GetApiKeyResponse newResponse() { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return GetApiKeyResponse::new; + } +} \ No newline at end of file diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java new file mode 100644 index 0000000000000..287ebcee4b6f2 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Request for get API key + */ +public final class GetApiKeyRequest extends ActionRequest { + + private final String realmName; + private final String userName; + private final String apiKeyId; + private final String apiKeyName; + + public GetApiKeyRequest() { + this(null, null, null, null); + } + + public GetApiKeyRequest(StreamInput in) throws IOException { + super(in); + realmName = in.readOptionalString(); + userName = in.readOptionalString(); + apiKeyId = in.readOptionalString(); + apiKeyName = in.readOptionalString(); + } + + public GetApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId, + @Nullable String apiKeyName) { + this.realmName = realmName; + this.userName = userName; + this.apiKeyId = apiKeyId; + this.apiKeyName = apiKeyName; + } + + public String getRealmName() { + return realmName; + } + + public String getUserName() { + return userName; + } + + public String getApiKeyId() { + return apiKeyId; + } + + public String getApiKeyName() { + return apiKeyName; + } + + /** + * Creates get API key request for given realm name + * @param realmName realm name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingRealmName(String realmName) { + return new GetApiKeyRequest(realmName, null, null, null); + } + + /** + * Creates get API key request for given user name + * @param userName user name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingUserName(String userName) { + return new GetApiKeyRequest(null, userName, null, null); + } + + /** + * Creates get API key request for given realm and user name + * @param realmName realm name + * @param userName user name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingRealmAndUserName(String realmName, String userName) { + return new GetApiKeyRequest(realmName, userName, null, null); + } + + /** + * Creates get API key request for given api key id + * @param apiKeyId api key id + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingApiKeyId(String apiKeyId) { + return new GetApiKeyRequest(null, null, apiKeyId, null); + } + + /** + * Creates get api key request for given api key name + * @param apiKeyName api key name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingApiKeyName(String apiKeyName) { + return new GetApiKeyRequest(null, null, null, apiKeyName); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false + && Strings.hasText(apiKeyName) == false) { + validationException = addValidationError("One of [api key id, api key name, username, realm name] must be specified", + validationException); + } + if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) { + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + validationException = addValidationError( + "username or realm name must not be specified when the api key id or api key name is specified", + validationException); + } + } + if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) { + validationException = addValidationError("only one of [api key id, api key name] can be specified", validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(realmName); + out.writeOptionalString(userName); + out.writeOptionalString(apiKeyId); + out.writeOptionalString(apiKeyName); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java new file mode 100644 index 0000000000000..97b8f380f6940 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for get API keys.
+ * The result contains information about the API keys that were found. + */ +public final class GetApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable { + + private final ApiKey[] foundApiKeysInfo; + + public GetApiKeyResponse(StreamInput in) throws IOException { + super(in); + this.foundApiKeysInfo = in.readArray(ApiKey::new, ApiKey[]::new); + } + + public GetApiKeyResponse(Collection foundApiKeysInfo) { + Objects.requireNonNull(foundApiKeysInfo, "found_api_keys_info must be provided"); + this.foundApiKeysInfo = foundApiKeysInfo.toArray(new ApiKey[0]); + } + + public static GetApiKeyResponse emptyResponse() { + return new GetApiKeyResponse(Collections.emptyList()); + } + + public ApiKey[] getApiKeyInfos() { + return foundApiKeysInfo; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .array("api_keys", (Object[]) foundApiKeysInfo); + return builder.endObject(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeArray(foundApiKeysInfo); + } + + @SuppressWarnings("unchecked") + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("get_api_key_response", args -> { + return (args[0] == null) ? GetApiKeyResponse.emptyResponse() : new GetApiKeyResponse((List) args[0]); + }); + static { + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ApiKey.fromXContent(p), new ParseField("api_keys")); + } + + public static GetApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "GetApiKeyResponse [foundApiKeysInfo=" + foundApiKeysInfo + "]"; + } + +} \ No newline at end of file diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java new file mode 100644 index 0000000000000..0f5c7e66e724c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.common.io.stream.Writeable; + +/** + * Action for invalidating API key + */ +public final class InvalidateApiKeyAction extends Action { + + public static final String NAME = "cluster:admin/xpack/security/api_key/invalidate"; + public static final InvalidateApiKeyAction INSTANCE = new InvalidateApiKeyAction(); + + private InvalidateApiKeyAction() { + super(NAME); + } + + @Override + public InvalidateApiKeyResponse newResponse() { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return InvalidateApiKeyResponse::new; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java new file mode 100644 index 0000000000000..f8815785d53d8 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Request for invalidating API key(s) so that it can no longer be used + */ +public final class InvalidateApiKeyRequest extends ActionRequest { + + private final String realmName; + private final String userName; + private final String id; + private final String name; + + public InvalidateApiKeyRequest() { + this(null, null, null, null); + } + + public InvalidateApiKeyRequest(StreamInput in) throws IOException { + super(in); + realmName = in.readOptionalString(); + userName = in.readOptionalString(); + id = in.readOptionalString(); + name = in.readOptionalString(); + } + + public InvalidateApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String id, + @Nullable String name) { + this.realmName = realmName; + this.userName = userName; + this.id = id; + this.name = name; + } + + public String getRealmName() { + return realmName; + } + + public String getUserName() { + return userName; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + /** + * Creates invalidate api key request for given realm name + * @param realmName realm name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingRealmName(String realmName) { + return new InvalidateApiKeyRequest(realmName, null, null, null); + } + + /** + * Creates invalidate API key request for given user name + * @param userName user name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingUserName(String userName) { + return new InvalidateApiKeyRequest(null, userName, null, null); + } + + /** + * Creates invalidate API key request for given realm and user name + * @param realmName realm name + * @param userName user name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingRealmAndUserName(String realmName, String userName) { + return new InvalidateApiKeyRequest(realmName, userName, null, null); + } + + /** + * Creates invalidate API key request for given api key id + * @param id api key id + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingApiKeyId(String id) { + return new InvalidateApiKeyRequest(null, null, id, null); + } + + /** + * Creates invalidate api key request for given api key name + * @param name api key name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingApiKeyName(String name) { + return new InvalidateApiKeyRequest(null, null, null, name); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(id) == false + && Strings.hasText(name) == false) { + validationException = addValidationError("One of [api key id, api key name, username, realm name] must be specified", + validationException); + } + if (Strings.hasText(id) || Strings.hasText(name)) { + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + validationException = addValidationError( + "username or realm name must not be specified when the api key id or api key name is specified", + validationException); + } + } + if (Strings.hasText(id) && Strings.hasText(name)) { + validationException = addValidationError("only one of [api key id, api key name] can be specified", validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(realmName); + out.writeOptionalString(userName); + out.writeOptionalString(id); + out.writeOptionalString(name); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java new file mode 100644 index 0000000000000..e9580c93d9086 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for invalidation of one or more API keys result.
+ * The result contains information about: + *

    + *
  • API key ids that were actually invalidated
    • + *
  • API key ids that were not invalidated in this request because they were already invalidated
    • + *
  • how many errors were encountered while invalidating API keys and the error details
    • + *
+ */ +public final class InvalidateApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable { + + private final List invalidatedApiKeys; + private final List previouslyInvalidatedApiKeys; + private final List errors; + + public InvalidateApiKeyResponse(StreamInput in) throws IOException { + super(in); + this.invalidatedApiKeys = in.readList(StreamInput::readString); + this.previouslyInvalidatedApiKeys = in.readList(StreamInput::readString); + this.errors = in.readList(StreamInput::readException); + } + + /** + * Constructor for API keys invalidation response + * @param invalidatedApiKeys list of invalidated API key ids + * @param previouslyInvalidatedApiKeys list of previously invalidated API key ids + * @param errors list of encountered errors while invalidating API keys + */ + public InvalidateApiKeyResponse(List invalidatedApiKeys, List previouslyInvalidatedApiKeys, + @Nullable List errors) { + this.invalidatedApiKeys = Objects.requireNonNull(invalidatedApiKeys, "invalidated_api_keys must be provided"); + this.previouslyInvalidatedApiKeys = Objects.requireNonNull(previouslyInvalidatedApiKeys, + "previously_invalidated_api_keys must be provided"); + if (null != errors) { + this.errors = errors; + } else { + this.errors = Collections.emptyList(); + } + } + + public static InvalidateApiKeyResponse emptyResponse() { + return new InvalidateApiKeyResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + } + + public List getInvalidatedApiKeys() { + return invalidatedApiKeys; + } + + public List getPreviouslyInvalidatedApiKeys() { + return previouslyInvalidatedApiKeys; + } + + public List getErrors() { + return errors; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .array("invalidated_api_keys", invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)) + .array("previously_invalidated_api_keys", previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)) + .field("error_count", errors.size()); + if (errors.isEmpty() == false) { + builder.field("error_details"); + builder.startArray(); + for (ElasticsearchException e : errors) { + builder.startObject(); + ElasticsearchException.generateThrowableXContent(builder, params, e); + builder.endObject(); + } + builder.endArray(); + } + return builder.endObject(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringCollection(invalidatedApiKeys); + out.writeStringCollection(previouslyInvalidatedApiKeys); + out.writeCollection(errors, StreamOutput::writeException); + } + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("invalidate_api_key_response", + args -> { + return new InvalidateApiKeyResponse((List) args[0], (List) args[1], (List) args[3]); + }); + static { + PARSER.declareStringArray(constructorArg(), new ParseField("invalidated_api_keys")); + PARSER.declareStringArray(constructorArg(), new ParseField("previously_invalidated_api_keys")); + // we parse error_count but ignore it while constructing response + PARSER.declareInt(constructorArg(), new ParseField("error_count")); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ElasticsearchException.fromXContent(p), + new ParseField("error_details")); + } + + public static InvalidateApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "InvalidateApiKeyResponse [invalidatedApiKeys=" + invalidatedApiKeys + ", previouslyInvalidatedApiKeys=" + + previouslyInvalidatedApiKeys + ", errors=" + errors + "]"; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java index 93c9d6bca9b64..27079eebcc36b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java @@ -37,7 +37,7 @@ public void readFrom(StreamInput in) throws IOException { int size = in.readVInt(); roles = new RoleDescriptor[size]; for (int i = 0; i < size; i++) { - roles[i] = RoleDescriptor.readFrom(in); + roles[i] = new RoleDescriptor(in); } } @@ -46,7 +46,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeVInt(roles.length); for (RoleDescriptor role : roles) { - RoleDescriptor.writeTo(role, out); + role.writeTo(out); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java index 22153ad0b1083..74984556dc1a0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges; import java.io.IOException; import java.util.Collection; @@ -49,7 +50,7 @@ public HasPrivilegesResponse(String username, boolean completeMatch, Map sorted(Collection resources) { - final Set set = new TreeSet<>(Comparator.comparing(o -> o.resource)); + final Set set = new TreeSet<>(Comparator.comparing(o -> o.getResource())); set.addAll(resources); return set; } @@ -116,11 +117,11 @@ public void readFrom(StreamInput in) throws IOException { private static Set readResourcePrivileges(StreamInput in) throws IOException { final int count = in.readVInt(); - final Set set = new TreeSet<>(Comparator.comparing(o -> o.resource)); + final Set set = new TreeSet<>(Comparator.comparing(o -> o.getResource())); for (int i = 0; i < count; i++) { final String index = in.readString(); final Map privileges = in.readMap(StreamInput::readString, StreamInput::readBoolean); - set.add(new ResourcePrivileges(index, privileges)); + set.add(ResourcePrivileges.builder(index).addPrivileges(privileges).build()); } return set; } @@ -144,8 +145,8 @@ public void writeTo(StreamOutput out) throws IOException { private static void writeResourcePrivileges(StreamOutput out, Set privileges) throws IOException { out.writeVInt(privileges.size()); for (ResourcePrivileges priv : privileges) { - out.writeString(priv.resource); - out.writeMap(priv.privileges, StreamOutput::writeString, StreamOutput::writeBoolean); + out.writeString(priv.getResource()); + out.writeMap(priv.getPrivileges(), StreamOutput::writeString, StreamOutput::writeBoolean); } } @@ -181,60 +182,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - private void appendResources(XContentBuilder builder, String field, Set privileges) + private void appendResources(XContentBuilder builder, String field, Set privileges) throws IOException { builder.startObject(field); - for (HasPrivilegesResponse.ResourcePrivileges privilege : privileges) { + for (ResourcePrivileges privilege : privileges) { builder.field(privilege.getResource()); builder.map(privilege.getPrivileges()); } builder.endObject(); } - - public static class ResourcePrivileges { - private final String resource; - private final Map privileges; - - public ResourcePrivileges(String resource, Map privileges) { - this.resource = Objects.requireNonNull(resource); - this.privileges = Collections.unmodifiableMap(privileges); - } - - public String getResource() { - return resource; - } - - public Map getPrivileges() { - return privileges; - } - - @Override - public String toString() { - return getClass().getSimpleName() + "{" + - "resource='" + resource + '\'' + - ", privileges=" + privileges + - '}'; - } - - @Override - public int hashCode() { - int result = resource.hashCode(); - result = 31 * result + privileges.hashCode(); - return result; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - final ResourcePrivileges other = (ResourcePrivileges) o; - return this.resource.equals(other.resource) && this.privileges.equals(other.privileges); - } - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index b9dbe0a948ff2..a93cc44aadb23 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -18,6 +18,8 @@ import java.io.IOException; import java.util.Base64; +import java.util.Collections; +import java.util.Map; import java.util.Objects; // TODO(hub-cap) Clean this up after moving User over - This class can re-inherit its field AUTHENTICATION_KEY in AuthenticationField. @@ -28,16 +30,25 @@ public class Authentication implements ToXContentObject { private final RealmRef authenticatedBy; private final RealmRef lookedUpBy; private final Version version; + private final AuthenticationType type; + private final Map metadata; public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy) { this(user, authenticatedBy, lookedUpBy, Version.CURRENT); } public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version) { + this(user, authenticatedBy, lookedUpBy, version, AuthenticationType.REALM, Collections.emptyMap()); + } + + public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version, + AuthenticationType type, Map metadata) { this.user = Objects.requireNonNull(user); this.authenticatedBy = Objects.requireNonNull(authenticatedBy); this.lookedUpBy = lookedUpBy; this.version = version; + this.type = type; + this.metadata = metadata; } public Authentication(StreamInput in) throws IOException { @@ -49,6 +60,13 @@ public Authentication(StreamInput in) throws IOException { this.lookedUpBy = null; } this.version = in.getVersion(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO change to V6_6 after backport + type = AuthenticationType.values()[in.readVInt()]; + metadata = in.readMap(); + } else { + type = AuthenticationType.REALM; + metadata = Collections.emptyMap(); + } } public User getUser() { @@ -67,8 +85,15 @@ public Version getVersion() { return version; } - public static Authentication readFromContext(ThreadContext ctx) - throws IOException, IllegalArgumentException { + public AuthenticationType getAuthenticationType() { + return type; + } + + public Map getMetadata() { + return metadata; + } + + public static Authentication readFromContext(ThreadContext ctx) throws IOException, IllegalArgumentException { Authentication authentication = ctx.getTransient(AuthenticationField.AUTHENTICATION_KEY); if (authentication != null) { assert ctx.getHeader(AuthenticationField.AUTHENTICATION_KEY) != null; @@ -107,8 +132,7 @@ public static Authentication decode(String header) throws IOException { * Writes the authentication to the context. There must not be an existing authentication in the context and if there is an * {@link IllegalStateException} will be thrown */ - public void writeToContext(ThreadContext ctx) - throws IOException, IllegalArgumentException { + public void writeToContext(ThreadContext ctx) throws IOException, IllegalArgumentException { ensureContextDoesNotContainAuthentication(ctx); String header = encode(); ctx.putTransient(AuthenticationField.AUTHENTICATION_KEY, this); @@ -141,28 +165,28 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO change to V6_6 after backport + out.writeVInt(type.ordinal()); + out.writeMap(metadata); + } } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Authentication that = (Authentication) o; - - if (!user.equals(that.user)) return false; - if (!authenticatedBy.equals(that.authenticatedBy)) return false; - if (lookedUpBy != null ? !lookedUpBy.equals(that.lookedUpBy) : that.lookedUpBy != null) return false; - return version.equals(that.version); + return user.equals(that.user) && + authenticatedBy.equals(that.authenticatedBy) && + Objects.equals(lookedUpBy, that.lookedUpBy) && + version.equals(that.version) && + type == that.type && + metadata.equals(that.metadata); } @Override public int hashCode() { - int result = user.hashCode(); - result = 31 * result + authenticatedBy.hashCode(); - result = 31 * result + (lookedUpBy != null ? lookedUpBy.hashCode() : 0); - result = 31 * result + version.hashCode(); - return result; + return Objects.hash(user, authenticatedBy, lookedUpBy, version, type, metadata); } @Override @@ -246,5 +270,13 @@ public int hashCode() { return result; } } + + public enum AuthenticationType { + REALM, + API_KEY, + TOKEN, + ANONYMOUS, + INTERNAL + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java index f27d95dc868ab..d8954501b8b83 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java @@ -68,10 +68,12 @@ private static Integer authSchemePriority(final String headerValue) { return 0; } else if (headerValue.regionMatches(true, 0, "bearer", 0, "bearer".length())) { return 1; - } else if (headerValue.regionMatches(true, 0, "basic", 0, "basic".length())) { + } else if (headerValue.regionMatches(true, 0, "apikey", 0, "apikey".length())) { return 2; - } else { + } else if (headerValue.regionMatches(true, 0, "basic", 0, "basic".length())) { return 3; + } else { + return 4; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java index 674711de88166..5705d7bf35723 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java @@ -43,7 +43,7 @@ * A holder for a Role that contains user-readable information about the Role * without containing the actual Role object. */ -public class RoleDescriptor implements ToXContentObject { +public class RoleDescriptor implements ToXContentObject, Writeable { public static final String ROLE_TYPE = "role"; @@ -110,6 +110,27 @@ public RoleDescriptor(String name, Collections.singletonMap("enabled", true); } + public RoleDescriptor(StreamInput in) throws IOException { + this.name = in.readString(); + this.clusterPrivileges = in.readStringArray(); + int size = in.readVInt(); + this.indicesPrivileges = new IndicesPrivileges[size]; + for (int i = 0; i < size; i++) { + indicesPrivileges[i] = new IndicesPrivileges(in); + } + this.runAs = in.readStringArray(); + this.metadata = in.readMap(); + this.transientMetadata = in.readMap(); + + if (in.getVersion().onOrAfter(Version.V_6_4_0)) { + this.applicationPrivileges = in.readArray(ApplicationResourcePrivileges::new, ApplicationResourcePrivileges[]::new); + this.conditionalClusterPrivileges = ConditionalClusterPrivileges.readArray(in); + } else { + this.applicationPrivileges = ApplicationResourcePrivileges.NONE; + this.conditionalClusterPrivileges = ConditionalClusterPrivileges.EMPTY_ARRAY; + } + } + public String getName() { return this.name; } @@ -232,46 +253,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, boolea return builder.endObject(); } - public static RoleDescriptor readFrom(StreamInput in) throws IOException { - String name = in.readString(); - String[] clusterPrivileges = in.readStringArray(); - int size = in.readVInt(); - IndicesPrivileges[] indicesPrivileges = new IndicesPrivileges[size]; - for (int i = 0; i < size; i++) { - indicesPrivileges[i] = new IndicesPrivileges(in); - } - String[] runAs = in.readStringArray(); - Map metadata = in.readMap(); - - final Map transientMetadata = in.readMap(); - - final ApplicationResourcePrivileges[] applicationPrivileges; - final ConditionalClusterPrivilege[] conditionalClusterPrivileges; - if (in.getVersion().onOrAfter(Version.V_6_4_0)) { - applicationPrivileges = in.readArray(ApplicationResourcePrivileges::new, ApplicationResourcePrivileges[]::new); - conditionalClusterPrivileges = ConditionalClusterPrivileges.readArray(in); - } else { - applicationPrivileges = ApplicationResourcePrivileges.NONE; - conditionalClusterPrivileges = ConditionalClusterPrivileges.EMPTY_ARRAY; - } - - return new RoleDescriptor(name, clusterPrivileges, indicesPrivileges, applicationPrivileges, conditionalClusterPrivileges, - runAs, metadata, transientMetadata); - } - - public static void writeTo(RoleDescriptor descriptor, StreamOutput out) throws IOException { - out.writeString(descriptor.name); - out.writeStringArray(descriptor.clusterPrivileges); - out.writeVInt(descriptor.indicesPrivileges.length); - for (IndicesPrivileges group : descriptor.indicesPrivileges) { + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeStringArray(clusterPrivileges); + out.writeVInt(indicesPrivileges.length); + for (IndicesPrivileges group : indicesPrivileges) { group.writeTo(out); } - out.writeStringArray(descriptor.runAs); - out.writeMap(descriptor.metadata); - out.writeMap(descriptor.transientMetadata); + out.writeStringArray(runAs); + out.writeMap(metadata); + out.writeMap(transientMetadata); if (out.getVersion().onOrAfter(Version.V_6_4_0)) { - out.writeArray(ApplicationResourcePrivileges::write, descriptor.applicationPrivileges); - ConditionalClusterPrivileges.writeArray(out, descriptor.getConditionalClusterPrivileges()); + out.writeArray(ApplicationResourcePrivileges::write, applicationPrivileges); + ConditionalClusterPrivileges.writeArray(out, getConditionalClusterPrivileges()); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java index 6df9ad834c1e5..8cdf099e676d8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java @@ -6,11 +6,13 @@ package org.elasticsearch.xpack.core.security.authz.accesscontrol; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; +import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -22,7 +24,7 @@ public class IndicesAccessControl { public static final IndicesAccessControl ALLOW_ALL = new IndicesAccessControl(true, Collections.emptyMap()); public static final IndicesAccessControl ALLOW_NO_INDICES = new IndicesAccessControl(true, Collections.singletonMap(IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER, - new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), null))); + new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), DocumentPermissions.allowAll()))); private final boolean granted; private final Map indexPermissions; @@ -55,12 +57,12 @@ public static class IndexAccessControl { private final boolean granted; private final FieldPermissions fieldPermissions; - private final Set queries; + private final DocumentPermissions documentPermissions; - public IndexAccessControl(boolean granted, FieldPermissions fieldPermissions, Set queries) { + public IndexAccessControl(boolean granted, FieldPermissions fieldPermissions, DocumentPermissions documentPermissions) { this.granted = granted; - this.fieldPermissions = fieldPermissions; - this.queries = queries; + this.fieldPermissions = (fieldPermissions == null) ? FieldPermissions.DEFAULT : fieldPermissions; + this.documentPermissions = (documentPermissions == null) ? DocumentPermissions.allowAll() : documentPermissions; } /** @@ -82,8 +84,33 @@ public FieldPermissions getFieldPermissions() { * then this means that there are no document level restrictions */ @Nullable - public Set getQueries() { - return queries; + public DocumentPermissions getDocumentPermissions() { + return documentPermissions; + } + + /** + * Returns a instance of {@link IndexAccessControl}, where the privileges for {@code this} object are constrained by the privileges + * contained in the provided parameter.
+ * Allowed fields for this index permission would be an intersection of allowed fields.
+ * Allowed documents for this index permission would be an intersection of allowed documents.
+ * + * @param limitedByIndexAccessControl {@link IndexAccessControl} + * @return {@link IndexAccessControl} + * @see FieldPermissions#limitFieldPermissions(FieldPermissions) + * @see DocumentPermissions#limitDocumentPermissions(DocumentPermissions) + */ + public IndexAccessControl limitIndexAccessControl(IndexAccessControl limitedByIndexAccessControl) { + final boolean granted; + if (this.granted == limitedByIndexAccessControl.granted) { + granted = this.granted; + } else { + granted = false; + } + FieldPermissions fieldPermissions = getFieldPermissions().limitFieldPermissions( + limitedByIndexAccessControl.fieldPermissions); + DocumentPermissions documentPermissions = getDocumentPermissions() + .limitDocumentPermissions(limitedByIndexAccessControl.getDocumentPermissions()); + return new IndexAccessControl(granted, fieldPermissions, documentPermissions); } @Override @@ -91,11 +118,38 @@ public String toString() { return "IndexAccessControl{" + "granted=" + granted + ", fieldPermissions=" + fieldPermissions + - ", queries=" + queries + + ", documentPermissions=" + documentPermissions + '}'; } } + /** + * Returns a instance of {@link IndicesAccessControl}, where the privileges for {@code this} + * object are constrained by the privileges contained in the provided parameter.
+ * + * @param limitedByIndicesAccessControl {@link IndicesAccessControl} + * @return {@link IndicesAccessControl} + */ + public IndicesAccessControl limitIndicesAccessControl(IndicesAccessControl limitedByIndicesAccessControl) { + final boolean granted; + if (this.granted == limitedByIndicesAccessControl.granted) { + granted = this.granted; + } else { + granted = false; + } + Set indexes = indexPermissions.keySet(); + Set otherIndexes = limitedByIndicesAccessControl.indexPermissions.keySet(); + Set commonIndexes = Sets.intersection(indexes, otherIndexes); + + Map indexPermissions = new HashMap<>(commonIndexes.size()); + for (String index : commonIndexes) { + IndexAccessControl indexAccessControl = getIndexPermissions(index); + IndexAccessControl limitedByIndexAccessControl = limitedByIndicesAccessControl.getIndexPermissions(index); + indexPermissions.put(index, indexAccessControl.limitIndexAccessControl(limitedByIndexAccessControl)); + } + return new IndicesAccessControl(granted, indexPermissions); + } + @Override public String toString() { return "IndicesAccessControl{" + diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java index a8651701448d2..56383909d846e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.core.security.authz.accesscontrol; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.BooleanQuery; @@ -18,64 +18,35 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.search.join.ToChildBlockJoinQuery; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.SparseFixedBitSet; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.LoggerMessageFormat; -import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.engine.EngineException; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.BoostingQueryBuilder; -import org.elasticsearch.index.query.ConstantScoreQueryBuilder; -import org.elasticsearch.index.query.GeoShapeQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.Rewriteable; -import org.elasticsearch.index.query.TermsQueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.search.NestedHelper; import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.script.TemplateScript; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader; +import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.support.Exceptions; import org.elasticsearch.xpack.core.security.user.User; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.function.Function; -import static org.apache.lucene.search.BooleanClause.Occur.FILTER; -import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; - /** * An {@link IndexSearcherWrapper} implementation that is used for field and document level security. *

@@ -107,7 +78,7 @@ public SecurityIndexSearcherWrapper(Function querySh } @Override - protected DirectoryReader wrap(DirectoryReader reader) { + protected DirectoryReader wrap(final DirectoryReader reader) { if (licenseState.isDocumentAndFieldLevelSecurityAllowed() == false) { return reader; } @@ -120,47 +91,22 @@ protected DirectoryReader wrap(DirectoryReader reader) { throw new IllegalStateException(LoggerMessageFormat.format("couldn't extract shardId from reader [{}]", reader)); } - IndicesAccessControl.IndexAccessControl permissions = indicesAccessControl.getIndexPermissions(shardId.getIndexName()); + final IndicesAccessControl.IndexAccessControl permissions = indicesAccessControl.getIndexPermissions(shardId.getIndexName()); // No permissions have been defined for an index, so don't intercept the index reader for access control if (permissions == null) { return reader; } - if (permissions.getQueries() != null) { - BooleanQuery.Builder filter = new BooleanQuery.Builder(); - for (BytesReference bytesReference : permissions.getQueries()) { - QueryShardContext queryShardContext = queryShardContextProvider.apply(shardId); - String templateResult = evaluateTemplate(bytesReference.utf8ToString()); - try (XContentParser parser = XContentFactory.xContent(templateResult) - .createParser(queryShardContext.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, templateResult)) { - QueryBuilder queryBuilder = queryShardContext.parseInnerQueryBuilder(parser); - verifyRoleQuery(queryBuilder); - failIfQueryUsesClient(queryBuilder, queryShardContext); - Query roleQuery = queryShardContext.toQuery(queryBuilder).query(); - filter.add(roleQuery, SHOULD); - if (queryShardContext.getMapperService().hasNested()) { - NestedHelper nestedHelper = new NestedHelper(queryShardContext.getMapperService()); - if (nestedHelper.mightMatchNestedDocs(roleQuery)) { - roleQuery = new BooleanQuery.Builder() - .add(roleQuery, FILTER) - .add(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()), FILTER) - .build(); - } - // If access is allowed on root doc then also access is allowed on all nested docs of that root document: - BitSetProducer rootDocs = queryShardContext.bitsetFilter( - Queries.newNonNestedFilter(queryShardContext.indexVersionCreated())); - ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs); - filter.add(includeNestedDocs, SHOULD); - } - } + DirectoryReader wrappedReader = reader; + DocumentPermissions documentPermissions = permissions.getDocumentPermissions(); + if (documentPermissions != null && documentPermissions.hasDocumentLevelPermissions()) { + BooleanQuery filterQuery = documentPermissions.filter(getUser(), scriptService, shardId, queryShardContextProvider); + if (filterQuery != null) { + wrappedReader = DocumentSubsetReader.wrap(wrappedReader, bitsetFilterCache, new ConstantScoreQuery(filterQuery)); } - - // at least one of the queries should match - filter.setMinimumNumberShouldMatch(1); - reader = DocumentSubsetReader.wrap(reader, bitsetFilterCache, new ConstantScoreQuery(filter.build())); } - return permissions.getFieldPermissions().filter(reader); + return permissions.getFieldPermissions().filter(wrappedReader); } catch (IOException e) { logger.error("Unable to apply field level security"); throw ExceptionsHelper.convertToElastic(e); @@ -255,48 +201,6 @@ static void intersectScorerAndRoleBits(Scorer scorer, SparseFixedBitSet roleBits } } - String evaluateTemplate(String querySource) throws IOException { - // EMPTY is safe here because we never use namedObject - try (XContentParser parser = XContentFactory.xContent(querySource).createParser(NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, querySource)) { - XContentParser.Token token = parser.nextToken(); - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("Unexpected token [" + token + "]"); - } - token = parser.nextToken(); - if (token != XContentParser.Token.FIELD_NAME) { - throw new ElasticsearchParseException("Unexpected token [" + token + "]"); - } - if ("template".equals(parser.currentName())) { - token = parser.nextToken(); - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("Unexpected token [" + token + "]"); - } - Script script = Script.parse(parser); - // Add the user details to the params - Map params = new HashMap<>(); - if (script.getParams() != null) { - params.putAll(script.getParams()); - } - User user = getUser(); - Map userModel = new HashMap<>(); - userModel.put("username", user.principal()); - userModel.put("full_name", user.fullName()); - userModel.put("email", user.email()); - userModel.put("roles", Arrays.asList(user.roles())); - userModel.put("metadata", Collections.unmodifiableMap(user.metadata())); - params.put("_user", userModel); - // Always enforce mustache script lang: - script = new Script(script.getType(), - script.getType() == ScriptType.STORED ? null : "mustache", script.getIdOrCode(), script.getOptions(), params); - TemplateScript compiledTemplate = scriptService.compile(script, TemplateScript.CONTEXT).newInstance(script.getParams()); - return compiledTemplate.execute(); - } else { - return querySource; - } - } - } - protected IndicesAccessControl getIndicesAccessControl() { IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); if (indicesAccessControl == null) { @@ -310,65 +214,4 @@ protected User getUser(){ return authentication.getUser(); } - /** - * Checks whether the role query contains queries we know can't be used as DLS role query. - */ - static void verifyRoleQuery(QueryBuilder queryBuilder) throws IOException { - if (queryBuilder instanceof TermsQueryBuilder) { - TermsQueryBuilder termsQueryBuilder = (TermsQueryBuilder) queryBuilder; - if (termsQueryBuilder.termsLookup() != null) { - throw new IllegalArgumentException("terms query with terms lookup isn't supported as part of a role query"); - } - } else if (queryBuilder instanceof GeoShapeQueryBuilder) { - GeoShapeQueryBuilder geoShapeQueryBuilder = (GeoShapeQueryBuilder) queryBuilder; - if (geoShapeQueryBuilder.shape() == null) { - throw new IllegalArgumentException("geoshape query referring to indexed shapes isn't support as part of a role query"); - } - } else if (queryBuilder.getName().equals("percolate")) { - // actually only if percolate query is referring to an existing document then this is problematic, - // a normal percolate query does work. However we can't check that here as this query builder is inside - // another module. So we don't allow the entire percolate query. I don't think users would ever use - // a percolate query as role query, so this restriction shouldn't prohibit anyone from using dls. - throw new IllegalArgumentException("percolate query isn't support as part of a role query"); - } else if (queryBuilder.getName().equals("has_child")) { - throw new IllegalArgumentException("has_child query isn't support as part of a role query"); - } else if (queryBuilder.getName().equals("has_parent")) { - throw new IllegalArgumentException("has_parent query isn't support as part of a role query"); - } else if (queryBuilder instanceof BoolQueryBuilder) { - BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder; - List clauses = new ArrayList<>(); - clauses.addAll(boolQueryBuilder.filter()); - clauses.addAll(boolQueryBuilder.must()); - clauses.addAll(boolQueryBuilder.mustNot()); - clauses.addAll(boolQueryBuilder.should()); - for (QueryBuilder clause : clauses) { - verifyRoleQuery(clause); - } - } else if (queryBuilder instanceof ConstantScoreQueryBuilder) { - verifyRoleQuery(((ConstantScoreQueryBuilder) queryBuilder).innerQuery()); - } else if (queryBuilder instanceof FunctionScoreQueryBuilder) { - verifyRoleQuery(((FunctionScoreQueryBuilder) queryBuilder).query()); - } else if (queryBuilder instanceof BoostingQueryBuilder) { - verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery()); - verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery()); - } - } - - /** - * Fall back validation that verifies that queries during rewrite don't use - * the client to make remote calls. In the case of DLS this can cause a dead - * lock if DLS is also applied on these remote calls. For example in the - * case of terms query with lookup, this can cause recursive execution of - * the DLS query until the get thread pool has been exhausted: - * https://github.com/elastic/x-plugins/issues/3145 - */ - static void failIfQueryUsesClient(QueryBuilder queryBuilder, QueryRewriteContext original) - throws IOException { - QueryRewriteContext copy = new QueryRewriteContext( - original.getXContentRegistry(), original.getWriteableRegistry(), null, original::nowInMillis); - Rewriteable.rewrite(queryBuilder, copy); - if (copy.hasAsyncActions()) { - throw new IllegalStateException("role queries are not allowed to execute additional requests"); - } - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java index 073e92f7faf44..0cd4e8a8b0ddc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java @@ -12,10 +12,12 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; +import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; import org.elasticsearch.xpack.core.security.support.Automatons; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -83,6 +85,40 @@ public boolean grants(ApplicationPrivilege other, String resource) { return matched; } + /** + * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a + * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to + * whether it is allowed or not. + * + * @param applicationName checks privileges for the provided application name + * @param checkForResources check permission grants for the set of resources + * @param checkForPrivilegeNames check permission grants for the set of privilege names + * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are + * performed + * @return an instance of {@link ResourcePrivilegesMap} + */ + public ResourcePrivilegesMap checkResourcePrivileges(final String applicationName, Set checkForResources, + Set checkForPrivilegeNames, + Collection storedPrivileges) { + final ResourcePrivilegesMap.Builder resourcePrivilegesMapBuilder = ResourcePrivilegesMap.builder(); + for (String checkResource : checkForResources) { + for (String checkPrivilegeName : checkForPrivilegeNames) { + final Set nameSet = Collections.singleton(checkPrivilegeName); + final ApplicationPrivilege checkPrivilege = ApplicationPrivilege.get(applicationName, nameSet, storedPrivileges); + assert checkPrivilege.getApplication().equals(applicationName) : "Privilege " + checkPrivilege + " should have application " + + applicationName; + assert checkPrivilege.name().equals(nameSet) : "Privilege " + checkPrivilege + " should have name " + nameSet; + + if (grants(checkPrivilege, checkResource)) { + resourcePrivilegesMapBuilder.addResourcePrivilege(checkResource, checkPrivilegeName, Boolean.TRUE); + } else { + resourcePrivilegesMapBuilder.addResourcePrivilege(checkResource, checkPrivilegeName, Boolean.FALSE); + } + } + } + return resourcePrivilegesMapBuilder.build(); + } + @Override public String toString() { return getClass().getSimpleName() + "{privileges=" + permissions + "}"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java index 3af016959d4ed..687798971399f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.security.authz.permission; +import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; @@ -33,6 +34,10 @@ public ClusterPrivilege privilege() { public abstract boolean check(String action, TransportRequest request); + public boolean grants(ClusterPrivilege clusterPrivilege) { + return Operations.subsetOf(clusterPrivilege.getAutomaton(), this.privilege().getAutomaton()); + } + public abstract List> privileges(); /** @@ -111,5 +116,10 @@ public List> privileges() { public boolean check(String action, TransportRequest request) { return children.stream().anyMatch(p -> p.check(action, request)); } + + @Override + public boolean grants(ClusterPrivilege clusterPrivilege) { + return children.stream().anyMatch(p -> p.grants(clusterPrivilege)); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java new file mode 100644 index 0000000000000..08d754b4e5357 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java @@ -0,0 +1,262 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.permission; + +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.search.join.ToChildBlockJoinQuery; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.GeoShapeQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.Rewriteable; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.index.search.NestedHelper; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.xpack.core.security.authz.support.SecurityQueryTemplateEvaluator; +import org.elasticsearch.xpack.core.security.user.User; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +import static org.apache.lucene.search.BooleanClause.Occur.FILTER; +import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; + +/** + * Stores document level permissions in the form queries that match all the accessible documents.
+ * The document level permissions may be limited by another set of queries in that case the limited + * queries are used as an additional filter. + */ +public final class DocumentPermissions { + private final Set queries; + private final Set limitedByQueries; + + private static DocumentPermissions ALLOW_ALL = new DocumentPermissions(); + + DocumentPermissions() { + this.queries = null; + this.limitedByQueries = null; + } + + DocumentPermissions(Set queries) { + this(queries, null); + } + + DocumentPermissions(Set queries, Set scopedByQueries) { + if (queries == null && scopedByQueries == null) { + throw new IllegalArgumentException("one of the queries or scoped queries must be provided"); + } + this.queries = (queries != null) ? Collections.unmodifiableSet(queries) : queries; + this.limitedByQueries = (scopedByQueries != null) ? Collections.unmodifiableSet(scopedByQueries) : scopedByQueries; + } + + public Set getQueries() { + return queries; + } + + public Set getLimitedByQueries() { + return limitedByQueries; + } + + /** + * @return {@code true} if either queries or scoped queries are present for document level security else returns {@code false} + */ + public boolean hasDocumentLevelPermissions() { + return queries != null || limitedByQueries != null; + } + + /** + * Creates a {@link BooleanQuery} to be used as filter to restrict access to documents.
+ * Document permission queries are used to create an boolean query.
+ * If the document permissions are limited, then there is an additional filter added restricting access to documents only allowed by the + * limited queries. + * + * @param user authenticated {@link User} + * @param scriptService {@link ScriptService} for evaluating query templates + * @param shardId {@link ShardId} + * @param queryShardContextProvider {@link QueryShardContext} + * @return {@link BooleanQuery} for the filter + * @throws IOException thrown if there is an exception during parsing + */ + public BooleanQuery filter(User user, ScriptService scriptService, ShardId shardId, + Function queryShardContextProvider) throws IOException { + if (hasDocumentLevelPermissions()) { + BooleanQuery.Builder filter; + if (queries != null && limitedByQueries != null) { + filter = new BooleanQuery.Builder(); + BooleanQuery.Builder scopedFilter = new BooleanQuery.Builder(); + buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, limitedByQueries, scopedFilter); + filter.add(scopedFilter.build(), FILTER); + + buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, queries, filter); + } else if (queries != null) { + filter = new BooleanQuery.Builder(); + buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, queries, filter); + } else if (limitedByQueries != null) { + filter = new BooleanQuery.Builder(); + buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, limitedByQueries, filter); + } else { + return null; + } + return filter.build(); + } + return null; + } + + private static void buildRoleQuery(User user, ScriptService scriptService, ShardId shardId, + Function queryShardContextProvider, Set queries, + BooleanQuery.Builder filter) throws IOException { + for (BytesReference bytesReference : queries) { + QueryShardContext queryShardContext = queryShardContextProvider.apply(shardId); + String templateResult = SecurityQueryTemplateEvaluator.evaluateTemplate(bytesReference.utf8ToString(), scriptService, user); + try (XContentParser parser = XContentFactory.xContent(templateResult).createParser(queryShardContext.getXContentRegistry(), + LoggingDeprecationHandler.INSTANCE, templateResult)) { + QueryBuilder queryBuilder = queryShardContext.parseInnerQueryBuilder(parser); + verifyRoleQuery(queryBuilder); + failIfQueryUsesClient(queryBuilder, queryShardContext); + Query roleQuery = queryShardContext.toQuery(queryBuilder).query(); + filter.add(roleQuery, SHOULD); + if (queryShardContext.getMapperService().hasNested()) { + NestedHelper nestedHelper = new NestedHelper(queryShardContext.getMapperService()); + if (nestedHelper.mightMatchNestedDocs(roleQuery)) { + roleQuery = new BooleanQuery.Builder().add(roleQuery, FILTER) + .add(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()), FILTER).build(); + } + // If access is allowed on root doc then also access is allowed on all nested docs of that root document: + BitSetProducer rootDocs = queryShardContext + .bitsetFilter(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated())); + ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs); + filter.add(includeNestedDocs, SHOULD); + } + } + } + // at least one of the queries should match + filter.setMinimumNumberShouldMatch(1); + } + + /** + * Checks whether the role query contains queries we know can't be used as DLS role query. + */ + static void verifyRoleQuery(QueryBuilder queryBuilder) throws IOException { + if (queryBuilder instanceof TermsQueryBuilder) { + TermsQueryBuilder termsQueryBuilder = (TermsQueryBuilder) queryBuilder; + if (termsQueryBuilder.termsLookup() != null) { + throw new IllegalArgumentException("terms query with terms lookup isn't supported as part of a role query"); + } + } else if (queryBuilder instanceof GeoShapeQueryBuilder) { + GeoShapeQueryBuilder geoShapeQueryBuilder = (GeoShapeQueryBuilder) queryBuilder; + if (geoShapeQueryBuilder.shape() == null) { + throw new IllegalArgumentException("geoshape query referring to indexed shapes isn't support as part of a role query"); + } + } else if (queryBuilder.getName().equals("percolate")) { + // actually only if percolate query is referring to an existing document then this is problematic, + // a normal percolate query does work. However we can't check that here as this query builder is inside + // another module. So we don't allow the entire percolate query. I don't think users would ever use + // a percolate query as role query, so this restriction shouldn't prohibit anyone from using dls. + throw new IllegalArgumentException("percolate query isn't support as part of a role query"); + } else if (queryBuilder.getName().equals("has_child")) { + throw new IllegalArgumentException("has_child query isn't support as part of a role query"); + } else if (queryBuilder.getName().equals("has_parent")) { + throw new IllegalArgumentException("has_parent query isn't support as part of a role query"); + } else if (queryBuilder instanceof BoolQueryBuilder) { + BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder; + List clauses = new ArrayList<>(); + clauses.addAll(boolQueryBuilder.filter()); + clauses.addAll(boolQueryBuilder.must()); + clauses.addAll(boolQueryBuilder.mustNot()); + clauses.addAll(boolQueryBuilder.should()); + for (QueryBuilder clause : clauses) { + verifyRoleQuery(clause); + } + } else if (queryBuilder instanceof ConstantScoreQueryBuilder) { + verifyRoleQuery(((ConstantScoreQueryBuilder) queryBuilder).innerQuery()); + } else if (queryBuilder instanceof FunctionScoreQueryBuilder) { + verifyRoleQuery(((FunctionScoreQueryBuilder) queryBuilder).query()); + } else if (queryBuilder instanceof BoostingQueryBuilder) { + verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery()); + verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery()); + } + } + + /** + * Fall back validation that verifies that queries during rewrite don't use + * the client to make remote calls. In the case of DLS this can cause a dead + * lock if DLS is also applied on these remote calls. For example in the + * case of terms query with lookup, this can cause recursive execution of + * the DLS query until the get thread pool has been exhausted: + * https://github.com/elastic/x-plugins/issues/3145 + */ + static void failIfQueryUsesClient(QueryBuilder queryBuilder, QueryRewriteContext original) + throws IOException { + QueryRewriteContext copy = new QueryRewriteContext( + original.getXContentRegistry(), original.getWriteableRegistry(), null, original::nowInMillis); + Rewriteable.rewrite(queryBuilder, copy); + if (copy.hasAsyncActions()) { + throw new IllegalStateException("role queries are not allowed to execute additional requests"); + } + } + + /** + * Create {@link DocumentPermissions} for given set of queries + * @param queries set of queries + * @return {@link DocumentPermissions} + */ + public static DocumentPermissions filteredBy(Set queries) { + if (queries == null || queries.isEmpty()) { + throw new IllegalArgumentException("null or empty queries not permitted"); + } + return new DocumentPermissions(queries); + } + + /** + * Create {@link DocumentPermissions} with no restriction. The {@link #getQueries()} + * will return {@code null} in this case and {@link #hasDocumentLevelPermissions()} + * will be {@code false} + * @return {@link DocumentPermissions} + */ + public static DocumentPermissions allowAll() { + return ALLOW_ALL; + } + + /** + * Create a document permissions, where the permissions for {@code this} are + * limited by the queries from other document permissions.
+ * + * @param limitedByDocumentPermissions {@link DocumentPermissions} used to limit the document level access + * @return instance of {@link DocumentPermissions} + */ + public DocumentPermissions limitDocumentPermissions( + DocumentPermissions limitedByDocumentPermissions) { + assert limitedByQueries == null + && limitedByDocumentPermissions.limitedByQueries == null : "nested scoping for document permissions is not permitted"; + if (queries == null && limitedByDocumentPermissions.queries == null) { + return DocumentPermissions.allowAll(); + } + return new DocumentPermissions(queries, limitedByDocumentPermissions.queries); + } + + @Override + public String toString() { + return "DocumentPermissions [queries=" + queries + ", scopedByQueries=" + limitedByQueries + "]"; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index 7e45b893fed6b..f58367dc43886 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -90,13 +90,15 @@ public FieldPermissions(FieldPermissionsDefinition fieldPermissionsDefinition) { long ramBytesUsed = BASE_FIELD_PERM_DEF_BYTES; - for (FieldGrantExcludeGroup group : fieldPermissionsDefinition.getFieldGrantExcludeGroups()) { - ramBytesUsed += BASE_FIELD_GROUP_BYTES + BASE_HASHSET_ENTRY_SIZE; - if (group.getGrantedFields() != null) { - ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getGrantedFields()); - } - if (group.getExcludedFields() != null) { - ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getExcludedFields()); + if (fieldPermissionsDefinition != null) { + for (FieldGrantExcludeGroup group : fieldPermissionsDefinition.getFieldGrantExcludeGroups()) { + ramBytesUsed += BASE_FIELD_GROUP_BYTES + BASE_HASHSET_ENTRY_SIZE; + if (group.getGrantedFields() != null) { + ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getGrantedFields()); + } + if (group.getExcludedFields() != null) { + ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getExcludedFields()); + } } } ramBytesUsed += permittedFieldsAutomaton.ramBytesUsed(); @@ -153,6 +155,28 @@ private static Automaton initializePermittedFieldsAutomaton(final String[] grant return grantedFieldsAutomaton; } + /** + * Returns a field permissions instance where it is limited by the given field permissions.
+ * If the current and the other field permissions have field level security then it takes + * an intersection of permitted fields.
+ * If none of the permissions have field level security enabled, then returns permissions + * instance where all fields are allowed. + * + * @param limitedBy {@link FieldPermissions} used to limit current field permissions + * @return {@link FieldPermissions} + */ + public FieldPermissions limitFieldPermissions(FieldPermissions limitedBy) { + if (hasFieldLevelSecurity() && limitedBy != null && limitedBy.hasFieldLevelSecurity()) { + Automaton permittedFieldsAutomaton = Automatons.intersectAndMinimize(getIncludeAutomaton(), limitedBy.getIncludeAutomaton()); + return new FieldPermissions(null, permittedFieldsAutomaton); + } else if (limitedBy != null && limitedBy.hasFieldLevelSecurity()) { + return new FieldPermissions(limitedBy.getFieldPermissionsDefinition(), limitedBy.getIncludeAutomaton()); + } else if (hasFieldLevelSecurity()) { + return new FieldPermissions(getFieldPermissionsDefinition(), getIncludeAutomaton()); + } + return FieldPermissions.DEFAULT; + } + /** * Returns true if this field permission policy allows access to the field and false if not. * fieldName can be a wildcard. @@ -178,7 +202,6 @@ public DirectoryReader filter(DirectoryReader reader) throws IOException { return FieldSubsetReader.wrap(reader, permittedFieldsAutomaton); } - // for testing only Automaton getIncludeAutomaton() { return originalAutomaton; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java index 27fa8b2cd9da0..006c6661d2c4c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.cluster.metadata.AliasOrIndex; @@ -23,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -123,6 +125,49 @@ public boolean check(String action) { return false; } + /** + * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap} + * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it + * is allowed or not. + * + * @param checkForIndexPatterns check permission grants for the set of index patterns + * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching + * @param checkForPrivileges check permission grants for the set of index privileges + * @return an instance of {@link ResourcePrivilegesMap} + */ + public ResourcePrivilegesMap checkResourcePrivileges(Set checkForIndexPatterns, boolean allowRestrictedIndices, + Set checkForPrivileges) { + final ResourcePrivilegesMap.Builder resourcePrivilegesMapBuilder = ResourcePrivilegesMap.builder(); + final Map predicateCache = new HashMap<>(); + for (String forIndexPattern : checkForIndexPatterns) { + final Automaton checkIndexAutomaton = IndicesPermission.Group.buildIndexMatcherAutomaton(allowRestrictedIndices, + forIndexPattern); + Automaton allowedIndexPrivilegesAutomaton = null; + for (Group group : groups) { + final Automaton groupIndexAutomaton = predicateCache.computeIfAbsent(group, + g -> IndicesPermission.Group.buildIndexMatcherAutomaton(g.allowRestrictedIndices(), g.indices())); + if (Operations.subsetOf(checkIndexAutomaton, groupIndexAutomaton)) { + if (allowedIndexPrivilegesAutomaton != null) { + allowedIndexPrivilegesAutomaton = Automatons + .unionAndMinimize(Arrays.asList(allowedIndexPrivilegesAutomaton, group.privilege().getAutomaton())); + } else { + allowedIndexPrivilegesAutomaton = group.privilege().getAutomaton(); + } + } + } + for (String privilege : checkForPrivileges) { + IndexPrivilege indexPrivilege = IndexPrivilege.get(Collections.singleton(privilege)); + if (allowedIndexPrivilegesAutomaton != null + && Operations.subsetOf(indexPrivilege.getAutomaton(), allowedIndexPrivilegesAutomaton)) { + resourcePrivilegesMapBuilder.addResourcePrivilege(forIndexPattern, privilege, Boolean.TRUE); + } else { + resourcePrivilegesMapBuilder.addResourcePrivilege(forIndexPattern, privilege, Boolean.FALSE); + } + } + } + return resourcePrivilegesMapBuilder.build(); + } + public Automaton allowedActionsMatcher(String index) { List automatonList = new ArrayList<>(); for (Group group : groups) { @@ -207,7 +252,8 @@ public Map authorize(String act } else { fieldPermissions = FieldPermissions.DEFAULT; } - indexPermissions.put(index, new IndicesAccessControl.IndexAccessControl(entry.getValue(), fieldPermissions, roleQueries)); + indexPermissions.put(index, new IndicesAccessControl.IndexAccessControl(entry.getValue(), fieldPermissions, + (roleQueries != null) ? DocumentPermissions.filteredBy(roleQueries) : DocumentPermissions.allowAll())); } return unmodifiableMap(indexPermissions); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java new file mode 100644 index 0000000000000..809b95965340e --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.permission; + +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; +import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; +import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; + +import java.util.Collection; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; + +/** + * A {@link Role} limited by another role.
+ * The effective permissions returned on {@link #authorize(String, Set, MetaData, FieldPermissionsCache)} call would be limited by the + * provided role. + */ +public final class LimitedRole extends Role { + private final Role limitedBy; + + LimitedRole(String[] names, ClusterPermission cluster, IndicesPermission indices, ApplicationPermission application, + RunAsPermission runAs, Role limitedBy) { + super(names, cluster, indices, application, runAs); + assert limitedBy != null : "limiting role is required"; + this.limitedBy = limitedBy; + } + + public Role limitedBy() { + return limitedBy; + } + + @Override + public IndicesAccessControl authorize(String action, Set requestedIndicesOrAliases, MetaData metaData, + FieldPermissionsCache fieldPermissionsCache) { + IndicesAccessControl indicesAccessControl = super.authorize(action, requestedIndicesOrAliases, metaData, fieldPermissionsCache); + IndicesAccessControl limitedByIndicesAccessControl = limitedBy.authorize(action, requestedIndicesOrAliases, metaData, + fieldPermissionsCache); + + return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl); + } + + /** + * @return A predicate that will match all the indices that this role and the limited by role has the privilege for executing the given + * action on. + */ + @Override + public Predicate allowedIndicesMatcher(String action) { + Predicate predicate = indices().allowedIndicesMatcher(action); + predicate = predicate.and(limitedBy.indices().allowedIndicesMatcher(action)); + return predicate; + } + + /** + * Check if indices permissions allow for the given action, also checks whether the limited by role allows the given actions + * + * @param action indices action + * @return {@code true} if action is allowed else returns {@code false} + */ + @Override + public boolean checkIndicesAction(String action) { + return super.checkIndicesAction(action) && limitedBy.checkIndicesAction(action); + } + + /** + * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap} + * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it + * is allowed or not.
+ * This one takes intersection of resource privileges with the resource privileges from the limited-by role. + * + * @param checkForIndexPatterns check permission grants for the set of index patterns + * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching + * @param checkForPrivileges check permission grants for the set of index privileges + * @return an instance of {@link ResourcePrivilegesMap} + */ + @Override + public ResourcePrivilegesMap checkIndicesPrivileges(Set checkForIndexPatterns, boolean allowRestrictedIndices, + Set checkForPrivileges) { + ResourcePrivilegesMap resourcePrivilegesMap = super.indices().checkResourcePrivileges(checkForIndexPatterns, allowRestrictedIndices, + checkForPrivileges); + ResourcePrivilegesMap resourcePrivilegesMapForLimitedRole = limitedBy.indices().checkResourcePrivileges(checkForIndexPatterns, + allowRestrictedIndices, checkForPrivileges); + return ResourcePrivilegesMap.intersection(resourcePrivilegesMap, resourcePrivilegesMapForLimitedRole); + } + + /** + * Check if cluster permissions allow for the given action, also checks whether the limited by role allows the given actions + * + * @param action cluster action + * @param request {@link TransportRequest} + * @return {@code true} if action is allowed else returns {@code false} + */ + @Override + public boolean checkClusterAction(String action, TransportRequest request) { + return super.checkClusterAction(action, request) && limitedBy.checkClusterAction(action, request); + } + + /** + * Check if cluster permissions grants the given cluster privilege, also checks whether the limited by role grants the given cluster + * privilege + * + * @param clusterPrivilege cluster privilege + * @return {@code true} if cluster privilege is allowed else returns {@code false} + */ + @Override + public boolean grants(ClusterPrivilege clusterPrivilege) { + return super.grants(clusterPrivilege) && limitedBy.grants(clusterPrivilege); + } + + /** + * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a + * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to + * whether it is allowed or not.
+ * This one takes intersection of resource privileges with the resource privileges from the limited-by role. + * + * @param applicationName checks privileges for the provided application name + * @param checkForResources check permission grants for the set of resources + * @param checkForPrivilegeNames check permission grants for the set of privilege names + * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are + * performed + * @return an instance of {@link ResourcePrivilegesMap} + */ + @Override + public ResourcePrivilegesMap checkApplicationResourcePrivileges(final String applicationName, Set checkForResources, + Set checkForPrivilegeNames, + Collection storedPrivileges) { + ResourcePrivilegesMap resourcePrivilegesMap = super.application().checkResourcePrivileges(applicationName, checkForResources, + checkForPrivilegeNames, storedPrivileges); + ResourcePrivilegesMap resourcePrivilegesMapForLimitedRole = limitedBy.application().checkResourcePrivileges(applicationName, + checkForResources, checkForPrivilegeNames, storedPrivileges); + return ResourcePrivilegesMap.intersection(resourcePrivilegesMap, resourcePrivilegesMapForLimitedRole); + } + + /** + * Create a new role defined by given role and the limited role. + * + * @param fromRole existing role {@link Role} + * @param limitedByRole restrict the newly formed role to the permissions defined by this limited {@link Role} + * @return {@link LimitedRole} + */ + public static LimitedRole createLimitedRole(Role fromRole, Role limitedByRole) { + Objects.requireNonNull(limitedByRole, "limited by role is required to create limited role"); + return new LimitedRole(fromRole.names(), fromRole.cluster(), fromRole.indices(), fromRole.application(), fromRole.runAs(), + limitedByRole); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java new file mode 100644 index 0000000000000..3c64cc4afa8a1 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.permission; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; + +/** + * A generic structure to encapsulate resource to privileges map. + */ +public final class ResourcePrivileges { + + private final String resource; + private final Map privileges; + + ResourcePrivileges(String resource, Map privileges) { + this.resource = Objects.requireNonNull(resource); + this.privileges = Collections.unmodifiableMap(privileges); + } + + public String getResource() { + return resource; + } + + public Map getPrivileges() { + return privileges; + } + + public boolean isAllowed(String privilege) { + return Boolean.TRUE.equals(privileges.get(privilege)); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{" + "resource='" + resource + '\'' + ", privileges=" + privileges + '}'; + } + + @Override + public int hashCode() { + int result = resource.hashCode(); + result = 31 * result + privileges.hashCode(); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + final ResourcePrivileges other = (ResourcePrivileges) o; + return this.resource.equals(other.resource) && this.privileges.equals(other.privileges); + } + + public static Builder builder(String resource) { + return new Builder(resource); + } + + public static final class Builder { + private final String resource; + private Map privileges = new HashMap<>(); + + private Builder(String resource) { + this.resource = resource; + } + + public Builder addPrivilege(String privilege, Boolean allowed) { + this.privileges.compute(privilege, (k, v) -> ((v == null) ? allowed : v && allowed)); + return this; + } + + public Builder addPrivileges(Map privileges) { + for (Entry entry : privileges.entrySet()) { + addPrivilege(entry.getKey(), entry.getValue()); + } + return this; + } + + public ResourcePrivileges build() { + return new ResourcePrivileges(resource, privileges); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java new file mode 100644 index 0000000000000..814a6ed29d39f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.permission; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * A generic structure to encapsulate resources to {@link ResourcePrivileges}. Also keeps track of whether the resource privileges allow + * permissions to all resources. + */ +public final class ResourcePrivilegesMap { + + private final boolean allAllowed; + private final Map resourceToResourcePrivileges; + + public ResourcePrivilegesMap(boolean allAllowed, Map resToResPriv) { + this.allAllowed = allAllowed; + this.resourceToResourcePrivileges = Collections.unmodifiableMap(Objects.requireNonNull(resToResPriv)); + } + + public boolean allAllowed() { + return allAllowed; + } + + public Map getResourceToResourcePrivileges() { + return resourceToResourcePrivileges; + } + + @Override + public int hashCode() { + return Objects.hash(allAllowed, resourceToResourcePrivileges); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final ResourcePrivilegesMap other = (ResourcePrivilegesMap) obj; + return allAllowed == other.allAllowed && Objects.equals(resourceToResourcePrivileges, other.resourceToResourcePrivileges); + } + + @Override + public String toString() { + return "ResourcePrivilegesMap [allAllowed=" + allAllowed + ", resourceToResourcePrivileges=" + resourceToResourcePrivileges + "]"; + } + + public static final class Builder { + private boolean allowAll = true; + private Map resourceToResourcePrivilegesBuilder = new LinkedHashMap<>(); + + public Builder addResourcePrivilege(String resource, String privilege, Boolean allowed) { + assert resource != null && privilege != null + && allowed != null : "resource, privilege and permission(allowed or denied) are required"; + ResourcePrivileges.Builder builder = resourceToResourcePrivilegesBuilder.computeIfAbsent(resource, ResourcePrivileges::builder); + builder.addPrivilege(privilege, allowed); + allowAll = allowAll && allowed; + return this; + } + + public Builder addResourcePrivilege(String resource, Map privilegePermissions) { + assert resource != null && privilegePermissions != null : "resource, privilege permissions(allowed or denied) are required"; + ResourcePrivileges.Builder builder = resourceToResourcePrivilegesBuilder.computeIfAbsent(resource, ResourcePrivileges::builder); + builder.addPrivileges(privilegePermissions); + allowAll = allowAll && privilegePermissions.values().stream().allMatch(b -> Boolean.TRUE.equals(b)); + return this; + } + + public Builder addResourcePrivilegesMap(ResourcePrivilegesMap resourcePrivilegesMap) { + resourcePrivilegesMap.getResourceToResourcePrivileges().entrySet().stream() + .forEach(e -> this.addResourcePrivilege(e.getKey(), e.getValue().getPrivileges())); + return this; + } + + public ResourcePrivilegesMap build() { + Map result = resourceToResourcePrivilegesBuilder.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().build())); + return new ResourcePrivilegesMap(allowAll, result); + } + } + + public static Builder builder() { + return new Builder(); + } + + /** + * Takes an intersection of resource privileges and returns a new instance of {@link ResourcePrivilegesMap}. If one of the resource + * privileges map does not allow access to a resource then the resulting map would also not allow access. + * + * @param left an instance of {@link ResourcePrivilegesMap} + * @param right an instance of {@link ResourcePrivilegesMap} + * @return a new instance of {@link ResourcePrivilegesMap}, an intersection of resource privileges. + */ + public static ResourcePrivilegesMap intersection(final ResourcePrivilegesMap left, final ResourcePrivilegesMap right) { + Objects.requireNonNull(left); + Objects.requireNonNull(right); + final ResourcePrivilegesMap.Builder builder = ResourcePrivilegesMap.builder(); + for (Entry leftResPrivsEntry : left.getResourceToResourcePrivileges().entrySet()) { + final ResourcePrivileges leftResPrivs = leftResPrivsEntry.getValue(); + final ResourcePrivileges rightResPrivs = right.getResourceToResourcePrivileges().get(leftResPrivsEntry.getKey()); + builder.addResourcePrivilege(leftResPrivsEntry.getKey(), leftResPrivs.getPrivileges()); + builder.addResourcePrivilege(leftResPrivsEntry.getKey(), rightResPrivs.getPrivileges()); + } + return builder.build(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java index 1f789e96d5a04..570fa02a9b5ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java @@ -10,9 +10,11 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; +import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; @@ -20,13 +22,15 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; -public final class Role { +public class Role { public static final Role EMPTY = Role.builder("__empty").build(); @@ -44,6 +48,7 @@ public final class Role { this.runAs = Objects.requireNonNull(runAs); } + public String[] names() { return names; } @@ -72,6 +77,79 @@ public static Builder builder(RoleDescriptor rd, FieldPermissionsCache fieldPerm return new Builder(rd, fieldPermissionsCache); } + /** + * @return A predicate that will match all the indices that this role + * has the privilege for executing the given action on. + */ + public Predicate allowedIndicesMatcher(String action) { + return indices().allowedIndicesMatcher(action); + } + + /** + * Check if indices permissions allow for the given action + * + * @param action indices action + * @return {@code true} if action is allowed else returns {@code false} + */ + public boolean checkIndicesAction(String action) { + return indices().check(action); + } + + + /** + * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap} + * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it + * is allowed or not. + * + * @param checkForIndexPatterns check permission grants for the set of index patterns + * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching + * @param checkForPrivileges check permission grants for the set of index privileges + * @return an instance of {@link ResourcePrivilegesMap} + */ + public ResourcePrivilegesMap checkIndicesPrivileges(Set checkForIndexPatterns, boolean allowRestrictedIndices, + Set checkForPrivileges) { + return indices().checkResourcePrivileges(checkForIndexPatterns, allowRestrictedIndices, checkForPrivileges); + } + + /** + * Check if cluster permissions allow for the given action + * + * @param action cluster action + * @param request {@link TransportRequest} + * @return {@code true} if action is allowed else returns {@code false} + */ + public boolean checkClusterAction(String action, TransportRequest request) { + return cluster().check(action, request); + } + + /** + * Check if cluster permissions grants the given cluster privilege + * + * @param clusterPrivilege cluster privilege + * @return {@code true} if cluster privilege is allowed else returns {@code false} + */ + public boolean grants(ClusterPrivilege clusterPrivilege) { + return cluster().grants(clusterPrivilege); + } + + /** + * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a + * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to + * whether it is allowed or not. + * + * @param applicationName checks privileges for the provided application name + * @param checkForResources check permission grants for the set of resources + * @param checkForPrivilegeNames check permission grants for the set of privilege names + * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are + * performed + * @return an instance of {@link ResourcePrivilegesMap} + */ + public ResourcePrivilegesMap checkApplicationResourcePrivileges(final String applicationName, Set checkForResources, + Set checkForPrivilegeNames, + Collection storedPrivileges) { + return application().checkResourcePrivileges(applicationName, checkForResources, checkForPrivilegeNames, storedPrivileges); + } + /** * Returns whether at least one group encapsulated by this indices permissions is authorized to execute the * specified action with the requested indices/aliases. At the same time if field and/or document level security @@ -204,4 +282,5 @@ static Tuple> convertApplicationPrivilege(Stri ), Sets.newHashSet(arp.getResources())); } } + } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java new file mode 100644 index 0000000000000..951c4acf10d0d --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.support; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.TemplateScript; +import org.elasticsearch.xpack.core.security.user.User; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Helper class that helps to evaluate the query source template. + */ +public final class SecurityQueryTemplateEvaluator { + + private SecurityQueryTemplateEvaluator() { + } + + /** + * If the query source is a template, then parses the script, compiles the + * script with user details parameters and then executes it to return the + * query string. + *

+ * Note: This method always enforces "mustache" script language for the + * template. + * + * @param querySource query string template to be evaluated. + * @param scriptService {@link ScriptService} + * @param user {@link User} details for user defined parameters in the + * script. + * @return resultant query string after compiling and executing the script. + * If the source does not contain template then it will return the query + * source without any modifications. + * @throws IOException thrown when there is any error parsing the query + * string. + */ + public static String evaluateTemplate(final String querySource, final ScriptService scriptService, final User user) throws IOException { + // EMPTY is safe here because we never use namedObject + try (XContentParser parser = XContentFactory.xContent(querySource).createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, querySource)) { + XContentParser.Token token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("Unexpected token [" + token + "]"); + } + token = parser.nextToken(); + if (token != XContentParser.Token.FIELD_NAME) { + throw new ElasticsearchParseException("Unexpected token [" + token + "]"); + } + if ("template".equals(parser.currentName())) { + token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("Unexpected token [" + token + "]"); + } + Script script = Script.parse(parser); + // Add the user details to the params + Map params = new HashMap<>(); + if (script.getParams() != null) { + params.putAll(script.getParams()); + } + Map userModel = new HashMap<>(); + userModel.put("username", user.principal()); + userModel.put("full_name", user.fullName()); + userModel.put("email", user.email()); + userModel.put("roles", Arrays.asList(user.roles())); + userModel.put("metadata", Collections.unmodifiableMap(user.metadata())); + params.put("_user", userModel); + // Always enforce mustache script lang: + script = new Script(script.getType(), script.getType() == ScriptType.STORED ? null : "mustache", script.getIdOrCode(), + script.getOptions(), params); + TemplateScript compiledTemplate = scriptService.compile(script, TemplateScript.CONTEXT).newInstance(script.getParams()); + return compiledTemplate.execute(); + } else { + return querySource; + } + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java index a7faf4d223108..4619035d0daaf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java @@ -10,6 +10,16 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.GetApiKeyAction; +import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.GetApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequestBuilder; import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction; @@ -334,6 +344,27 @@ public void invalidateToken(InvalidateTokenRequest request, ActionListener listener) { + client.execute(CreateApiKeyAction.INSTANCE, request, listener); + } + + public void invalidateApiKey(InvalidateApiKeyRequest request, ActionListener listener) { + client.execute(InvalidateApiKeyAction.INSTANCE, request, listener); + } + + public void getApiKey(GetApiKeyRequest request, ActionListener listener) { + client.execute(GetApiKeyAction.INSTANCE, request, listener); + } + public SamlAuthenticateRequestBuilder prepareSamlAuthenticate(byte[] xmlContent, List validIds) { final SamlAuthenticateRequestBuilder builder = new SamlAuthenticateRequestBuilder(client); builder.saml(xmlContent); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java index 87a0099580b5f..7e6fd7ca46283 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java @@ -26,6 +26,7 @@ import static org.apache.lucene.util.automaton.MinimizationOperations.minimize; import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES; import static org.apache.lucene.util.automaton.Operations.concatenate; +import static org.apache.lucene.util.automaton.Operations.intersection; import static org.apache.lucene.util.automaton.Operations.minus; import static org.apache.lucene.util.automaton.Operations.union; import static org.elasticsearch.common.Strings.collectionToDelimitedString; @@ -173,6 +174,11 @@ public static Automaton minusAndMinimize(Automaton a1, Automaton a2) { return minimize(res, maxDeterminizedStates); } + public static Automaton intersectAndMinimize(Automaton a1, Automaton a2) { + Automaton res = intersection(a1, a2); + return minimize(res, maxDeterminizedStates); + } + public static Predicate predicate(String... patterns) { return predicate(Arrays.asList(patterns)); } diff --git a/x-pack/plugin/core/src/main/resources/security-index-template.json b/x-pack/plugin/core/src/main/resources/security-index-template.json index 3723aff9054de..183ffff4ea534 100644 --- a/x-pack/plugin/core/src/main/resources/security-index-template.json +++ b/x-pack/plugin/core/src/main/resources/security-index-template.json @@ -152,6 +152,40 @@ "type" : "date", "format" : "epoch_millis" }, + "api_key_hash" : { + "type" : "keyword", + "index": false, + "doc_values": false + }, + "api_key_invalidated" : { + "type" : "boolean" + }, + "role_descriptors" : { + "type" : "object", + "enabled": false + }, + "limited_by_role_descriptors" : { + "type" : "object", + "enabled": false + }, + "version" : { + "type" : "integer" + }, + "creator" : { + "type" : "object", + "properties" : { + "principal" : { + "type": "keyword" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + }, + "realm" : { + "type" : "keyword" + } + } + }, "rules" : { "type" : "object", "dynamic" : true diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java new file mode 100644 index 0000000000000..fb4f87089e8e7 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class CreateApiKeyRequestBuilderTests extends ESTestCase { + + public void testParserAndCreateApiRequestBuilder() throws IOException { + boolean withExpiration = randomBoolean(); + final String json = "{ \"name\" : \"my-api-key\", " + + ((withExpiration) ? " \"expiration\": \"1d\", " : "") + +" \"role_descriptors\": { \"role-a\": {\"cluster\":[\"a-1\", \"a-2\"]," + + " \"index\": [{\"names\": [\"indx-a\"], \"privileges\": [\"read\"] }] }, " + + " \"role-b\": {\"cluster\":[\"b\"]," + + " \"index\": [{\"names\": [\"indx-b\"], \"privileges\": [\"read\"] }] } " + + "} }"; + final BytesArray source = new BytesArray(json); + final NodeClient mockClient = mock(NodeClient.class); + final CreateApiKeyRequest request = new CreateApiKeyRequestBuilder(mockClient).source(source, XContentType.JSON).request(); + final List actualRoleDescriptors = request.getRoleDescriptors(); + assertThat(request.getName(), equalTo("my-api-key")); + assertThat(actualRoleDescriptors.size(), is(2)); + for (RoleDescriptor rd : actualRoleDescriptors) { + String[] clusters = null; + IndicesPrivileges indicesPrivileges = null; + if (rd.getName().equals("role-a")) { + clusters = new String[] { "a-1", "a-2" }; + indicesPrivileges = RoleDescriptor.IndicesPrivileges.builder().indices("indx-a").privileges("read").build(); + } else if (rd.getName().equals("role-b")){ + clusters = new String[] { "b" }; + indicesPrivileges = RoleDescriptor.IndicesPrivileges.builder().indices("indx-b").privileges("read").build(); + } else { + fail("unexpected role name"); + } + assertThat(rd.getClusterPrivileges(), arrayContainingInAnyOrder(clusters)); + assertThat(rd.getIndicesPrivileges(), + arrayContainingInAnyOrder(indicesPrivileges)); + } + if (withExpiration) { + assertThat(request.getExpiration(), equalTo(TimeValue.parseTimeValue("1d", "expiration"))); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java new file mode 100644 index 0000000000000..654d56b42130e --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +public class CreateApiKeyRequestTests extends ESTestCase { + + public void testNameValidation() { + final String name = randomAlphaOfLengthBetween(1, 256); + CreateApiKeyRequest request = new CreateApiKeyRequest(); + + ActionRequestValidationException ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name is required")); + + request.setName(name); + ve = request.validate(); + assertNull(ve); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> request.setName("")); + assertThat(e.getMessage(), containsString("name must not be null or empty")); + + e = expectThrows(IllegalArgumentException.class, () -> request.setName(null)); + assertThat(e.getMessage(), containsString("name must not be null or empty")); + + request.setName(randomAlphaOfLength(257)); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not be more than 256 characters long")); + + request.setName(" leading space"); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + + request.setName("trailing space "); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + + request.setName(" leading and trailing space "); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + + request.setName("inner space"); + ve = request.validate(); + assertNull(ve); + + request.setName("_foo"); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not begin with an underscore")); + } + + public void testSerialization() throws IOException { + final String name = randomAlphaOfLengthBetween(1, 256); + final TimeValue expiration = randomBoolean() ? null : + TimeValue.parseTimeValue(randomTimeValue(), "test serialization of create api key"); + final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values()); + final int numDescriptors = randomIntBetween(0, 4); + final List descriptorList = new ArrayList<>(); + for (int i = 0; i < numDescriptors; i++) { + descriptorList.add(new RoleDescriptor("role_" + i, new String[] { "all" }, null, null)); + } + + final CreateApiKeyRequest request = new CreateApiKeyRequest(); + request.setName(name); + request.setExpiration(expiration); + + if (refreshPolicy != request.getRefreshPolicy() || randomBoolean()) { + request.setRefreshPolicy(refreshPolicy); + } + if (descriptorList.isEmpty() == false || randomBoolean()) { + request.setRoleDescriptors(descriptorList); + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + request.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + final CreateApiKeyRequest serialized = new CreateApiKeyRequest(in); + assertEquals(name, serialized.getName()); + assertEquals(expiration, serialized.getExpiration()); + assertEquals(refreshPolicy, serialized.getRefreshPolicy()); + assertEquals(descriptorList, serialized.getRoleDescriptors()); + } + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java new file mode 100644 index 0000000000000..20ff4bc251d15 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.hamcrest.Matchers.equalTo; + +public class CreateApiKeyResponseTests extends AbstractXContentTestCase { + + @Override + protected CreateApiKeyResponse doParseInstance(XContentParser parser) throws IOException { + return CreateApiKeyResponse.fromXContent(parser); + } + + @Override + protected CreateApiKeyResponse createTestInstance() { + final String name = randomAlphaOfLengthBetween(1, 256); + final SecureString key = new SecureString(UUIDs.randomBase64UUID().toCharArray()); + final Instant expiration = randomBoolean() ? Instant.now().plus(7L, ChronoUnit.DAYS) : null; + final String id = randomAlphaOfLength(100); + return new CreateApiKeyResponse(name, id, key, expiration); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + public void testSerialization() throws IOException { + final CreateApiKeyResponse response = createTestInstance(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + response.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + CreateApiKeyResponse serialized = new CreateApiKeyResponse(in); + assertThat(serialized, equalTo(response)); + } + } + } + + public void testEqualsHashCode() { + CreateApiKeyResponse createApiKeyResponse = createTestInstance(); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> { + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> { + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration()); + }, CreateApiKeyResponseTests::mutateTestItem); + } + + private static CreateApiKeyResponse mutateTestItem(CreateApiKeyResponse original) { + switch (randomIntBetween(0, 3)) { + case 0: + return new CreateApiKeyResponse(randomAlphaOfLength(5), original.getId(), original.getKey(), original.getExpiration()); + case 1: + return new CreateApiKeyResponse(original.getName(), randomAlphaOfLength(5), original.getKey(), original.getExpiration()); + case 2: + return new CreateApiKeyResponse(original.getName(), original.getId(), new SecureString(UUIDs.randomBase64UUID().toCharArray()), + original.getExpiration()); + case 3: + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), Instant.now()); + default: + return new CreateApiKeyResponse(randomAlphaOfLength(5), original.getId(), original.getKey(), original.getExpiration()); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java new file mode 100644 index 0000000000000..27be0d88eb82c --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class GetApiKeyRequestTests extends ESTestCase { + + public void testRequestValidation() { + GetApiKeyRequest request = GetApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5)); + ActionRequestValidationException ve = request.validate(); + assertNull(ve); + request = GetApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = GetApiKeyRequest.usingRealmName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = GetApiKeyRequest.usingUserName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = GetApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7)); + ve = request.validate(); + assertNull(ve); + } + + public void testRequestValidationFailureScenarios() throws IOException { + class Dummy extends ActionRequest { + String realm; + String user; + String apiKeyId; + String apiKeyName; + + Dummy(String[] a) { + realm = a[0]; + user = a[1]; + apiKeyId = a[2]; + apiKeyName = a[3]; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(realm); + out.writeOptionalString(user); + out.writeOptionalString(apiKeyId); + out.writeOptionalString(apiKeyName); + } + } + + String[][] inputs = new String[][] { + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), + randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" }, + { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" }, + { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } }; + String[][] expectedErrorMessages = new String[][] { { "One of [api key id, api key name, username, realm name] must be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified" }, + { "only one of [api key id, api key name] can be specified" } }; + + for (int caseNo = 0; caseNo < inputs.length; caseNo++) { + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + OutputStreamStreamOutput osso = new OutputStreamStreamOutput(bos)) { + Dummy d = new Dummy(inputs[caseNo]); + d.writeTo(osso); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + InputStreamStreamInput issi = new InputStreamStreamInput(bis); + + GetApiKeyRequest request = new GetApiKeyRequest(issi); + ActionRequestValidationException ve = request.validate(); + assertNotNull(ve); + assertEquals(expectedErrorMessages[caseNo].length, ve.validationErrors().size()); + assertThat(ve.validationErrors(), containsInAnyOrder(expectedErrorMessages[caseNo])); + } + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java new file mode 100644 index 0000000000000..c278c135edaf8 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public class GetApiKeyResponseTests extends ESTestCase { + + public void testSerialization() throws IOException { + boolean withExpiration = randomBoolean(); + ApiKey apiKeyInfo = createApiKeyInfo(randomAlphaOfLength(4), randomAlphaOfLength(5), Instant.now(), + (withExpiration) ? Instant.now() : null, false, randomAlphaOfLength(4), randomAlphaOfLength(5)); + GetApiKeyResponse response = new GetApiKeyResponse(Collections.singletonList(apiKeyInfo)); + try (BytesStreamOutput output = new BytesStreamOutput()) { + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + GetApiKeyResponse serialized = new GetApiKeyResponse(input); + assertThat(serialized.getApiKeyInfos(), equalTo(response.getApiKeyInfos())); + } + } + } + + public void testToXContent() throws IOException { + ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false, + "user-a", "realm-x"); + ApiKey apiKeyInfo2 = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true, + "user-b", "realm-y"); + GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1, apiKeyInfo2)); + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertThat(Strings.toString(builder), equalTo( + "{" + + "\"api_keys\":[" + + "{\"id\":\"id-1\",\"name\":\"name1\",\"creation\":100000,\"expiration\":10000000,\"invalidated\":false," + + "\"username\":\"user-a\",\"realm\":\"realm-x\"}," + + "{\"id\":\"id-2\",\"name\":\"name2\",\"creation\":100000,\"expiration\":10000000,\"invalidated\":true," + + "\"username\":\"user-b\",\"realm\":\"realm-y\"}" + + "]" + + "}")); + } + + private ApiKey createApiKeyInfo(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, + String realm) { + return new ApiKey(name, id, creation, expiration, invalidated, username, realm); + } +} + diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java new file mode 100644 index 0000000000000..3d7fd90234286 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class InvalidateApiKeyRequestTests extends ESTestCase { + + public void testRequestValidation() { + InvalidateApiKeyRequest request = InvalidateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5)); + ActionRequestValidationException ve = request.validate(); + assertNull(ve); + request = InvalidateApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = InvalidateApiKeyRequest.usingRealmName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = InvalidateApiKeyRequest.usingUserName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = InvalidateApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7)); + ve = request.validate(); + assertNull(ve); + } + + public void testRequestValidationFailureScenarios() throws IOException { + class Dummy extends ActionRequest { + String realm; + String user; + String apiKeyId; + String apiKeyName; + + Dummy(String[] a) { + realm = a[0]; + user = a[1]; + apiKeyId = a[2]; + apiKeyName = a[3]; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(realm); + out.writeOptionalString(user); + out.writeOptionalString(apiKeyId); + out.writeOptionalString(apiKeyName); + } + } + + String[][] inputs = new String[][] { + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), + randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" }, + { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" }, + { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } }; + String[][] expectedErrorMessages = new String[][] { { "One of [api key id, api key name, username, realm name] must be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified" }, + { "only one of [api key id, api key name] can be specified" } }; + + + for (int caseNo = 0; caseNo < inputs.length; caseNo++) { + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + OutputStreamStreamOutput osso = new OutputStreamStreamOutput(bos)) { + Dummy d = new Dummy(inputs[caseNo]); + d.writeTo(osso); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + InputStreamStreamInput issi = new InputStreamStreamInput(bis); + + InvalidateApiKeyRequest request = new InvalidateApiKeyRequest(issi); + ActionRequestValidationException ve = request.validate(); + assertNotNull(ve); + assertEquals(expectedErrorMessages[caseNo].length, ve.validationErrors().size()); + assertThat(ve.validationErrors(), containsInAnyOrder(expectedErrorMessages[caseNo])); + } + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java new file mode 100644 index 0000000000000..f4606a4f20f1b --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class InvalidateApiKeyResponseTests extends ESTestCase { + + public void testSerialization() throws IOException { + InvalidateApiKeyResponse response = new InvalidateApiKeyResponse(Arrays.asList("api-key-id-1"), + Arrays.asList("api-key-id-2", "api-key-id-3"), + Arrays.asList(new ElasticsearchException("error1"), + new ElasticsearchException("error2"))); + try (BytesStreamOutput output = new BytesStreamOutput()) { + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + InvalidateApiKeyResponse serialized = new InvalidateApiKeyResponse(input); + assertThat(serialized.getInvalidatedApiKeys(), equalTo(response.getInvalidatedApiKeys())); + assertThat(serialized.getPreviouslyInvalidatedApiKeys(), + equalTo(response.getPreviouslyInvalidatedApiKeys())); + assertThat(serialized.getErrors().size(), equalTo(response.getErrors().size())); + assertThat(serialized.getErrors().get(0).toString(), containsString("error1")); + assertThat(serialized.getErrors().get(1).toString(), containsString("error2")); + } + } + + response = new InvalidateApiKeyResponse(Arrays.asList(generateRandomStringArray(20, 15, false)), + Arrays.asList(generateRandomStringArray(20, 15, false)), + Collections.emptyList()); + try (BytesStreamOutput output = new BytesStreamOutput()) { + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + InvalidateApiKeyResponse serialized = new InvalidateApiKeyResponse(input); + assertThat(serialized.getInvalidatedApiKeys(), equalTo(response.getInvalidatedApiKeys())); + assertThat(serialized.getPreviouslyInvalidatedApiKeys(), + equalTo(response.getPreviouslyInvalidatedApiKeys())); + assertThat(serialized.getErrors().size(), equalTo(response.getErrors().size())); + } + } + } + + public void testToXContent() throws IOException { + InvalidateApiKeyResponse response = new InvalidateApiKeyResponse(Arrays.asList("api-key-id-1"), + Arrays.asList("api-key-id-2", "api-key-id-3"), + Arrays.asList(new ElasticsearchException("error1", new IllegalArgumentException("msg - 1")), + new ElasticsearchException("error2", new IllegalArgumentException("msg - 2")))); + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertThat(Strings.toString(builder), + equalTo("{" + + "\"invalidated_api_keys\":[\"api-key-id-1\"]," + + "\"previously_invalidated_api_keys\":[\"api-key-id-2\",\"api-key-id-3\"]," + + "\"error_count\":2," + + "\"error_details\":[" + + "{\"type\":\"exception\"," + + "\"reason\":\"error1\"," + + "\"caused_by\":{" + + "\"type\":\"illegal_argument_exception\"," + + "\"reason\":\"msg - 1\"}" + + "}," + + "{\"type\":\"exception\"," + + "\"reason\":\"error2\"," + + "\"caused_by\":" + + "{\"type\":\"illegal_argument_exception\"," + + "\"reason\":\"msg - 2\"}" + + "}" + + "]" + + "}")); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java index 0481e01e74ac3..a605917f01c2d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.protocol.AbstractHlrcStreamableXContentTestCase; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges; import org.hamcrest.Matchers; import java.io.IOException; @@ -59,16 +60,17 @@ public void testSerializationV63() throws IOException { } public void testToXContent() throws Exception { - final HasPrivilegesResponse response = new HasPrivilegesResponse("daredevil", false, - Collections.singletonMap("manage", true), - Arrays.asList( - new HasPrivilegesResponse.ResourcePrivileges("staff", - MapBuilder.newMapBuilder(new LinkedHashMap<>()) - .put("read", true).put("index", true).put("delete", false).put("manage", false).map()), - new HasPrivilegesResponse.ResourcePrivileges("customers", - MapBuilder.newMapBuilder(new LinkedHashMap<>()) - .put("read", true).put("index", true).put("delete", true).put("manage", false).map()) - ), Collections.emptyMap()); + final HasPrivilegesResponse response = new HasPrivilegesResponse("daredevil", false, Collections.singletonMap("manage", true), + Arrays.asList( + ResourcePrivileges.builder("staff") + .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<>()).put("read", true) + .put("index", true).put("delete", false).put("manage", false).map()) + .build(), + ResourcePrivileges.builder("customers") + .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<>()).put("read", true) + .put("index", true).put("delete", true).put("manage", false).map()) + .build()), + Collections.emptyMap()); final XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()); response.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -120,9 +122,9 @@ public HasPrivilegesResponse convertHlrcToInternal(org.elasticsearch.client.secu ); } - private static List toResourcePrivileges(Map> map) { + private static List toResourcePrivileges(Map> map) { return map.entrySet().stream() - .map(e -> new HasPrivilegesResponse.ResourcePrivileges(e.getKey(), e.getValue())) + .map(e -> ResourcePrivileges.builder(e.getKey()).addPrivileges(e.getValue()).build()) .collect(Collectors.toList()); } @@ -146,23 +148,23 @@ private HasPrivilegesResponse randomResponse() { for (String priv : randomArray(1, 6, String[]::new, () -> randomAlphaOfLengthBetween(3, 12))) { cluster.put(priv, randomBoolean()); } - final Collection index = randomResourcePrivileges(); - final Map> application = new HashMap<>(); + final Collection index = randomResourcePrivileges(); + final Map> application = new HashMap<>(); for (String app : randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 6).toLowerCase(Locale.ROOT))) { application.put(app, randomResourcePrivileges()); } return new HasPrivilegesResponse(username, randomBoolean(), cluster, index, application); } - private Collection randomResourcePrivileges() { - final Collection list = new ArrayList<>(); + private Collection randomResourcePrivileges() { + final Collection list = new ArrayList<>(); // Use hash set to force a unique set of resources for (String resource : Sets.newHashSet(randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(2, 6)))) { final Map privileges = new HashMap<>(); for (String priv : randomArray(1, 5, String[]::new, () -> randomAlphaOfLengthBetween(3, 8))) { privileges.put(priv, randomBoolean()); } - list.add(new HasPrivilegesResponse.ResourcePrivileges(resource, privileges)); + list.add(ResourcePrivileges.builder(resource).addPrivileges(privileges).build()); } return list; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java index ae7798815731b..9b5bf450924a2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java @@ -122,8 +122,9 @@ public void testSortsWWWAuthenticateHeaderValues() { final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""; final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\""; final String negotiateAuthScheme = randomFrom("Negotiate", "Negotiate Ijoijksdk"); + final String apiKeyAuthScheme = "ApiKey"; final Map> failureResponeHeaders = new HashMap<>(); - final List supportedSchemes = Arrays.asList(basicAuthScheme, bearerAuthScheme, negotiateAuthScheme); + final List supportedSchemes = Arrays.asList(basicAuthScheme, bearerAuthScheme, negotiateAuthScheme, apiKeyAuthScheme); Collections.shuffle(supportedSchemes, random()); failureResponeHeaders.put("WWW-Authenticate", supportedSchemes); final DefaultAuthenticationFailureHandler failuerHandler = new DefaultAuthenticationFailureHandler(failureResponeHeaders); @@ -134,7 +135,7 @@ public void testSortsWWWAuthenticateHeaderValues() { assertThat(ese, is(notNullValue())); assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue())); assertThat(ese.getMessage(), equalTo("error attempting to authenticate request")); - assertWWWAuthenticateWithSchemes(ese, negotiateAuthScheme, bearerAuthScheme, basicAuthScheme); + assertWWWAuthenticateWithSchemes(ese, negotiateAuthScheme, bearerAuthScheme, apiKeyAuthScheme, basicAuthScheme); } private void assertWWWAuthenticateWithSchemes(final ElasticsearchSecurityException ese, final String... schemes) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java index ff132894af8ed..5eccd4090e8bf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java @@ -8,6 +8,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -16,12 +17,14 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Accountable; import org.elasticsearch.client.Client; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -36,14 +39,21 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.AbstractBuilderTestCase; import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.AuthenticationField; +import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; +import org.elasticsearch.xpack.core.security.user.User; import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; @@ -52,7 +62,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -public class SecurityIndexSearcherWrapperIntegrationTests extends ESTestCase { +public class SecurityIndexSearcherWrapperIntegrationTests extends AbstractBuilderTestCase { public void testDLS() throws Exception { ShardId shardId = new ShardId("_index", "_na_", 0); @@ -63,9 +73,12 @@ public void testDLS() throws Exception { .then(invocationOnMock -> Collections.singletonList((String) invocationOnMock.getArguments()[0])); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + final Authentication authentication = mock(Authentication.class); + when(authentication.getUser()).thenReturn(mock(User.class)); + threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), - singleton(new BytesArray("{\"match_all\" : {}}"))); + DocumentPermissions.filteredBy(singleton(new BytesArray("{\"match_all\" : {}}")))); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -158,4 +171,116 @@ protected IndicesAccessControl getIndicesAccessControl() { directoryReader.close(); directory.close(); } + + public void testDLSWithLimitedPermissions() throws Exception { + ShardId shardId = new ShardId("_index", "_na_", 0); + MapperService mapperService = mock(MapperService.class); + ScriptService scriptService = mock(ScriptService.class); + when(mapperService.documentMapper()).thenReturn(null); + when(mapperService.simpleMatchToFullName(anyString())) + .then(invocationOnMock -> Collections.singletonList((String) invocationOnMock.getArguments()[0])); + + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + final Authentication authentication = mock(Authentication.class); + when(authentication.getUser()).thenReturn(mock(User.class)); + threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); + final boolean noFilteredIndexPermissions = randomBoolean(); + boolean restrictiveLimitedIndexPermissions = false; + if (noFilteredIndexPermissions == false) { + restrictiveLimitedIndexPermissions = randomBoolean(); + } + Set queries = new HashSet<>(); + queries.add(new BytesArray("{\"terms\" : { \"f2\" : [\"fv22\"] } }")); + queries.add(new BytesArray("{\"terms\" : { \"f2\" : [\"fv32\"] } }")); + IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new + FieldPermissions(), + DocumentPermissions.filteredBy(queries)); + queries = singleton(new BytesArray("{\"terms\" : { \"f1\" : [\"fv11\", \"fv21\", \"fv31\"] } }")); + if (restrictiveLimitedIndexPermissions) { + queries = singleton(new BytesArray("{\"terms\" : { \"f1\" : [\"fv11\", \"fv31\"] } }")); + } + IndicesAccessControl.IndexAccessControl limitedIndexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new + FieldPermissions(), + DocumentPermissions.filteredBy(queries)); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY); + Client client = mock(Client.class); + when(client.settings()).thenReturn(Settings.EMPTY); + final long nowInMillis = randomNonNegativeLong(); + QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, null, null, mapperService, null, + null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null); + QueryShardContext queryShardContext = spy(realQueryShardContext); + IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() { + @Override + public void onCache(ShardId shardId, Accountable accountable) { + } + + @Override + public void onRemoval(ShardId shardId, Accountable accountable) { + } + }); + + XPackLicenseState licenseState = mock(XPackLicenseState.class); + when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true); + SecurityIndexSearcherWrapper wrapper = new SecurityIndexSearcherWrapper(s -> queryShardContext, + bitsetFilterCache, threadContext, licenseState, scriptService) { + + @Override + protected IndicesAccessControl getIndicesAccessControl() { + IndicesAccessControl indicesAccessControl = new IndicesAccessControl(true, singletonMap("_index", indexAccessControl)); + if (noFilteredIndexPermissions) { + return indicesAccessControl; + } + IndicesAccessControl limitedByIndicesAccessControl = new IndicesAccessControl(true, + singletonMap("_index", limitedIndexAccessControl)); + return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl); + } + }; + + Directory directory = newDirectory(); + IndexWriter iw = new IndexWriter( + directory, + new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE) + ); + + Document doc1 = new Document(); + doc1.add(new StringField("f1", "fv11", Store.NO)); + doc1.add(new StringField("f2", "fv12", Store.NO)); + iw.addDocument(doc1); + Document doc2 = new Document(); + doc2.add(new StringField("f1", "fv21", Store.NO)); + doc2.add(new StringField("f2", "fv22", Store.NO)); + iw.addDocument(doc2); + Document doc3 = new Document(); + doc3.add(new StringField("f1", "fv31", Store.NO)); + doc3.add(new StringField("f2", "fv32", Store.NO)); + iw.addDocument(doc3); + iw.commit(); + iw.close(); + + DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId); + DirectoryReader wrappedDirectoryReader = wrapper.wrap(directoryReader); + IndexSearcher indexSearcher = wrapper.wrap(new IndexSearcher(wrappedDirectoryReader)); + + ScoreDoc[] hits = indexSearcher.search(new MatchAllDocsQuery(), 1000).scoreDocs; + Set actualDocIds = new HashSet<>(); + for (ScoreDoc doc : hits) { + actualDocIds.add(doc.doc); + } + + if (noFilteredIndexPermissions) { + assertThat(actualDocIds, containsInAnyOrder(1, 2)); + } else { + if (restrictiveLimitedIndexPermissions) { + assertThat(actualDocIds, containsInAnyOrder(2)); + } else { + assertThat(actualDocIds, containsInAnyOrder(1, 2)); + } + } + + bitsetFilterCache.close(); + directoryReader.close(); + directory.close(); + } + } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java index 06838ac6ffae1..7900eaba4c848 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java @@ -28,21 +28,16 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.FixedBitSet; -import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.SparseFixedBitSet; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -50,59 +45,35 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.BoostingQueryBuilder; -import org.elasticsearch.index.query.ConstantScoreQueryBuilder; -import org.elasticsearch.index.query.GeoShapeQueryBuilder; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.TermsQueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.TermsLookup; -import org.elasticsearch.join.query.HasChildQueryBuilder; -import org.elasticsearch.join.query.HasParentQueryBuilder; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.script.TemplateScript; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader; +import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; -import org.elasticsearch.xpack.core.security.user.User; import org.junit.After; import org.junit.Before; -import org.mockito.ArgumentCaptor; import java.io.IOException; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; -import java.util.Map; import java.util.Set; import static java.util.Collections.singletonMap; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapper.intersectScorerAndRoleBits; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase { @@ -136,7 +107,7 @@ public void setup() throws Exception { IndexShard indexShard = mock(IndexShard.class); when(indexShard.shardId()).thenReturn(shardId); - Directory directory = new RAMDirectory(); + Directory directory = new MMapDirectory(createTempDir()); IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig()); writer.close(); @@ -156,7 +127,7 @@ public void testDefaultMetaFields() throws Exception { @Override protected IndicesAccessControl getIndicesAccessControl() { IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, - new FieldPermissions(fieldPermissionDef(new String[]{}, null)), null); + new FieldPermissions(fieldPermissionDef(new String[]{}, null)), DocumentPermissions.allowAll()); return new IndicesAccessControl(true, singletonMap("_index", indexAccessControl)); } }; @@ -423,66 +394,6 @@ public void testIndexSearcherWrapperDenseWithDeletions() throws IOException { doTestIndexSearcherWrapper(false, true); } - public void testTemplating() throws Exception { - User user = new User("_username", new String[]{"role1", "role2"}, "_full_name", "_email", - Collections.singletonMap("key", "value"), true); - securityIndexSearcherWrapper = - new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService) { - - @Override - protected User getUser() { - return user; - } - }; - - TemplateScript.Factory compiledTemplate = templateParams -> - new TemplateScript(templateParams) { - @Override - public String execute() { - return "rendered_text"; - } - }; - - when(scriptService.compile(any(Script.class), eq(TemplateScript.CONTEXT))).thenReturn(compiledTemplate); - - XContentBuilder builder = jsonBuilder(); - String query = Strings.toString(new TermQueryBuilder("field", "{{_user.username}}").toXContent(builder, ToXContent.EMPTY_PARAMS)); - Script script = new Script(ScriptType.INLINE, "mustache", query, Collections.singletonMap("custom", "value")); - builder = jsonBuilder().startObject().field("template"); - script.toXContent(builder, ToXContent.EMPTY_PARAMS); - String querySource = Strings.toString(builder.endObject()); - - securityIndexSearcherWrapper.evaluateTemplate(querySource); - ArgumentCaptor