From ae6765b4a5a06c4c613faf73ba6ef4d91c6bab78 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 7 Aug 2023 15:36:34 +0200 Subject: [PATCH 1/4] [Docs] For CCS and CCR local cluster determines priviliges of API key (#98205) This PR adds a short call out to our CCS & CCR docs for the existing, certificate-based security model: when API keys are used for authentication, the privileges of the API key are determined by the local cluster, instead of the remote. This is a recurring source of confusion for customers, and generally un-intuitive behavior. I'm opting for a brief call out, instead of diving into too much detail. To fully explain (or justify) this behavior, we would likely need a lot of text and more context around how API keys work. Keeping it short gives users a pointer in the right direction, without distracting from the main documentation of CCS and CCR. LMWYT! --- .../remote-clusters-privileges.asciidoc | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc b/x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc index d0899494140a7..25adf65442720 100644 --- a/x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc +++ b/x-pack/docs/en/security/authentication/remote-clusters-privileges.asciidoc @@ -17,7 +17,7 @@ retrieve roles dynamically. When you use the APIs to manage roles in the The following requests use the <>. You must have at least the -`manage_security` cluster privilege to use this API. +`manage_security` cluster privilege to use this API. [[remote-clusters-privileges-ccr]] //tag::configure-ccr-privileges[] @@ -33,8 +33,11 @@ On the remote cluster that contains the leader index, the {ccr} role requires the `read_ccr` cluster privilege, and `monitor` and `read` privileges on the leader index. -NOTE: If requests will be issued <>, -then the authenticating user must have the `run_as` privilege on the remote +NOTE: If requests are authenticated with an <>, the API key +requires the above privileges on the **local** cluster, instead of the remote. + +NOTE: If requests are issued <>, +then the authenticating user must have the `run_as` privilege on the remote cluster. The following request creates a `remote-replication` role on the remote cluster: @@ -99,7 +102,7 @@ POST /_security/role/remote-replication } ---- -After creating the `remote-replication` role on each cluster, use the +After creating the `remote-replication` role on each cluster, use the <> to create a user on the local cluster cluster and assign the `remote-replication` role. For example, the following request assigns the `remote-replication` role to a user @@ -133,8 +136,11 @@ local and remote clusters, and then create a user with the required roles. On the remote cluster, the {ccs} role requires the `read` and `read_cross_cluster` privileges for the target indices. -NOTE: If requests will be issued <>, -then the authenticating user must have the `run_as` privilege on the remote +NOTE: If requests are authenticated with an <>, the API key +requires the above privileges on the **local** cluster, instead of the remote. + +NOTE: If requests are issued <>, +then the authenticating user must have the `run_as` privilege on the remote cluster. The following request creates a `remote-search` role on the remote cluster: @@ -180,7 +186,7 @@ POST /_security/role/remote-search {} ---- -After creating the `remote-search` role on each cluster, use the +After creating the `remote-search` role on each cluster, use the <> to create a user on the local cluster and assign the `remote-search` role. For example, the following request assigns the `remote-search` role to a user named `cross-search-user`: @@ -263,7 +269,7 @@ Assign your {kib} users a role that grants PUT /_security/user/cross-cluster-kibana { "password" : "l0ng-r4nd0m-p@ssw0rd", - "roles" : [ + "roles" : [ "logstash-reader", "kibana-access" ] From 9acf5c266245dc966ea4f0926f6cd246a94e8ec7 Mon Sep 17 00:00:00 2001 From: Matt Culbreth Date: Mon, 7 Aug 2023 10:24:45 -0400 Subject: [PATCH 2/4] Updating SLM APIs to INTERNAL for serverless (#98261) * Updating SLM APIs to INTERNAL for serverless * Remove scope from `ReservedSnapshotAction`; spotless fixes --- .../xpack/slm/action/RestDeleteSnapshotLifecycleAction.java | 3 +++ .../xpack/slm/action/RestExecuteSnapshotLifecycleAction.java | 3 +++ .../xpack/slm/action/RestExecuteSnapshotRetentionAction.java | 3 +++ .../elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java | 3 +++ .../xpack/slm/action/RestGetSnapshotLifecycleAction.java | 3 +++ .../xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java | 3 +++ .../xpack/slm/action/RestPutSnapshotLifecycleAction.java | 3 +++ .../org/elasticsearch/xpack/slm/action/RestStartSLMAction.java | 3 +++ .../org/elasticsearch/xpack/slm/action/RestStopSLMAction.java | 3 +++ 9 files changed, 27 insertions(+) diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestDeleteSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestDeleteSnapshotLifecycleAction.java index 65562c4609e3a..bebf7b176600f 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestDeleteSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestDeleteSnapshotLifecycleAction.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.slm.action.DeleteSnapshotLifecycleAction; @@ -17,6 +19,7 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE; +@ServerlessScope(Scope.INTERNAL) public class RestDeleteSnapshotLifecycleAction extends BaseRestHandler { @Override diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotLifecycleAction.java index 02b9b29ca1f40..58e86392e0083 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotLifecycleAction.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotLifecycleAction; @@ -18,6 +20,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; +@ServerlessScope(Scope.INTERNAL) public class RestExecuteSnapshotLifecycleAction extends BaseRestHandler { @Override diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java index 246ed0bec4d96..63ae6576be2fa 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestExecuteSnapshotRetentionAction.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.slm.action.ExecuteSnapshotRetentionAction; @@ -17,6 +19,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; +@ServerlessScope(Scope.INTERNAL) public class RestExecuteSnapshotRetentionAction extends BaseRestHandler { @Override diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java index 1a5bbe01b8c88..a6a208fbf3105 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSLMStatusAction.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.slm.action.GetSLMStatusAction; @@ -17,6 +19,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; +@ServerlessScope(Scope.INTERNAL) public class RestGetSLMStatusAction extends BaseRestHandler { @Override diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleAction.java index 61da343a47e2a..aec1ab4c4ebd2 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleAction.java @@ -11,6 +11,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleAction; @@ -18,6 +20,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; +@ServerlessScope(Scope.INTERNAL) public class RestGetSnapshotLifecycleAction extends BaseRestHandler { @Override diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java index c33a206e1dfc4..56646800db871 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestGetSnapshotLifecycleStatsAction.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.slm.action.GetSnapshotLifecycleStatsAction; @@ -17,6 +19,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; +@ServerlessScope(Scope.INTERNAL) public class RestGetSnapshotLifecycleStatsAction extends BaseRestHandler { @Override diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestPutSnapshotLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestPutSnapshotLifecycleAction.java index fb8169f60b057..ddd86639c9551 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestPutSnapshotLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestPutSnapshotLifecycleAction.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.slm.action.PutSnapshotLifecycleAction; @@ -19,6 +21,7 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; +@ServerlessScope(Scope.INTERNAL) public class RestPutSnapshotLifecycleAction extends BaseRestHandler { @Override diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestStartSLMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestStartSLMAction.java index fffafda42f3f1..c33a31cb7888e 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestStartSLMAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestStartSLMAction.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.slm.action.StartSLMAction; @@ -17,6 +19,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; +@ServerlessScope(Scope.INTERNAL) public class RestStartSLMAction extends BaseRestHandler { @Override diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestStopSLMAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestStopSLMAction.java index 3d8893565e60c..aeeaca2f1b237 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestStopSLMAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/slm/action/RestStopSLMAction.java @@ -10,6 +10,8 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.slm.action.StopSLMAction; @@ -17,6 +19,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; +@ServerlessScope(Scope.INTERNAL) public class RestStopSLMAction extends BaseRestHandler { @Override From 4367c3f31c8ae7a7c4a791fa42a2e67a21026dbb Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Mon, 7 Aug 2023 10:38:12 -0400 Subject: [PATCH 3/4] Add query rulesets counts to enterprise search telemetry (#98071) --- .../apis/list-query-rulesets.asciidoc | 23 +++- docs/reference/rest-api/usage.asciidoc | 6 + .../org/elasticsearch/TransportVersion.java | 3 +- .../EnterpriseSearchFeatureSetUsage.java | 45 ++++++-- ...rchFeatureSetUsageBWCSerializingTests.java | 57 +++++++++ ...SearchFeatureSetUsageSerializingTests.java | 47 -------- .../test/entsearch/100_usage.yml | 109 +++++++++++++++++- .../test/entsearch/225_query_ruleset_list.yml | 102 ++++++++++++++-- .../EnterpriseSearchUsageTransportAction.java | 99 ++++++++++++++-- .../rules/QueryRulesIndexService.java | 25 +++- .../rules/QueryRulesetListItem.java | 53 +++++++-- .../rules/QueryRulesIndexServiceTests.java | 20 +++- ...setsActionResponseBWCSerializingTests.java | 21 +++- 13 files changed, 507 insertions(+), 103 deletions(-) create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsageBWCSerializingTests.java delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsageSerializingTests.java diff --git a/docs/reference/query-rules/apis/list-query-rulesets.asciidoc b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc index b22d253ef9135..96e987033e133 100644 --- a/docs/reference/query-rules/apis/list-query-rulesets.asciidoc +++ b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc @@ -58,16 +58,33 @@ A sample response: "results": [ { "ruleset_id": "ruleset-1", - "rules_count": 10 + "rule_total_count": 10, + "rule_criteria_types_counts: { + "exact": 5, + "fuzzy": 5 + } }, { "ruleset_id": "ruleset-2", - "rules_count": 15 + "rule_total_count": 15, + "rule_criteria_types_counts: { + "exact": 5, + "fuzzy": 10, + "gt": 4 + } }, { "ruleset_id": "ruleset-3", - "rules_count": 5 + "rule_total_count": 5, + "rule_criteria_types_counts: { + "exact": 1, + "contains": 4 + } } ] } ---- +// TEST[skip:TBD] + +[NOTE] +The counts in `rule_criteria_types_counts` may be larger than the value of `rule_total_count`, because a rule may have multiple criteria. diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 7515b69a7bc0c..d3f2df5977c06 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -427,6 +427,12 @@ GET /_xpack/usage }, "analytics_collections": { "count": 0 + }, + "query_rulesets": { + "total_rule_count": 0, + "total_count": 0, + "min_rule_count": 0, + "max_rule_count": 0 } } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index af3e9907b8664..cf956f0a89e75 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -174,9 +174,10 @@ private static TransportVersion registerTransportVersion(int id, String uniqueId public static final TransportVersion V_8_500_049 = registerTransportVersion(8_500_049, "828bb6ce-2fbb-11ee-be56-0242ac120002"); public static final TransportVersion V_8_500_050 = registerTransportVersion(8_500_050, "69722fa2-7c0a-4227-86fb-6d6a9a0a0321"); public static final TransportVersion V_8_500_051 = registerTransportVersion(8_500_051, "a28b43bc-bb5f-4406-afcf-26900aa98a71"); + public static final TransportVersion V_8_500_052 = registerTransportVersion(8_500_052, "2d382b3d-9838-4cce-84c8-4142113e5c2b"); private static class CurrentHolder { - private static final TransportVersion CURRENT = findCurrent(V_8_500_051); + private static final TransportVersion CURRENT = findCurrent(V_8_500_052); // finds the pluggable current version, or uses the given fallback private static TransportVersion findCurrent(TransportVersion fallback) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsage.java index 1d8332eec01cc..0d49c9e2a8c84 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsage.java @@ -15,46 +15,67 @@ import org.elasticsearch.xpack.core.XPackField; import java.io.IOException; -import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Objects; public class EnterpriseSearchFeatureSetUsage extends XPackFeatureSet.Usage { + static final TransportVersion BEHAVIORAL_ANALYTICS_TRANSPORT_VERSION = TransportVersion.V_8_8_1; + static final TransportVersion QUERY_RULES_TRANSPORT_VERSION = TransportVersion.V_8_500_046; + public static final String SEARCH_APPLICATIONS = "search_applications"; public static final String ANALYTICS_COLLECTIONS = "analytics_collections"; + public static final String QUERY_RULESETS = "query_rulesets"; public static final String COUNT = "count"; + public static final String TOTAL_COUNT = "total_count"; + public static final String TOTAL_RULE_COUNT = "total_rule_count"; + public static final String MIN_RULE_COUNT = "min_rule_count"; + public static final String MAX_RULE_COUNT = "max_rule_count"; + public static final String RULE_CRITERIA_TOTAL_COUNTS = "rule_criteria_total_counts"; + private final Map searchApplicationsUsage; private final Map analyticsCollectionsUsage; + private final Map queryRulesUsage; public EnterpriseSearchFeatureSetUsage( boolean available, boolean enabled, Map searchApplicationsUsage, - Map analyticsCollectionsUsage + Map analyticsCollectionsUsage, + Map queryRulesUsage ) { super(XPackField.ENTERPRISE_SEARCH, available, enabled); this.searchApplicationsUsage = Objects.requireNonNull(searchApplicationsUsage); this.analyticsCollectionsUsage = Objects.requireNonNull(analyticsCollectionsUsage); + this.queryRulesUsage = Objects.requireNonNull(queryRulesUsage); } public EnterpriseSearchFeatureSetUsage(StreamInput in) throws IOException { super(in); this.searchApplicationsUsage = in.readMap(); - if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_8_1)) { - this.analyticsCollectionsUsage = in.readMap(); - } else { - this.analyticsCollectionsUsage = Collections.emptyMap(); + Map analyticsCollectionsUsage = new HashMap<>(); + Map queryRulesUsage = new HashMap<>(); + if (in.getTransportVersion().onOrAfter(QUERY_RULES_TRANSPORT_VERSION)) { + analyticsCollectionsUsage = in.readMap(); + queryRulesUsage = in.readMap(); + } else if (in.getTransportVersion().onOrAfter(BEHAVIORAL_ANALYTICS_TRANSPORT_VERSION)) { + analyticsCollectionsUsage = in.readMap(); } + this.analyticsCollectionsUsage = analyticsCollectionsUsage; + this.queryRulesUsage = queryRulesUsage; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeGenericMap(searchApplicationsUsage); - if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_8_1)) { + if (out.getTransportVersion().onOrAfter(BEHAVIORAL_ANALYTICS_TRANSPORT_VERSION)) { out.writeGenericMap(analyticsCollectionsUsage); } + if (out.getTransportVersion().onOrAfter(QUERY_RULES_TRANSPORT_VERSION)) { + out.writeGenericMap(queryRulesUsage); + } } @Override @@ -67,6 +88,7 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx super.innerXContent(builder, params); builder.field(SEARCH_APPLICATIONS, searchApplicationsUsage); builder.field(ANALYTICS_COLLECTIONS, analyticsCollectionsUsage); + builder.field(QUERY_RULESETS, queryRulesUsage); } @Override @@ -75,12 +97,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; EnterpriseSearchFeatureSetUsage that = (EnterpriseSearchFeatureSetUsage) o; return Objects.equals(searchApplicationsUsage, that.searchApplicationsUsage) - && Objects.equals(analyticsCollectionsUsage, that.analyticsCollectionsUsage); + && Objects.equals(analyticsCollectionsUsage, that.analyticsCollectionsUsage) + && Objects.equals(queryRulesUsage, that.queryRulesUsage); } @Override public int hashCode() { - return Objects.hash(searchApplicationsUsage, analyticsCollectionsUsage); + return Objects.hash(searchApplicationsUsage, analyticsCollectionsUsage, queryRulesUsage); } public Map getSearchApplicationsUsage() { @@ -90,4 +113,8 @@ public Map getSearchApplicationsUsage() { public Map getAnalyticsCollectionsUsage() { return analyticsCollectionsUsage; } + + public Map getQueryRulesUsage() { + return queryRulesUsage; + } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsageBWCSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsageBWCSerializingTests.java new file mode 100644 index 0000000000000..04684ed324829 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsageBWCSerializingTests.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.application; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage.BEHAVIORAL_ANALYTICS_TRANSPORT_VERSION; +import static org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage.QUERY_RULES_TRANSPORT_VERSION; + +public class EnterpriseSearchFeatureSetUsageBWCSerializingTests extends AbstractBWCWireSerializationTestCase< + EnterpriseSearchFeatureSetUsage> { + + @Override + protected EnterpriseSearchFeatureSetUsage createTestInstance() { + Map searchApplicationsStats = Map.of(EnterpriseSearchFeatureSetUsage.COUNT, randomLongBetween(0, 100000)); + Map analyticsCollectionsStats = Map.of(EnterpriseSearchFeatureSetUsage.COUNT, randomLongBetween(0, 100000)); + Map queryRulesStats = Map.of(EnterpriseSearchFeatureSetUsage.COUNT, randomLongBetween(0, 100000)); + return new EnterpriseSearchFeatureSetUsage(true, true, searchApplicationsStats, analyticsCollectionsStats, queryRulesStats); + } + + @Override + protected EnterpriseSearchFeatureSetUsage mutateInstance(EnterpriseSearchFeatureSetUsage instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected Writeable.Reader instanceReader() { + return EnterpriseSearchFeatureSetUsage::new; + } + + @Override + protected EnterpriseSearchFeatureSetUsage mutateInstanceForVersion(EnterpriseSearchFeatureSetUsage instance, TransportVersion version) { + if (version.onOrAfter(QUERY_RULES_TRANSPORT_VERSION)) { + return instance; + } else if (version.onOrAfter(BEHAVIORAL_ANALYTICS_TRANSPORT_VERSION)) { + return new EnterpriseSearchFeatureSetUsage( + true, + true, + instance.getSearchApplicationsUsage(), + instance.getAnalyticsCollectionsUsage(), + Map.of() + ); + } else { + return new EnterpriseSearchFeatureSetUsage(true, true, instance.getSearchApplicationsUsage(), Map.of(), Map.of()); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsageSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsageSerializingTests.java deleted file mode 100644 index 036c83212dbbd..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/application/EnterpriseSearchFeatureSetUsageSerializingTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.application; - -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class EnterpriseSearchFeatureSetUsageSerializingTests extends AbstractWireSerializingTestCase { - - @Override - protected EnterpriseSearchFeatureSetUsage createTestInstance() { - Map searchApplicationsStats = new HashMap<>(); - Map analyticsCollectionsStats = new HashMap<>(); - searchApplicationsStats.put(EnterpriseSearchFeatureSetUsage.COUNT, randomLongBetween(0, 100000)); - analyticsCollectionsStats.put(EnterpriseSearchFeatureSetUsage.COUNT, randomLongBetween(0, 100000)); - return new EnterpriseSearchFeatureSetUsage(true, true, searchApplicationsStats, analyticsCollectionsStats); - } - - @Override - protected EnterpriseSearchFeatureSetUsage mutateInstance(EnterpriseSearchFeatureSetUsage instance) throws IOException { - long searchApplicationsCount = (long) instance.getSearchApplicationsUsage().get(EnterpriseSearchFeatureSetUsage.COUNT); - searchApplicationsCount = randomValueOtherThan(searchApplicationsCount, () -> randomLongBetween(0, 100000)); - long analyticsCollectionsCount = (long) instance.getAnalyticsCollectionsUsage().get(EnterpriseSearchFeatureSetUsage.COUNT); - analyticsCollectionsCount = randomValueOtherThan(analyticsCollectionsCount, () -> randomLongBetween(0, 100000)); - - Map searchApplicationsStats = new HashMap<>(); - Map analyticsCollectionsStats = new HashMap<>(); - searchApplicationsStats.put("count", searchApplicationsCount); - analyticsCollectionsStats.put("count", analyticsCollectionsCount); - - return new EnterpriseSearchFeatureSetUsage(true, true, searchApplicationsStats, analyticsCollectionsStats); - } - - @Override - protected Writeable.Reader instanceReader() { - return EnterpriseSearchFeatureSetUsage::new; - } -} diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/100_usage.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/100_usage.yml index 37c17947a3bb9..2d7b56bc175eb 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/100_usage.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/100_usage.yml @@ -35,7 +35,8 @@ teardown: enabled: true, available: true, search_applications: { count: 0 }, - analytics_collections: { count: 0 } + analytics_collections: { count: 0 }, + query_rulesets: { total_count: 0, total_rule_count: 0, min_rule_count: 0, max_rule_count: 0 } } } @@ -59,7 +60,8 @@ teardown: enabled: true, available: true, search_applications: { count: 1 }, - analytics_collections: { count: 0 } + analytics_collections: { count: 0 }, + query_rulesets: { total_count: 0, total_rule_count: 0, min_rule_count: 0, max_rule_count: 0 } } } @@ -87,7 +89,8 @@ teardown: enabled: true, available: true, search_applications: { count: 2 }, - analytics_collections: { count: 1 } + analytics_collections: { count: 1 }, + query_rulesets: { total_count: 0, total_rule_count: 0, min_rule_count: 0, max_rule_count: 0 } } } @@ -103,7 +106,8 @@ teardown: enabled: true, available: true, search_applications: { count: 1 }, - analytics_collections: { count: 1 } + analytics_collections: { count: 1 }, + query_rulesets: { total_count: 0, total_rule_count: 0, min_rule_count: 0, max_rule_count: 0 } } } @@ -119,6 +123,101 @@ teardown: enabled: true, available: true, search_applications: { count: 1 }, - analytics_collections: { count: 0 } + analytics_collections: { count: 0 }, + query_rulesets: { total_count: 0, total_rule_count: 0, min_rule_count: 0, max_rule_count: 0 } } } + + - do: + query_ruleset.put: + ruleset_id: test-query-ruleset + body: + rules: + - rule_id: query-rule-id1 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ puggles ] + actions: + ids: + - 'id1' + - 'id2' + - rule_id: query-rule-id2 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ pugs ] + actions: + ids: + - 'id3' + - 'id4' + + - do: + query_ruleset.put: + ruleset_id: test-query-ruleset2 + body: + rules: + - rule_id: query-rule-id1 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ beagles ] + actions: + ids: + - 'id1' + - 'id2' + - rule_id: query-rule-id2 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ pugs ] + actions: + ids: + - 'id3' + - 'id4' + - rule_id: query-rule-id3 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ puggles ] + actions: + ids: + - 'id4' + - 'id5' + + - do: + xpack.usage: { } + + - match: { + enterprise_search: { + enabled: true, + available: true, + search_applications: { count: 1 }, + analytics_collections: { count: 0 }, + query_rulesets: { total_count: 2, total_rule_count: 5, min_rule_count: 2, max_rule_count: 3, rule_criteria_total_counts: { exact: 5 } } + } + } + + - do: + query_ruleset.delete: + ruleset_id: test-query-ruleset2 + + - do: + xpack.usage: { } + + - match: { + enterprise_search: { + enabled: true, + available: true, + search_applications: { count: 1 }, + analytics_collections: { count: 0 }, + query_rulesets: { total_count: 1, total_rule_count: 2, min_rule_count: 2, max_rule_count: 2, rule_criteria_total_counts: { exact: 2 } } + } + } + + diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/225_query_ruleset_list.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/225_query_ruleset_list.yml index 8829aec093273..4d9022e04206b 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/225_query_ruleset_list.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/225_query_ruleset_list.yml @@ -119,13 +119,16 @@ setup: # Alphabetical order by ruleset_id for results - match: { results.0.ruleset_id: "test-query-ruleset-1" } - - match: { results.0.rules_count: 3 } + - match: { results.0.rule_total_count: 3 } + - match: { results.0.rule_criteria_types_counts: { exact: 3 } } - match: { results.1.ruleset_id: "test-query-ruleset-2" } - - match: { results.1.rules_count: 4 } + - match: { results.1.rule_total_count: 4 } + - match: { results.1.rule_criteria_types_counts: { exact: 4 } } - match: { results.2.ruleset_id: "test-query-ruleset-3" } - - match: { results.2.rules_count: 2 } + - match: { results.2.rule_total_count: 2 } + - match: { results.2.rule_criteria_types_counts: { exact: 2 } } --- "List Query Rulesets - with from": @@ -137,10 +140,12 @@ setup: # Alphabetical order by ruleset_id for results - match: { results.0.ruleset_id: "test-query-ruleset-2" } - - match: { results.0.rules_count: 4 } + - match: { results.0.rule_total_count: 4 } + - match: { results.0.rule_criteria_types_counts: { exact: 4 } } - match: { results.1.ruleset_id: "test-query-ruleset-3" } - - match: { results.1.rules_count: 2 } + - match: { results.1.rule_total_count: 2 } + - match: { results.1.rule_criteria_types_counts: { exact: 2 } } --- "List Query Rulesets - with size": @@ -152,10 +157,12 @@ setup: # Alphabetical order by ruleset_id for results - match: { results.0.ruleset_id: "test-query-ruleset-1" } - - match: { results.0.rules_count: 3 } + - match: { results.0.rule_total_count: 3 } + - match: { results.0.rule_criteria_types_counts: { exact: 3 } } - match: { results.1.ruleset_id: "test-query-ruleset-2" } - - match: { results.1.rules_count: 4 } + - match: { results.1.rule_total_count: 4 } + - match: { results.1.rule_criteria_types_counts: { exact: 4 } } --- "List Query Rulesets - empty": @@ -175,3 +182,84 @@ setup: query_ruleset.list: { } - match: { count: 0 } + +--- +"List Query Rulesets with multiple rules": + - do: + query_ruleset.put: + ruleset_id: a-test-query-ruleset-with-lots-of-criteria + body: + rules: + - rule_id: query-rule-id1 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ puggles ] + - type: gt + metadata: year + values: [ 2023 ] + actions: + ids: + - 'id1' + - 'id2' + - rule_id: query-rule-id2 + type: pinned + criteria: + - type: exact + metadata: query_string + values: [ pug ] + actions: + ids: + - 'id3' + - 'id4' + - rule_id: query-rule-id3 + type: pinned + criteria: + - type: fuzzy + metadata: query_string + values: [ puggles ] + actions: + ids: + - 'id5' + - 'id6' + - rule_id: query-rule-id4 + type: pinned + criteria: + - type: always + actions: + ids: + - 'id7' + - 'id8' + - rule_id: query-rule-id5 + type: pinned + criteria: + - type: prefix + metadata: query_string + values: [ pug ] + - type: suffix + metadata: query_string + values: [ gle ] + actions: + ids: + - 'id9' + - 'id10' + + - do: + query_ruleset.list: + from: 0 + size: 1 + + - match: { count: 4 } + + # Alphabetical order by ruleset_id for results + - match: { results.0.ruleset_id: "a-test-query-ruleset-with-lots-of-criteria" } + - match: { results.0.rule_total_count: 5 } + - match: + results.0.rule_criteria_types_counts: + exact: 2 + gt: 1 + fuzzy: 1 + prefix: 1 + suffix: 1 + always: 1 diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java index 485beb8ba5945..4d5ea0b5a3d01 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java @@ -8,8 +8,13 @@ package org.elasticsearch.xpack.application; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.IndicesAdminClient; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -24,6 +29,10 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.application.analytics.action.GetAnalyticsCollectionAction; +import org.elasticsearch.xpack.application.rules.QueryRuleCriteriaType; +import org.elasticsearch.xpack.application.rules.QueryRulesIndexService; +import org.elasticsearch.xpack.application.rules.QueryRulesetListItem; +import org.elasticsearch.xpack.application.rules.action.ListQueryRulesetsAction; import org.elasticsearch.xpack.application.search.action.ListSearchApplicationAction; import org.elasticsearch.xpack.application.utils.LicenseUtils; import org.elasticsearch.xpack.core.XPackSettings; @@ -35,14 +44,23 @@ import java.util.Collections; import java.util.HashMap; +import java.util.IntSummaryStatistics; +import java.util.List; +import java.util.Locale; import java.util.Map; import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; +import static org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage.MAX_RULE_COUNT; +import static org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage.MIN_RULE_COUNT; +import static org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage.RULE_CRITERIA_TOTAL_COUNTS; +import static org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage.TOTAL_COUNT; +import static org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage.TOTAL_RULE_COUNT; public class EnterpriseSearchUsageTransportAction extends XPackUsageFeatureTransportAction { private static final Logger logger = LogManager.getLogger(EnterpriseSearchUsageTransportAction.class); private final XPackLicenseState licenseState; private final OriginSettingClient clientWithOrigin; + private final IndicesAdminClient indicesAdminClient; private final boolean enabled; @@ -67,6 +85,7 @@ public EnterpriseSearchUsageTransportAction( ); this.licenseState = licenseState; this.clientWithOrigin = new OriginSettingClient(client, ENT_SEARCH_ORIGIN); + this.indicesAdminClient = clientWithOrigin.admin().indices(); this.enabled = XPackSettings.ENTERPRISE_SEARCH_ENABLED.get(settings); } @@ -82,6 +101,7 @@ protected void masterOperation( LicenseUtils.LICENSED_ENT_SEARCH_FEATURE.checkWithoutTracking(licenseState), enabled, Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap() ); listener.onResponse(new XPackUsageFeatureResponse(usage)); @@ -90,8 +110,9 @@ protected void masterOperation( Map searchApplicationsUsage = new HashMap<>(); Map analyticsCollectionsUsage = new HashMap<>(); + Map queryRulesUsage = new HashMap<>(); - // Step 2: Fetch search applications count and return usage + // Step 3: Fetch search applications count and return usage ListSearchApplicationAction.Request searchApplicationsCountRequest = new ListSearchApplicationAction.Request( "*", new PageParams(0, 0) @@ -104,7 +125,8 @@ protected void masterOperation( LicenseUtils.LICENSED_ENT_SEARCH_FEATURE.checkWithoutTracking(licenseState), enabled, searchApplicationsUsage, - analyticsCollectionsUsage + analyticsCollectionsUsage, + queryRulesUsage ) ) ); @@ -115,18 +137,17 @@ protected void masterOperation( LicenseUtils.LICENSED_ENT_SEARCH_FEATURE.checkWithoutTracking(licenseState), enabled, Collections.emptyMap(), - analyticsCollectionsUsage + analyticsCollectionsUsage, + queryRulesUsage ) ) ); }); - // Step 1: Fetch analytics collections count - GetAnalyticsCollectionAction.Request analyticsCollectionsCountRequest = new GetAnalyticsCollectionAction.Request( - new String[] { "*" } - ); - ActionListener analyticsCollectionsCountListener = ActionListener.wrap(response -> { - addAnalyticsCollectionsUsage(response, analyticsCollectionsUsage); + // Step 2: Fetch query rules stats + + ActionListener listQueryRulesetsListener = ActionListener.wrap(response -> { + addQueryRulesetUsage(response, queryRulesUsage); clientWithOrigin.execute(ListSearchApplicationAction.INSTANCE, searchApplicationsCountRequest, searchApplicationsCountListener); }, e -> { @@ -138,6 +159,43 @@ protected void masterOperation( } ); + IndicesStatsRequest indicesStatsRequest = indicesAdminClient.prepareStats(QueryRulesIndexService.QUERY_RULES_ALIAS_NAME) + .setDocs(true) + .request(); + + // Step 1: Fetch analytics collections count + GetAnalyticsCollectionAction.Request analyticsCollectionsCountRequest = new GetAnalyticsCollectionAction.Request( + new String[] { "*" } + ); + + ActionListener analyticsCollectionsCountListener = ActionListener.wrap(response -> { + addAnalyticsCollectionsUsage(response, analyticsCollectionsUsage); + indicesAdminClient.execute(IndicesStatsAction.INSTANCE, indicesStatsRequest, new ActionListener<>() { + @Override + public void onResponse(IndicesStatsResponse indicesStatsResponse) { + Map indicesStats = indicesStatsResponse.getIndices(); + int queryRulesetCount = indicesStats.values() + .stream() + .mapToInt(indexShardStats -> (int) indexShardStats.getPrimaries().getDocs().getCount()) + .sum(); + + ListQueryRulesetsAction.Request queryRulesetsCountRequest = new ListQueryRulesetsAction.Request( + new PageParams(0, queryRulesetCount) + ); + clientWithOrigin.execute(ListQueryRulesetsAction.INSTANCE, queryRulesetsCountRequest, listQueryRulesetsListener); + } + + @Override + public void onFailure(Exception e) { + ListQueryRulesetsAction.Request queryRulesetsCountRequest = new ListQueryRulesetsAction.Request(new PageParams(0, 0)); + clientWithOrigin.execute(ListQueryRulesetsAction.INSTANCE, queryRulesetsCountRequest, listQueryRulesetsListener); + } + }); + }, e -> { + ListQueryRulesetsAction.Request queryRulesetsCountRequest = new ListQueryRulesetsAction.Request(new PageParams(0, 0)); + clientWithOrigin.execute(ListQueryRulesetsAction.INSTANCE, queryRulesetsCountRequest, listQueryRulesetsListener); + }); + // Step 0: Kick off requests clientWithOrigin.execute( GetAnalyticsCollectionAction.INSTANCE, @@ -148,7 +206,6 @@ protected void masterOperation( private void addSearchApplicationsUsage(ListSearchApplicationAction.Response response, Map searchApplicationsUsage) { long count = response.queryPage().count(); - searchApplicationsUsage.put(EnterpriseSearchFeatureSetUsage.COUNT, count); } @@ -157,7 +214,27 @@ private void addAnalyticsCollectionsUsage( Map analyticsCollectionsUsage ) { long count = response.getAnalyticsCollections().size(); - analyticsCollectionsUsage.put(EnterpriseSearchFeatureSetUsage.COUNT, count); } + + private void addQueryRulesetUsage(ListQueryRulesetsAction.Response response, Map queryRulesUsage) { + List results = response.queryPage().results(); + IntSummaryStatistics ruleStats = results.stream().mapToInt(QueryRulesetListItem::ruleTotalCount).summaryStatistics(); + + Map criteriaTypeCountMap = new HashMap<>(); + results.stream() + .flatMap(result -> result.criteriaTypeToCountMap().entrySet().stream()) + .forEach(entry -> criteriaTypeCountMap.merge(entry.getKey(), entry.getValue(), Integer::sum)); + + Map rulesTypeCountMap = new HashMap<>(); + criteriaTypeCountMap.forEach((criteriaType, count) -> rulesTypeCountMap.put(criteriaType.name().toLowerCase(Locale.ROOT), count)); + + queryRulesUsage.put(TOTAL_COUNT, response.queryPage().count()); + queryRulesUsage.put(TOTAL_RULE_COUNT, ruleStats.getSum()); + queryRulesUsage.put(MIN_RULE_COUNT, results.isEmpty() ? 0 : ruleStats.getMin()); + queryRulesUsage.put(MAX_RULE_COUNT, results.isEmpty() ? 0 : ruleStats.getMax()); + if (rulesTypeCountMap.isEmpty() == false) { + queryRulesUsage.put(RULE_CRITERIA_TOTAL_COUNTS, rulesTypeCountMap); + } + } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java index 67ba6e6fce0fc..12ab6f6542fd8 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java @@ -44,6 +44,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -278,7 +280,13 @@ public void listQueryRulesets(int from, int size, ActionListener() { @@ -312,9 +320,20 @@ private static QueryRulesetListItem hitToQueryRulesetListItem(SearchHit searchHi final Map sourceMap = searchHit.getSourceAsMap(); final String rulesetId = (String) sourceMap.get(QueryRuleset.ID_FIELD.getPreferredName()); @SuppressWarnings("unchecked") - final int numRules = ((List) sourceMap.get(QueryRuleset.RULES_FIELD.getPreferredName())).size(); + final List> rules = ((List>) sourceMap.get(QueryRuleset.RULES_FIELD.getPreferredName())); + final int numRules = rules.size(); + final Map queryRuleCriteriaTypeToCountMap = new HashMap<>(); + for (LinkedHashMap rule : rules) { + @SuppressWarnings("unchecked") + List> criteriaList = ((List>) rule.get(QueryRule.CRITERIA_FIELD.getPreferredName())); + for (LinkedHashMap criteria : criteriaList) { + final String criteriaType = ((String) criteria.get(QueryRuleCriteria.TYPE_FIELD.getPreferredName())); + final QueryRuleCriteriaType queryRuleCriteriaType = QueryRuleCriteriaType.type(criteriaType); + queryRuleCriteriaTypeToCountMap.compute(queryRuleCriteriaType, (k, v) -> v == null ? 1 : v + 1); + } + } - return new QueryRulesetListItem(rulesetId, numRules); + return new QueryRulesetListItem(rulesetId, numRules, queryRuleCriteriaTypeToCountMap); } static class DelegatingIndexNotFoundActionListener extends DelegatingActionListener { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java index 56f1fadaa0306..bedfaae57498f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesetListItem.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.application.rules; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -15,6 +16,8 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Locale; +import java.util.Map; import java.util.Objects; /** @@ -23,34 +26,51 @@ */ public class QueryRulesetListItem implements Writeable, ToXContentObject { + // TODO we need to actually bump transport version, but there's no point until main is merged. Placeholder for now. + public static final TransportVersion EXPANDED_RULESET_COUNT_TRANSPORT_VERSION = TransportVersion.V_8_500_052; + public static final ParseField RULESET_ID_FIELD = new ParseField("ruleset_id"); - public static final ParseField NUM_RULES_FIELD = new ParseField("rules_count"); + public static final ParseField RULE_TOTAL_COUNT_FIELD = new ParseField("rule_total_count"); + public static final ParseField RULE_CRITERIA_TYPE_COUNTS_FIELD = new ParseField("rule_criteria_types_counts"); private final String rulesetId; - private final int numRules; + private final int ruleTotalCount; + private final Map criteriaTypeToCountMap; /** * Constructs a QueryRulesetListItem. * * @param rulesetId The unique identifier for the ruleset - * @param numRules The number of rules contained within the ruleset. + * @param ruleTotalCount The number of rules contained within the ruleset. + * @param criteriaTypeToCountMap A map of criteria type to the number of rules of that type. */ - public QueryRulesetListItem(String rulesetId, int numRules) { + public QueryRulesetListItem(String rulesetId, int ruleTotalCount, Map criteriaTypeToCountMap) { Objects.requireNonNull(rulesetId, "rulesetId cannot be null on a QueryRuleListItem"); this.rulesetId = rulesetId; - this.numRules = numRules; + this.ruleTotalCount = ruleTotalCount; + this.criteriaTypeToCountMap = criteriaTypeToCountMap; } public QueryRulesetListItem(StreamInput in) throws IOException { this.rulesetId = in.readString(); - this.numRules = in.readInt(); + this.ruleTotalCount = in.readInt(); + if (in.getTransportVersion().onOrAfter(EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) { + this.criteriaTypeToCountMap = in.readMap(m -> in.readEnum(QueryRuleCriteriaType.class), StreamInput::readInt); + } else { + this.criteriaTypeToCountMap = Map.of(); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(RULESET_ID_FIELD.getPreferredName(), rulesetId); - builder.field(NUM_RULES_FIELD.getPreferredName(), numRules); + builder.field(RULE_TOTAL_COUNT_FIELD.getPreferredName(), ruleTotalCount); + builder.startObject(RULE_CRITERIA_TYPE_COUNTS_FIELD.getPreferredName()); + for (QueryRuleCriteriaType criteriaType : criteriaTypeToCountMap.keySet()) { + builder.field(criteriaType.name().toLowerCase(Locale.ROOT), criteriaTypeToCountMap.get(criteriaType)); + } + builder.endObject(); builder.endObject(); return builder; } @@ -58,7 +78,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(rulesetId); - out.writeInt(numRules); + out.writeInt(ruleTotalCount); + if (out.getTransportVersion().onOrAfter(EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) { + out.writeMap(criteriaTypeToCountMap, StreamOutput::writeEnum, StreamOutput::writeInt); + } } /** @@ -75,8 +98,12 @@ public String rulesetId() { * * @return the total number of rules. */ - public int numRules() { - return numRules; + public int ruleTotalCount() { + return ruleTotalCount; + } + + public Map criteriaTypeToCountMap() { + return criteriaTypeToCountMap; } @Override @@ -84,11 +111,13 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; QueryRulesetListItem that = (QueryRulesetListItem) o; - return numRules == that.numRules && Objects.equals(rulesetId, that.rulesetId); + return ruleTotalCount == that.ruleTotalCount + && Objects.equals(rulesetId, that.rulesetId) + && Objects.equals(criteriaTypeToCountMap, that.criteriaTypeToCountMap); } @Override public int hashCode() { - return Objects.hash(rulesetId, numRules); + return Objects.hash(rulesetId, ruleTotalCount, criteriaTypeToCountMap); } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexServiceTests.java index 7fbddf6d89fc4..1e8fa5953606b 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexServiceTests.java @@ -33,6 +33,8 @@ import static org.elasticsearch.xpack.application.rules.QueryRule.QueryRuleType; import static org.elasticsearch.xpack.application.rules.QueryRuleCriteriaType.EXACT; +import static org.elasticsearch.xpack.application.rules.QueryRuleCriteriaType.FUZZY; +import static org.elasticsearch.xpack.application.rules.QueryRuleCriteriaType.GTE; import static org.elasticsearch.xpack.application.rules.QueryRulesIndexService.QUERY_RULES_CONCRETE_INDEX_NAME; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.equalTo; @@ -108,13 +110,19 @@ public void testListQueryRulesets() throws Exception { new QueryRule( "my_rule_" + i, QueryRuleType.PINNED, - List.of(new QueryRuleCriteria(EXACT, "query_string", List.of("foo" + i))), + List.of( + new QueryRuleCriteria(EXACT, "query_string", List.of("foo" + i)), + new QueryRuleCriteria(GTE, "query_string", List.of(i)) + ), Map.of("ids", List.of("id1", "id2")) ), new QueryRule( "my_rule_" + i + "_" + (i + 1), QueryRuleType.PINNED, - List.of(new QueryRuleCriteria(EXACT, "query_string", List.of("bar" + i))), + List.of( + new QueryRuleCriteria(FUZZY, "query_string", List.of("bar" + i)), + new QueryRuleCriteria(GTE, "user.age", List.of(i)) + ), Map.of("ids", List.of("id3", "id4")) ) ); @@ -147,8 +155,14 @@ public void testListQueryRulesets() throws Exception { for (int i = 0; i < 5; i++) { int index = i + 5; - String rulesetId = rulesets.get(i).rulesetId(); + QueryRulesetListItem ruleset = rulesets.get(i); + String rulesetId = ruleset.rulesetId(); assertThat(rulesetId, equalTo("my_ruleset_" + index)); + Map criteriaTypeCountMap = ruleset.criteriaTypeToCountMap(); + assertThat(criteriaTypeCountMap.size(), equalTo(3)); + assertThat(criteriaTypeCountMap.get(EXACT), equalTo(1)); + assertThat(criteriaTypeCountMap.get(FUZZY), equalTo(1)); + assertThat(criteriaTypeCountMap.get(GTE), equalTo(2)); } } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java index 0b3cbdab9a946..1613e31f94206 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionResponseBWCSerializingTests.java @@ -9,11 +9,16 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.application.rules.QueryRuleCriteriaType; import org.elasticsearch.xpack.application.rules.QueryRuleset; import org.elasticsearch.xpack.application.rules.QueryRulesetListItem; import org.elasticsearch.xpack.application.search.SearchApplicationTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + public class ListQueryRulesetsActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< ListQueryRulesetsAction.Response> { @@ -25,7 +30,11 @@ protected Writeable.Reader instanceReader() { private static ListQueryRulesetsAction.Response randomQueryRulesetListItem() { return new ListQueryRulesetsAction.Response(randomList(10, () -> { QueryRuleset queryRuleset = SearchApplicationTestUtils.randomQueryRuleset(); - return new QueryRulesetListItem(queryRuleset.id(), queryRuleset.rules().size()); + Map criteriaTypeToCountMap = Map.of( + randomFrom(QueryRuleCriteriaType.values()), + randomIntBetween(0, 10) + ); + return new QueryRulesetListItem(queryRuleset.id(), queryRuleset.rules().size(), criteriaTypeToCountMap); }), randomLongBetween(0, 1000)); } @@ -44,6 +53,14 @@ protected ListQueryRulesetsAction.Response mutateInstanceForVersion( ListQueryRulesetsAction.Response instance, TransportVersion version ) { - return instance; + if (version.onOrAfter(QueryRulesetListItem.EXPANDED_RULESET_COUNT_TRANSPORT_VERSION)) { + return instance; + } else { + List updatedResults = new ArrayList<>(); + for (QueryRulesetListItem listItem : instance.queryPage.results()) { + updatedResults.add(new QueryRulesetListItem(listItem.rulesetId(), listItem.ruleTotalCount(), Map.of())); + } + return new ListQueryRulesetsAction.Response(updatedResults, instance.queryPage.count()); + } } } From 1395273c82410aad0123c5fc4754cb47c7d9d18f Mon Sep 17 00:00:00 2001 From: Alyosha Karamazov Date: Mon, 7 Aug 2023 16:05:01 +0100 Subject: [PATCH 4/4] Fix transform incorrectly calculating date bucket on updating old data (#97992) This pull request fixes #97101, where a continuous pivot transform was incorrectly calculating a metric aggregation on a date bucket when a document from that bucket was updated. This was due to the transform taking the updated document's timestamp field as an upper bound in the transforms optimization query and, consequently, ignoring all documents with a later timestamp in the updated metric aggregation calculation for the bucket. contributed by @alyokaz closes #97101 --- docs/changelog/97992.yaml | 6 + .../integration/TransformPivotRestIT.java | 120 ++++++++++++ .../CompositeBucketsChangeCollector.java | 2 +- .../CompositeBucketsChangeCollectorTests.java | 121 ------------ .../DateHistogramFieldCollectorTests.java | 178 ++++++++++++++++++ 5 files changed, 305 insertions(+), 122 deletions(-) create mode 100644 docs/changelog/97992.yaml create mode 100644 x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/DateHistogramFieldCollectorTests.java diff --git a/docs/changelog/97992.yaml b/docs/changelog/97992.yaml new file mode 100644 index 0000000000000..6f5746c04b852 --- /dev/null +++ b/docs/changelog/97992.yaml @@ -0,0 +1,6 @@ +pr: 97401 +summary: Fix transform incorrectly calculating date bucket on updating old data +area: Transform +type: bug +issues: + - 97101 diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java index 77c7272c3b60d..f13d3c5d200b7 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java @@ -1000,6 +1000,126 @@ private void assertDateHistogramPivot(String indexName) throws Exception { Map indexStats = getAsMap(transformIndex + "/_stats"); assertEquals(104, XContentMapValues.extractValue("_all.total.docs.count", indexStats)); assertOnePivotValue(transformIndex + "/_search?q=by_hr:1484499600000", 4.0833333333); + + } + + // test that docs in same date bucket with a later date than the updated doc are not ignored by the transform. + @SuppressWarnings("unchecked") + public void testContinuousDateHistogramPivot() throws Exception { + String indexName = "continuous_reviews_date_histogram"; + + // ingest timestamp used to allow grouped on timestamp field to remain the same on update + Request createIngestPipeLine = new Request("PUT", "/_ingest/pipeline/es-timeadd"); + createIngestPipeLine.setJsonEntity(""" + { + "processors":[ + { + "set":{ + "field":"_source.es_timestamp", + "value":"{{_ingest.timestamp}}" + } + } + ] + }"""); + client().performRequest(createIngestPipeLine); + + Request setDefaultPipeline = new Request("PUT", indexName); + setDefaultPipeline.setJsonEntity(""" + { + "settings":{ + "index.default_pipeline":"es-timeadd" + } + }"""); + client().performRequest(setDefaultPipeline); + + Request createIndex = new Request("PUT", indexName + "/_doc/1"); + createIndex.setJsonEntity(""" + { + "user_id" : "user_1", + "timestamp": "2023-07-24T17:10:00.000Z", + "stars": 5 + }"""); + client().performRequest(createIndex); + + var putRequest = new Request("PUT", indexName + "/_doc/2"); + putRequest.setJsonEntity(""" + { + "user_id" : "user_2", + "timestamp": "2023-07-24T17:55:00.000Z", + "stars": 5 + }"""); + client().performRequest(putRequest); + + String transformId = "continuous_date_histogram_pivot"; + String transformIndex = "pivot_reviews_via_date_continuous_histogram"; + setupDataAccessRole(DATA_ACCESS_ROLE, indexName, transformIndex); + + final Request createTransformRequest = createRequestWithAuth( + "PUT", + getTransformEndpoint() + transformId, + BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS + ); + + String config = Strings.format(""" + { + "source": { + "index": "%s" + }, + "dest": { + "index": "%s" + }, + "frequency": "1s", + "sync": { + "time": { + "field": "es_timestamp", + "delay": "1s" + } + }, + "pivot": { + "group_by": { + "by_hr": { + "date_histogram": { + "fixed_interval": "1h", + "field": "timestamp" + } + } + }, + "aggregations": { + "total_rating": { + "sum": { + "field": "stars" + } + } + } + } + }""", indexName, transformIndex); + + createTransformRequest.setJsonEntity(config); + var createTransformResponse = entityAsMap(client().performRequest(createTransformRequest)); + assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE)); + + startAndWaitForContinuousTransform(transformId, transformIndex, null); + assertTrue(indexExists(transformIndex)); + + // update stars field in first doc + Request updateDoc = new Request("PUT", indexName + "/_doc/1"); + updateDoc.setJsonEntity(""" + { + "user_id" : "user_1", + "timestamp": "2023-07-24T17:10:00.000Z", + "stars": 6 + }"""); + updateDoc.addParameter("refresh", "true"); + client().performRequest(updateDoc); + + waitForTransformCheckpoint(transformId, 2); + stopTransform(transformId, false); + refreshIndex(transformIndex); + + var searchResponse = getAsMap(transformIndex + "/_search"); + var hits = ((List>) XContentMapValues.extractValue("hits.hits", searchResponse)).get(0); + var totalStars = (double) XContentMapValues.extractValue("_source.total_rating", hits); + assertEquals(11, totalStars, 0); } @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java index 5c479629a1995..0636555459632 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollector.java @@ -408,7 +408,7 @@ public boolean collectChangesFromAggregations(Aggregations aggregations) { if (lowerBoundResult != null && upperBoundResult != null) { // we only need to round the lower bound, because the checkpoint will not contain new data for the upper bound lowerBound = rounding.round((long) lowerBoundResult.value()); - upperBound = (long) upperBoundResult.value(); + upperBound = rounding.nextRoundingValue((long) upperBoundResult.value()); return false; } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java index a924286978959..926f872125ce3 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java @@ -11,17 +11,13 @@ import org.elasticsearch.action.search.SearchResponseSections; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation.SingleValue; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; -import org.elasticsearch.xpack.core.transform.transforms.pivot.DateHistogramGroupSource; import org.elasticsearch.xpack.core.transform.transforms.pivot.GeoTileGroupSourceTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfigTests; @@ -35,7 +31,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -43,7 +38,6 @@ import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -132,121 +126,6 @@ public void testTermsFieldCollector() throws IOException { assertThat(((TermsQueryBuilder) queryBuilder).values(), containsInAnyOrder("id1", "id2", "id3")); } - public void testDateHistogramFieldCollector() throws IOException { - Map groups = new LinkedHashMap<>(); - - SingleGroupSource groupBy = new DateHistogramGroupSource( - "timestamp", - null, - false, - new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.MINUTE), - null, - null - ); - groups.put("output_timestamp", groupBy); - - ChangeCollector collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, "timestamp"); - - QueryBuilder queryBuilder = collector.buildFilterQuery( - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 66_666L), - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 200_222L) - ); - - assertNotNull(queryBuilder); - assertThat(queryBuilder, instanceOf(RangeQueryBuilder.class)); - // rounded down - assertThat(((RangeQueryBuilder) queryBuilder).from(), equalTo(Long.valueOf(60_000))); - assertTrue(((RangeQueryBuilder) queryBuilder).includeLower()); - assertThat(((RangeQueryBuilder) queryBuilder).fieldName(), equalTo("timestamp")); - - // timestamp field does not match - collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, "sync_timestamp"); - - SingleValue minTimestamp = mock(SingleValue.class); - when(minTimestamp.getName()).thenReturn("_transform_change_collector.output_timestamp.min"); - when(minTimestamp.value()).thenReturn(122_633.0); - - SingleValue maxTimestamp = mock(SingleValue.class); - when(maxTimestamp.getName()).thenReturn("_transform_change_collector.output_timestamp.max"); - when(maxTimestamp.value()).thenReturn(302_523.0); - - // simulate the agg response, that should inject - Aggregations aggs = new Aggregations(Arrays.asList(minTimestamp, maxTimestamp)); - SearchResponseSections sections = new SearchResponseSections(null, aggs, null, false, null, null, 1); - SearchResponse response = new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null); - collector.processSearchResponse(response); - - // provide checkpoints, although they don't matter in this case - queryBuilder = collector.buildFilterQuery( - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 66_666L), - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 200_222L) - ); - - assertNotNull(queryBuilder); - assertThat(queryBuilder, instanceOf(RangeQueryBuilder.class)); - // rounded down - assertThat(((RangeQueryBuilder) queryBuilder).from(), equalTo(Long.valueOf(120_000))); - assertTrue(((RangeQueryBuilder) queryBuilder).includeLower()); - // the upper bound is not rounded - assertThat(((RangeQueryBuilder) queryBuilder).to(), equalTo(Long.valueOf(302_523))); - assertTrue(((RangeQueryBuilder) queryBuilder).includeUpper()); - assertThat(((RangeQueryBuilder) queryBuilder).fieldName(), equalTo("timestamp")); - - // field does not match, but output field equals sync field - collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, "output_timestamp"); - - when(minTimestamp.getName()).thenReturn("_transform_change_collector.output_timestamp.min"); - when(minTimestamp.value()).thenReturn(242_633.0); - - when(maxTimestamp.getName()).thenReturn("_transform_change_collector.output_timestamp.max"); - when(maxTimestamp.value()).thenReturn(602_523.0); - - // simulate the agg response, that should inject - collector.processSearchResponse(response); - queryBuilder = collector.buildFilterQuery( - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 66_666L), - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 200_222L) - ); - - assertNotNull(queryBuilder); - - assertThat(queryBuilder, instanceOf(RangeQueryBuilder.class)); - // rounded down - assertThat(((RangeQueryBuilder) queryBuilder).from(), equalTo(Long.valueOf(240_000))); - assertTrue(((RangeQueryBuilder) queryBuilder).includeLower()); - // the upper bound is not rounded - assertThat(((RangeQueryBuilder) queryBuilder).to(), equalTo(Long.valueOf(602_523))); - assertTrue(((RangeQueryBuilder) queryBuilder).includeUpper()); - assertThat(((RangeQueryBuilder) queryBuilder).fieldName(), equalTo("timestamp")); - - // missing bucket disables optimization - groupBy = new DateHistogramGroupSource( - "timestamp", - null, - true, - new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.MINUTE), - null, - null - ); - groups.put("output_timestamp", groupBy); - - collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, "timestamp"); - - queryBuilder = collector.buildFilterQuery( - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 66_666L), - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 200_222L) - ); - assertNull(queryBuilder); - - collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, "sync_timestamp"); - - queryBuilder = collector.buildFilterQuery( - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 66_666L), - new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 200_222L) - ); - assertNull(queryBuilder); - } - public void testNoTermsFieldCollectorForScripts() throws IOException { Map groups = new LinkedHashMap<>(); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/DateHistogramFieldCollectorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/DateHistogramFieldCollectorTests.java new file mode 100644 index 0000000000000..55b8928148b40 --- /dev/null +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/DateHistogramFieldCollectorTests.java @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.transforms.pivot; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation.SingleValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; +import org.elasticsearch.xpack.core.transform.transforms.pivot.DateHistogramGroupSource; +import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource; +import org.elasticsearch.xpack.transform.transforms.Function.ChangeCollector; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DateHistogramFieldCollectorTests extends ESTestCase { + private Map groups; + + private SingleValue minTimestamp; + private SingleValue maxTimestamp; + + private static final String TIMESTAMP = "timestamp"; + private static final String OUTPUT_TIMESTAMP = "output_timestamp"; + private static final String SYNC_TIMESTAMP = "sync_timestamp"; + + private static final SingleGroupSource groupBy = new DateHistogramGroupSource( + TIMESTAMP, + null, + false, + new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.MINUTE), + null, + null + ); + + private static final double MIN_TIMESTAMP_VALUE = 122_633; + private static final double MAX_TIMESTAMP_VALUE = 302_525; + private static final double EXPECTED_LOWER_BOUND = 120_000; + + private static final double EXPECTED_UPPER_BOUND = 360_000; + + @Before + public void setupDateHistogramFieldCollectorTest() { + minTimestamp = mock(NumericMetricsAggregation.SingleValue.class); + maxTimestamp = mock(NumericMetricsAggregation.SingleValue.class); + + when(minTimestamp.getName()).thenReturn("_transform_change_collector.output_timestamp.min"); + when(maxTimestamp.getName()).thenReturn("_transform_change_collector.output_timestamp.max"); + when(minTimestamp.value()).thenReturn(MIN_TIMESTAMP_VALUE); + when(maxTimestamp.value()).thenReturn(MAX_TIMESTAMP_VALUE); + + groups = new HashMap<>(); + } + + public void testWhenFieldAndSyncFieldSame() { + groups.put(OUTPUT_TIMESTAMP, groupBy); + ChangeCollector collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, TIMESTAMP); + QueryBuilder queryBuilder = buildFilterQuery(collector); + + assertQuery(queryBuilder, 60_000.0, TIMESTAMP); + } + + public void testWhenFieldAndSyncFieldDifferent() { + groups.put(OUTPUT_TIMESTAMP, groupBy); + ChangeCollector collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, SYNC_TIMESTAMP); + + // simulate the agg response, that should inject + SearchResponse response = buildSearchResponse(minTimestamp, maxTimestamp); + collector.processSearchResponse(response); + + // checkpoints are provided although are not used in this case + QueryBuilder queryBuilder = buildFilterQuery(collector); + + assertQuery(queryBuilder, EXPECTED_LOWER_BOUND, EXPECTED_UPPER_BOUND, TIMESTAMP); + } + + public void testWhenOutputAndSyncFieldSame() { + groups.put(OUTPUT_TIMESTAMP, groupBy); + ChangeCollector collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, SYNC_TIMESTAMP); + + // simulate the agg response, that should inject + SearchResponse response = buildSearchResponse(minTimestamp, maxTimestamp); + collector.processSearchResponse(response); + QueryBuilder queryBuilder = buildFilterQuery(collector); + + assertQuery(queryBuilder, EXPECTED_LOWER_BOUND, EXPECTED_UPPER_BOUND, TIMESTAMP); + } + + public void testMissingBucketDisablesOptimization() { + // missing bucket disables optimization + DateHistogramGroupSource groupBy = new DateHistogramGroupSource( + TIMESTAMP, + null, + true, + new DateHistogramGroupSource.FixedInterval(DateHistogramInterval.MINUTE), + null, + null + ); + groups.put(OUTPUT_TIMESTAMP, groupBy); + + // field and sync_field are the same + ChangeCollector collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, TIMESTAMP); + QueryBuilder queryBuilder = buildFilterQuery(collector); + + assertNull(queryBuilder); + + // field and sync_field are different + collector = CompositeBucketsChangeCollector.buildChangeCollector(groups, SYNC_TIMESTAMP); + queryBuilder = buildFilterQuery(collector); + + assertNull(queryBuilder); + } + + private static void assertQuery( + QueryBuilder queryBuilder, + Double expectedLowerBound, + Double expectedUpperBound, + String expectedFieldName + ) { + assertQuery(queryBuilder, expectedLowerBound, expectedFieldName); + + // the upper bound is rounded up to the nearest time unit + assertThat(((RangeQueryBuilder) queryBuilder).to(), equalTo(expectedUpperBound.longValue())); + assertTrue(((RangeQueryBuilder) queryBuilder).includeUpper()); + } + + private static void assertQuery(QueryBuilder queryBuilder, Double expectedLowerBound, String expectedFieldName) { + assertNotNull(queryBuilder); + assertThat(queryBuilder, instanceOf(RangeQueryBuilder.class)); + + // lower bound is rounded down to the nearest time unit + assertThat(((RangeQueryBuilder) queryBuilder).from(), equalTo(expectedLowerBound.longValue())); + assertTrue(((RangeQueryBuilder) queryBuilder).includeLower()); + + assertThat(((RangeQueryBuilder) queryBuilder).fieldName(), equalTo(expectedFieldName)); + } + + // Util methods + private static QueryBuilder buildFilterQuery(ChangeCollector collector) { + return collector.buildFilterQuery( + new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 66_666L), + new TransformCheckpoint("t_id", 42L, 42L, Collections.emptyMap(), 200_222L) + ); + } + + private static SearchResponse buildSearchResponse(SingleValue minTimestamp, SingleValue maxTimestamp) { + SearchResponseSections sections = new SearchResponseSections( + null, + new Aggregations(Arrays.asList(minTimestamp, maxTimestamp)), + null, + false, + null, + null, + 1 + ); + return new SearchResponse(sections, null, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY, null); + } + +}