diff --git a/docs/changelog/106714.yaml b/docs/changelog/106714.yaml new file mode 100644 index 000000000000..65b0acd77d76 --- /dev/null +++ b/docs/changelog/106714.yaml @@ -0,0 +1,5 @@ +pr: 106714 +summary: Add non-indexed fields to ecs templates +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/107046.yaml b/docs/changelog/107046.yaml new file mode 100644 index 000000000000..6c1373e09d17 --- /dev/null +++ b/docs/changelog/107046.yaml @@ -0,0 +1,6 @@ +pr: 107046 +summary: "[Security Solution] Add `read` permission for third party agent indices\ + \ for `kibana_system`" +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/107054.yaml b/docs/changelog/107054.yaml new file mode 100644 index 000000000000..6511cb518549 --- /dev/null +++ b/docs/changelog/107054.yaml @@ -0,0 +1,6 @@ +pr: 107054 +summary: Query API Keys support for both `aggs` and `aggregations` keywords +area: Security +type: enhancement +issues: + - 106839 diff --git a/docs/reference/rest-api/security/query-api-key.asciidoc b/docs/reference/rest-api/security/query-api-key.asciidoc index 1888a110e072..ad4184ec34a2 100644 --- a/docs/reference/rest-api/security/query-api-key.asciidoc +++ b/docs/reference/rest-api/security/query-api-key.asciidoc @@ -232,7 +232,7 @@ simply mentioning `metadata` (not followed by any dot and sub-field name). NOTE: You cannot query the role descriptors of an API key. ==== -`aggs`:: +`aggs` or `aggregations`:: (Optional, object) Any <> to run over the corpus of returned API keys. Aggregations and queries work together. Aggregations are computed only on the API keys that match the query. This supports only a subset of aggregation types, namely: <>, diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java new file mode 100644 index 000000000000..b171c6e6f035 --- /dev/null +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java @@ -0,0 +1,284 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.FeatureFlag; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.RestTestLegacyFeatures; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.equalTo; + +public class FullClusterRestartDownsampleIT extends ParameterizedFullClusterRestartTestCase { + + private static final String FIXED_INTERVAL = "1h"; + private String index; + private String policy; + private String dataStream; + + private static TemporaryFolder repoDirectory = new TemporaryFolder(); + + protected static LocalClusterConfigProvider clusterConfig = c -> {}; + + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(2) + .setting("xpack.security.enabled", "false") + .setting("indices.lifecycle.poll_interval", "5s") + .apply(() -> clusterConfig) + .feature(FeatureFlag.TIME_SERIES_MODE) + .feature(FeatureFlag.FAILURE_STORE_ENABLED) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(repoDirectory).around(cluster); + + public FullClusterRestartDownsampleIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) { + super(upgradeStatus); + } + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + + private static final String POLICY = """ + { + "policy": { + "phases": { + "hot": { + "actions": { + "rollover" : { + "max_age": "30s" + }, + "downsample": { + "fixed_interval": "$interval" + } + } + } + } + } + } + """; + + private static final String TEMPLATE = """ + { + "index_patterns": ["%s*"], + "template": { + "settings":{ + "index": { + "number_of_replicas": 0, + "number_of_shards": 1, + "time_series": { + "start_time": "2010-01-01T00:00:00.000Z", + "end_time": "2022-01-01T00:00:00.000Z" + }, + "routing_path": ["metricset"], + "mode": "time_series", + "look_ahead_time": "1m", + "lifecycle.name": "%s" + } + }, + "mappings":{ + "properties": { + "@timestamp" : { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + }, + "volume": { + "type": "double", + "time_series_metric": "gauge" + } + } + } + }, + "data_stream": { } + }"""; + + private static final String TEMPLATE_NO_TIME_BOUNDARIES = """ + { + "index_patterns": ["%s*"], + "template": { + "settings":{ + "index": { + "number_of_replicas": 0, + "number_of_shards": 1, + "routing_path": ["metricset"], + "mode": "time_series", + "lifecycle.name": "%s" + } + }, + "mappings":{ + "properties": { + "@timestamp" : { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + }, + "volume": { + "type": "double", + "time_series_metric": "gauge" + } + } + } + }, + "data_stream": { } + }"""; + + private static final String BULK = """ + {"create": {}} + {"@timestamp": "2020-01-01T05:10:00Z", "metricset": "pod", "volume" : 10} + {"create": {}} + {"@timestamp": "2020-01-01T05:20:00Z", "metricset": "pod", "volume" : 20} + {"create": {}} + {"@timestamp": "2020-01-01T05:30:00Z", "metricset": "pod", "volume" : 30} + {"create": {}} + {"@timestamp": "2020-01-01T05:40:00Z", "metricset": "pod", "volume" : 40} + {"create": {}} + {"@timestamp": "2020-01-01T06:10:00Z", "metricset": "pod", "volume" : 50} + {"create": {}} + {"@timestamp": "2020-01-01T07:10:00Z", "metricset": "pod", "volume" : 60} + {"create": {}} + {"@timestamp": "2020-01-01T09:10:00Z", "metricset": "pod", "volume" : 70} + {"create": {}} + {"@timestamp": "2020-01-01T09:20:00Z", "metricset": "pod", "volume" : 80} + """; + + @Before + public void refreshAbstractions() { + policy = "policy-" + randomAlphaOfLength(5); + dataStream = "ds-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + index = ".ds-" + dataStream; + logger.info("--> running [{}] with index [{}], data stream [{}], and policy [{}]", getTestName(), index, dataStream, policy); + } + + private void createIndex() throws IOException { + var putIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putIndexTemplateRequest.setJsonEntity(Strings.format(TEMPLATE, dataStream, policy)); + assertOK(client().performRequest(putIndexTemplateRequest)); + } + + private void bulk() throws IOException { + var bulkRequest = new Request("POST", "/" + dataStream + "/_bulk"); + bulkRequest.setJsonEntity(BULK); + bulkRequest.addParameter("refresh", "true"); + var response = client().performRequest(bulkRequest); + assertOK(response); + var responseBody = entityAsMap(response); + assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); + } + + private void createIlmPolicy() throws IOException { + Request request = new Request("PUT", "_ilm/policy/" + policy); + request.setJsonEntity(POLICY.replace("$interval", FIXED_INTERVAL)); + client().performRequest(request); + } + + private void startDownsampling() throws Exception { + // Update template to not contain time boundaries anymore (rollover is blocked otherwise due to index time + // boundaries overlapping after rollover) + Request updateIndexTemplateRequest = new Request("POST", "/_index_template/1"); + updateIndexTemplateRequest.setJsonEntity(Strings.format(TEMPLATE_NO_TIME_BOUNDARIES, dataStream, policy)); + assertOK(client().performRequest(updateIndexTemplateRequest)); + + // Manual rollover the original index such that it's not the write index in the data stream anymore + Request rolloverRequest = new Request("POST", "/" + dataStream + "/_rollover"); + rolloverRequest.setJsonEntity(""" + { + "conditions": { + "max_docs": "1" + } + }"""); + client().performRequest(rolloverRequest); + logger.info("rollover complete"); + } + + private void runQuery() throws Exception { + String rollup = waitAndGetRollupIndexName(); + assertFalse(rollup.isEmpty()); + + // Retry until the downsample index is populated. + assertBusy(() -> { + Request request = new Request("POST", "/" + dataStream + "/_search"); + var map = entityAsMap(client().performRequest(request)); + var hits = (List) ((Map) map.get("hits")).get("hits"); + assertEquals(4, hits.size()); + for (var hit : hits) { + assertEquals(rollup, ((Map) hit).get("_index")); + } + }, 30, TimeUnit.SECONDS); + } + + private String waitAndGetRollupIndexName() throws InterruptedException, IOException { + final String[] rollupIndexName = new String[1]; + waitUntil(() -> { + try { + rollupIndexName[0] = getRollupIndexName(); + return rollupIndexName[0] != null; + } catch (IOException e) { + return false; + } + }, 120, TimeUnit.SECONDS); + if (rollupIndexName[0] == null) { + logger.warn("--> rollup index name is NULL"); + } else { + logger.info("--> original index name is [{}], rollup index name is [{}]", index, rollupIndexName[0]); + } + return rollupIndexName[0]; + } + + private String getRollupIndexName() throws IOException { + String endpoint = "/downsample-" + FIXED_INTERVAL + "-" + index + "-*/?expand_wildcards=all"; + Response response = client().performRequest(new Request("GET", endpoint)); + Map asMap = responseAsMap(response); + if (asMap.size() == 1) { + return (String) asMap.keySet().toArray()[0]; + } + logger.warn("--> No matching rollup name for path [%s]", endpoint); + return null; + } + + public void testRollupIndex() throws Exception { + assumeTrue( + "Downsample got many stability improvements in 8.10.0", + oldClusterHasFeature(RestTestLegacyFeatures.TSDB_DOWNSAMPLING_STABLE) + ); + if (isRunningAgainstOldCluster()) { + createIlmPolicy(); + createIndex(); + bulk(); + startDownsampling(); + } else { + runQuery(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java new file mode 100644 index 000000000000..f1fc107df5f6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreDefinition.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.RoutingFieldMapper; + +import java.io.IOException; + +/** + * A utility class that contains the mappings and settings logic for failure store indices that are a part of data streams. + */ +public class DataStreamFailureStoreDefinition { + + public static final CompressedXContent DATA_STREAM_FAILURE_STORE_MAPPING; + + static { + try { + /* + * The data stream failure store mapping. The JSON content is as follows: + * { + * "_doc": { + * "dynamic": false, + * "_routing": { + * "required": false + * }, + * "properties": { + * "@timestamp": { + * "type": "date", + * "ignore_malformed": false + * }, + * "document": { + * "properties": { + * "id": { + * "type": "keyword" + * }, + * "routing": { + * "type": "keyword" + * }, + * "index": { + * "type": "keyword" + * } + * } + * }, + * "error": { + * "properties": { + * "message": { + * "type": "wildcard" + * }, + * "stack_trace": { + * "type": "text" + * }, + * "type": { + * "type": "keyword" + * }, + * "pipeline": { + * "type": "keyword" + * }, + * "pipeline_trace": { + * "type": "keyword" + * }, + * "processor": { + * "type": "keyword" + * } + * } + * } + * } + * } + * } + */ + DATA_STREAM_FAILURE_STORE_MAPPING = new CompressedXContent( + (builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME) + .field("dynamic", false) + .startObject(RoutingFieldMapper.NAME) + .field("required", false) + .endObject() + .startObject("properties") + .startObject(MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD) + .field("type", DateFieldMapper.CONTENT_TYPE) + .field("ignore_malformed", false) + .endObject() + .startObject("document") + .startObject("properties") + // document.source is unmapped so that it can be persisted in source only without worrying that the document might cause + // a mapping error + .startObject("id") + .field("type", "keyword") + .endObject() + .startObject("routing") + .field("type", "keyword") + .endObject() + .startObject("index") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .startObject("error") + .startObject("properties") + .startObject("message") + .field("type", "wildcard") + .endObject() + .startObject("stack_trace") + .field("type", "text") + .endObject() + .startObject("type") + .field("type", "keyword") + .endObject() + .startObject("pipeline") + .field("type", "keyword") + .endObject() + .startObject("pipeline_trace") + .field("type", "keyword") + .endObject() + .startObject("processor") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ); + } catch (IOException e) { + throw new AssertionError(e); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 1e2e15a6300c..0daa12b7ed71 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -92,8 +92,6 @@ public class MetadataIndexTemplateService { private static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING; - private static final CompressedXContent DATA_STREAM_FAILURE_STORE_MAPPING; - static { final Map> defaultTimestampField = Map.of( DEFAULT_TIMESTAMP_FIELD, @@ -122,110 +120,6 @@ public class MetadataIndexTemplateService { .map(defaultTimestampField) .endObject() ); - /* - * The data stream failure store mapping. The JSON content is as follows: - * { - * "_doc": { - * "dynamic": false, - * "_routing": { - * "required": false - * }, - * "properties": { - * "@timestamp": { - * "type": "date", - * "ignore_malformed": false - * }, - * "document": { - * "properties": { - * "id": { - * "type": "keyword" - * }, - * "routing": { - * "type": "keyword" - * }, - * "index": { - * "type": "keyword" - * } - * } - * }, - * "error": { - * "properties": { - * "message": { - * "type": "wildcard" - * }, - * "stack_trace": { - * "type": "text" - * }, - * "type": { - * "type": "keyword" - * }, - * "pipeline": { - * "type": "keyword" - * }, - * "pipeline_trace": { - * "type": "keyword" - * }, - * "processor": { - * "type": "keyword" - * } - * } - * } - * } - * } - * } - */ - DATA_STREAM_FAILURE_STORE_MAPPING = new CompressedXContent( - (builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME) - .field("dynamic", false) - .startObject(RoutingFieldMapper.NAME) - .field("required", false) - .endObject() - .startObject("properties") - .startObject(DEFAULT_TIMESTAMP_FIELD) - .field("type", DateFieldMapper.CONTENT_TYPE) - .field("ignore_malformed", false) - .endObject() - .startObject("document") - .startObject("properties") - // document.source is unmapped so that it can be persisted in source only without worrying that the document might cause - // a mapping error - .startObject("id") - .field("type", "keyword") - .endObject() - .startObject("routing") - .field("type", "keyword") - .endObject() - .startObject("index") - .field("type", "keyword") - .endObject() - .endObject() - .endObject() - .startObject("error") - .startObject("properties") - .startObject("message") - .field("type", "wildcard") - .endObject() - .startObject("stack_trace") - .field("type", "text") - .endObject() - .startObject("type") - .field("type", "keyword") - .endObject() - .startObject("pipeline") - .field("type", "keyword") - .endObject() - .startObject("pipeline_trace") - .field("type", "keyword") - .endObject() - .startObject("processor") - .field("type", "keyword") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); - } catch (IOException e) { throw new AssertionError(e); } @@ -1446,7 +1340,10 @@ public static List collectMappings( Objects.requireNonNull(template, "Composable index template must be provided"); // Check if this is a failure store index, and if it is, discard any template mappings. Failure store mappings are predefined. if (template.getDataStreamTemplate() != null && indexName.startsWith(DataStream.FAILURE_STORE_PREFIX)) { - return List.of(DATA_STREAM_FAILURE_STORE_MAPPING, ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET); + return List.of( + DataStreamFailureStoreDefinition.DATA_STREAM_FAILURE_STORE_MAPPING, + ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET + ); } List mappings = template.composedOf() .stream() diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 3c76734b794d..cdb7f44d41e4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -272,6 +272,12 @@ static RoleDescriptor kibanaSystem(String name) { .indices(".logs-osquery_manager.actions-*") .privileges("auto_configure", "create_index", "read", "index", "write", "delete") .build(), + + // Third party agent (that use non-Elastic Defend integrations) info logs indices. + // Kibana reads from these to display agent status/info to the user. + // These are indices that filebeat writes to, and the data in these indices are ingested by Fleet integrations + // in order to provide support for response actions related to malicious events for such agents. + RoleDescriptor.IndicesPrivileges.builder().indices("logs-sentinel_one.*", "logs-crowdstrike.*").privileges("read").build(), // For ILM policy for APM, Endpoint, & Synthetics packages that have delete action RoleDescriptor.IndicesPrivileges.builder() .indices( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index b0d25949947e..39a94e4a2f0b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -993,6 +993,37 @@ public void testKibanaSystemRole() { assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(true)); }); + // Tests for third-party agent indices that `kibana_system` has only `read` access + Arrays.asList( + "logs-sentinel_one." + randomAlphaOfLength(randomIntBetween(0, 13)), + "logs-crowdstrike." + randomAlphaOfLength(randomIntBetween(0, 13)) + ).forEach((index) -> { + final IndexAbstraction indexAbstraction = mockIndexAbstraction(index); + assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false)); + assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(indexAbstraction), is(false)); + assertThat( + kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), + is(false) + ); + assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true)); + assertThat( + kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), + is(false) + ); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(indexAbstraction), is(false)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(indexAbstraction), is(false)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportMultiSearchAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(indexAbstraction), is(false)); + assertThat( + kibanaRole.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(indexAbstraction), + is(true) + ); + assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(true)); + assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(true)); + }); + // Index for Endpoint specific actions Arrays.asList(".logs-endpoint.actions-" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((index) -> { final IndexAbstraction indexAbstraction = mockIndexAbstraction(index); diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json index 7eaf37ba1d95..3eae6c1fa4f5 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ecs@mappings.json @@ -23,6 +23,30 @@ "unmatch_mapping_type": "object" } }, + { + "ecs_non_indexed_keyword": { + "mapping": { + "type": "keyword", + "index": false, + "doc_values": false + }, + "path_match": [ + "event.original" + ] + } + }, + { + "ecs_non_indexed_long": { + "mapping": { + "type": "long", + "index": false, + "doc_values": false + }, + "path_match": [ + "*.x509.public_key_exponent" + ] + } + }, { "ecs_ip": { "mapping": { diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ApiKeyAggsIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ApiKeyAggsIT.java index 427d918fd64d..f9d5c42affcf 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ApiKeyAggsIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ApiKeyAggsIT.java @@ -98,7 +98,7 @@ public void testFiltersAggs() throws IOException { // other bucket assertAggs(API_KEY_USER_AUTH_HEADER, typedAggs, """ { - "aggs": { + "aggregations": { "only_user_keys": { "filters": { "other_bucket_key": "other_user_keys", @@ -267,7 +267,7 @@ public void testFiltersAggs() throws IOException { "good-api-key-invalidated": { "term": {"invalidated": false}} } }, - "aggs": { + "aggregations": { "wrong-field": { "filters": { "filters": { @@ -487,7 +487,7 @@ public void testFilterAggs() throws IOException { { "usernames": { "terms": { "field": "username" } } } ] }, - "aggs": { + "aggregations": { "not_expired": { "filter": { "range": { @@ -564,7 +564,7 @@ public void testDisallowedAggTypes() { ); request.setJsonEntity(""" { - "aggs": { + "aggregations": { "all_.security_docs": { "global": {}, "aggs": { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyAction.java index 77c2a080dbb5..59992e42d88d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyAction.java @@ -36,6 +36,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.search.aggregations.AggregatorFactories.parseAggregators; +import static org.elasticsearch.search.builder.SearchSourceBuilder.AGGREGATIONS_FIELD; +import static org.elasticsearch.search.builder.SearchSourceBuilder.AGGS_FIELD; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; /** @@ -47,19 +49,27 @@ public final class RestQueryApiKeyAction extends ApiKeyBaseRestHandler { @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "query_api_key_request_payload", - a -> new Payload( - (QueryBuilder) a[0], - (AggregatorFactories.Builder) a[1], - (Integer) a[2], - (Integer) a[3], - (List) a[4], - (SearchAfterBuilder) a[5] - ) + a -> { + if (a[1] != null && a[2] != null) { + throw new IllegalArgumentException("Duplicate 'aggs' or 'aggregations' field"); + } else { + return new Payload( + (QueryBuilder) a[0], + (AggregatorFactories.Builder) (a[1] != null ? a[1] : a[2]), + (Integer) a[3], + (Integer) a[4], + (List) a[5], + (SearchAfterBuilder) a[6] + ); + } + } ); static { PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseTopLevelQuery(p), new ParseField("query")); - PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseAggregators(p), new ParseField("aggs")); + // only one of aggs or aggregations is allowed + PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseAggregators(p), AGGREGATIONS_FIELD); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseAggregators(p), AGGS_FIELD); PARSER.declareInt(optionalConstructorArg(), new ParseField("from")); PARSER.declareInt(optionalConstructorArg(), new ParseField("size")); PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyActionTests.java index 74d1203fd52e..2240b72c1a96 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -35,6 +36,7 @@ import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; @@ -48,6 +50,7 @@ import java.util.Map; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -145,6 +148,69 @@ public void doE assertNotNull(responseSetOnce.get()); } + public void testAggsAndAggregationsTogether() { + String agg1; + String agg2; + if (randomBoolean()) { + agg1 = "aggs"; + agg2 = "aggregations"; + } else { + agg1 = "aggregations"; + agg2 = "aggs"; + } + final String requestBody = Strings.format(""" + { + "%s": { + "all_keys_by_type": { + "composite": { + "sources": [ + { "type": { "terms": { "field": "type" } } } + ] + } + } + }, + "%s": { + "type_cardinality": { + "cardinality": { + "field": "type" + } + } + } + }""", agg1, agg2); + + final FakeRestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry()).withContent( + new BytesArray(requestBody), + XContentType.JSON + ).build(); + final SetOnce responseSetOnce = new SetOnce<>(); + final RestChannel restChannel = new AbstractRestChannel(restRequest, randomBoolean()) { + @Override + public void sendResponse(RestResponse restResponse) { + responseSetOnce.set(restResponse); + } + }; + final var client = new NodeClient(Settings.EMPTY, threadPool) { + @SuppressWarnings("unchecked") + @Override + public void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + fail("TEST failed, request parsing should've failed"); + listener.onResponse((Response) QueryApiKeyResponse.EMPTY); + } + }; + RestQueryApiKeyAction restQueryApiKeyAction = new RestQueryApiKeyAction(Settings.EMPTY, mockLicenseState); + XContentParseException ex = expectThrows( + XContentParseException.class, + () -> restQueryApiKeyAction.handleRequest(restRequest, restChannel, client) + ); + assertThat(ex.getCause().getMessage(), containsString("Duplicate 'aggs' or 'aggregations' field")); + assertThat(ex.getMessage(), containsString("Failed to build [query_api_key_request_payload]")); + assertNull(responseSetOnce.get()); + } + public void testParsingSearchParameters() throws Exception { final String requestBody = """ { diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java index 09e9a6090c48..8bdf7b30a999 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -191,6 +191,11 @@ public void testNumericMessage() throws IOException { verifyEcsMappings(indexName); } + private void assertType(String expectedType, Map actualMappings) throws IOException { + assertNotNull("expected to get non-null mappings for field", actualMappings); + assertEquals(expectedType, actualMappings.get("type")); + } + public void testUsage() throws IOException { String indexName = "test-usage"; createTestIndex(indexName); @@ -205,13 +210,13 @@ public void testUsage() throws IOException { indexDocument(indexName, fieldsMap); final Map rawMappings = getMappings(indexName); - final Map flatFieldMappings = new HashMap<>(); + final Map> flatFieldMappings = new HashMap<>(); processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), ""); - assertEquals("scaled_float", flatFieldMappings.get("host.cpu.usage")); - assertEquals("scaled_float", flatFieldMappings.get("string.usage")); - assertEquals("long", flatFieldMappings.get("usage")); - assertEquals("long", flatFieldMappings.get("root.usage.long")); - assertEquals("float", flatFieldMappings.get("root.usage.float")); + assertType("scaled_float", flatFieldMappings.get("host.cpu.usage")); + assertType("scaled_float", flatFieldMappings.get("string.usage")); + assertType("long", flatFieldMappings.get("usage")); + assertType("long", flatFieldMappings.get("root.usage.long")); + assertType("float", flatFieldMappings.get("root.usage.float")); } public void testOnlyMatchLeafFields() throws IOException { @@ -230,16 +235,16 @@ public void testOnlyMatchLeafFields() throws IOException { indexDocument(indexName, fieldsMap); final Map rawMappings = getMappings(indexName); - final Map flatFieldMappings = new HashMap<>(); + final Map> flatFieldMappings = new HashMap<>(); processRawMappingsSubtree(rawMappings, flatFieldMappings, new HashMap<>(), ""); - assertEquals("long", flatFieldMappings.get("foo.message.bar")); - assertEquals("long", flatFieldMappings.get("foo.url.path.bar")); - assertEquals("long", flatFieldMappings.get("foo.url.full.bar")); - assertEquals("long", flatFieldMappings.get("foo.stack_trace.bar")); - assertEquals("long", flatFieldMappings.get("foo.user_agent.original.bar")); - assertEquals("long", flatFieldMappings.get("foo.created.bar")); - assertEquals("float", flatFieldMappings.get("foo._score.bar")); - assertEquals("long", flatFieldMappings.get("foo.structured_data")); + assertType("long", flatFieldMappings.get("foo.message.bar")); + assertType("long", flatFieldMappings.get("foo.url.path.bar")); + assertType("long", flatFieldMappings.get("foo.url.full.bar")); + assertType("long", flatFieldMappings.get("foo.stack_trace.bar")); + assertType("long", flatFieldMappings.get("foo.user_agent.original.bar")); + assertType("long", flatFieldMappings.get("foo.created.bar")); + assertType("float", flatFieldMappings.get("foo._score.bar")); + assertType("long", flatFieldMappings.get("foo.structured_data")); } private static void indexDocument(String indexName, Map flattenedFieldsMap) throws IOException { @@ -364,28 +369,26 @@ private Map getMappings(String indexName) throws IOException { private void processRawMappingsSubtree( final Map fieldSubtrees, - final Map flatFieldMappings, - final Map flatMultiFieldsMappings, + final Map> flatFieldMappings, + final Map> flatMultiFieldsMappings, final String subtreePrefix ) { fieldSubtrees.forEach((fieldName, fieldMappings) -> { String fieldFullPath = subtreePrefix + fieldName; Map fieldMappingsMap = ((Map) fieldMappings); - String type = (String) fieldMappingsMap.get("type"); - if (type != null) { - flatFieldMappings.put(fieldFullPath, type); + if (fieldMappingsMap.get("type") != null) { + flatFieldMappings.put(fieldFullPath, fieldMappingsMap); } Map subfields = (Map) fieldMappingsMap.get("properties"); if (subfields != null) { processRawMappingsSubtree(subfields, flatFieldMappings, flatMultiFieldsMappings, fieldFullPath + "."); } - Map> fields = (Map>) fieldMappingsMap.get("fields"); + Map> fields = (Map>) fieldMappingsMap.get("fields"); if (fields != null) { fields.forEach((subFieldName, multiFieldMappings) -> { String subFieldFullPath = fieldFullPath + "." + subFieldName; - String subFieldType = Objects.requireNonNull(multiFieldMappings.get("type")); - flatMultiFieldsMappings.put(subFieldFullPath, subFieldType); + flatMultiFieldsMappings.put(subFieldFullPath, multiFieldMappings); }); } }); @@ -393,34 +396,44 @@ private void processRawMappingsSubtree( private void verifyEcsMappings(String indexName) throws IOException { final Map rawMappings = getMappings(indexName); - final Map flatFieldMappings = new HashMap<>(); - final Map flatMultiFieldsMappings = new HashMap<>(); + final Map> flatFieldMappings = new HashMap<>(); + final Map> flatMultiFieldsMappings = new HashMap<>(); processRawMappingsSubtree(rawMappings, flatFieldMappings, flatMultiFieldsMappings, ""); Map> shallowFieldMapCopy = new HashMap<>(ecsFlatFieldDefinitions); logger.info("Testing mapping of {} ECS fields", shallowFieldMapCopy.size()); List nonEcsFields = new ArrayList<>(); Map fieldToWrongMappingType = new HashMap<>(); - flatFieldMappings.forEach((fieldName, actualMappingType) -> { + List wronglyIndexedFields = new ArrayList<>(); + List wronglyDocValuedFields = new ArrayList<>(); + flatFieldMappings.forEach((fieldName, actualMappings) -> { Map expectedMappings = shallowFieldMapCopy.remove(fieldName); if (expectedMappings == null) { nonEcsFields.add(fieldName); } else { String expectedType = (String) expectedMappings.get("type"); + String actualMappingType = (String) actualMappings.get("type"); if (actualMappingType.equals(expectedType) == false) { fieldToWrongMappingType.put(fieldName, actualMappingType); } + if (expectedMappings.get("index") != actualMappings.get("index")) { + wronglyIndexedFields.add(fieldName); + } + if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) { + wronglyDocValuedFields.add(fieldName); + } } }); Map shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions); logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size()); - flatMultiFieldsMappings.forEach((fieldName, actualMappingType) -> { + flatMultiFieldsMappings.forEach((fieldName, actualMappings) -> { String expectedType = shallowMultiFieldMapCopy.remove(fieldName); if (expectedType != null) { // not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to // ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead // to multi-field mapping for ECS fields for which such are not defined + String actualMappingType = (String) actualMappings.get("type"); if (actualMappingType.equals(expectedType) == false) { fieldToWrongMappingType.put(fieldName, actualMappingType); } @@ -457,6 +470,8 @@ private void verifyEcsMappings(String indexName) throws IOException { ); }); nonEcsFields.forEach(field -> logger.error("The test document contains '{}', which is not an ECS field", field)); + wronglyIndexedFields.forEach(fieldName -> logger.error("ECS field '{}' should be mapped with \"index: false\"", fieldName)); + wronglyDocValuedFields.forEach(fieldName -> logger.error("ECS field '{}' should be mapped with \"doc_values: false\"", fieldName)); assertTrue("ECS is not fully covered by the current ECS dynamic templates, see details above", shallowFieldMapCopy.isEmpty()); assertTrue( @@ -468,5 +483,14 @@ private void verifyEcsMappings(String indexName) throws IOException { fieldToWrongMappingType.isEmpty() ); assertTrue("The test document contains non-ECS fields, see details above", nonEcsFields.isEmpty()); + assertTrue( + "At least one field was not mapped with \"index: false\" as it should according to its ECS definitions, see details above", + wronglyIndexedFields.isEmpty() + ); + assertTrue( + "At least one field was not mapped with \"doc_values: false\" as it should according to its ECS definitions, see " + + "details above", + wronglyDocValuedFields.isEmpty() + ); } } diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index b21e8c0c1581..3930cfe6cd94 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -47,7 +47,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { // The stack template registry version. This number must be incremented when we make changes // to built-in templates. - public static final int REGISTRY_VERSION = 8; + public static final int REGISTRY_VERSION = 9; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting(