From e19cd0d20756f8e0a65a2d35f73dcbb014f36300 Mon Sep 17 00:00:00 2001 From: Florian Lehner Date: Tue, 20 Aug 2024 12:06:22 +0200 Subject: [PATCH 01/32] [Profiling] add container.id field to event index template (#111969) * [Profiling] add container.id field to event index template This PR adds a new container.id field to the index template of the profiling-events data-stream. The field name was chosen in [accordance with ECS](https://www.elastic.co/guide/en/ecs/current/ecs-container.html#field-container-id) and also matches what the field is called in the APM indices. Signed-off-by: Florian Lehner * Update docs/changelog/111969.yaml --------- Signed-off-by: Florian Lehner --- docs/changelog/111969.yaml | 5 +++++ .../profiling/component-template/profiling-events.json | 3 +++ .../persistence/ProfilingIndexTemplateRegistry.java | 5 +++-- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/111969.yaml diff --git a/docs/changelog/111969.yaml b/docs/changelog/111969.yaml new file mode 100644 index 0000000000000..2d276850c4988 --- /dev/null +++ b/docs/changelog/111969.yaml @@ -0,0 +1,5 @@ +pr: 111969 +summary: "[Profiling] add `container.id` field to event index template" +area: Application +type: enhancement +issues: [] diff --git a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json index 9b90f97682306..8f50ebd334f16 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/profiling/component-template/profiling-events.json @@ -77,6 +77,9 @@ }, "service.name": { "type": "keyword" + }, + "container.id": { + "type": "keyword" } } } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java index 3b361748abf67..7d8a474453c4c 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/persistence/ProfilingIndexTemplateRegistry.java @@ -53,10 +53,11 @@ public class ProfilingIndexTemplateRegistry extends IndexTemplateRegistry { // version 10: changed mapping profiling-events @timestamp to 'date_nanos' from 'date' // version 11: Added 'profiling.agent.protocol' keyword mapping to profiling-hosts // version 12: Added 'profiling.agent.env_https_proxy' keyword mapping to profiling-hosts - public static final int INDEX_TEMPLATE_VERSION = 12; + // version 13: Added 'container.id' keyword mapping to profiling-events + public static final int INDEX_TEMPLATE_VERSION = 13; // history for individual indices / index templates. Only bump these for breaking changes that require to create a new index - public static final int PROFILING_EVENTS_VERSION = 4; + public static final int PROFILING_EVENTS_VERSION = 5; public static final int PROFILING_EXECUTABLES_VERSION = 1; public static final int PROFILING_METRICS_VERSION = 2; public static final int PROFILING_HOSTS_VERSION = 2; From 9ab86652355bebd4909409a66166596531b66005 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Tue, 20 Aug 2024 13:23:36 +0300 Subject: [PATCH 02/32] No error when `store_array_source` is used without synthetic source (#111966) * No error for store_array_source in standard mode * Update docs/changelog/111966.yaml * nested object test * restore noop tests * spotless fix --- docs/changelog/111966.yaml | 5 +++++ .../java/org/elasticsearch/index/mapper/ObjectMapper.java | 3 --- .../index/mapper/NestedObjectMapperTests.java | 8 ++++---- .../org/elasticsearch/index/mapper/ObjectMapperTests.java | 8 ++++---- 4 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/111966.yaml diff --git a/docs/changelog/111966.yaml b/docs/changelog/111966.yaml new file mode 100644 index 0000000000000..facf0a61c4d8a --- /dev/null +++ b/docs/changelog/111966.yaml @@ -0,0 +1,5 @@ +pr: 111966 +summary: No error when `store_array_source` is used without synthetic source +area: Mapping +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 843fc3b15a6df..2c78db6bc8b0d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -473,9 +473,6 @@ public final boolean storeArraySource() { @Override public void validate(MappingLookup mappers) { - if (storeArraySource() && mappers.isSourceSynthetic() == false) { - throw new MapperParsingException("Parameter [" + STORE_ARRAY_SOURCE_PARAM + "] can only be set in synthetic source mode."); - } for (Mapper mapper : this.mappers.values()) { mapper.validate(mappers); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 4fba22101df03..13bd5955d67a5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1575,11 +1575,11 @@ public void testStoreArraySourceinSyntheticSourceMode() throws IOException { assertNotNull(mapper.mapping().getRoot().getMapper("o")); } - public void testStoreArraySourceThrowsInNonSyntheticSourceMode() { - var exception = expectThrows(MapperParsingException.class, () -> createDocumentMapper(mapping(b -> { + public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { b.startObject("o").field("type", "nested").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); - }))); - assertEquals("Parameter [store_array_source] can only be set in synthetic source mode.", exception.getMessage()); + })); + assertNotNull(mapper.mapping().getRoot().getMapper("o")); } public void testSyntheticNestedWithObject() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 6687a28883716..3c81f833985dd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -546,11 +546,11 @@ public void testStoreArraySourceinSyntheticSourceMode() throws IOException { assertNotNull(mapper.mapping().getRoot().getMapper("o")); } - public void testStoreArraySourceThrowsInNonSyntheticSourceMode() { - var exception = expectThrows(MapperParsingException.class, () -> createDocumentMapper(mapping(b -> { + public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOException { + DocumentMapper mapper = createDocumentMapper(mapping(b -> { b.startObject("o").field("type", "object").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); - }))); - assertEquals("Parameter [store_array_source] can only be set in synthetic source mode.", exception.getMessage()); + })); + assertNotNull(mapper.mapping().getRoot().getMapper("o")); } public void testNestedObjectWithMultiFieldsgetTotalFieldsCount() { From 3f49509f04b307cf84abd63f01a7b83819e17184 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Tue, 20 Aug 2024 12:56:31 +0200 Subject: [PATCH 03/32] Make error.grouping_name script compatible with synthetic _source (#112009) --- .../component-templates/logs-apm.error@mappings.yaml | 11 ++++++++--- .../rest-api-spec/test/20_error_grouping.yml | 7 ++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml index 1e2a6a679dc30..c1d004b4e7bf4 100644 --- a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml @@ -28,9 +28,14 @@ template: return; } def exception = params['_source'].error?.exception; - def exceptionMessage = exception != null && exception.length > 0 ? exception[0]?.message : null; - if (exceptionMessage != null && exceptionMessage != "") { - emit(exception[0].message); + if (exception != null && exception.isEmpty() == false) { + def exceptionMessage = exception instanceof Map ? exception?.message : exception[0]?.message; + if (exceptionMessage instanceof List) { + exceptionMessage = exceptionMessage[0] + } + if (exceptionMessage != null && exceptionMessage != "") { + emit(exceptionMessage); + } } # http.* diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml index f7cd386227fe8..37a1651da562b 100644 --- a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_error_grouping.yml @@ -39,6 +39,10 @@ setup: - create: {} - '{"@timestamp": "2017-06-22", "error": {"log": {"message": ""}, "exception": [{"message": "exception_used"}]}}' + # Non-empty error.exception.message used from array + - create: {} + - '{"@timestamp": "2017-06-22", "error": {"log": {"message": ""}, "exception": [{"message": "first_exception_used"}, {"message": "2_ignored"}]}}' + - is_false: errors - do: @@ -46,7 +50,7 @@ setup: index: logs-apm.error-testing body: fields: ["error.grouping_name"] - - length: { hits.hits: 7 } + - length: { hits.hits: 8 } - match: { hits.hits.0.fields: null } - match: { hits.hits.1.fields: null } - match: { hits.hits.2.fields: null } @@ -54,3 +58,4 @@ setup: - match: { hits.hits.4.fields: null } - match: { hits.hits.5.fields: {"error.grouping_name": ["log_used"]} } - match: { hits.hits.6.fields: {"error.grouping_name": ["exception_used"]} } + - match: { hits.hits.7.fields: {"error.grouping_name": ["first_exception_used"]} } From dd49c33479d5f27cd708a2a71b0407af970d6e94 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Tue, 20 Aug 2024 13:40:59 +0200 Subject: [PATCH 04/32] ESQL: BUCKET: allow numerical spans as whole numbers (#111874) This laxes the check on numerical spans to allow them be specified as whole numbers. So far it was required that they be provided as a double. This also expands the tests for date ranges to include string types. Resolves #109340, resolves #104646, resolves #105375. --- docs/changelog/111874.yaml | 8 + .../esql/functions/examples/bucket.asciidoc | 4 - .../functions/kibana/definition/bucket.json | 460 +++++++++++++++--- .../esql/functions/parameters/bucket.asciidoc | 4 +- .../esql/functions/types/bucket.asciidoc | 14 + .../src/main/resources/bucket.csv-spec | 26 + .../src/main/resources/meta.csv-spec | 8 +- .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../expression/function/grouping/Bucket.java | 50 +- .../xpack/esql/analysis/VerifierTests.java | 64 ++- .../function/grouping/BucketTests.java | 53 +- .../optimizer/LogicalPlanOptimizerTests.java | 43 ++ 12 files changed, 632 insertions(+), 109 deletions(-) create mode 100644 docs/changelog/111874.yaml diff --git a/docs/changelog/111874.yaml b/docs/changelog/111874.yaml new file mode 100644 index 0000000000000..26ec90aa6cd4c --- /dev/null +++ b/docs/changelog/111874.yaml @@ -0,0 +1,8 @@ +pr: 111874 +summary: "ESQL: BUCKET: allow numerical spans as whole numbers" +area: ES|QL +type: enhancement +issues: + - 104646 + - 109340 + - 105375 diff --git a/docs/reference/esql/functions/examples/bucket.asciidoc b/docs/reference/esql/functions/examples/bucket.asciidoc index e1bba0529d7db..4afea30660339 100644 --- a/docs/reference/esql/functions/examples/bucket.asciidoc +++ b/docs/reference/esql/functions/examples/bucket.asciidoc @@ -86,10 +86,6 @@ include::{esql-specs}/bucket.csv-spec[tag=docsBucketNumericWithSpan] |=== include::{esql-specs}/bucket.csv-spec[tag=docsBucketNumericWithSpan-result] |=== - -NOTE: When providing the bucket size as the second parameter, it must be -of a floating point type. - Create hourly buckets for the last 24 hours, and calculate the number of events per hour: [source.merge.styled,esql] ---- diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json index 7141ca4c27443..14bd74c1c20f3 100644 --- a/docs/reference/esql/functions/kibana/definition/bucket.json +++ b/docs/reference/esql/functions/kibana/definition/bucket.json @@ -40,13 +40,253 @@ "name" : "from", "type" : "datetime", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "datetime", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field", + "type" : "datetime", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + }, + { + "name" : "from", + "type" : "datetime", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field", + "type" : "datetime", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + }, + { + "name" : "from", + "type" : "datetime", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field", + "type" : "datetime", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "datetime", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field", + "type" : "datetime", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field", + "type" : "datetime", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field", + "type" : "datetime", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "datetime", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field", + "type" : "datetime", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "datetime" + }, + { + "params" : [ + { + "name" : "field", + "type" : "datetime", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -88,6 +328,24 @@ "variadic" : false, "returnType" : "double" }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + } + ], + "variadic" : false, + "returnType" : "double" + }, { "params" : [ { @@ -106,13 +364,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -136,13 +394,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -166,13 +424,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -196,13 +454,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -226,13 +484,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -256,13 +514,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -286,13 +544,13 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -316,13 +574,13 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -346,13 +604,31 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "long", + "optional" : false, + "description" : "Target number of buckets." } ], "variadic" : false, @@ -376,6 +652,24 @@ "variadic" : false, "returnType" : "double" }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + } + ], + "variadic" : false, + "returnType" : "double" + }, { "params" : [ { @@ -394,13 +688,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -424,13 +718,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -454,13 +748,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -484,13 +778,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -514,13 +808,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -544,13 +838,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -574,13 +868,13 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -604,13 +898,13 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -634,13 +928,31 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "long", + "optional" : false, + "description" : "Target number of buckets." } ], "variadic" : false, @@ -664,6 +976,24 @@ "variadic" : false, "returnType" : "double" }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets." + } + ], + "variadic" : false, + "returnType" : "double" + }, { "params" : [ { @@ -682,13 +1012,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -712,13 +1042,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -742,13 +1072,13 @@ "name" : "from", "type" : "double", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -772,13 +1102,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -802,13 +1132,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -832,13 +1162,13 @@ "name" : "from", "type" : "integer", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -862,13 +1192,13 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "double", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -892,13 +1222,13 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "integer", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, @@ -922,13 +1252,31 @@ "name" : "from", "type" : "long", "optional" : true, - "description" : "Start of the range. Can be a number or a date expressed as a string." + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", "type" : "long", "optional" : true, - "description" : "End of the range. Can be a number or a date expressed as a string." + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "long", + "optional" : false, + "description" : "Target number of buckets." } ], "variadic" : false, diff --git a/docs/reference/esql/functions/parameters/bucket.asciidoc b/docs/reference/esql/functions/parameters/bucket.asciidoc index 39aac14aaa36d..342ea560aaa0b 100644 --- a/docs/reference/esql/functions/parameters/bucket.asciidoc +++ b/docs/reference/esql/functions/parameters/bucket.asciidoc @@ -9,7 +9,7 @@ Numeric or date expression from which to derive buckets. Target number of buckets. `from`:: -Start of the range. Can be a number or a date expressed as a string. +Start of the range. Can be a number, a date or a date expressed as a string. `to`:: -End of the range. Can be a number or a date expressed as a string. +End of the range. Can be a number, a date or a date expressed as a string. diff --git a/docs/reference/esql/functions/types/bucket.asciidoc b/docs/reference/esql/functions/types/bucket.asciidoc index d1ce8e499eb07..1cbfad14ca379 100644 --- a/docs/reference/esql/functions/types/bucket.asciidoc +++ b/docs/reference/esql/functions/types/bucket.asciidoc @@ -7,6 +7,14 @@ field | buckets | from | to | result datetime | date_period | | | datetime datetime | integer | datetime | datetime | datetime +datetime | integer | datetime | keyword | datetime +datetime | integer | datetime | text | datetime +datetime | integer | keyword | datetime | datetime +datetime | integer | keyword | keyword | datetime +datetime | integer | keyword | text | datetime +datetime | integer | text | datetime | datetime +datetime | integer | text | keyword | datetime +datetime | integer | text | text | datetime datetime | time_duration | | | datetime double | double | | | double double | integer | double | double | double @@ -18,6 +26,8 @@ double | integer | integer | long | double double | integer | long | double | double double | integer | long | integer | double double | integer | long | long | double +double | integer | | | double +double | long | | | double integer | double | | | double integer | integer | double | double | double integer | integer | double | integer | double @@ -28,6 +38,8 @@ integer | integer | integer | long | double integer | integer | long | double | double integer | integer | long | integer | double integer | integer | long | long | double +integer | integer | | | double +integer | long | | | double long | double | | | double long | integer | double | double | double long | integer | double | integer | double @@ -38,4 +50,6 @@ long | integer | integer | long | double long | integer | long | double | double long | integer | long | integer | double long | integer | long | long | double +long | integer | | | double +long | long | | | double |=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec index 7e2afb9267e5b..b8569ead94509 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/bucket.csv-spec @@ -314,6 +314,21 @@ FROM sample_data 3 |2025-10-01T00:00:00.000Z ; +bucketByYearLowBucketCount#[skip:-8.13.99, reason:BUCKET extended in 8.14] +FROM employees +| WHERE hire_date >= "1985-02-18T00:00:00.000Z" AND hire_date <= "1988-10-18T00:00:00.000Z" +| STATS c = COUNT(*) BY b = BUCKET(hire_date, 3, "1985-02-18T00:00:00.000Z", "1988-10-18T00:00:00.000Z") +| SORT b +; + +// Note: we don't bucket to anything longer than 1 year (like 2 years), so even if requesting 3 buckets, we still get 4 + c:long | b:date +11 |1985-01-01T00:00:00.000Z +11 |1986-01-01T00:00:00.000Z +15 |1987-01-01T00:00:00.000Z +9 |1988-01-01T00:00:00.000Z +; + // // Numeric bucketing // @@ -393,6 +408,17 @@ ROW long = TO_LONG(100), double = 99., int = 100 99.0 |0.0 |99.0 ; +// identical results as above +bucketNumericMixedTypesIntegerSpans +required_capability: bucket_whole_number_as_span +ROW long = TO_LONG(100), double = 99., int = 100 +| STATS BY b1 = BUCKET(long, double::int), b2 = BUCKET(double, long), b3 = BUCKET(int, 49.5) +; + + b1:double| b2:double| b3:double +99.0 |0.0 |99.0 +; + bucketWithFloats#[skip:-8.13.99, reason:BUCKET renamed in 8.14] FROM employees | WHERE hire_date >= "1985-01-01T00:00:00Z" AND hire_date < "1986-01-01T00:00:00Z" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 35c852d6ba2fe..951545a546826 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -9,8 +9,8 @@ synopsis:keyword "double atan(number:double|integer|long|unsigned_long)" "double atan2(y_coordinate:double|integer|long|unsigned_long, x_coordinate:double|integer|long|unsigned_long)" "double avg(number:double|integer|long)" -"double|date bin(field:integer|long|double|date, buckets:integer|double|date_period|time_duration, ?from:integer|long|double|date, ?to:integer|long|double|date)" -"double|date bucket(field:integer|long|double|date, buckets:integer|double|date_period|time_duration, ?from:integer|long|double|date, ?to:integer|long|double|date)" +"double|date bin(field:integer|long|double|date, buckets:integer|long|double|date_period|time_duration, ?from:integer|long|double|date|keyword|text, ?to:integer|long|double|date|keyword|text)" +"double|date bucket(field:integer|long|double|date, buckets:integer|long|double|date_period|time_duration, ?from:integer|long|double|date|keyword|text, ?to:integer|long|double|date|keyword|text)" "boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version case(condition:boolean, trueValue...:boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version)" "double cbrt(number:double|integer|long|unsigned_long)" "double|integer|long|unsigned_long ceil(number:double|integer|long|unsigned_long)" @@ -132,8 +132,8 @@ asin |number |"double|integer|long|unsigne atan |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. atan2 |[y_coordinate, x_coordinate] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |[y coordinate. If `null`\, the function returns `null`., x coordinate. If `null`\, the function returns `null`.] avg |number |"double|integer|long" |[""] -bin |[field, buckets, from, to] |["integer|long|double|date", "integer|double|date_period|time_duration", "integer|long|double|date", "integer|long|double|date"] |[Numeric or date expression from which to derive buckets., Target number of buckets., Start of the range. Can be a number or a date expressed as a string., End of the range. Can be a number or a date expressed as a string.] -bucket |[field, buckets, from, to] |["integer|long|double|date", "integer|double|date_period|time_duration", "integer|long|double|date", "integer|long|double|date"] |[Numeric or date expression from which to derive buckets., Target number of buckets., Start of the range. Can be a number or a date expressed as a string., End of the range. Can be a number or a date expressed as a string.] +bin |[field, buckets, from, to] |["integer|long|double|date", "integer|long|double|date_period|time_duration", "integer|long|double|date|keyword|text", "integer|long|double|date|keyword|text"] |[Numeric or date expression from which to derive buckets., Target number of buckets\, or desired bucket size if `from` and `to` parameters are omitted., Start of the range. Can be a number\, a date or a date expressed as a string., End of the range. Can be a number\, a date or a date expressed as a string.] +bucket |[field, buckets, from, to] |["integer|long|double|date", "integer|long|double|date_period|time_duration", "integer|long|double|date|keyword|text", "integer|long|double|date|keyword|text"] |[Numeric or date expression from which to derive buckets., Target number of buckets\, or desired bucket size if `from` and `to` parameters are omitted., Start of the range. Can be a number\, a date or a date expressed as a string., End of the range. Can be a number\, a date or a date expressed as a string.] case |[condition, trueValue] |[boolean, "boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version"] |[A condition., The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches.] cbrt |number |"double|integer|long|unsigned_long" |"Numeric expression. If `null`, the function returns `null`." ceil |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 996c5ac2ea319..0477167cd7315 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -234,7 +234,12 @@ public enum Cap { /** * Changed error messages for fields with conflicting types in different indices. */ - SHORT_ERROR_MESSAGES_FOR_UNSUPPORTED_FIELDS; + SHORT_ERROR_MESSAGES_FOR_UNSUPPORTED_FIELDS, + + /** + * Support for the whole number spans in BUCKET function. + */ + BUCKET_WHOLE_NUMBER_AS_SPAN; private final boolean snapshotOnly; private final FeatureFlag featureFlag; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 712eee8672bf3..5fabfe0e03d89 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -144,9 +145,7 @@ another in which the bucket size is provided directly (two parameters). ), @Example(description = """ The range can be omitted if the desired bucket size is known in advance. Simply - provide it as the second argument:""", file = "bucket", tag = "docsBucketNumericWithSpan", explanation = """ - NOTE: When providing the bucket size as the second parameter, it must be - of a floating point type."""), + provide it as the second argument:""", file = "bucket", tag = "docsBucketNumericWithSpan"), @Example( description = "Create hourly buckets for the last 24 hours, and calculate the number of events per hour:", file = "bucket", @@ -176,23 +175,23 @@ public Bucket( ) Expression field, @Param( name = "buckets", - type = { "integer", "double", "date_period", "time_duration" }, - description = "Target number of buckets." + type = { "integer", "long", "double", "date_period", "time_duration" }, + description = "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." ) Expression buckets, @Param( name = "from", - type = { "integer", "long", "double", "date" }, + type = { "integer", "long", "double", "date", "keyword", "text" }, optional = true, - description = "Start of the range. Can be a number or a date expressed as a string." + description = "Start of the range. Can be a number, a date or a date expressed as a string." ) Expression from, @Param( name = "to", - type = { "integer", "long", "double", "date" }, + type = { "integer", "long", "double", "date", "keyword", "text" }, optional = true, - description = "End of the range. Can be a number or a date expressed as a string." + description = "End of the range. Can be a number, a date or a date expressed as a string." ) Expression to ) { - super(source, from != null && to != null ? List.of(field, buckets, from, to) : List.of(field, buckets)); + super(source, fields(field, buckets, from, to)); this.field = field; this.buckets = buckets; this.from = from; @@ -209,6 +208,19 @@ private Bucket(StreamInput in) throws IOException { ); } + private static List fields(Expression field, Expression buckets, Expression from, Expression to) { + List list = new ArrayList<>(4); + list.add(field); + list.add(buckets); + if (from != null) { + list.add(from); + if (to != null) { + list.add(to); + } + } + return list; + } + @Override public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); @@ -251,7 +263,6 @@ public ExpressionEvaluator.Factory toEvaluator(Function isNumeric(from, sourceText(), THIRD)).and(() -> isNumeric(to, sourceText(), FOURTH)) - : isNumeric(buckets, sourceText(), SECOND).and(checkArgsCount(2)); + return isNumeric(buckets, sourceText(), SECOND).and(() -> { + if (bucketsType.isRationalNumber()) { + return checkArgsCount(2); + } else { // second arg is a whole number: either a span, but as a whole, or count, and we must expect a range + var resolution = checkArgsCount(2); + if (resolution.resolved() == false) { + resolution = checkArgsCount(4).and(() -> isNumeric(from, sourceText(), THIRD)) + .and(() -> isNumeric(to, sourceText(), FOURTH)); + } + return resolution; + } + }); } return isType(field, e -> false, sourceText(), FIRST, "datetime", "numeric"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index ab216e10b674c..bdea0807a78c4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -394,6 +394,66 @@ public void testGroupingInsideGrouping() { ); } + public void testInvalidBucketCalls() { + assertThat( + error("from test | stats max(emp_no) by bucket(emp_no, 5, \"2000-01-01\")"), + containsString( + "function expects exactly four arguments when the first one is of type [INTEGER] and the second of type [INTEGER]" + ) + ); + + assertThat( + error("from test | stats max(emp_no) by bucket(emp_no, 1 week, \"2000-01-01\")"), + containsString( + "second argument of [bucket(emp_no, 1 week, \"2000-01-01\")] must be [numeric], found value [1 week] type [date_period]" + ) + ); + + assertThat( + error("from test | stats max(emp_no) by bucket(hire_date, 5.5, \"2000-01-01\")"), + containsString( + "second argument of [bucket(hire_date, 5.5, \"2000-01-01\")] must be [integral, date_period or time_duration], " + + "found value [5.5] type [double]" + ) + ); + + assertThat( + error("from test | stats max(emp_no) by bucket(hire_date, 5, 1 day, 1 month)"), + containsString( + "third argument of [bucket(hire_date, 5, 1 day, 1 month)] must be [datetime or string], " + + "found value [1 day] type [date_period]" + ) + ); + + assertThat( + error("from test | stats max(emp_no) by bucket(hire_date, 5, \"2000-01-01\", 1 month)"), + containsString( + "fourth argument of [bucket(hire_date, 5, \"2000-01-01\", 1 month)] must be [datetime or string], " + + "found value [1 month] type [date_period]" + ) + ); + + assertThat( + error("from test | stats max(emp_no) by bucket(hire_date, 5, \"2000-01-01\")"), + containsString( + "function expects exactly four arguments when the first one is of type [DATETIME] and the second of type [INTEGER]" + ) + ); + + assertThat( + error("from test | stats max(emp_no) by bucket(emp_no, \"5\")"), + containsString("second argument of [bucket(emp_no, \"5\")] must be [numeric], found value [\"5\"] type [keyword]") + ); + + assertThat( + error("from test | stats max(emp_no) by bucket(hire_date, \"5\")"), + containsString( + "second argument of [bucket(hire_date, \"5\")] must be [integral, date_period or time_duration], " + + "found value [\"5\"] type [keyword]" + ) + ); + } + public void testAggsWithInvalidGrouping() { assertEquals( "1:35: column [languages] cannot be used as an aggregate once declared in the STATS BY grouping key [l = languages % 3]", @@ -748,9 +808,9 @@ public void testAggsResolutionWithUnresolvedGroupings() { ); assertThat(error("FROM tests | STATS " + agg_func + "(foobar) by foobar"), matchesRegex("1:\\d+: Unknown column \\[foobar]")); assertThat( - error("FROM tests | STATS " + agg_func + "(foobar) by BUCKET(languages, 10)"), + error("FROM tests | STATS " + agg_func + "(foobar) by BUCKET(hire_date, 10)"), matchesRegex( - "1:\\d+: function expects exactly four arguments when the first one is of type \\[INTEGER]" + "1:\\d+: function expects exactly four arguments when the first one is of type \\[DATETIME]" + " and the second of type \\[INTEGER]\n" + "line 1:\\d+: Unknown column \\[foobar]" ) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java index 4c7b812111450..a26504b8ced9a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java @@ -73,7 +73,7 @@ public static Iterable parameters() { } // TODO once we cast above the functions we can drop these - private static final DataType[] DATE_BOUNDS_TYPE = new DataType[] { DataType.DATETIME }; + private static final DataType[] DATE_BOUNDS_TYPE = new DataType[] { DataType.DATETIME, DataType.KEYWORD, DataType.TEXT }; private static void dateCases(List suppliers, String name, LongSupplier date) { for (DataType fromType : DATE_BOUNDS_TYPE) { @@ -89,7 +89,7 @@ private static void dateCases(List suppliers, String name, Lon args, "DateTruncEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding[DAY_OF_MONTH in Z][fixed to midnight]]", DataType.DATETIME, - dateResultsMatcher(args) + resultsMatcher(args) ); })); // same as above, but a low bucket count and datetime bounds that match it (at hour span) @@ -136,7 +136,7 @@ private static void dateCasesWithSpan( args, "DateTruncEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding" + spanStr + "]", DataType.DATETIME, - dateResultsMatcher(args) + resultsMatcher(args) ); })); } @@ -167,7 +167,7 @@ private static void numberCases(List suppliers, String name, D + ", " + "rhs=LiteralsEvaluator[lit=50.0]]], rhs=LiteralsEvaluator[lit=50.0]]", DataType.DOUBLE, - dateResultsMatcher(args) + resultsMatcher(args) ); })); } @@ -187,26 +187,29 @@ private static TestCaseSupplier.TypedData numericBound(String name, DataType typ } private static void numberCasesWithSpan(List suppliers, String name, DataType numberType, Supplier number) { - suppliers.add(new TestCaseSupplier(name, List.of(numberType, DataType.DOUBLE), () -> { - List args = new ArrayList<>(); - args.add(new TestCaseSupplier.TypedData(number.get(), "field")); - args.add(new TestCaseSupplier.TypedData(50., DataType.DOUBLE, "span").forceLiteral()); - String attr = "Attribute[channel=0]"; - if (numberType == DataType.INTEGER) { - attr = "CastIntToDoubleEvaluator[v=" + attr + "]"; - } else if (numberType == DataType.LONG) { - attr = "CastLongToDoubleEvaluator[v=" + attr + "]"; - } - return new TestCaseSupplier.TestCase( - args, - "MulDoublesEvaluator[lhs=FloorDoubleEvaluator[val=DivDoublesEvaluator[lhs=" - + attr - + ", " - + "rhs=LiteralsEvaluator[lit=50.0]]], rhs=LiteralsEvaluator[lit=50.0]]", - DataType.DOUBLE, - dateResultsMatcher(args) - ); - })); + for (Number span : List.of(50, 50L, 50d)) { + DataType spanType = DataType.fromJava(span); + suppliers.add(new TestCaseSupplier(name, List.of(numberType, spanType), () -> { + List args = new ArrayList<>(); + args.add(new TestCaseSupplier.TypedData(number.get(), "field")); + args.add(new TestCaseSupplier.TypedData(span, spanType, "span").forceLiteral()); + String attr = "Attribute[channel=0]"; + if (numberType == DataType.INTEGER) { + attr = "CastIntToDoubleEvaluator[v=" + attr + "]"; + } else if (numberType == DataType.LONG) { + attr = "CastLongToDoubleEvaluator[v=" + attr + "]"; + } + return new TestCaseSupplier.TestCase( + args, + "MulDoublesEvaluator[lhs=FloorDoubleEvaluator[val=DivDoublesEvaluator[lhs=" + + attr + + ", " + + "rhs=LiteralsEvaluator[lit=50.0]]], rhs=LiteralsEvaluator[lit=50.0]]", + DataType.DOUBLE, + resultsMatcher(args) + ); + })); + } } @@ -214,7 +217,7 @@ private static TestCaseSupplier.TypedData keywordDateLiteral(String name, DataTy return new TestCaseSupplier.TypedData(date, type, name).forceLiteral(); } - private static Matcher dateResultsMatcher(List typedData) { + private static Matcher resultsMatcher(List typedData) { if (typedData.get(0).type() == DataType.DATETIME) { long millis = ((Number) typedData.get(0).data()).longValue(); return equalTo(Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(millis)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index c6b12eb0dc23f..a294f33ece5c3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -3514,6 +3514,49 @@ public void testBucketWithAggExpression() { assertThat(agg.groupings().get(0), is(ref)); } + public void testBucketWithNonFoldingArgs() { + assertThat( + typesError("from types | stats max(integer) by bucket(date, integer, \"2000-01-01\", \"2000-01-02\")"), + containsString( + "second argument of [bucket(date, integer, \"2000-01-01\", \"2000-01-02\")] must be a constant, " + "received [integer]" + ) + ); + + assertThat( + typesError("from types | stats max(integer) by bucket(date, 2, date, \"2000-01-02\")"), + containsString("third argument of [bucket(date, 2, date, \"2000-01-02\")] must be a constant, " + "received [date]") + ); + + assertThat( + typesError("from types | stats max(integer) by bucket(date, 2, \"2000-01-02\", date)"), + containsString("fourth argument of [bucket(date, 2, \"2000-01-02\", date)] must be a constant, " + "received [date]") + ); + + assertThat( + typesError("from types | stats max(integer) by bucket(integer, long, 4, 5)"), + containsString("second argument of [bucket(integer, long, 4, 5)] must be a constant, " + "received [long]") + ); + + assertThat( + typesError("from types | stats max(integer) by bucket(integer, 3, long, 5)"), + containsString("third argument of [bucket(integer, 3, long, 5)] must be a constant, " + "received [long]") + ); + + assertThat( + typesError("from types | stats max(integer) by bucket(integer, 3, 4, long)"), + containsString("fourth argument of [bucket(integer, 3, 4, long)] must be a constant, " + "received [long]") + ); + } + + private String typesError(String query) { + VerificationException e = expectThrows(VerificationException.class, () -> planTypes(query)); + String message = e.getMessage(); + assertTrue(message.startsWith("Found ")); + String pattern = "\nline "; + int index = message.indexOf(pattern); + return message.substring(index + pattern.length()); + } + /** * Expects * Project[[x{r}#5]] From 47d331662cc8801336c36d83fa7b0ce7b6959a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Tue, 20 Aug 2024 14:42:44 +0200 Subject: [PATCH 05/32] Fix query roles test by setting license synchronously (#112002) Relates: https://github.com/elastic/elasticsearch/issues/110729 The `testQueryDLSFLSRolesShowAsDisabled` failed intermittently and my theory is that it's because applying the license of the cluster to cluster state has `NORMAL` priority and therefore sometimes (very rarely) takes more than 10 seconds. There are some related discussions to this, see: https://github.com/elastic/elasticsearch/pull/67182, https://github.com/elastic/elasticsearch/issues/64578 Since we're not testing the actual license lifecycle in this test, but instead how an applied license impacts the query roles API, I changed the approach to use the synchronous `/_license/start_trial` API in a `@before` so we can be sure the license was applied before we start testing. An alternative to this fix could be to increase the timeout. --- muted-tests.yml | 3 -- .../xpack/security/LicenseDLSFLSRoleIT.java | 44 ++++++++----------- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 95fb4a32b4227..c2e0d48c31a20 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -65,9 +65,6 @@ tests: - class: "org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests" issue: "https://github.com/elastic/elasticsearch/issues/110408" method: "testCreateAndRestorePartialSearchableSnapshot" -- class: org.elasticsearch.xpack.security.LicenseDLSFLSRoleIT - method: testQueryDLSFLSRolesShowAsDisabled - issue: https://github.com/elastic/elasticsearch/issues/110729 - class: org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests method: testPopulationOfCacheWhenLoadingPrivilegesForAllApplications issue: https://github.com/elastic/elasticsearch/issues/110789 diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java index f81bab4866bdf..552e9f5cba578 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/LicenseDLSFLSRoleIT.java @@ -9,7 +9,6 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.SecureString; @@ -21,6 +20,8 @@ import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.junit.After; +import org.junit.Before; import org.junit.ClassRule; import java.io.IOException; @@ -50,8 +51,6 @@ public final class LicenseDLSFLSRoleIT extends ESRestTestCase { public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .nodes(1) .distribution(DistributionType.DEFAULT) - // start as "trial" - .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") .setting("xpack.security.http.ssl.enabled", "false") .setting("xpack.security.transport.ssl.enabled", "false") @@ -61,6 +60,23 @@ public final class LicenseDLSFLSRoleIT extends ESRestTestCase { .user(READ_SECURITY_USER, READ_SECURITY_PASSWORD.toString(), "read_security_user_role", false) .build(); + @Before + public void setupLicense() throws IOException { + // start with trial license + Request request = new Request("POST", "/_license/start_trial?acknowledge=true"); + Response response = adminClient().performRequest(request); + assertOK(response); + assertTrue((boolean) responseAsMap(response).get("trial_was_started")); + } + + @After + public void removeLicense() throws IOException { + // start with trial license + Request request = new Request("DELETE", "/_license"); + Response response = adminClient().performRequest(request); + assertOK(response); + } + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); @@ -78,10 +94,7 @@ protected Settings restClientSettings() { return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); } - @SuppressWarnings("unchecked") public void testQueryDLSFLSRolesShowAsDisabled() throws Exception { - // auto-generated "trial" - waitForLicense(adminClient(), "trial"); // neither DLS nor FLS role { RoleDescriptor.IndicesPrivileges[] indicesPrivileges = new RoleDescriptor.IndicesPrivileges[] { @@ -138,7 +151,6 @@ public void testQueryDLSFLSRolesShowAsDisabled() throws Exception { Map responseMap = responseAsMap(response); assertTrue(((Boolean) responseMap.get("basic_was_started"))); assertTrue(((Boolean) responseMap.get("acknowledged"))); - waitForLicense(adminClient(), "basic"); // now the same roles show up as disabled ("enabled" is "false") assertQuery(client(), "", 4, roles -> { roles.sort(Comparator.comparing(o -> ((String) o.get("name")))); @@ -175,22 +187,4 @@ private static void assertRoleEnabled(Map roleMap, boolean enabl assertThat(roleMap.get("transient_metadata"), instanceOf(Map.class)); assertThat(((Map) roleMap.get("transient_metadata")).get("enabled"), equalTo(enabled)); } - - @SuppressWarnings("unchecked") - private static void waitForLicense(RestClient adminClient, String type) throws Exception { - final Request request = new Request("GET", "_license"); - assertBusy(() -> { - Response response; - try { - response = adminClient.performRequest(request); - } catch (ResponseException e) { - throw new AssertionError("license not yet installed", e); - } - assertOK(response); - Map responseMap = responseAsMap(response); - assertTrue(responseMap.containsKey("license")); - assertThat(((Map) responseMap.get("license")).get("status"), equalTo("active")); - assertThat(((Map) responseMap.get("license")).get("type"), equalTo(type)); - }); - } } From 5a10545d371c9665892a431fd7e036b500c6f3ba Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 20 Aug 2024 06:07:08 -0700 Subject: [PATCH 06/32] Upgrade xcontent to Jackson 2.17.0 (#111948) --- docs/changelog/111948.yaml | 5 +++ gradle/verification-metadata.xml | 35 +++++++++++++------ libs/x-content/impl/build.gradle | 2 +- .../provider/json/JsonXContentImpl.java | 2 ++ .../search/MultiSearchRequestTests.java | 12 ++++--- .../index/mapper/DocumentParserTests.java | 2 +- .../HuggingFaceElserResponseEntityTests.java | 2 +- 7 files changed, 42 insertions(+), 18 deletions(-) create mode 100644 docs/changelog/111948.yaml diff --git a/docs/changelog/111948.yaml b/docs/changelog/111948.yaml new file mode 100644 index 0000000000000..a3a592abaf1ca --- /dev/null +++ b/docs/changelog/111948.yaml @@ -0,0 +1,5 @@ +pr: 111948 +summary: Upgrade xcontent to Jackson 2.17.0 +area: Infra/Core +type: upgrade +issues: [] diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 00f1caec24cf7..1001ab2b709dd 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -306,6 +306,11 @@ + + + + + @@ -336,6 +341,11 @@ + + + + + @@ -346,6 +356,11 @@ + + + + + @@ -361,6 +376,11 @@ + + + + + @@ -953,11 +973,6 @@ - - - - - @@ -1746,16 +1761,16 @@ - - - - - + + + + + diff --git a/libs/x-content/impl/build.gradle b/libs/x-content/impl/build.gradle index 41b65044735ca..829b75524baeb 100644 --- a/libs/x-content/impl/build.gradle +++ b/libs/x-content/impl/build.gradle @@ -12,7 +12,7 @@ base { archivesName = "x-content-impl" } -String jacksonVersion = "2.15.0" +String jacksonVersion = "2.17.0" dependencies { compileOnly project(':libs:elasticsearch-core') diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java index ae494796c88cb..4e04230a7486e 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentImpl.java @@ -54,6 +54,8 @@ public static final XContent jsonXContent() { jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false); jsonFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); jsonFactory.configure(JsonParser.Feature.USE_FAST_DOUBLE_PARSER, true); + // keeping existing behavior of including source, for now + jsonFactory.configure(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION, true); jsonXContent = new JsonXContentImpl(); } diff --git a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java index a45730a82dbc2..67c8599f47029 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java @@ -45,6 +45,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -582,11 +583,12 @@ public void testFailOnExtraCharacters() throws IOException { """, null); fail("should have caught second line; extra closing brackets"); } catch (XContentParseException e) { - assertEquals( - "[1:31] Unexpected close marker '}': expected ']' (for root starting at " - + "[Source: (byte[])\"{ \"query\": {\"match_all\": {}}}}}}different error message\"; line: 1, column: 0])\n " - + "at [Source: (byte[])\"{ \"query\": {\"match_all\": {}}}}}}different error message\"; line: 1, column: 31]", - e.getMessage() + assertThat( + e.getMessage(), + containsString( + "Unexpected close marker '}': expected ']' (for root starting at " + + "[Source: (byte[])\"{ \"query\": {\"match_all\": {}}}}}}different error message\"" + ) ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 7fa08acd53882..1a0e2376797b8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2642,7 +2642,7 @@ same name need to be part of the same mappings (hence the same document). If th } public void testDeeplyNestedDocument() throws Exception { - int depth = 10000; + int depth = 20; DocumentMapper docMapper = createMapperService(Settings.builder().put(getIndexSettings()).build(), mapping(b -> {})) .documentMapper(); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java index c3c416d8fe65e..e350a539ba928 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceElserResponseEntityTests.java @@ -310,7 +310,7 @@ public void testFails_ResponseIsInvalidJson_MissingSquareBracket() { ) ); - assertThat(thrownException.getMessage(), containsString("expected close marker for Array (start marker at [Source: (byte[])")); + assertThat(thrownException.getMessage(), containsString("expected close marker for Array (start marker at")); } public void testFails_ResponseIsInvalidJson_MissingField() { From 3de2587f939b38fa7532844406010822ef3ec260 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 20 Aug 2024 09:23:35 -0400 Subject: [PATCH 07/32] ESQL: Test on-the-wire size for some plan nodes (#111980) This adds some very highly specified tests for the on-the-wire size for plan nodes, especially those with mapping conflicts. We've been having some trouble with this being *very* large on the wire and we'd like to take more care in the future to keep these from growing. The plan is that we'll lower these limits as we go, "ratcheting" the serialization size down as we make improvements. The test will make sure we don't make things worse. --- .../test/ByteSizeEqualsMatcher.java | 43 +++++ .../xpack/esql/analysis/Analyzer.java | 5 +- .../esql/index/EsIndexSerializationTests.java | 102 +++++++++++ .../ExchangeSinkExecSerializationTests.java | 159 ++++++++++++++++++ 4 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/ByteSizeEqualsMatcher.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/ByteSizeEqualsMatcher.java b/test/framework/src/main/java/org/elasticsearch/test/ByteSizeEqualsMatcher.java new file mode 100644 index 0000000000000..172d5f2076a0f --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/ByteSizeEqualsMatcher.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; + +/** + * Equality matcher for {@link ByteSizeValue} that has a nice description of failures. + */ +public class ByteSizeEqualsMatcher extends TypeSafeMatcher { + public static ByteSizeEqualsMatcher byteSizeEquals(ByteSizeValue expected) { + return new ByteSizeEqualsMatcher(expected); + } + + private final ByteSizeValue expected; + + private ByteSizeEqualsMatcher(ByteSizeValue expected) { + this.expected = expected; + } + + @Override + protected boolean matchesSafely(ByteSizeValue byteSizeValue) { + return expected.equals(byteSizeValue); + } + + @Override + public void describeTo(Description description) { + description.appendValue(expected.toString()).appendText(" (").appendValue(expected.getBytes()).appendText(" bytes)"); + } + + @Override + protected void describeMismatchSafely(ByteSizeValue item, Description mismatchDescription) { + mismatchDescription.appendValue(item.toString()).appendText(" (").appendValue(item.getBytes()).appendText(" bytes)"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 4a116fd102cd0..3ffb4acbe6455 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -212,8 +212,11 @@ protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) { * Specific flattening method, different from the default EsRelation that: * 1. takes care of data type widening (for certain types) * 2. drops the object and keyword hierarchy + *

+ * Public for testing. + *

*/ - private static List mappingAsAttributes(Source source, Map mapping) { + public static List mappingAsAttributes(Source source, Map mapping) { var list = new ArrayList(); mappingAsAttributes(list, source, null, mapping); list.sort(Comparator.comparing(Attribute::name)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java index 1ac61a2adf68e..e1b56d61a211c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexSerializationTests.java @@ -7,17 +7,26 @@ package org.elasticsearch.xpack.esql.index; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.EsFieldTests; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import java.io.IOException; import java.util.HashMap; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import static org.elasticsearch.test.ByteSizeEqualsMatcher.byteSizeEquals; public class EsIndexSerializationTests extends AbstractWireSerializingTestCase { public static EsIndex randomEsIndex() { @@ -73,4 +82,97 @@ protected EsIndex mutateInstance(EsIndex instance) throws IOException { protected NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(EsField.getNamedWriteables()); } + + /** + * Build an {@link EsIndex} with many conflicting fields across many indices. + */ + public static EsIndex indexWithManyConflicts(boolean withParent) { + /* + * The number of fields with a mapping conflict. + */ + int conflictingCount = 250; + /* + * The number of indices that map conflicting fields are "keyword". + * One other index will map the field as "text" + */ + int keywordIndicesCount = 600; + /* + * The number of fields that don't have a mapping conflict. + */ + int nonConflictingCount = 7000; + + Set keywordIndices = new TreeSet<>(); + for (int i = 0; i < keywordIndicesCount; i++) { + keywordIndices.add(String.format(Locale.ROOT, ".ds-logs-apache.access-external-2024.08.09-%08d", i)); + } + + Set textIndices = Set.of("logs-endpoint.events.imported"); + + Map fields = new TreeMap<>(); + for (int i = 0; i < conflictingCount; i++) { + String name = String.format(Locale.ROOT, "blah.blah.blah.blah.blah.blah.conflict.name%04d", i); + Map> conflicts = Map.of("text", textIndices, "keyword", keywordIndices); + fields.put(name, new InvalidMappedField(name, conflicts)); + } + for (int i = 0; i < nonConflictingCount; i++) { + String name = String.format(Locale.ROOT, "blah.blah.blah.blah.blah.blah.nonconflict.name%04d", i); + fields.put(name, new EsField(name, DataType.KEYWORD, Map.of(), true)); + } + + if (withParent) { + EsField parent = new EsField("parent", DataType.OBJECT, Map.copyOf(fields), false); + fields.put("parent", parent); + } + + TreeSet concrete = new TreeSet<>(); + concrete.addAll(keywordIndices); + concrete.addAll(textIndices); + + return new EsIndex("name", fields, concrete); + } + + /** + * Test the size of serializing an index with many conflicts at the root level. + * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more. + */ + public void testManyTypeConflicts() throws IOException { + testManyTypeConflicts(false, ByteSizeValue.ofBytes(976591)); + } + + /** + * Test the size of serializing an index with many conflicts inside a "parent" object. + * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more. + */ + public void testManyTypeConflictsWithParent() throws IOException { + testManyTypeConflicts(true, ByteSizeValue.ofBytes(1921374)); + /* + * History: + * 16.9mb - start + * 1.8mb - shorten error messages for UnsupportedAttributes #111973 + */ + } + + /** + * Test the size of serializing an index with many conflicts. Callers of + * this method intentionally use a very precise size for the serialized + * data so a programmer making changes has to think when this size changes. + *

+ * In general, shrinking the over the wire size is great and the precise + * size should just ratchet downwards. Small upwards movement is fine so + * long as you understand why the change is happening and you think it's + * worth it for the data node request for a big index to grow. + *

+ *

+ * Large upwards movement in the size is not fine! Folks frequently make + * requests across large clusters with many fields and these requests can + * really clog up the network interface. Super large results here can make + * ESQL impossible to use at all for big mappings with many conflicts. + *

+ */ + private void testManyTypeConflicts(boolean withParent, ByteSizeValue expected) throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + indexWithManyConflicts(withParent).writeTo(out); + assertThat(ByteSizeValue.ofBytes(out.bytes().length()), byteSizeEquals(expected)); + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java new file mode 100644 index 0000000000000..237f8d6a9c580 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/physical/ExchangeSinkExecSerializationTests.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plan.physical; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.analysis.Analyzer; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.EsIndexSerializationTests; +import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.Limit; +import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.session.Configuration; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.ByteSizeEqualsMatcher.byteSizeEquals; +import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; +import static org.hamcrest.Matchers.equalTo; + +public class ExchangeSinkExecSerializationTests extends ESTestCase { + // TODO port this to AbstractPhysicalPlanSerializationTests when implementing NamedWriteable + private Configuration config; + + public static Source randomSource() { + int lineNumber = between(0, EXAMPLE_QUERY.length - 1); + String line = EXAMPLE_QUERY[lineNumber]; + int offset = between(0, line.length() - 2); + int length = between(1, line.length() - offset - 1); + String text = line.substring(offset, offset + length); + return new Source(lineNumber + 1, offset, text); + } + + /** + * Test the size of serializing a plan with many conflicts. + * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more. + */ + public void testManyTypeConflicts() throws IOException { + testManyTypeConflicts(false, ByteSizeValue.ofBytes(2444252)); + } + + /** + * Test the size of serializing a plan with many conflicts. + * See {@link #testManyTypeConflicts(boolean, ByteSizeValue)} for more. + */ + public void testManyTypeConflictsWithParent() throws IOException { + testManyTypeConflicts(true, ByteSizeValue.ofBytes(5885765)); + /* + * History: + * 2 gb+ - start + * 43.3mb - Cache attribute subclasses #111447 + * 5.6mb - shorten error messages for UnsupportedAttributes #111973 + */ + } + + /** + * Test the size of serializing a plan with many conflicts. Callers of + * this method intentionally use a very precise size for the serialized + * data so a programmer making changes has to think when this size changes. + *

+ * In general, shrinking the over the wire size is great and the precise + * size should just ratchet downwards. Small upwards movement is fine so + * long as you understand why the change is happening and you think it's + * worth it for the data node request for a big index to grow. + *

+ *

+ * Large upwards movement in the size is not fine! Folks frequently make + * requests across large clusters with many fields and these requests can + * really clog up the network interface. Super large results here can make + * ESQL impossible to use at all for big mappings with many conflicts. + *

+ */ + private void testManyTypeConflicts(boolean withParent, ByteSizeValue expected) throws IOException { + EsIndex index = EsIndexSerializationTests.indexWithManyConflicts(withParent); + List attributes = Analyzer.mappingAsAttributes(randomSource(), index.mapping()); + EsRelation relation = new EsRelation(randomSource(), index, attributes, IndexMode.STANDARD); + Limit limit = new Limit(randomSource(), new Literal(randomSource(), 10, DataType.INTEGER), relation); + Project project = new Project(randomSource(), limit, limit.output()); + FragmentExec fragmentExec = new FragmentExec(project); + ExchangeSinkExec exchangeSinkExec = new ExchangeSinkExec(randomSource(), fragmentExec.output(), false, fragmentExec); + try ( + BytesStreamOutput out = new BytesStreamOutput(); + PlanStreamOutput pso = new PlanStreamOutput(out, new PlanNameRegistry(), configuration()) + ) { + pso.writePhysicalPlanNode(exchangeSinkExec); + assertThat(ByteSizeValue.ofBytes(out.bytes().length()), byteSizeEquals(expected)); + try ( + PlanStreamInput psi = new PlanStreamInput( + out.bytes().streamInput(), + new PlanNameRegistry(), + getNamedWriteableRegistry(), + configuration() + ) + ) { + assertThat(psi.readPhysicalPlanNode(), equalTo(exchangeSinkExec)); + } + } + } + + private NamedWriteableRegistry getNamedWriteableRegistry() { + List entries = new ArrayList<>(); + entries.addAll(PhysicalPlan.getNamedWriteables()); + entries.addAll(LogicalPlan.getNamedWriteables()); + entries.addAll(AggregateFunction.getNamedWriteables()); + entries.addAll(Expression.getNamedWriteables()); + entries.addAll(Attribute.getNamedWriteables()); + entries.addAll(EsField.getNamedWriteables()); + entries.addAll(Block.getNamedWriteables()); + entries.addAll(NamedExpression.getNamedWriteables()); + entries.addAll(new SearchModule(Settings.EMPTY, List.of()).getNamedWriteables()); + return new NamedWriteableRegistry(entries); + } + + private Configuration configuration() { + return config; + } + + private static final String[] EXAMPLE_QUERY = new String[] { + "I am the very model of a modern Major-Gineral,", + "I've information vegetable, animal, and mineral,", + "I know the kings of England, and I quote the fights historical", + "From Marathon to Waterloo, in order categorical;", + "I'm very well acquainted, too, with matters mathematical,", + "I understand equations, both the simple and quadratical,", + "About binomial theorem I'm teeming with a lot o' news,", + "With many cheerful facts about the square of the hypotenuse." }; + + @Before + public void initConfig() { + config = randomConfiguration(String.join("\n", EXAMPLE_QUERY), Map.of()); + } +} From e3f378ebd289876fb15afa3e03aabd502fae7d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Tue, 20 Aug 2024 15:24:55 +0200 Subject: [PATCH 08/32] ESQL: Strings support for MAX and MIN aggregations (#111544) Support Version, Keyword and Text in Max an Min aggregations. The current implementation of both max and min does: For non-grouping: - Store a BytesRef - When there's a max/min, copy it to the internal array. Grow it if needed For grouping: - Keep an array of BytesRef (null by default: there's no "initial/default value" here, as there's no "MAX" value for a string) - Each BytesRef stores their own array, which will be grown as needed to copy the new max/min Some notes: - It's not shrinking the arrays, as to avoid having to copy, and potentially grow it again - It's using raw arrays. But maybe it should use BigArrays to compute in the circuit breaker? Part of https://github.com/elastic/elasticsearch/issues/110346 --- docs/changelog/111544.yaml | 5 + .../esql/functions/kibana/definition/max.json | 36 +++ .../esql/functions/kibana/definition/min.json | 36 +++ .../esql/functions/types/max.asciidoc | 3 + .../esql/functions/types/min.asciidoc | 3 + .../compute/gen/AggregatorImplementer.java | 2 +- .../org/elasticsearch/compute/gen/Types.java | 1 + .../MaxBytesRefAggregatorFunction.java | 133 +++++++++++ ...MaxBytesRefAggregatorFunctionSupplier.java | 38 ++++ ...MaxBytesRefGroupingAggregatorFunction.java | 210 ++++++++++++++++++ .../MinBytesRefAggregatorFunction.java | 133 +++++++++++ ...MinBytesRefAggregatorFunctionSupplier.java | 38 ++++ ...MinBytesRefGroupingAggregatorFunction.java | 210 ++++++++++++++++++ .../aggregation/AbstractArrayState.java | 2 +- .../aggregation/BytesRefArrayState.java | 153 +++++++++++++ .../aggregation/MaxBytesRefAggregator.java | 149 +++++++++++++ .../aggregation/MinBytesRefAggregator.java | 149 +++++++++++++ .../operator/BreakingBytesRefBuilder.java | 10 +- .../MaxBytesRefAggregatorFunctionTests.java | 53 +++++ ...tesRefGroupingAggregatorFunctionTests.java | 62 ++++++ .../MinBytesRefAggregatorFunctionTests.java | 53 +++++ ...tesRefGroupingAggregatorFunctionTests.java | 62 ++++++ .../BreakingBytesRefBuilderTests.java | 26 ++- .../src/main/resources/meta.csv-spec | 12 +- .../src/main/resources/stats.csv-spec | 160 +++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../expression/function/aggregate/Max.java | 24 +- .../expression/function/aggregate/Min.java | 24 +- .../xpack/esql/planner/AggregateMapper.java | 2 +- .../xpack/esql/analysis/AnalyzerTests.java | 20 +- .../xpack/esql/analysis/VerifierTests.java | 4 +- .../function/aggregate/MaxTests.java | 40 +++- .../function/aggregate/MinTests.java | 40 +++- 33 files changed, 1848 insertions(+), 50 deletions(-) create mode 100644 docs/changelog/111544.yaml create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunctionTests.java diff --git a/docs/changelog/111544.yaml b/docs/changelog/111544.yaml new file mode 100644 index 0000000000000..d4c46f485e664 --- /dev/null +++ b/docs/changelog/111544.yaml @@ -0,0 +1,5 @@ +pr: 111544 +summary: "ESQL: Strings support for MAX and MIN aggregations" +area: ES|QL +type: feature +issues: [] diff --git a/docs/reference/esql/functions/kibana/definition/max.json b/docs/reference/esql/functions/kibana/definition/max.json index 853cb9f9a97c3..725b42763816d 100644 --- a/docs/reference/esql/functions/kibana/definition/max.json +++ b/docs/reference/esql/functions/kibana/definition/max.json @@ -64,6 +64,18 @@ "variadic" : false, "returnType" : "ip" }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { @@ -75,6 +87,30 @@ ], "variadic" : false, "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "text" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "version" } ], "examples" : [ diff --git a/docs/reference/esql/functions/kibana/definition/min.json b/docs/reference/esql/functions/kibana/definition/min.json index 1c0c02eb9860f..68dfdd6cfd8c0 100644 --- a/docs/reference/esql/functions/kibana/definition/min.json +++ b/docs/reference/esql/functions/kibana/definition/min.json @@ -64,6 +64,18 @@ "variadic" : false, "returnType" : "ip" }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { @@ -75,6 +87,30 @@ ], "variadic" : false, "returnType" : "long" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "text" + }, + { + "params" : [ + { + "name" : "field", + "type" : "version", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "version" } ], "examples" : [ diff --git a/docs/reference/esql/functions/types/max.asciidoc b/docs/reference/esql/functions/types/max.asciidoc index 5b7293d4a4293..705745d76dbab 100644 --- a/docs/reference/esql/functions/types/max.asciidoc +++ b/docs/reference/esql/functions/types/max.asciidoc @@ -10,5 +10,8 @@ datetime | datetime double | double integer | integer ip | ip +keyword | keyword long | long +text | text +version | version |=== diff --git a/docs/reference/esql/functions/types/min.asciidoc b/docs/reference/esql/functions/types/min.asciidoc index 5b7293d4a4293..705745d76dbab 100644 --- a/docs/reference/esql/functions/types/min.asciidoc +++ b/docs/reference/esql/functions/types/min.asciidoc @@ -10,5 +10,8 @@ datetime | datetime double | double integer | integer ip | ip +keyword | keyword long | long +text | text +version | version |=== diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index b3d32a82cc7a9..914724905541d 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -102,7 +102,7 @@ public AggregatorImplementer(Elements elements, TypeElement declarationType, Int this.createParameters = init.getParameters() .stream() .map(Parameter::from) - .filter(f -> false == f.type().equals(BIG_ARRAYS)) + .filter(f -> false == f.type().equals(BIG_ARRAYS) && false == f.type().equals(DRIVER_CONTEXT)) .toList(); this.implementation = ClassName.get( diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index 3150741ddcb05..2b42adc67d71a 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -34,6 +34,7 @@ public class Types { static final TypeName BLOCK_ARRAY = ArrayTypeName.of(BLOCK); static final ClassName VECTOR = ClassName.get(DATA_PACKAGE, "Vector"); + static final ClassName CIRCUIT_BREAKER = ClassName.get("org.elasticsearch.common.breaker", "CircuitBreaker"); static final ClassName BIG_ARRAYS = ClassName.get("org.elasticsearch.common.util", "BigArrays"); static final ClassName BOOLEAN_BLOCK = ClassName.get(DATA_PACKAGE, "BooleanBlock"); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java new file mode 100644 index 0000000000000..62897c61ea80e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java @@ -0,0 +1,133 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MaxBytesRefAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxBytesRefAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final MaxBytesRefAggregator.SingleState state; + + private final List channels; + + public MaxBytesRefAggregatorFunction(DriverContext driverContext, List channels, + MaxBytesRefAggregator.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MaxBytesRefAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MaxBytesRefAggregatorFunction(driverContext, channels, MaxBytesRefAggregator.initSingle(driverContext)); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + BytesRefBlock block = page.getBlock(channels.get(0)); + BytesRefVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(BytesRefVector vector) { + BytesRef scratch = new BytesRef(); + for (int i = 0; i < vector.getPositionCount(); i++) { + MaxBytesRefAggregator.combine(state, vector.getBytesRef(i, scratch)); + } + } + + private void addRawBlock(BytesRefBlock block) { + BytesRef scratch = new BytesRef(); + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + MaxBytesRefAggregator.combine(state, block.getBytesRef(i, scratch)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + BytesRefVector max = ((BytesRefBlock) maxUncast).asVector(); + assert max.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + MaxBytesRefAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = MaxBytesRefAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..7c8af2e0c7e6d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MaxBytesRefAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxBytesRefAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MaxBytesRefAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MaxBytesRefAggregatorFunction aggregator(DriverContext driverContext) { + return MaxBytesRefAggregatorFunction.create(driverContext, channels); + } + + @Override + public MaxBytesRefGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return MaxBytesRefGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "max of bytes"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..1720a8863a613 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java @@ -0,0 +1,210 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MaxBytesRefAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxBytesRefGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final MaxBytesRefAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public MaxBytesRefGroupingAggregatorFunction(List channels, + MaxBytesRefAggregator.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MaxBytesRefGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new MaxBytesRefGroupingAggregatorFunction(channels, MaxBytesRefAggregator.initGrouping(driverContext), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + BytesRefBlock valuesBlock = page.getBlock(channels.get(0)); + BytesRefVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MaxBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MaxBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MaxBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + MaxBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + BytesRefVector max = ((BytesRefBlock) maxUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert max.getPositionCount() == seen.getPositionCount(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + MaxBytesRefAggregator.GroupingState inState = ((MaxBytesRefGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + MaxBytesRefAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = MaxBytesRefAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java new file mode 100644 index 0000000000000..3346dd762f17f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java @@ -0,0 +1,133 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MinBytesRefAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinBytesRefAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("min", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final MinBytesRefAggregator.SingleState state; + + private final List channels; + + public MinBytesRefAggregatorFunction(DriverContext driverContext, List channels, + MinBytesRefAggregator.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MinBytesRefAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MinBytesRefAggregatorFunction(driverContext, channels, MinBytesRefAggregator.initSingle(driverContext)); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + BytesRefBlock block = page.getBlock(channels.get(0)); + BytesRefVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(BytesRefVector vector) { + BytesRef scratch = new BytesRef(); + for (int i = 0; i < vector.getPositionCount(); i++) { + MinBytesRefAggregator.combine(state, vector.getBytesRef(i, scratch)); + } + } + + private void addRawBlock(BytesRefBlock block) { + BytesRef scratch = new BytesRef(); + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + MinBytesRefAggregator.combine(state, block.getBytesRef(i, scratch)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block minUncast = page.getBlock(channels.get(0)); + if (minUncast.areAllValuesNull()) { + return; + } + BytesRefVector min = ((BytesRefBlock) minUncast).asVector(); + assert min.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + MinBytesRefAggregator.combineIntermediate(state, min.getBytesRef(0, scratch), seen.getBoolean(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = MinBytesRefAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..cb6ab0d06d401 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MinBytesRefAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinBytesRefAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MinBytesRefAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MinBytesRefAggregatorFunction aggregator(DriverContext driverContext) { + return MinBytesRefAggregatorFunction.create(driverContext, channels); + } + + @Override + public MinBytesRefGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return MinBytesRefGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "min of bytes"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..eb309614fcf3c --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java @@ -0,0 +1,210 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MinBytesRefAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinBytesRefGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("min", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final MinBytesRefAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public MinBytesRefGroupingAggregatorFunction(List channels, + MinBytesRefAggregator.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MinBytesRefGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new MinBytesRefGroupingAggregatorFunction(channels, MinBytesRefAggregator.initGrouping(driverContext), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + BytesRefBlock valuesBlock = page.getBlock(channels.get(0)); + BytesRefVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MinBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MinBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MinBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + MinBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block minUncast = page.getBlock(channels.get(0)); + if (minUncast.areAllValuesNull()) { + return; + } + BytesRefVector min = ((BytesRefBlock) minUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert min.getPositionCount() == seen.getPositionCount(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + MinBytesRefAggregator.GroupingState inState = ((MinBytesRefGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + MinBytesRefAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = MinBytesRefAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java index 0dc008cb22396..1573efdd81059 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java @@ -21,7 +21,7 @@ public AbstractArrayState(BigArrays bigArrays) { this.bigArrays = bigArrays; } - final boolean hasValue(int groupId) { + boolean hasValue(int groupId) { return seen == null || seen.get(groupId); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java new file mode 100644 index 0000000000000..eb0a992c8610f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/BytesRefArrayState.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ObjectArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of BytesRefs. It is created in a mode where it + * won't track the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

+ * This class is a specialized version of the {@code X-ArrayState.java.st} template. + *

+ */ +public final class BytesRefArrayState implements GroupingAggregatorState, Releasable { + private final BigArrays bigArrays; + private final CircuitBreaker breaker; + private final String breakerLabel; + private ObjectArray values; + /** + * If false, no group id is expected to have nulls. + * If true, they may have nulls. + */ + private boolean groupIdTrackingEnabled; + + BytesRefArrayState(BigArrays bigArrays, CircuitBreaker breaker, String breakerLabel) { + this.bigArrays = bigArrays; + this.breaker = breaker; + this.breakerLabel = breakerLabel; + this.values = bigArrays.newObjectArray(0); + } + + BytesRef get(int groupId) { + return values.get(groupId).bytesRefView(); + } + + void set(int groupId, BytesRef value) { + ensureCapacity(groupId); + + var currentBuilder = values.get(groupId); + if (currentBuilder == null) { + currentBuilder = new BreakingBytesRefBuilder(breaker, breakerLabel, value.length); + values.set(groupId, currentBuilder); + } + + currentBuilder.copyBytes(value); + } + + Block toValuesBlock(IntVector selected, DriverContext driverContext) { + if (false == groupIdTrackingEnabled) { + try (var builder = driverContext.blockFactory().newBytesRefVectorBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + var value = get(group); + builder.appendBytesRef(value); + } + return builder.build().asBlock(); + } + } + try (var builder = driverContext.blockFactory().newBytesRefBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group)) { + var value = get(group); + builder.appendBytesRef(value); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + var minSize = groupId + 1; + if (minSize > values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, minSize); + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + assert blocks.length >= offset + 2; + try ( + var valuesBuilder = driverContext.blockFactory().newBytesRefVectorBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + var emptyBytesRef = new BytesRef(); + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group)) { + var value = get(group); + valuesBuilder.appendBytesRef(value); + } else { + valuesBuilder.appendBytesRef(emptyBytesRef); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + } + blocks[offset] = valuesBuilder.build().asBlock(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + } + } + + boolean hasValue(int groupId) { + return groupId < values.size() && values.get(groupId) != null; + } + + /** + * Switches this array state into tracking which group ids are set. This is + * idempotent and fast if already tracking so it's safe to, say, call it once + * for every block of values that arrives containing {@code null}. + * + *

+ * This class tracks seen group IDs differently from {@code AbstractArrayState}, as it just + * stores a flag to know if optimizations can be made. + *

+ */ + void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + this.groupIdTrackingEnabled = true; + } + + @Override + public void close() { + for (int i = 0; i < values.size(); i++) { + Releasables.closeWhileHandlingException(values.get(i)); + } + + Releasables.close(values); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java new file mode 100644 index 0000000000000..144214f93571e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregator.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator for `Max`, that works with BytesRef values. + * Gets the biggest BytesRef value, based on its bytes natural order (Delegated to {@link BytesRef#compareTo}). + */ +@Aggregator({ @IntermediateState(name = "max", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class MaxBytesRefAggregator { + private static boolean isBetter(BytesRef value, BytesRef otherValue) { + return value.compareTo(otherValue) > 0; + } + + public static SingleState initSingle(DriverContext driverContext) { + return new SingleState(driverContext.breaker()); + } + + public static void combine(SingleState state, BytesRef value) { + state.add(value); + } + + public static void combineIntermediate(SingleState state, BytesRef value, boolean seen) { + if (seen) { + combine(state, value); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext); + } + + public static GroupingState initGrouping(DriverContext driverContext) { + return new GroupingState(driverContext.bigArrays(), driverContext.breaker()); + } + + public static void combine(GroupingState state, int groupId, BytesRef value) { + state.add(groupId, value); + } + + public static void combineIntermediate(GroupingState state, int groupId, BytesRef value, boolean seen) { + if (seen) { + state.add(groupId, value); + } + } + + public static void combineStates(GroupingState state, int groupId, GroupingState otherState, int otherGroupId) { + state.combine(groupId, otherState, otherGroupId); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(selected, driverContext); + } + + public static class GroupingState implements Releasable { + private final BytesRefArrayState internalState; + + private GroupingState(BigArrays bigArrays, CircuitBreaker breaker) { + this.internalState = new BytesRefArrayState(bigArrays, breaker, "max_bytes_ref_grouping_aggregator"); + } + + public void add(int groupId, BytesRef value) { + if (internalState.hasValue(groupId) == false || isBetter(value, internalState.get(groupId))) { + internalState.set(groupId, value); + } + } + + public void combine(int groupId, GroupingState otherState, int otherGroupId) { + if (otherState.internalState.hasValue(otherGroupId)) { + add(groupId, otherState.internalState.get(otherGroupId)); + } + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + internalState.toIntermediate(blocks, offset, selected, driverContext); + } + + Block toBlock(IntVector selected, DriverContext driverContext) { + return internalState.toValuesBlock(selected, driverContext); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + internalState.enableGroupIdTracking(seen); + } + + @Override + public void close() { + Releasables.close(internalState); + } + } + + public static class SingleState implements Releasable { + private final BreakingBytesRefBuilder internalState; + private boolean seen; + + private SingleState(CircuitBreaker breaker) { + this.internalState = new BreakingBytesRefBuilder(breaker, "max_bytes_ref_aggregator"); + this.seen = false; + } + + public void add(BytesRef value) { + if (seen == false || isBetter(value, internalState.bytesRefView())) { + seen = true; + + internalState.grow(value.length); + internalState.setLength(value.length); + + System.arraycopy(value.bytes, value.offset, internalState.bytes(), 0, value.length); + } + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + } + + Block toBlock(DriverContext driverContext) { + if (seen == false) { + return driverContext.blockFactory().newConstantNullBlock(1); + } + + return driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); + } + + @Override + public void close() { + Releasables.close(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java new file mode 100644 index 0000000000000..830900702a371 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregator.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator for `Min`, that works with BytesRef values. + * Gets the smallest BytesRef value, based on its bytes natural order (Delegated to {@link BytesRef#compareTo}). + */ +@Aggregator({ @IntermediateState(name = "min", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class MinBytesRefAggregator { + private static boolean isBetter(BytesRef value, BytesRef otherValue) { + return value.compareTo(otherValue) < 0; + } + + public static SingleState initSingle(DriverContext driverContext) { + return new SingleState(driverContext.breaker()); + } + + public static void combine(SingleState state, BytesRef value) { + state.add(value); + } + + public static void combineIntermediate(SingleState state, BytesRef value, boolean seen) { + if (seen) { + combine(state, value); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext); + } + + public static GroupingState initGrouping(DriverContext driverContext) { + return new GroupingState(driverContext.bigArrays(), driverContext.breaker()); + } + + public static void combine(GroupingState state, int groupId, BytesRef value) { + state.add(groupId, value); + } + + public static void combineIntermediate(GroupingState state, int groupId, BytesRef value, boolean seen) { + if (seen) { + state.add(groupId, value); + } + } + + public static void combineStates(GroupingState state, int groupId, GroupingState otherState, int otherGroupId) { + state.combine(groupId, otherState, otherGroupId); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(selected, driverContext); + } + + public static class GroupingState implements Releasable { + private final BytesRefArrayState internalState; + + private GroupingState(BigArrays bigArrays, CircuitBreaker breaker) { + this.internalState = new BytesRefArrayState(bigArrays, breaker, "min_bytes_ref_grouping_aggregator"); + } + + public void add(int groupId, BytesRef value) { + if (internalState.hasValue(groupId) == false || isBetter(value, internalState.get(groupId))) { + internalState.set(groupId, value); + } + } + + public void combine(int groupId, GroupingState otherState, int otherGroupId) { + if (otherState.internalState.hasValue(otherGroupId)) { + add(groupId, otherState.internalState.get(otherGroupId)); + } + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + internalState.toIntermediate(blocks, offset, selected, driverContext); + } + + Block toBlock(IntVector selected, DriverContext driverContext) { + return internalState.toValuesBlock(selected, driverContext); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + internalState.enableGroupIdTracking(seen); + } + + @Override + public void close() { + Releasables.close(internalState); + } + } + + public static class SingleState implements Releasable { + private final BreakingBytesRefBuilder internalState; + private boolean seen; + + private SingleState(CircuitBreaker breaker) { + this.internalState = new BreakingBytesRefBuilder(breaker, "min_bytes_ref_aggregator"); + this.seen = false; + } + + public void add(BytesRef value) { + if (seen == false || isBetter(value, internalState.bytesRefView())) { + seen = true; + + internalState.grow(value.length); + internalState.setLength(value.length); + + System.arraycopy(value.bytes, value.offset, internalState.bytes(), 0, value.length); + } + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + } + + Block toBlock(DriverContext driverContext) { + if (seen == false) { + return driverContext.blockFactory().newConstantNullBlock(1); + } + + return driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); + } + + @Override + public void close() { + Releasables.close(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java index 17e67335919b1..2578452ad9062 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilder.java @@ -131,6 +131,14 @@ public void append(BytesRef bytes) { append(bytes.bytes, bytes.offset, bytes.length); } + /** + * Set the content of the builder to the given bytes. + */ + public void copyBytes(BytesRef newBytes) { + clear(); + append(newBytes); + } + /** * Reset the builder to an empty bytes array. Doesn't deallocate any memory. */ @@ -141,7 +149,7 @@ public void clear() { /** * Returns a view of the data added as a {@link BytesRef}. Importantly, this does not * copy the bytes and any further modification to the {@link BreakingBytesRefBuilder} - * will modify the returned {@link BytesRef}. The called must copy the bytes + * will modify the returned {@link BytesRef}. The caller must copy the bytes * if they wish to keep them. */ public BytesRef bytesRefView() { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionTests.java new file mode 100644 index 0000000000000..adc891a6a977d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunctionTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MaxBytesRefAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceBytesRefBlockSourceOperator( + blockFactory, + IntStream.range(0, size).mapToObj(l -> new BytesRef(randomAlphaOfLengthBetween(0, 100))) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MaxBytesRefAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "max of bytes"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Optional max = input.stream().flatMap(b -> allBytesRefs(b)).max(Comparator.naturalOrder()); + if (max.isEmpty()) { + assertThat(result.isNull(0), equalTo(true)); + return; + } + assertThat(result.isNull(0), equalTo(false)); + assertThat(BlockUtils.toJavaObject(result, 0), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..75a6a839ea62d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunctionTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MaxBytesRefGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongBytesRefTupleBlockSourceOperator( + blockFactory, + IntStream.range(0, size).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(randomAlphaOfLengthBetween(0, 100)))) + ); + } + + @Override + protected DataType acceptedDataType() { + return DataType.IP; + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MaxBytesRefAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "max of bytes"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Optional max = input.stream().flatMap(p -> allBytesRefs(p, group)).max(Comparator.naturalOrder()); + if (max.isEmpty()) { + assertThat(result.isNull(position), equalTo(true)); + return; + } + assertThat(result.isNull(position), equalTo(false)); + assertThat(BlockUtils.toJavaObject(result, position), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionTests.java new file mode 100644 index 0000000000000..b4383d6b0f56e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunctionTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceBytesRefBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MinBytesRefAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceBytesRefBlockSourceOperator( + blockFactory, + IntStream.range(0, size).mapToObj(l -> new BytesRef(randomAlphaOfLengthBetween(0, 100))) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MinBytesRefAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "min of bytes"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Optional max = input.stream().flatMap(b -> allBytesRefs(b)).min(Comparator.naturalOrder()); + if (max.isEmpty()) { + assertThat(result.isNull(0), equalTo(true)); + return; + } + assertThat(result.isNull(0), equalTo(false)); + assertThat(BlockUtils.toJavaObject(result, 0), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..d4cfca819f3b7 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunctionTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MinBytesRefGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongBytesRefTupleBlockSourceOperator( + blockFactory, + IntStream.range(0, size).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), new BytesRef(randomAlphaOfLengthBetween(0, 100)))) + ); + } + + @Override + protected DataType acceptedDataType() { + return DataType.IP; + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MinBytesRefAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "min of bytes"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Optional max = input.stream().flatMap(p -> allBytesRefs(p, group)).min(Comparator.naturalOrder()); + if (max.isEmpty()) { + assertThat(result.isNull(position), equalTo(true)); + return; + } + assertThat(result.isNull(position), equalTo(false)); + assertThat(BlockUtils.toJavaObject(result, position), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java index 24f5297a0d6fe..266c17febc5b3 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/BreakingBytesRefBuilderTests.java @@ -32,7 +32,7 @@ public void testBreakOnBuild() { public void testAddByte() { testAgainstOracle(() -> new TestIteration() { - byte b = randomByte(); + final byte b = randomByte(); @Override public int size() { @@ -53,7 +53,7 @@ public void applyToOracle(BytesRefBuilder oracle) { public void testAddBytesRef() { testAgainstOracle(() -> new TestIteration() { - BytesRef ref = new BytesRef(randomAlphaOfLengthBetween(1, 100)); + final BytesRef ref = new BytesRef(randomAlphaOfLengthBetween(1, 100)); @Override public int size() { @@ -72,10 +72,23 @@ public void applyToOracle(BytesRefBuilder oracle) { }); } + public void testCopyBytes() { + CircuitBreaker breaker = new MockBigArrays.LimitedBreaker(CircuitBreaker.REQUEST, ByteSizeValue.ofBytes(300)); + try (BreakingBytesRefBuilder builder = new BreakingBytesRefBuilder(breaker, "test")) { + String initialValue = randomAlphaOfLengthBetween(1, 50); + builder.copyBytes(new BytesRef(initialValue)); + assertThat(builder.bytesRefView().utf8ToString(), equalTo(initialValue)); + + String newValue = randomAlphaOfLengthBetween(350, 500); + Exception e = expectThrows(CircuitBreakingException.class, () -> builder.copyBytes(new BytesRef(newValue))); + assertThat(e.getMessage(), equalTo("over test limit")); + } + } + public void testGrow() { testAgainstOracle(() -> new TestIteration() { - int length = between(1, 100); - byte b = randomByte(); + final int length = between(1, 100); + final byte b = randomByte(); @Override public int size() { @@ -118,10 +131,11 @@ private void testAgainstOracle(Supplier iterations) { assertThat(builder.bytesRefView(), equalTo(oracle.get())); while (true) { TestIteration iteration = iterations.get(); - boolean willResize = builder.length() + iteration.size() >= builder.bytes().length; + int targetSize = builder.length() + iteration.size(); + boolean willResize = targetSize >= builder.bytes().length; if (willResize) { long resizeMemoryUsage = BreakingBytesRefBuilder.SHALLOW_SIZE + ramForArray(builder.bytes().length); - resizeMemoryUsage += ramForArray(ArrayUtil.oversize(builder.length() + iteration.size(), Byte.BYTES)); + resizeMemoryUsage += ramForArray(ArrayUtil.oversize(targetSize, Byte.BYTES)); if (resizeMemoryUsage > limit) { Exception e = expectThrows(CircuitBreakingException.class, () -> iteration.applyToBuilder(builder)); assertThat(e.getMessage(), equalTo("over test limit")); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 951545a546826..be3ab86d3e04f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -40,10 +40,10 @@ double e() "double log(?base:integer|unsigned_long|long|double, number:integer|unsigned_long|long|double)" "double log10(number:double|integer|long|unsigned_long)" "keyword|text ltrim(string:keyword|text)" -"boolean|double|integer|long|date|ip max(field:boolean|double|integer|long|date|ip)" +"boolean|double|integer|long|date|ip|keyword|text|long|version max(field:boolean|double|integer|long|date|ip|keyword|text|long|version)" "double median(number:double|integer|long)" "double median_absolute_deviation(number:double|integer|long)" -"boolean|double|integer|long|date|ip min(field:boolean|double|integer|long|date|ip)" +"boolean|double|integer|long|date|ip|keyword|text|long|version min(field:boolean|double|integer|long|date|ip|keyword|text|long|version)" "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_append(field1:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, field2:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)" "double mv_avg(number:double|integer|long|unsigned_long)" "keyword mv_concat(string:text|keyword, delim:text|keyword)" @@ -163,10 +163,10 @@ locate |[string, substring, start] |["keyword|text", "keyword|te log |[base, number] |["integer|unsigned_long|long|double", "integer|unsigned_long|long|double"] |["Base of logarithm. If `null`\, the function returns `null`. If not provided\, this function returns the natural logarithm (base e) of a value.", "Numeric expression. If `null`\, the function returns `null`."] log10 |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. ltrim |string |"keyword|text" |String expression. If `null`, the function returns `null`. -max |field |"boolean|double|integer|long|date|ip" |[""] +max |field |"boolean|double|integer|long|date|ip|keyword|text|long|version" |[""] median |number |"double|integer|long" |[""] median_absolut|number |"double|integer|long" |[""] -min |field |"boolean|double|integer|long|date|ip" |[""] +min |field |"boolean|double|integer|long|date|ip|keyword|text|long|version" |[""] mv_append |[field1, field2] |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"] | ["", ""] mv_avg |number |"double|integer|long|unsigned_long" |Multivalue expression. mv_concat |[string, delim] |["text|keyword", "text|keyword"] |[Multivalue expression., Delimiter.] @@ -411,10 +411,10 @@ locate |integer log |double |[true, false] |false |false log10 |double |false |false |false ltrim |"keyword|text" |false |false |false -max |"boolean|double|integer|long|date|ip" |false |false |true +max |"boolean|double|integer|long|date|ip|keyword|text|long|version" |false |false |true median |double |false |false |true median_absolut|double |false |false |true -min |"boolean|double|integer|long|date|ip" |false |false |true +min |"boolean|double|integer|long|date|ip|keyword|text|long|version" |false |false |true mv_append |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |[false, false] |false |false mv_avg |double |false |false |false mv_concat |keyword |[false, false] |false |false diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index eb373b6ddef6b..fc607edf4d212 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -76,6 +76,166 @@ fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec0 | fe82::cae2:65ff:fece:fec fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe80::cae2:65ff:fece:feb9 | fe81::cae2:65ff:fece:feb9 | gamma ; +maxOfVersion +required_capability: agg_max_min_string_support +from apps +| eval x = version +| where id > 2 +| stats max(version), a = max(version), b = max(x), c = max(case(name == "iiiii", "100.0.0"::version, version)); + +max(version):version | a:version | b:version | c:version +bad | bad | bad | 100.0.0 +; + +maxOfVersionGrouping +required_capability: agg_max_min_string_support +from apps +| eval x = version +| where id > 2 +| stats max(version), a = max(version), b = max(x), c = max(case(name == "ccccc", "100.0.0"::version, version)) by name +| sort name asc +| limit 3; + +max(version):version | a:version | b:version | c:version | name:keyword +1.2.3.4 | 1.2.3.4 | 1.2.3.4 | 1.2.3.4 | aaaaa +2.3.4 | 2.3.4 | 2.3.4 | 100.0.0 | ccccc +2.12.0 | 2.12.0 | 2.12.0 | 2.12.0 | ddddd +; + +maxOfKeyword +required_capability: agg_max_min_string_support +from airports +| eval x = abbrev +| where scalerank >= 9 +| stats max(abbrev), a = max(abbrev), b = max(x), c = max(case(mv_first(type) == "small", "___"::keyword, abbrev)); + +max(abbrev):keyword | a:keyword | b:keyword | c:keyword +ZAH | ZAH | ZAH | ___ +; + +maxOfKeywordGrouping +required_capability: agg_max_min_string_support +from airports +| eval x = abbrev +| where scalerank >= 9 +| stats max(abbrev), a = max(abbrev), b = max(x), c = max(case(mv_first(type) == "small", "___"::keyword, abbrev)) by type +| sort type asc +| limit 4; + +max(abbrev):keyword | a:keyword | b:keyword | c:keyword | type:keyword +IXC | IXC | IXC | IXC | major +ZAH | ZAH | ZAH | ZAH | mid +VIBY | VIBY | VIBY | VIBY | military +OPQS | OPQS | OPQS | ___ | small +; + +maxOfText +required_capability: agg_max_min_string_support +from airports +| eval x = name +| where scalerank >= 9 +| stats max(name), a = max(name), b = max(x); + +max(name):text | a:text | b:text +Zaporozhye Int'l | Zaporozhye Int'l | Zaporozhye Int'l +; + +maxOfTextGrouping +required_capability: agg_max_min_string_support +from airports +| eval x = name +| where scalerank >= 9 +| stats max(name), a = max(name), b = max(x) by type +| sort type asc +| limit 4; + +max(name):text | a:text | b:text | type:keyword +Cheongju Int'l | Cheongju Int'l | Cheongju Int'l | major +Zaporozhye Int'l | Zaporozhye Int'l | Zaporozhye Int'l | mid +Zaporozhye Int'l | Zaporozhye Int'l | Zaporozhye Int'l | military +Sahnewal | Sahnewal | Sahnewal | small +; + +minOfVersion +required_capability: agg_max_min_string_support +from apps +| eval x = version +| where id > 2 +| stats min(version), a = min(version), b = min(x), c = min(case(name == "iiiii", "1.0"::version, version)); + +min(version):version | a:version | b:version | c:version +1.2.3.4 | 1.2.3.4 | 1.2.3.4 | 1.0 +; + +minOfVersionGrouping +required_capability: agg_max_min_string_support +from apps +| eval x = version +| where id > 2 +| stats min(version), a = min(version), b = min(x), c = min(case(name == "ccccc", "100.0.0"::version, version)) by name +| sort name asc +| limit 3; + +min(version):version | a:version | b:version | c:version | name:keyword +1.2.3.4 | 1.2.3.4 | 1.2.3.4 | 1.2.3.4 | aaaaa +2.3.4 | 2.3.4 | 2.3.4 | 100.0.0 | ccccc +2.12.0 | 2.12.0 | 2.12.0 | 2.12.0 | ddddd +; + +minOfKeyword +required_capability: agg_max_min_string_support +from airports +| eval x = abbrev +| where scalerank >= 9 +| stats min(abbrev), a = min(abbrev), b = min(x), c = max(case(mv_first(type) == "small", "___"::keyword, abbrev)); + +min(abbrev):keyword | a:keyword | b:keyword | c:keyword +AWZ | AWZ | AWZ | ___ +; + +minOfKeywordGrouping +required_capability: agg_max_min_string_support +from airports +| eval x = abbrev +| where scalerank >= 9 +| stats min(abbrev), a = min(abbrev), b = min(x), c = min(case(mv_first(type) == "small", "___"::keyword, abbrev)) by type +| sort type asc +| limit 4; + +min(abbrev):keyword | a:keyword | b:keyword | c:keyword | type:keyword +CJJ | CJJ | CJJ | CJJ | major +AWZ | AWZ | AWZ | AWZ | mid +GWL | GWL | GWL | GWL | military +LUH | LUH | LUH | ___ | small +; + +minOfText +required_capability: agg_max_min_string_support +from airports +| eval x = name +| where scalerank >= 9 +| stats min(name), a = min(name), b = min(x); + +min(name):text | a:text | b:text +Abdul Rachman Saleh | Abdul Rachman Saleh | Abdul Rachman Saleh +; + +minOfTextGrouping +required_capability: agg_max_min_string_support +from airports +| eval x = name +| where scalerank >= 9 +| stats min(name), a = min(name), b = min(x) by type +| sort type asc +| limit 4; + +min(name):text | a:text | b:text | type:keyword +Chandigarh Int'l | Chandigarh Int'l | Chandigarh Int'l | major +Abdul Rachman Saleh | Abdul Rachman Saleh | Abdul Rachman Saleh | mid +Abdul Rachman Saleh | Abdul Rachman Saleh | Abdul Rachman Saleh | military +Dhamial | Dhamial | Dhamial | small +; + minOfBooleanExpression required_capability: agg_max_min_boolean_support from employees diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 0477167cd7315..7937ae67c70bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -67,6 +67,11 @@ public enum Cap { */ AGG_MAX_MIN_IP_SUPPORT, + /** + * Support for strings in aggregations {@code MAX} and {@code MIN}. + */ + AGG_MAX_MIN_STRING_SUPPORT, + /** * Support for booleans in {@code TOP} aggregation. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index 22224628e23ad..e7f790f90803a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxBooleanAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.MaxBytesRefAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxIpAggregatorFunctionSupplier; @@ -32,12 +33,15 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; +import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial; public class Max extends AggregateFunction implements ToAggregator, SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Max", Max::new); @FunctionInfo( - returnType = { "boolean", "double", "integer", "long", "date", "ip" }, + returnType = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" }, description = "The maximum value of a field.", isAggregation = true, examples = { @@ -50,7 +54,13 @@ public class Max extends AggregateFunction implements ToAggregator, SurrogateExp tag = "docsStatsMaxNestedExpression" ) } ) - public Max(Source source, @Param(name = "field", type = { "boolean", "double", "integer", "long", "date", "ip" }) Expression field) { + public Max( + Source source, + @Param( + name = "field", + type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" } + ) Expression field + ) { super(source, field); } @@ -77,13 +87,10 @@ public Max replaceChildren(List newChildren) { protected TypeResolution resolveType() { return TypeResolutions.isType( field(), - e -> e == DataType.BOOLEAN || e == DataType.DATETIME || e == DataType.IP || (e.isNumeric() && e != DataType.UNSIGNED_LONG), + t -> isRepresentable(t) && t != UNSIGNED_LONG && isSpatial(t) == false, sourceText(), DEFAULT, - "boolean", - "datetime", - "ip", - "numeric except unsigned_long or counter types" + "representable except unsigned_long and spatial types" ); } @@ -110,6 +117,9 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { if (type == DataType.IP) { return new MaxIpAggregatorFunctionSupplier(inputChannels); } + if (type == DataType.VERSION || type == DataType.KEYWORD || type == DataType.TEXT) { + return new MaxBytesRefAggregatorFunctionSupplier(inputChannels); + } throw EsqlIllegalArgumentException.illegalDataType(type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index 8e7bb6bc3e799..6866811995059 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinBooleanAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.MinBytesRefAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinIpAggregatorFunctionSupplier; @@ -32,12 +33,15 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; +import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatial; public class Min extends AggregateFunction implements ToAggregator, SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Min", Min::new); @FunctionInfo( - returnType = { "boolean", "double", "integer", "long", "date", "ip" }, + returnType = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" }, description = "The minimum value of a field.", isAggregation = true, examples = { @@ -50,7 +54,13 @@ public class Min extends AggregateFunction implements ToAggregator, SurrogateExp tag = "docsStatsMinNestedExpression" ) } ) - public Min(Source source, @Param(name = "field", type = { "boolean", "double", "integer", "long", "date", "ip" }) Expression field) { + public Min( + Source source, + @Param( + name = "field", + type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" } + ) Expression field + ) { super(source, field); } @@ -77,13 +87,10 @@ public Min replaceChildren(List newChildren) { protected TypeResolution resolveType() { return TypeResolutions.isType( field(), - e -> e == DataType.BOOLEAN || e == DataType.DATETIME || e == DataType.IP || (e.isNumeric() && e != DataType.UNSIGNED_LONG), + t -> isRepresentable(t) && t != UNSIGNED_LONG && isSpatial(t) == false, sourceText(), DEFAULT, - "boolean", - "datetime", - "ip", - "numeric except unsigned_long or counter types" + "representable except unsigned_long and spatial types" ); } @@ -110,6 +117,9 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { if (type == DataType.IP) { return new MinIpAggregatorFunctionSupplier(inputChannels); } + if (type == DataType.VERSION || type == DataType.KEYWORD || type == DataType.TEXT) { + return new MinBytesRefAggregatorFunctionSupplier(inputChannels); + } throw EsqlIllegalArgumentException.illegalDataType(type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 213d7266a0b1e..60bf4be1d2b03 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -160,7 +160,7 @@ private static Stream, Tuple>> typeAndNames(Class if (NumericAggregate.class.isAssignableFrom(clazz)) { types = NUMERIC; } else if (Max.class.isAssignableFrom(clazz) || Min.class.isAssignableFrom(clazz)) { - types = List.of("Boolean", "Int", "Long", "Double", "Ip"); + types = List.of("Boolean", "Int", "Long", "Double", "Ip", "BytesRef"); } else if (clazz == Count.class) { types = List.of(""); // no extra type distinction } else if (SpatialAggregateFunction.class.isAssignableFrom(clazz)) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index f663002a51d68..3fb4b80d3974e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1809,13 +1809,13 @@ public void testUnsupportedTypesInStats() { found value [x] type [unsigned_long] line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long, _source, or counter types],\ found value [x] type [unsigned_long] - line 2:39: argument of [max(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\ + line 2:39: argument of [max(x)] must be [representable except unsigned_long and spatial types],\ found value [x] type [unsigned_long] line 2:47: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [unsigned_long] line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [unsigned_long] - line 2:88: argument of [min(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\ + line 2:88: argument of [min(x)] must be [representable except unsigned_long and spatial types],\ found value [x] type [unsigned_long] line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long],\ found value [x] type [unsigned_long] @@ -1824,21 +1824,17 @@ public void testUnsupportedTypesInStats() { verifyUnsupported(""" row x = to_version("1.2") - | stats avg(x), max(x), median(x), median_absolute_deviation(x), min(x), percentile(x, 10), sum(x) + | stats avg(x), median(x), median_absolute_deviation(x), percentile(x, 10), sum(x) """, """ - Found 7 problems + Found 5 problems line 2:10: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [version] - line 2:18: argument of [max(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\ + line 2:18: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [version] - line 2:26: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ + line 2:29: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [version] - line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ - found value [x] type [version] - line 2:67: argument of [min(x)] must be [boolean, datetime, ip or numeric except unsigned_long or counter types],\ - found value [x] type [version] - line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version] - line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]"""); + line 2:59: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version] + line 2:78: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]"""); } public void testInOnText() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index bdea0807a78c4..e2403505921a9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -766,7 +766,7 @@ public void testAggregateOnCounter() { error("FROM tests | STATS min(network.bytes_in)", tsdb), equalTo( "1:20: argument of [min(network.bytes_in)] must be" - + " [boolean, datetime, ip or numeric except unsigned_long or counter types]," + + " [representable except unsigned_long and spatial types]," + " found value [network.bytes_in] type [counter_long]" ) ); @@ -775,7 +775,7 @@ public void testAggregateOnCounter() { error("FROM tests | STATS max(network.bytes_in)", tsdb), equalTo( "1:20: argument of [max(network.bytes_in)] must be" - + " [boolean, datetime, ip or numeric except unsigned_long or counter types]," + + " [representable except unsigned_long and spatial types]," + " found value [network.bytes_in] type [counter_long]" ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java index 52e908a51dd1e..ce2bf7e262ae9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.versionfield.Version; import java.util.ArrayList; import java.util.Comparator; @@ -44,7 +45,10 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), MultiRowTestCaseSupplier.dateCases(1, 1000), MultiRowTestCaseSupplier.booleanCases(1, 1000), - MultiRowTestCaseSupplier.ipCases(1, 1000) + MultiRowTestCaseSupplier.ipCases(1, 1000), + MultiRowTestCaseSupplier.versionCases(1, 1000), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) ).flatMap(List::stream).map(MaxTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); suppliers.addAll( @@ -109,14 +113,44 @@ public static Iterable parameters() { DataType.IP, equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))) ) - ) + ), + new TestCaseSupplier(List.of(DataType.KEYWORD), () -> { + var value = new BytesRef(randomAlphaOfLengthBetween(0, 50)); + return new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.KEYWORD, "field")), + "Max[field=Attribute[channel=0]]", + DataType.KEYWORD, + equalTo(value) + ); + }), + new TestCaseSupplier(List.of(DataType.TEXT), () -> { + var value = new BytesRef(randomAlphaOfLengthBetween(0, 50)); + return new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.TEXT, "field")), + "Max[field=Attribute[channel=0]]", + DataType.TEXT, + equalTo(value) + ); + }), + new TestCaseSupplier(List.of(DataType.VERSION), () -> { + var value = randomBoolean() + ? new Version(randomAlphaOfLengthBetween(1, 10)).toBytesRef() + : new Version(randomIntBetween(0, 100) + "." + randomIntBetween(0, 100) + "." + randomIntBetween(0, 100)) + .toBytesRef(); + return new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.VERSION, "field")), + "Max[field=Attribute[channel=0]]", + DataType.VERSION, + equalTo(value) + ); + }) ) ); return parameterSuppliersFromTypedDataWithDefaultChecks( suppliers, false, - (v, p) -> "boolean, datetime, ip or numeric except unsigned_long or counter types" + (v, p) -> "representable except unsigned_long and spatial types" ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java index 9514c817df497..7250072cd2003 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.versionfield.Version; import java.util.ArrayList; import java.util.Comparator; @@ -44,7 +45,10 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true), MultiRowTestCaseSupplier.dateCases(1, 1000), MultiRowTestCaseSupplier.booleanCases(1, 1000), - MultiRowTestCaseSupplier.ipCases(1, 1000) + MultiRowTestCaseSupplier.ipCases(1, 1000), + MultiRowTestCaseSupplier.versionCases(1, 1000), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) ).flatMap(List::stream).map(MinTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); suppliers.addAll( @@ -109,14 +113,44 @@ public static Iterable parameters() { DataType.IP, equalTo(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("127.0.0.1")))) ) - ) + ), + new TestCaseSupplier(List.of(DataType.KEYWORD), () -> { + var value = new BytesRef(randomAlphaOfLengthBetween(0, 50)); + return new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.KEYWORD, "field")), + "Min[field=Attribute[channel=0]]", + DataType.KEYWORD, + equalTo(value) + ); + }), + new TestCaseSupplier(List.of(DataType.TEXT), () -> { + var value = new BytesRef(randomAlphaOfLengthBetween(0, 50)); + return new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.TEXT, "field")), + "Min[field=Attribute[channel=0]]", + DataType.TEXT, + equalTo(value) + ); + }), + new TestCaseSupplier(List.of(DataType.VERSION), () -> { + var value = randomBoolean() + ? new Version(randomAlphaOfLengthBetween(1, 10)).toBytesRef() + : new Version(randomIntBetween(0, 100) + "." + randomIntBetween(0, 100) + "." + randomIntBetween(0, 100)) + .toBytesRef(); + return new TestCaseSupplier.TestCase( + List.of(TestCaseSupplier.TypedData.multiRow(List.of(value), DataType.VERSION, "field")), + "Min[field=Attribute[channel=0]]", + DataType.VERSION, + equalTo(value) + ); + }) ) ); return parameterSuppliersFromTypedDataWithDefaultChecks( suppliers, false, - (v, p) -> "boolean, datetime, ip or numeric except unsigned_long or counter types" + (v, p) -> "representable except unsigned_long and spatial types" ); } From 65ce50c60a20918dc34183456c160eb7454a2479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Tue, 20 Aug 2024 15:29:19 +0200 Subject: [PATCH 09/32] ESQL: Added mv_percentile function (#111749) - Added the `mv_percentile(values, percentile)` function - Used as a surrogate in the `percentile(column, percentile)` aggregation - Updated docs to specify that the surrogate _should_ be implemented if possible The same way as mv_median does, this yields exact results (Ignoring double operations error). For that, some decisions were made, specially in the long evaluator (Check the comments in context in `MvPercentile.java`) Closes https://github.com/elastic/elasticsearch/issues/111591 --- docs/changelog/111749.yaml | 6 + .../description/mv_percentile.asciidoc | 5 + .../functions/examples/mv_percentile.asciidoc | 13 + .../kibana/definition/mv_percentile.json | 173 +++++++ .../functions/kibana/docs/mv_percentile.md | 11 + .../functions/layout/mv_percentile.asciidoc | 15 + .../parameters/mv_percentile.asciidoc | 9 + .../functions/signature/mv_percentile.svg | 1 + .../functions/types/mv_percentile.asciidoc | 17 + .../compute/data/BlockUtils.java | 2 + .../src/main/resources/meta.csv-spec | 6 +- .../src/main/resources/mv_percentile.csv-spec | 163 ++++++ .../main/resources/stats_percentile.csv-spec | 36 ++ .../MvPercentileDoubleEvaluator.java | 125 +++++ .../MvPercentileIntegerEvaluator.java | 126 +++++ .../multivalue/MvPercentileLongEvaluator.java | 126 +++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../function/EsqlFunctionRegistry.java | 2 + .../function/aggregate/Percentile.java | 16 +- .../function/aggregate/package-info.java | 18 +- .../AbstractMultivalueFunction.java | 1 + .../scalar/multivalue/MvPercentile.java | 446 +++++++++++++++++ .../function/AbstractAggregationTestCase.java | 7 +- .../AbstractScalarFunctionTestCase.java | 27 + .../function/MultivalueTestCaseSupplier.java | 325 ++++++++++++ .../expression/function/TestCaseSupplier.java | 2 +- .../function/aggregate/PercentileTests.java | 2 +- .../scalar/multivalue/MvPercentileTests.java | 466 ++++++++++++++++++ 28 files changed, 2135 insertions(+), 16 deletions(-) create mode 100644 docs/changelog/111749.yaml create mode 100644 docs/reference/esql/functions/description/mv_percentile.asciidoc create mode 100644 docs/reference/esql/functions/examples/mv_percentile.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/mv_percentile.json create mode 100644 docs/reference/esql/functions/kibana/docs/mv_percentile.md create mode 100644 docs/reference/esql/functions/layout/mv_percentile.asciidoc create mode 100644 docs/reference/esql/functions/parameters/mv_percentile.asciidoc create mode 100644 docs/reference/esql/functions/signature/mv_percentile.svg create mode 100644 docs/reference/esql/functions/types/mv_percentile.asciidoc create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_percentile.csv-spec create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileDoubleEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileIntegerEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileLongEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultivalueTestCaseSupplier.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java diff --git a/docs/changelog/111749.yaml b/docs/changelog/111749.yaml new file mode 100644 index 0000000000000..77e0c65005dd6 --- /dev/null +++ b/docs/changelog/111749.yaml @@ -0,0 +1,6 @@ +pr: 111749 +summary: "ESQL: Added `mv_percentile` function" +area: ES|QL +type: feature +issues: + - 111591 diff --git a/docs/reference/esql/functions/description/mv_percentile.asciidoc b/docs/reference/esql/functions/description/mv_percentile.asciidoc new file mode 100644 index 0000000000000..3e731f6525cec --- /dev/null +++ b/docs/reference/esql/functions/description/mv_percentile.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur. diff --git a/docs/reference/esql/functions/examples/mv_percentile.asciidoc b/docs/reference/esql/functions/examples/mv_percentile.asciidoc new file mode 100644 index 0000000000000..9b20a5bef5e0d --- /dev/null +++ b/docs/reference/esql/functions/examples/mv_percentile.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/mv_percentile.csv-spec[tag=example] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/mv_percentile.csv-spec[tag=example-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/mv_percentile.json b/docs/reference/esql/functions/kibana/definition/mv_percentile.json new file mode 100644 index 0000000000000..dad611122f0db --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/mv_percentile.json @@ -0,0 +1,173 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "mv_percentile", + "description" : "Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "double", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "integer", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "long", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "double", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "integer", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "long", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "double", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "integer", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "Multivalue expression." + }, + { + "name" : "percentile", + "type" : "long", + "optional" : false, + "description" : "The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead." + } + ], + "variadic" : false, + "returnType" : "long" + } + ], + "examples" : [ + "ROW values = [5, 5, 10, 12, 5000]\n| EVAL p50 = MV_PERCENTILE(values, 50), median = MV_MEDIAN(values)" + ] +} diff --git a/docs/reference/esql/functions/kibana/docs/mv_percentile.md b/docs/reference/esql/functions/kibana/docs/mv_percentile.md new file mode 100644 index 0000000000000..560a0aefa1dc3 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/mv_percentile.md @@ -0,0 +1,11 @@ + + +### MV_PERCENTILE +Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur. + +``` +ROW values = [5, 5, 10, 12, 5000] +| EVAL p50 = MV_PERCENTILE(values, 50), median = MV_MEDIAN(values) +``` diff --git a/docs/reference/esql/functions/layout/mv_percentile.asciidoc b/docs/reference/esql/functions/layout/mv_percentile.asciidoc new file mode 100644 index 0000000000000..a86c4a136b5cd --- /dev/null +++ b/docs/reference/esql/functions/layout/mv_percentile.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-mv_percentile]] +=== `MV_PERCENTILE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/mv_percentile.svg[Embedded,opts=inline] + +include::../parameters/mv_percentile.asciidoc[] +include::../description/mv_percentile.asciidoc[] +include::../types/mv_percentile.asciidoc[] +include::../examples/mv_percentile.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/mv_percentile.asciidoc b/docs/reference/esql/functions/parameters/mv_percentile.asciidoc new file mode 100644 index 0000000000000..57804185e191a --- /dev/null +++ b/docs/reference/esql/functions/parameters/mv_percentile.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number`:: +Multivalue expression. + +`percentile`:: +The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead. diff --git a/docs/reference/esql/functions/signature/mv_percentile.svg b/docs/reference/esql/functions/signature/mv_percentile.svg new file mode 100644 index 0000000000000..b4d623636572f --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_percentile.svg @@ -0,0 +1 @@ +MV_PERCENTILE(number,percentile) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/mv_percentile.asciidoc b/docs/reference/esql/functions/types/mv_percentile.asciidoc new file mode 100644 index 0000000000000..99a58b9c3d2e2 --- /dev/null +++ b/docs/reference/esql/functions/types/mv_percentile.asciidoc @@ -0,0 +1,17 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number | percentile | result +double | double | double +double | integer | double +double | long | double +integer | double | integer +integer | integer | integer +integer | long | integer +long | double | long +long | integer | long +long | long | long +|=== diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index a697a3f6c15fa..3df389135e9d3 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -87,6 +87,8 @@ public static Block[] fromListRow(BlockFactory blockFactory, List row, i } else { wrapper.builder.mvOrdering(Block.MvOrdering.DEDUPLICATED_UNORDERD); } + } else if (isAscending(listVal) && random.nextBoolean()) { + wrapper.builder.mvOrdering(Block.MvOrdering.SORTED_ASCENDING); } blocks[i] = wrapper.builder.build(); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index be3ab86d3e04f..f1f66a9cb990c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -54,6 +54,7 @@ double e() "boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version mv_max(field:boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version)" "double|integer|long|unsigned_long mv_median(number:double|integer|long|unsigned_long)" "boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version mv_min(field:boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version)" +"double|integer|long mv_percentile(number:double|integer|long, percentile:double|integer|long)" "double mv_pseries_weighted_sum(number:double, p:double)" "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_slice(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, start:integer, ?end:integer)" "boolean|date|double|integer|ip|keyword|long|text|version mv_sort(field:boolean|date|double|integer|ip|keyword|long|text|version, ?order:keyword)" @@ -177,6 +178,7 @@ mv_last |field |"boolean|cartesian_point|car mv_max |field |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version" |Multivalue expression. mv_median |number |"double|integer|long|unsigned_long" |Multivalue expression. mv_min |field |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version" |Multivalue expression. +mv_percentile |[number, percentile] |["double|integer|long", "double|integer|long"] |[Multivalue expression., The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead.] mv_pseries_wei|[number, p] |[double, double] |[Multivalue expression., It is a constant number that represents the 'p' parameter in the P-Series. It impacts every element's contribution to the weighted sum.] mv_slice |[field, start, end] |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", integer, integer]|[Multivalue expression. If `null`\, the function returns `null`., Start position. If `null`\, the function returns `null`. The start argument can be negative. An index of -1 is used to specify the last value in the list., End position(included). Optional; if omitted\, the position at `start` is returned. The end argument can be negative. An index of -1 is used to specify the last value in the list.] mv_sort |[field, order] |["boolean|date|double|integer|ip|keyword|long|text|version", keyword] |[Multivalue expression. If `null`\, the function returns `null`., Sort order. The valid options are ASC and DESC\, the default is ASC.] @@ -300,6 +302,7 @@ mv_last |Converts a multivalue expression into a single valued column cont mv_max |Converts a multivalued expression into a single valued column containing the maximum value. mv_median |Converts a multivalued field into a single valued field containing the median value. mv_min |Converts a multivalued expression into a single valued column containing the minimum value. +mv_percentile |Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur. mv_pseries_wei|Converts a multivalued expression into a single-valued column by multiplying every element on the input list by its corresponding term in P-Series and computing the sum. mv_slice |Returns a subset of the multivalued field using the start and end index values. mv_sort |Sorts a multivalued field in lexicographical order. @@ -425,6 +428,7 @@ mv_last |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|g mv_max |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version" |false |false |false mv_median |"double|integer|long|unsigned_long" |false |false |false mv_min |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version" |false |false |false +mv_percentile |"double|integer|long" |[false, false] |false |false mv_pseries_wei|"double" |[false, false] |false |false mv_slice |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |[false, false, true] |false |false mv_sort |"boolean|date|double|integer|ip|keyword|long|text|version" |[false, true] |false |false @@ -504,5 +508,5 @@ countFunctions#[skip:-8.15.99] meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -114 | 114 | 114 +115 | 115 | 115 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_percentile.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_percentile.csv-spec new file mode 100644 index 0000000000000..e22b40c7ecad8 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_percentile.csv-spec @@ -0,0 +1,163 @@ +default +required_capability: fn_mv_percentile + +// tag::example[] +ROW values = [5, 5, 10, 12, 5000] +| EVAL p50 = MV_PERCENTILE(values, 50), median = MV_MEDIAN(values) +// end::example[] +; + +// tag::example-result[] +values:integer | p50:integer | median:integer +[5, 5, 10, 12, 5000] | 10 | 10 +// end::example-result[] +; + +p0 +required_capability: fn_mv_percentile + +ROW a = [5, 5, 10, 12, 5000] +| EVAL pInt = MV_PERCENTILE(a, 0), pLong = MV_PERCENTILE(a, 0::long), pDouble = MV_PERCENTILE(a, 0.0) +| KEEP pInt, pLong, pDouble +; + +pInt:integer | pLong:integer | pDouble:integer +5 | 5 | 5 +; + +p100 +required_capability: fn_mv_percentile + +ROW a = [5, 5, 10, 12, 5000] +| EVAL pInt = MV_PERCENTILE(a, 100), pLong = MV_PERCENTILE(a, 100::long), pDouble = MV_PERCENTILE(a, 100.0) +| KEEP pInt, pLong, pDouble +; + +pInt:integer | pLong:integer | pDouble:integer +5000 | 5000 | 5000 +; + +fractionInt +required_capability: fn_mv_percentile + +ROW a = [0, 10] +| EVAL pInt = MV_PERCENTILE(a, 75), pLong = MV_PERCENTILE(a, 75::long), pDouble = MV_PERCENTILE(a, 75.0) +| KEEP pInt, pLong, pDouble +; + +pInt:integer | pLong:integer | pDouble:integer +7 | 7 | 7 +; + +fractionLong +required_capability: fn_mv_percentile + +ROW a = to_long([0, 10]) +| EVAL pInt = MV_PERCENTILE(a, 75), pLong = MV_PERCENTILE(a, 75::long), pDouble = MV_PERCENTILE(a, 75.0) +| KEEP pInt, pLong, pDouble +; + +pInt:long | pLong:long | pDouble:long +7 | 7 | 7 +; + +fractionDouble +required_capability: fn_mv_percentile + +ROW a = [0., 10.] +| EVAL pInt = MV_PERCENTILE(a, 75), pLong = MV_PERCENTILE(a, 75::long), pDouble = MV_PERCENTILE(a, 75.0) +| KEEP pInt, pLong, pDouble +; + +pInt:double | pLong:double | pDouble:double +7.5 | 7.5 | 7.5 +; + +singleValue +required_capability: fn_mv_percentile + +ROW integer = 5, long = 5::long, double = 5.0 +| EVAL + integer = MV_PERCENTILE(integer, 75), + long = MV_PERCENTILE(long, 75), + double = MV_PERCENTILE(double, 75) +; + +integer:integer | long:long | double:double +5 | 5 | 5 +; + +fromIndex +required_capability: fn_mv_percentile + +FROM employees +| EVAL + integer = MV_PERCENTILE(salary_change.int, 75), + long = MV_PERCENTILE(salary_change.long, 75), + double = MV_PERCENTILE(salary_change, 75) +| KEEP integer, long, double +| SORT double +| LIMIT 3 +; + +integer:integer | long:long | double:double +-8 | -8 | -8.46 +-7 | -7 | -7.08 +-6 | -6 | -6.9 +; + +fromIndexPercentile +required_capability: fn_mv_percentile + +FROM employees +| SORT emp_no +| LIMIT 1 +| EVAL + integer = MV_PERCENTILE(salary_change.int, languages), + long = MV_PERCENTILE(salary_change.long, languages.long), + double = MV_PERCENTILE(salary_change, height), + null_value = MV_PERCENTILE(salary_change, emp_no) +| KEEP integer, long, double, null_value +; +warning:Line 8:14: evaluation of [MV_PERCENTILE(salary_change, emp_no)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 8:14: java.lang.IllegalArgumentException: Percentile parameter must be a number between 0 and 100, found [10001.0] + +integer:integer | long:long | double:double | null_value:double +1 | 1 | 1.19 | null +; + +multipleExpressions +required_capability: fn_mv_percentile + +ROW x = [0, 5, 10] +| EVAL + MV_PERCENTILE(x, 75), + a = MV_PERCENTILE(x, 75), + b = MV_PERCENTILE(TO_DOUBLE([0, 5, 10]), 75), + c = MV_PERCENTILE(CASE(true, x, [0, 1]), 75) +; + +x:integer | MV_PERCENTILE(x, 75):integer | a:integer | b:double | c:integer +[0, 5, 10] | 7 | 7 | 7.5 | 7 +; + +nullsAndFolds +required_capability: fn_mv_percentile + +ROW x = [5, 5, 10, 12, 5000], n = null, y = 50 +| EVAL evalNull = null / 2, evalValue = 31 + 1 +| LIMIT 1 +| EVAL + a = mv_percentile(y, 90), + b = mv_percentile(x, y), + c = mv_percentile(null, null), + d = mv_percentile(null, y), + e = mv_percentile(evalNull, y), + f = mv_percentile(evalValue, y), + g = mv_percentile(n, y) +| KEEP a, b, c, d, e, f, g +; + +a:integer | b:integer | c:null | d:null | e:integer | f:integer | g:null +50 | 10 | null | null | null | 32 | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_percentile.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_percentile.csv-spec index db386e877b9c3..2ac7a0cf6217a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_percentile.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_percentile.csv-spec @@ -195,3 +195,39 @@ p80_max_salary_change:double 12.132 // end::docsStatsPercentileNestedExpression-result[] ; + +constantsFrom +required_capability: fn_mv_percentile +from employees +| eval single = 7, mv = [1, 7, 10] +| stats + eval_single = percentile(single, 50), + eval_mv = percentile(mv, 50), + constant_single = percentile(5, 50), + constant_mv = percentile([1, 5, 10], 50); + +eval_single:double | eval_mv:double | constant_single:double | constant_mv:double +7 | 7 | 5 | 5 +; + +constantsRow +required_capability: fn_mv_percentile +row single=7, mv=[1, 7, 10] +| stats + eval_single = percentile(single, 50), + eval_mv = percentile(mv, 50), + constant_single = percentile(5, 50), + constant_mv = percentile([1, 5, 10], 50); + +eval_single:double | eval_mv:double | constant_single:double | constant_mv:double +7 | 7 | 5 | 5 +; + +singleConstant +required_capability: fn_mv_percentile +row a=0 +| stats constant_single = percentile(5, 50); + +constant_single:double +5 +; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileDoubleEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileDoubleEvaluator.java new file mode 100644 index 0000000000000..dd370e90b2c86 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileDoubleEvaluator.java @@ -0,0 +1,125 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.util.function.Function; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvPercentile}. + * This class is generated. Do not edit it. + */ +public final class MvPercentileDoubleEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator values; + + private final EvalOperator.ExpressionEvaluator percentile; + + private final MvPercentile.DoubleSortingScratch scratch; + + private final DriverContext driverContext; + + public MvPercentileDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator values, + EvalOperator.ExpressionEvaluator percentile, MvPercentile.DoubleSortingScratch scratch, + DriverContext driverContext) { + this.values = values; + this.percentile = percentile; + this.scratch = scratch; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (DoubleBlock valuesBlock = (DoubleBlock) values.eval(page)) { + try (DoubleBlock percentileBlock = (DoubleBlock) percentile.eval(page)) { + return eval(page.getPositionCount(), valuesBlock, percentileBlock); + } + } + } + + public DoubleBlock eval(int positionCount, DoubleBlock valuesBlock, DoubleBlock percentileBlock) { + try(DoubleBlock.Builder result = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!valuesBlock.isNull(p)) { + allBlocksAreNulls = false; + } + if (percentileBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (percentileBlock.getValueCount(p) != 1) { + if (percentileBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + try { + MvPercentile.process(result, p, valuesBlock, percentileBlock.getDouble(percentileBlock.getFirstValueIndex(p)), this.scratch); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvPercentileDoubleEvaluator[" + "values=" + values + ", percentile=" + percentile + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(values, percentile); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory values; + + private final EvalOperator.ExpressionEvaluator.Factory percentile; + + private final Function scratch; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory values, + EvalOperator.ExpressionEvaluator.Factory percentile, + Function scratch) { + this.source = source; + this.values = values; + this.percentile = percentile; + this.scratch = scratch; + } + + @Override + public MvPercentileDoubleEvaluator get(DriverContext context) { + return new MvPercentileDoubleEvaluator(source, values.get(context), percentile.get(context), scratch.apply(context), context); + } + + @Override + public String toString() { + return "MvPercentileDoubleEvaluator[" + "values=" + values + ", percentile=" + percentile + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileIntegerEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileIntegerEvaluator.java new file mode 100644 index 0000000000000..93dda414c7b33 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileIntegerEvaluator.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.util.function.Function; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvPercentile}. + * This class is generated. Do not edit it. + */ +public final class MvPercentileIntegerEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator values; + + private final EvalOperator.ExpressionEvaluator percentile; + + private final MvPercentile.IntSortingScratch scratch; + + private final DriverContext driverContext; + + public MvPercentileIntegerEvaluator(Source source, EvalOperator.ExpressionEvaluator values, + EvalOperator.ExpressionEvaluator percentile, MvPercentile.IntSortingScratch scratch, + DriverContext driverContext) { + this.values = values; + this.percentile = percentile; + this.scratch = scratch; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (IntBlock valuesBlock = (IntBlock) values.eval(page)) { + try (DoubleBlock percentileBlock = (DoubleBlock) percentile.eval(page)) { + return eval(page.getPositionCount(), valuesBlock, percentileBlock); + } + } + } + + public IntBlock eval(int positionCount, IntBlock valuesBlock, DoubleBlock percentileBlock) { + try(IntBlock.Builder result = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!valuesBlock.isNull(p)) { + allBlocksAreNulls = false; + } + if (percentileBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (percentileBlock.getValueCount(p) != 1) { + if (percentileBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + try { + MvPercentile.process(result, p, valuesBlock, percentileBlock.getDouble(percentileBlock.getFirstValueIndex(p)), this.scratch); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvPercentileIntegerEvaluator[" + "values=" + values + ", percentile=" + percentile + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(values, percentile); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory values; + + private final EvalOperator.ExpressionEvaluator.Factory percentile; + + private final Function scratch; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory values, + EvalOperator.ExpressionEvaluator.Factory percentile, + Function scratch) { + this.source = source; + this.values = values; + this.percentile = percentile; + this.scratch = scratch; + } + + @Override + public MvPercentileIntegerEvaluator get(DriverContext context) { + return new MvPercentileIntegerEvaluator(source, values.get(context), percentile.get(context), scratch.apply(context), context); + } + + @Override + public String toString() { + return "MvPercentileIntegerEvaluator[" + "values=" + values + ", percentile=" + percentile + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileLongEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileLongEvaluator.java new file mode 100644 index 0000000000000..10d0b7c3283b2 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileLongEvaluator.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.util.function.Function; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link MvPercentile}. + * This class is generated. Do not edit it. + */ +public final class MvPercentileLongEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator values; + + private final EvalOperator.ExpressionEvaluator percentile; + + private final MvPercentile.LongSortingScratch scratch; + + private final DriverContext driverContext; + + public MvPercentileLongEvaluator(Source source, EvalOperator.ExpressionEvaluator values, + EvalOperator.ExpressionEvaluator percentile, MvPercentile.LongSortingScratch scratch, + DriverContext driverContext) { + this.values = values; + this.percentile = percentile; + this.scratch = scratch; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (LongBlock valuesBlock = (LongBlock) values.eval(page)) { + try (DoubleBlock percentileBlock = (DoubleBlock) percentile.eval(page)) { + return eval(page.getPositionCount(), valuesBlock, percentileBlock); + } + } + } + + public LongBlock eval(int positionCount, LongBlock valuesBlock, DoubleBlock percentileBlock) { + try(LongBlock.Builder result = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + position: for (int p = 0; p < positionCount; p++) { + boolean allBlocksAreNulls = true; + if (!valuesBlock.isNull(p)) { + allBlocksAreNulls = false; + } + if (percentileBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (percentileBlock.getValueCount(p) != 1) { + if (percentileBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (allBlocksAreNulls) { + result.appendNull(); + continue position; + } + try { + MvPercentile.process(result, p, valuesBlock, percentileBlock.getDouble(percentileBlock.getFirstValueIndex(p)), this.scratch); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "MvPercentileLongEvaluator[" + "values=" + values + ", percentile=" + percentile + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(values, percentile); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory values; + + private final EvalOperator.ExpressionEvaluator.Factory percentile; + + private final Function scratch; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory values, + EvalOperator.ExpressionEvaluator.Factory percentile, + Function scratch) { + this.source = source; + this.values = values; + this.percentile = percentile; + this.scratch = scratch; + } + + @Override + public MvPercentileLongEvaluator get(DriverContext context) { + return new MvPercentileLongEvaluator(source, values.get(context), percentile.get(context), scratch.apply(context), context); + } + + @Override + public String toString() { + return "MvPercentileLongEvaluator[" + "values=" + values + ", percentile=" + percentile + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 7937ae67c70bc..913eb382a5daf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -37,6 +37,11 @@ public enum Cap { */ FN_MV_APPEND, + /** + * Support for {@code MV_PERCENTILE} function. + */ + FN_MV_PERCENTILE, + /** * Support for function {@code IP_PREFIX}. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 6e23f4445b564..c64cbdbd2a9ed 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -96,6 +96,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMedian; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvPSeriesWeightedSum; +import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvPercentile; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSlice; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSort; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; @@ -362,6 +363,7 @@ private FunctionDefinition[][] functions() { def(MvMax.class, MvMax::new, "mv_max"), def(MvMedian.class, MvMedian::new, "mv_median"), def(MvMin.class, MvMin::new, "mv_min"), + def(MvPercentile.class, MvPercentile::new, "mv_percentile"), def(MvPSeriesWeightedSum.class, MvPSeriesWeightedSum::new, "mv_pseries_weighted_sum"), def(MvSort.class, MvSort::new, "mv_sort"), def(MvSlice.class, MvSlice::new, "mv_slice"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java index 54cebc7daad5d..0d5dd4b66501c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java @@ -18,9 +18,12 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.SurrogateExpression; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; +import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvPercentile; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -31,7 +34,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -public class Percentile extends NumericAggregate { +public class Percentile extends NumericAggregate implements SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "Percentile", @@ -152,4 +155,15 @@ protected AggregatorFunctionSupplier doubleSupplier(List inputChannels) private int percentileValue() { return ((Number) percentile.fold()).intValue(); } + + @Override + public Expression surrogate() { + var field = field(); + + if (field.foldable()) { + return new MvPercentile(source(), new ToDouble(source(), field), percentile()); + } + + return null; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java index 055e34ad5a633..1c10c7d2fa9ef 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java @@ -68,18 +68,18 @@ * {@code dataType}: This will return the datatype of your function. * May be based on its current parameters. * - * - * - * Finally, you may want to implement some interfaces. - * Check their JavaDocs to see if they are suitable for your function: - *
    - *
  • - * {@link org.elasticsearch.xpack.esql.planner.ToAggregator}: (More information about aggregators below) - *
    • *
  • - * {@link org.elasticsearch.xpack.esql.expression.SurrogateExpression} + * Implement {@link org.elasticsearch.xpack.esql.expression.SurrogateExpression}, and its required + * {@link org.elasticsearch.xpack.esql.expression.SurrogateExpression#surrogate()} method. + *

    + * It's used to be able to fold the aggregation when it receives only literals, + * or when the aggregation can be simplified. + *

    *
    • *
+ * + * Finally, implement {@link org.elasticsearch.xpack.esql.planner.ToAggregator} (More information about aggregators below). + * The only case when this interface is not required is when it always returns another function in its surrogate. * *
  • * To introduce your aggregation to the engine: diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java index 90810d282ca52..cb0f9fdd8d5db 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java @@ -44,6 +44,7 @@ public static List getNamedWriteables() { MvMax.ENTRY, MvMedian.ENTRY, MvMin.ENTRY, + MvPercentile.ENTRY, MvPSeriesWeightedSum.ENTRY, MvSlice.ENTRY, MvSort.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java new file mode 100644 index 0000000000000..b1e710b9b2a40 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java @@ -0,0 +1,446 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.ann.Fixed; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; + +public class MvPercentile extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "MvPercentile", + MvPercentile::new + ); + + /** + * 2^52 is the smallest integer where it and all smaller integers can be represented exactly as double + */ + private static final double MAX_SAFE_LONG_DOUBLE = Double.longBitsToDouble(0x4330000000000000L); + + private final Expression field; + private final Expression percentile; + + @FunctionInfo( + returnType = { "double", "integer", "long" }, + description = "Converts a multivalued field into a single valued field containing " + + "the value at which a certain percentage of observed values occur.", + examples = @Example(file = "mv_percentile", tag = "example") + ) + public MvPercentile( + Source source, + @Param(name = "number", type = { "double", "integer", "long" }, description = "Multivalue expression.") Expression field, + @Param( + name = "percentile", + type = { "double", "integer", "long" }, + description = "The percentile to calculate. Must be a number between 0 and 100. " + + "Numbers out of range will return a null instead." + ) Expression percentile + ) { + super(source, List.of(field, percentile)); + this.field = field; + this.percentile = percentile; + } + + private MvPercentile(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(field); + out.writeNamedWriteable(percentile); + } + + @Override + protected Expression.TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + return isType(field, dt -> dt.isNumeric() && dt != UNSIGNED_LONG, sourceText(), FIRST, "numeric except unsigned_long").and( + isType(percentile, dt -> dt.isNumeric() && dt != UNSIGNED_LONG, sourceText(), SECOND, "numeric except unsigned_long") + ); + } + + @Override + public boolean foldable() { + return field.foldable() && percentile.foldable(); + } + + public final Expression field() { + return field; + } + + @Override + public DataType dataType() { + return field.dataType(); + } + + @Override + public final ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + var fieldEval = toEvaluator.apply(field); + var percentileEval = Cast.cast(source(), percentile.dataType(), DOUBLE, toEvaluator.apply(percentile)); + + return switch (PlannerUtils.toElementType(field.dataType())) { + case INT -> new MvPercentileIntegerEvaluator.Factory(source(), fieldEval, percentileEval, (d) -> new IntSortingScratch()); + case LONG -> new MvPercentileLongEvaluator.Factory(source(), fieldEval, percentileEval, (d) -> new LongSortingScratch()); + case DOUBLE -> new MvPercentileDoubleEvaluator.Factory(source(), fieldEval, percentileEval, (d) -> new DoubleSortingScratch()); + default -> throw EsqlIllegalArgumentException.illegalDataType(field.dataType()); + }; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new MvPercentile(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, MvPercentile::new, field, percentile); + } + + static class DoubleSortingScratch { + private static final double[] EMPTY = new double[0]; + + public double[] values = EMPTY; + } + + static class IntSortingScratch { + private static final int[] EMPTY = new int[0]; + + public int[] values = EMPTY; + } + + static class LongSortingScratch { + private static final long[] EMPTY = new long[0]; + + public long[] values = EMPTY; + } + + // Evaluators + + @Evaluator(extraName = "Double", warnExceptions = IllegalArgumentException.class) + static void process( + DoubleBlock.Builder builder, + int position, + DoubleBlock values, + double percentile, + @Fixed(includeInToString = false, build = true) DoubleSortingScratch scratch + ) { + int valueCount = values.getValueCount(position); + int firstValueIndex = values.getFirstValueIndex(position); + + if (valueCount == 0) { + builder.appendNull(); + return; + } + + if (percentile < 0 || percentile > 100) { + throw new IllegalArgumentException("Percentile parameter must be a number between 0 and 100, found [" + percentile + "]"); + } + + builder.appendDouble(calculateDoublePercentile(values, firstValueIndex, valueCount, percentile, scratch)); + } + + @Evaluator(extraName = "Integer", warnExceptions = IllegalArgumentException.class) + static void process( + IntBlock.Builder builder, + int position, + IntBlock values, + double percentile, + @Fixed(includeInToString = false, build = true) IntSortingScratch scratch + ) { + int valueCount = values.getValueCount(position); + int firstValueIndex = values.getFirstValueIndex(position); + + if (valueCount == 0) { + builder.appendNull(); + return; + } + + if (percentile < 0 || percentile > 100) { + throw new IllegalArgumentException("Percentile parameter must be a number between 0 and 100, found [" + percentile + "]"); + } + + builder.appendInt(calculateIntPercentile(values, firstValueIndex, valueCount, percentile, scratch)); + } + + @Evaluator(extraName = "Long", warnExceptions = IllegalArgumentException.class) + static void process( + LongBlock.Builder builder, + int position, + LongBlock values, + double percentile, + @Fixed(includeInToString = false, build = true) LongSortingScratch scratch + ) { + int valueCount = values.getValueCount(position); + int firstValueIndex = values.getFirstValueIndex(position); + + if (valueCount == 0) { + builder.appendNull(); + return; + } + + if (percentile < 0 || percentile > 100) { + throw new IllegalArgumentException("Percentile parameter must be a number between 0 and 100, found [" + percentile + "]"); + } + + builder.appendLong(calculateLongPercentile(values, firstValueIndex, valueCount, percentile, scratch)); + } + + // Percentile calculators + + private static double calculateDoublePercentile( + DoubleBlock valuesBlock, + int firstValueIndex, + int valueCount, + double percentile, + DoubleSortingScratch scratch + ) { + if (valueCount == 1) { + return valuesBlock.getDouble(firstValueIndex); + } + + var p = percentile / 100.0; + var index = p * (valueCount - 1); + var lowerIndex = (int) index; + var upperIndex = lowerIndex + 1; + var fraction = index - lowerIndex; + + if (valuesBlock.mvSortedAscending()) { + if (percentile == 0) { + return valuesBlock.getDouble(0); + } else if (percentile == 100) { + return valuesBlock.getDouble(valueCount - 1); + } else { + assert lowerIndex >= 0 && upperIndex < valueCount; + return calculateDoublePercentile(fraction, valuesBlock.getDouble(lowerIndex), valuesBlock.getDouble(upperIndex)); + } + } + + if (percentile == 0) { + double min = Double.POSITIVE_INFINITY; + for (int i = 0; i < valueCount; i++) { + min = Math.min(min, valuesBlock.getDouble(firstValueIndex + i)); + } + return min; + } else if (percentile == 100) { + double max = Double.NEGATIVE_INFINITY; + for (int i = 0; i < valueCount; i++) { + max = Math.max(max, valuesBlock.getDouble(firstValueIndex + i)); + } + return max; + } + + if (scratch.values.length < valueCount) { + scratch.values = new double[ArrayUtil.oversize(valueCount, Double.BYTES)]; + } + + for (int i = 0; i < valueCount; i++) { + scratch.values[i] = valuesBlock.getDouble(firstValueIndex + i); + } + + Arrays.sort(scratch.values, 0, valueCount); + + assert lowerIndex >= 0 && upperIndex < valueCount; + return calculateDoublePercentile(fraction, scratch.values[lowerIndex], scratch.values[upperIndex]); + } + + private static int calculateIntPercentile( + IntBlock valuesBlock, + int firstValueIndex, + int valueCount, + double percentile, + IntSortingScratch scratch + ) { + if (valueCount == 1) { + return valuesBlock.getInt(firstValueIndex); + } + + var p = percentile / 100.0; + var index = p * (valueCount - 1); + var lowerIndex = (int) index; + var upperIndex = lowerIndex + 1; + var fraction = index - lowerIndex; + + if (valuesBlock.mvSortedAscending()) { + if (percentile == 0) { + return valuesBlock.getInt(0); + } else if (percentile == 100) { + return valuesBlock.getInt(valueCount - 1); + } else { + assert lowerIndex >= 0 && upperIndex < valueCount; + var lowerValue = valuesBlock.getInt(lowerIndex); + var upperValue = valuesBlock.getInt(upperIndex); + var difference = (long) upperValue - lowerValue; + return lowerValue + (int) (fraction * difference); + } + } + + if (percentile == 0) { + int min = Integer.MAX_VALUE; + for (int i = 0; i < valueCount; i++) { + min = Math.min(min, valuesBlock.getInt(firstValueIndex + i)); + } + return min; + } else if (percentile == 100) { + int max = Integer.MIN_VALUE; + for (int i = 0; i < valueCount; i++) { + max = Math.max(max, valuesBlock.getInt(firstValueIndex + i)); + } + return max; + } + + if (scratch.values.length < valueCount) { + scratch.values = new int[ArrayUtil.oversize(valueCount, Integer.BYTES)]; + } + + for (int i = 0; i < valueCount; i++) { + scratch.values[i] = valuesBlock.getInt(firstValueIndex + i); + } + + Arrays.sort(scratch.values, 0, valueCount); + + assert lowerIndex >= 0 && upperIndex < valueCount; + var lowerValue = scratch.values[lowerIndex]; + var upperValue = scratch.values[upperIndex]; + var difference = (long) upperValue - lowerValue; + return lowerValue + (int) (fraction * difference); + } + + private static long calculateLongPercentile( + LongBlock valuesBlock, + int firstValueIndex, + int valueCount, + double percentile, + LongSortingScratch scratch + ) { + if (valueCount == 1) { + return valuesBlock.getLong(firstValueIndex); + } + + var p = percentile / 100.0; + var index = p * (valueCount - 1); + var lowerIndex = (int) index; + var upperIndex = lowerIndex + 1; + var fraction = index - lowerIndex; + + if (valuesBlock.mvSortedAscending()) { + if (percentile == 0) { + return valuesBlock.getLong(0); + } else if (percentile == 100) { + return valuesBlock.getLong(valueCount - 1); + } else { + assert lowerIndex >= 0 && upperIndex < valueCount; + return calculateLongPercentile(fraction, valuesBlock.getLong(lowerIndex), valuesBlock.getLong(upperIndex)); + } + } + + if (percentile == 0) { + long min = Long.MAX_VALUE; + for (int i = 0; i < valueCount; i++) { + min = Math.min(min, valuesBlock.getLong(firstValueIndex + i)); + } + return min; + } else if (percentile == 100) { + long max = Long.MIN_VALUE; + for (int i = 0; i < valueCount; i++) { + max = Math.max(max, valuesBlock.getLong(firstValueIndex + i)); + } + return max; + } + + if (scratch.values.length < valueCount) { + scratch.values = new long[ArrayUtil.oversize(valueCount, Long.BYTES)]; + } + + for (int i = 0; i < valueCount; i++) { + scratch.values[i] = valuesBlock.getLong(firstValueIndex + i); + } + + Arrays.sort(scratch.values, 0, valueCount); + + assert lowerIndex >= 0 && upperIndex < valueCount; + return calculateLongPercentile(fraction, scratch.values[lowerIndex], scratch.values[upperIndex]); + } + + /** + * Calculates a percentile for a long avoiding overflows and double precision issues. + *

    + * To do that, if the values are over the limit of the representable double integers, + * it uses instead BigDecimals for the calculations. + *

    + */ + private static long calculateLongPercentile(double fraction, long lowerValue, long upperValue) { + if (upperValue < MAX_SAFE_LONG_DOUBLE && lowerValue > -MAX_SAFE_LONG_DOUBLE) { + var difference = upperValue - lowerValue; + return lowerValue + (long) (fraction * difference); + } + + var lowerValueBigDecimal = new BigDecimal(lowerValue); + var upperValueBigDecimal = new BigDecimal(upperValue); + var difference = upperValueBigDecimal.subtract(lowerValueBigDecimal); + var fractionBigDecimal = new BigDecimal(fraction); + return lowerValueBigDecimal.add(fractionBigDecimal.multiply(difference)).longValue(); + } + + /** + * Calculates a percentile for a double avoiding overflows. + *

    + * If the values are too separated (negative + positive), it uses a slightly different approach. + * This approach would fail if the values are big but not separated, so it's only used in this case. + *

    + */ + private static double calculateDoublePercentile(double fraction, double lowerValue, double upperValue) { + if (lowerValue < 0 && upperValue > 0) { + // Order is required to avoid `upper - lower` overflows + return (lowerValue + fraction * upperValue) - fraction * lowerValue; + } + + var difference = upperValue - lowerValue; + return lowerValue + fraction * difference; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java index 65425486ea4e0..f3c87e0e9d1d7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java @@ -301,14 +301,15 @@ private void resolveExpression(Expression expression, Consumer onAgg } expression = resolveSurrogates(expression); + // As expressions may be composed of multiple functions, we need to fold nulls bottom-up + expression = expression.transformUp(e -> new FoldNull().rule(e)); + assertThat(expression.dataType(), equalTo(testCase.expectedType())); + Expression.TypeResolution resolution = expression.typeResolved(); if (resolution.unresolved()) { throw new AssertionError("expected resolved " + resolution.message()); } - expression = new FoldNull().rule(expression); - assertThat(expression.dataType(), equalTo(testCase.expectedType())); - assumeTrue( "Surrogate expression with non-trivial children cannot be evaluated", expression.children() diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java index 66b587f257e2e..3ef2a7f821457 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java @@ -74,6 +74,30 @@ protected static Iterable parameterSuppliersFromTypedDataWithDefaultCh ); } + /** + * Converts a list of test cases into a list of parameter suppliers. + * Also, adds a default set of extra test cases. + *

    + * Use if possible, as this method may get updated with new checks in the future. + *

    + * + * @param nullsExpectedType See {@link #anyNullIsNull(List, ExpectedType, ExpectedEvaluatorToString)} + * @param evaluatorToString See {@link #anyNullIsNull(List, ExpectedType, ExpectedEvaluatorToString)} + */ + protected static Iterable parameterSuppliersFromTypedDataWithDefaultChecks( + ExpectedType nullsExpectedType, + ExpectedEvaluatorToString evaluatorToString, + List suppliers, + PositionalErrorMessageSupplier positionalErrorMessageSupplier + ) { + return parameterSuppliersFromTypedData( + errorsForCasesWithoutExamples( + anyNullIsNull(randomizeBytesRefsOffset(suppliers), nullsExpectedType, evaluatorToString), + positionalErrorMessageSupplier + ) + ); + } + public final void testEvaluate() { assumeTrue("Can't build evaluator", testCase.canBuildEvaluator()); boolean readFloating = randomBoolean(); @@ -97,6 +121,7 @@ public final void testEvaluate() { Object result; try (ExpressionEvaluator evaluator = evaluator(expression).get(driverContext())) { try (Block block = evaluator.eval(row(testCase.getDataValues()))) { + assertThat(block.getPositionCount(), is(1)); result = toJavaObjectUnsignedLongAware(block, 0); } } @@ -217,6 +242,7 @@ private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext con ExpressionEvaluator eval = evaluator(expression).get(context); Block block = eval.eval(new Page(positions, manyPositionsBlocks)) ) { + assertThat(block.getPositionCount(), is(positions)); for (int p = 0; p < positions; p++) { if (nullPositions.contains(p)) { assertThat(toJavaObject(block, p), allNullsMatcher()); @@ -260,6 +286,7 @@ public final void testEvaluateInManyThreads() throws ExecutionException, Interru try (EvalOperator.ExpressionEvaluator eval = evalSupplier.get(driverContext())) { for (int c = 0; c < count; c++) { try (Block block = eval.eval(page)) { + assertThat(block.getPositionCount(), is(1)); assertThat(toJavaObjectUnsignedLongAware(block, 0), testCase.getMatcher()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultivalueTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultivalueTestCaseSupplier.java new file mode 100644 index 0000000000000..01c73e9ef0482 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultivalueTestCaseSupplier.java @@ -0,0 +1,325 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; + +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.elasticsearch.test.ESTestCase.randomList; +import static org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier.TypedDataSupplier; + +/** + * Extension of {@link TestCaseSupplier} that provided multivalue test cases. + */ +public final class MultivalueTestCaseSupplier { + + private static final int MIN_VALUES = 1; + private static final int MAX_VALUES = 1000; + + private MultivalueTestCaseSupplier() {} + + public static List intCases(int min, int max, boolean includeZero) { + List cases = new ArrayList<>(); + + for (Block.MvOrdering ordering : Block.MvOrdering.values()) { + if (0 <= max && 0 >= min && includeZero) { + cases.add( + new TypedDataSupplier( + "<0 mv " + ordering + " ints>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> 0), ordering), + DataType.INTEGER + ) + ); + } + + if (max != 0) { + cases.add( + new TypedDataSupplier( + "<" + max + " mv " + ordering + " ints>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> max), ordering), + DataType.INTEGER + ) + ); + } + + if (min != 0 && min != max) { + cases.add( + new TypedDataSupplier( + "<" + min + " mv " + ordering + " ints>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> min), ordering), + DataType.INTEGER + ) + ); + } + + int lower = Math.max(min, 1); + int upper = Math.min(max, Integer.MAX_VALUE); + if (lower < upper) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomIntBetween(lower, upper)), ordering), + DataType.INTEGER + ) + ); + } + + int lower1 = Math.max(min, Integer.MIN_VALUE); + int upper1 = Math.min(max, -1); + if (lower1 < upper1) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomIntBetween(lower1, upper1)), ordering), + DataType.INTEGER + ) + ); + } + + if (min < 0 && max > 0) { + cases.add( + new TypedDataSupplier("", () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> { + if (includeZero) { + return ESTestCase.randomIntBetween(min, max); + } + return randomBoolean() ? ESTestCase.randomIntBetween(min, -1) : ESTestCase.randomIntBetween(1, max); + }), ordering), DataType.INTEGER) + ); + } + } + + return cases; + } + + public static List longCases(long min, long max, boolean includeZero) { + List cases = new ArrayList<>(); + + for (Block.MvOrdering ordering : Block.MvOrdering.values()) { + if (0 <= max && 0 >= min && includeZero) { + cases.add( + new TypedDataSupplier( + "<0 mv " + ordering + " longs>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> 0L), ordering), + DataType.LONG + ) + ); + } + + if (max != 0) { + cases.add( + new TypedDataSupplier( + "<" + max + " mv " + ordering + " longs>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> max), ordering), + DataType.LONG + ) + ); + } + + if (min != 0 && min != max) { + cases.add( + new TypedDataSupplier( + "<" + min + " mv " + ordering + " longs>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> min), ordering), + DataType.LONG + ) + ); + } + + long lower = Math.max(min, 1); + long upper = Math.min(max, Long.MAX_VALUE); + if (lower < upper) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomLongBetween(lower, upper)), ordering), + DataType.LONG + ) + ); + } + + long lower1 = Math.max(min, Long.MIN_VALUE); + long upper1 = Math.min(max, -1); + if (lower1 < upper1) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomLongBetween(lower1, upper1)), ordering), + DataType.LONG + ) + ); + } + + if (min < 0 && max > 0) { + cases.add( + new TypedDataSupplier("", () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> { + if (includeZero) { + return ESTestCase.randomLongBetween(min, max); + } + return randomBoolean() ? ESTestCase.randomLongBetween(min, -1) : ESTestCase.randomLongBetween(1, max); + }), ordering), DataType.LONG) + ); + } + } + + return cases; + } + + public static List doubleCases(double min, double max, boolean includeZero) { + List cases = new ArrayList<>(); + + for (Block.MvOrdering ordering : Block.MvOrdering.values()) { + if (0d <= max && 0d >= min && includeZero) { + cases.add( + new TypedDataSupplier( + "<0 mv " + ordering + " doubles>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> 0d), ordering), + DataType.DOUBLE + ) + ); + cases.add( + new TypedDataSupplier( + "<-0 mv " + ordering + " doubles>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> -0d), ordering), + DataType.DOUBLE + ) + ); + } + + if (max != 0d) { + cases.add( + new TypedDataSupplier( + "<" + max + " mv " + ordering + " doubles>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> max), ordering), + DataType.DOUBLE + ) + ); + } + + if (min != 0d && min != max) { + cases.add( + new TypedDataSupplier( + "<" + min + " mv " + ordering + " doubles>", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> min), ordering), + DataType.DOUBLE + ) + ); + } + + double lower1 = Math.max(min, 0d); + double upper1 = Math.min(max, 1d); + if (lower1 < upper1) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder( + randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomDoubleBetween(lower1, upper1, true)), + ordering + ), + DataType.DOUBLE + ) + ); + } + + double lower2 = Math.max(min, -1d); + double upper2 = Math.min(max, 0d); + if (lower2 < upper2) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder( + randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomDoubleBetween(lower2, upper2, true)), + ordering + ), + DataType.DOUBLE + ) + ); + } + + double lower3 = Math.max(min, 1d); + double upper3 = Math.min(max, Double.MAX_VALUE); + if (lower3 < upper3) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder( + randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomDoubleBetween(lower3, upper3, true)), + ordering + ), + DataType.DOUBLE + ) + ); + } + + double lower4 = Math.max(min, -Double.MAX_VALUE); + double upper4 = Math.min(max, -1d); + if (lower4 < upper4) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder( + randomList(MIN_VALUES, MAX_VALUES, () -> ESTestCase.randomDoubleBetween(lower4, upper4, true)), + ordering + ), + DataType.DOUBLE + ) + ); + } + + if (min < 0 && max > 0) { + cases.add( + new TypedDataSupplier( + "", + () -> putInOrder(randomList(MIN_VALUES, MAX_VALUES, () -> { + if (includeZero) { + return ESTestCase.randomDoubleBetween(min, max, true); + } + return randomBoolean() + ? ESTestCase.randomDoubleBetween(min, -1, true) + : ESTestCase.randomDoubleBetween(1, max, true); + }), ordering), + DataType.DOUBLE + ) + ); + } + } + + return cases; + } + + private static > List putInOrder(List mvData, Block.MvOrdering ordering) { + switch (ordering) { + case UNORDERED -> { + } + case DEDUPLICATED_UNORDERD -> { + var dedup = new LinkedHashSet<>(mvData); + mvData.clear(); + mvData.addAll(dedup); + } + case DEDUPLICATED_AND_SORTED_ASCENDING -> { + var dedup = new HashSet<>(mvData); + mvData.clear(); + mvData.addAll(dedup); + Collections.sort(mvData); + } + case SORTED_ASCENDING -> { + Collections.sort(mvData); + } + default -> throw new UnsupportedOperationException("unsupported ordering [" + ordering + "]"); + } + + return mvData; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 5ef71e7ae30fb..a1caa784c9787 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -1289,7 +1289,7 @@ private static String castToUnsignedLongEvaluator(String original, DataType curr throw new UnsupportedOperationException(); } - private static String castToDoubleEvaluator(String original, DataType current) { + public static String castToDoubleEvaluator(String original, DataType current) { if (current == DataType.DOUBLE) { return original; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java index 5271431bd43b8..be11515876966 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PercentileTests.java @@ -52,7 +52,7 @@ public static Iterable parameters() { } } - return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers, false, (v, p) -> "numeric except unsigned_long"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java new file mode 100644 index 0000000000000..3410b95458302 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentileTests.java @@ -0,0 +1,466 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.StringUtils; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.MultivalueTestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class MvPercentileTests extends AbstractScalarFunctionTestCase { + public MvPercentileTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List cases = new ArrayList<>(); + + var fieldSuppliers = Stream.of( + MultivalueTestCaseSupplier.intCases(Integer.MIN_VALUE, Integer.MAX_VALUE, true), + MultivalueTestCaseSupplier.longCases(Long.MIN_VALUE, Long.MAX_VALUE, true), + MultivalueTestCaseSupplier.doubleCases(-Double.MAX_VALUE, Double.MAX_VALUE, true) + ).flatMap(List::stream).toList(); + + var percentileSuppliers = Stream.of( + TestCaseSupplier.intCases(0, 100, true), + TestCaseSupplier.longCases(0, 100, true), + TestCaseSupplier.doubleCases(0, 100, true) + ).flatMap(List::stream).toList(); + + for (var fieldSupplier : fieldSuppliers) { + for (var percentileSupplier : percentileSuppliers) { + cases.add(makeSupplier(fieldSupplier, percentileSupplier)); + } + } + + for (var percentileType : List.of(INTEGER, LONG, DataType.DOUBLE)) { + cases.addAll( + List.of( + // Doubles + new TestCaseSupplier( + "median double", + List.of(DOUBLE, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10., 5., 10.), DOUBLE, "field"), + percentileWithType(50, percentileType) + ), + evaluatorString(DOUBLE, percentileType), + DOUBLE, + equalTo(5.) + ) + ), + new TestCaseSupplier( + "single value double", + List.of(DOUBLE, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(55.), DOUBLE, "field"), + percentileWithType(randomIntBetween(0, 100), percentileType) + ), + evaluatorString(DOUBLE, percentileType), + DOUBLE, + equalTo(55.) + ) + ), + new TestCaseSupplier( + "p0 double", + List.of(DOUBLE, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10., 5., 10.), DOUBLE, "field"), + percentileWithType(0, percentileType) + ), + evaluatorString(DOUBLE, percentileType), + DOUBLE, + equalTo(-10.) + ) + ), + new TestCaseSupplier( + "p100 double", + List.of(DOUBLE, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10., 5., 10.), DOUBLE, "field"), + percentileWithType(100, percentileType) + ), + evaluatorString(DOUBLE, percentileType), + DOUBLE, + equalTo(10.) + ) + ), + new TestCaseSupplier( + "averaged double", + List.of(DOUBLE, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10., 5., 10.), DOUBLE, "field"), + percentileWithType(75, percentileType) + ), + evaluatorString(DOUBLE, percentileType), + DOUBLE, + equalTo(7.5) + ) + ), + new TestCaseSupplier( + "big double difference", + List.of(DOUBLE, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-Double.MAX_VALUE, Double.MAX_VALUE), DOUBLE, "field"), + percentileWithType(50, percentileType) + ), + evaluatorString(DOUBLE, percentileType), + DOUBLE, + closeTo(0, 0.0000001) + ) + ), + + // Int + new TestCaseSupplier( + "median int", + List.of(INTEGER, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10, 5, 10), INTEGER, "field"), + percentileWithType(50, percentileType) + ), + evaluatorString(INTEGER, percentileType), + INTEGER, + equalTo(5) + ) + ), + new TestCaseSupplier( + "single value int", + List.of(INTEGER, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(55), INTEGER, "field"), + percentileWithType(randomIntBetween(0, 100), percentileType) + ), + evaluatorString(INTEGER, percentileType), + INTEGER, + equalTo(55) + ) + ), + new TestCaseSupplier( + "p0 int", + List.of(INTEGER, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10, 5, 10), INTEGER, "field"), + percentileWithType(0, percentileType) + ), + evaluatorString(INTEGER, percentileType), + INTEGER, + equalTo(-10) + ) + ), + new TestCaseSupplier( + "p100 int", + List.of(INTEGER, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10, 5, 10), INTEGER, "field"), + percentileWithType(100, percentileType) + ), + evaluatorString(INTEGER, percentileType), + INTEGER, + equalTo(10) + ) + ), + new TestCaseSupplier( + "averaged int", + List.of(INTEGER, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10, 5, 10), INTEGER, "field"), + percentileWithType(75, percentileType) + ), + evaluatorString(INTEGER, percentileType), + INTEGER, + equalTo(7) + ) + ), + new TestCaseSupplier( + "big int difference", + List.of(INTEGER, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(Integer.MIN_VALUE, Integer.MAX_VALUE), INTEGER, "field"), + percentileWithType(50, percentileType) + ), + evaluatorString(INTEGER, percentileType), + INTEGER, + equalTo(-1) // Negative max is 1 smaller than positive max + ) + ), + + // Long + new TestCaseSupplier( + "median long", + List.of(LONG, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10L, 5L, 10L), LONG, "field"), + percentileWithType(50, percentileType) + ), + evaluatorString(LONG, percentileType), + LONG, + equalTo(5L) + ) + ), + new TestCaseSupplier( + "single value long", + List.of(LONG, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(55L), LONG, "field"), + percentileWithType(randomIntBetween(0, 100), percentileType) + ), + evaluatorString(LONG, percentileType), + LONG, + equalTo(55L) + ) + ), + new TestCaseSupplier( + "p0 long", + List.of(LONG, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10L, 5L, 10L), LONG, "field"), + percentileWithType(0, percentileType) + ), + evaluatorString(LONG, percentileType), + LONG, + equalTo(-10L) + ) + ), + new TestCaseSupplier( + "p100 long", + List.of(LONG, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10L, 5L, 10L), LONG, "field"), + percentileWithType(100, percentileType) + ), + evaluatorString(LONG, percentileType), + LONG, + equalTo(10L) + ) + ), + new TestCaseSupplier( + "averaged long", + List.of(LONG, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(-10L, 5L, 10L), LONG, "field"), + percentileWithType(75, percentileType) + ), + evaluatorString(LONG, percentileType), + LONG, + equalTo(7L) + ) + ), + new TestCaseSupplier( + "big long difference", + List.of(LONG, percentileType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(List.of(Long.MIN_VALUE, Long.MAX_VALUE), LONG, "field"), + percentileWithType(50, percentileType) + ), + evaluatorString(LONG, percentileType), + LONG, + equalTo(0L) + ) + ) + ) + ); + + for (var fieldType : List.of(INTEGER, LONG, DataType.DOUBLE)) { + cases.add( + new TestCaseSupplier( + "out of bounds percentile <" + fieldType + ", " + percentileType + ">", + List.of(fieldType, percentileType), + () -> { + var percentile = numberWithType( + randomBoolean() ? randomIntBetween(Integer.MIN_VALUE, -1) : randomIntBetween(101, Integer.MAX_VALUE), + percentileType + ); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(numberWithType(0, fieldType), fieldType, "field"), + new TestCaseSupplier.TypedData(percentile, percentileType, "percentile") + ), + evaluatorString(fieldType, percentileType), + fieldType, + nullValue() + ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") + .withWarning( + "Line -1:-1: java.lang.IllegalArgumentException: Percentile parameter must be " + + "a number between 0 and 100, found [" + + percentile.doubleValue() + + "]" + ); + } + ) + ); + } + } + + return parameterSuppliersFromTypedDataWithDefaultChecks( + (nullPosition, nullValueDataType, original) -> nullValueDataType == DataType.NULL && nullPosition == 0 + ? DataType.NULL + : original.expectedType(), + (nullPosition, nullData, original) -> original, + cases, + (v, p) -> "numeric except unsigned_long" + ); + } + + @SuppressWarnings("unchecked") + private static TestCaseSupplier makeSupplier( + TestCaseSupplier.TypedDataSupplier fieldSupplier, + TestCaseSupplier.TypedDataSupplier percentileSupplier + ) { + return new TestCaseSupplier( + "field: " + fieldSupplier.name() + ", percentile: " + percentileSupplier.name(), + List.of(fieldSupplier.type(), percentileSupplier.type()), + () -> { + var fieldTypedData = fieldSupplier.get(); + var percentileTypedData = percentileSupplier.get(); + + var values = (List) fieldTypedData.data(); + var percentile = ((Number) percentileTypedData.data()).doubleValue(); + + var expected = calculatePercentile(values, percentile); + + return new TestCaseSupplier.TestCase( + List.of(fieldTypedData, percentileTypedData), + evaluatorString(fieldSupplier.type(), percentileSupplier.type()), + fieldSupplier.type(), + expected instanceof Double expectedDouble + ? closeTo(expectedDouble, Math.abs(expectedDouble * 0.0000001)) + : equalTo(expected) + ); + } + ); + } + + private static Number calculatePercentile(List rawValues, double percentile) { + if (rawValues.isEmpty() || percentile < 0 || percentile > 100) { + return null; + } + + if (rawValues.size() == 1) { + return rawValues.get(0); + } + + int valueCount = rawValues.size(); + var p = percentile / 100.0; + var index = p * (valueCount - 1); + var lowerIndex = (int) index; + var upperIndex = lowerIndex + 1; + var fraction = index - lowerIndex; + + if (rawValues.get(0) instanceof Integer) { + var values = rawValues.stream().mapToInt(Number::intValue).sorted().toArray(); + + if (percentile == 0) { + return values[0]; + } else if (percentile == 100) { + return values[valueCount - 1]; + } else { + assert lowerIndex >= 0 && upperIndex < valueCount; + var difference = (long) values[upperIndex] - values[lowerIndex]; + return values[lowerIndex] + (int) (fraction * difference); + } + } + + if (rawValues.get(0) instanceof Long) { + var values = rawValues.stream().mapToLong(Number::longValue).sorted().toArray(); + + if (percentile == 0) { + return values[0]; + } else if (percentile == 100) { + return values[valueCount - 1]; + } else { + assert lowerIndex >= 0 && upperIndex < valueCount; + return calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])).longValue(); + } + } + + if (rawValues.get(0) instanceof Double) { + var values = rawValues.stream().mapToDouble(Number::doubleValue).sorted().toArray(); + + if (percentile == 0) { + return values[0]; + } else if (percentile == 100) { + return values[valueCount - 1]; + } else { + assert lowerIndex >= 0 && upperIndex < valueCount; + return calculatePercentile(fraction, new BigDecimal(values[lowerIndex]), new BigDecimal(values[upperIndex])).doubleValue(); + } + } + + throw new IllegalArgumentException("Unsupported type: " + rawValues.get(0).getClass()); + } + + private static BigDecimal calculatePercentile(double fraction, BigDecimal lowerValue, BigDecimal upperValue) { + return lowerValue.add(new BigDecimal(fraction).multiply(upperValue.subtract(lowerValue))); + } + + private static TestCaseSupplier.TypedData percentileWithType(Number value, DataType type) { + return new TestCaseSupplier.TypedData(numberWithType(value, type), type, "percentile"); + } + + private static Number numberWithType(Number value, DataType type) { + return switch (type) { + case INTEGER -> value.intValue(); + case LONG -> value.longValue(); + default -> value.doubleValue(); + }; + } + + private static String evaluatorString(DataType fieldDataType, DataType percentileDataType) { + var fieldTypeName = StringUtils.underscoreToLowerCamelCase(fieldDataType.name()); + + fieldTypeName = fieldTypeName.substring(0, 1).toUpperCase(Locale.ROOT) + fieldTypeName.substring(1); + + var percentileEvaluator = TestCaseSupplier.castToDoubleEvaluator("Attribute[channel=1]", percentileDataType); + + return "MvPercentile" + fieldTypeName + "Evaluator[values=Attribute[channel=0], percentile=" + percentileEvaluator + "]"; + } + + @Override + protected final Expression build(Source source, List args) { + return new MvPercentile(source, args.get(0), args.get(1)); + } +} From 0dab4b0571c263aa786234c42395390d62bf89cd Mon Sep 17 00:00:00 2001 From: Stef Nestor <26751266+stefnestor@users.noreply.github.com> Date: Tue, 20 Aug 2024 08:22:22 -0600 Subject: [PATCH 10/32] (Doc+) Removing "current_node" from Allocation Explain API under Fix Watermark Errors (#111946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 👋 howdy, team! This just simplifies the Allocation Explain API request to not need to include the `current_node` which may not be known when troubleshooting the [Fix Watermark Errors](https://www.elastic.co/guide/en/elasticsearch/reference/current/fix-watermark-errors.html) guide. TIA! Stef --- .../common-issues/disk-usage-exceeded.asciidoc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc index 728d805db7a30..7eb27d5428956 100644 --- a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc +++ b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc @@ -44,13 +44,11 @@ GET _cluster/allocation/explain { "index": "my-index", "shard": 0, - "primary": false, - "current_node": "my-node" + "primary": false } ---- // TEST[s/^/PUT my-index\n/] // TEST[s/"primary": false,/"primary": false/] -// TEST[s/"current_node": "my-node"//] [[fix-watermark-errors-temporary]] ==== Temporary Relief From 30408ce9145ba8325ab3726a347ff1eb0b468de3 Mon Sep 17 00:00:00 2001 From: Siddharth Rayabharam Date: Tue, 20 Aug 2024 11:58:43 -0400 Subject: [PATCH 11/32] Disable tests if adaptive allocation feature flag is disabled (#111942) --- .../xpack/inference/integration/ModelRegistryIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index 5157683f2dce9..d776f3963c2ca 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -25,6 +25,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsFeatureFlag; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.services.elser.ElserInternalModel; @@ -101,6 +102,7 @@ public void testStoreModelWithUnknownFields() throws Exception { } public void testGetModel() throws Exception { + assumeTrue("Only if 'inference_adaptive_allocations' feature flag is enabled", AdaptiveAllocationsFeatureFlag.isEnabled()); String inferenceEntityId = "test-get-model"; Model model = buildElserModelConfig(inferenceEntityId, TaskType.SPARSE_EMBEDDING); AtomicReference putModelHolder = new AtomicReference<>(); From d8e705d5da0db38b0cfc488f503eec4728a2e30f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 20 Aug 2024 11:59:13 -0400 Subject: [PATCH 12/32] ESQL: Document `date` instead of `datetime` (#111985) This changes the generated types tables in the docs to say `date` instead of `datetime`. That's the name of the field in Elasticsearch so it's a lot less confusing to call it that. Closes #111650 --- .../description/to_datetime.asciidoc | 2 +- .../esql/functions/kibana/definition/add.json | 26 +-- .../functions/kibana/definition/bucket.json | 150 +++++++++--------- .../functions/kibana/definition/case.json | 4 +- .../functions/kibana/definition/coalesce.json | 6 +- .../functions/kibana/definition/count.json | 2 +- .../kibana/definition/count_distinct.json | 8 +- .../kibana/definition/date_diff.json | 8 +- .../kibana/definition/date_extract.json | 4 +- .../kibana/definition/date_format.json | 4 +- .../kibana/definition/date_parse.json | 8 +- .../kibana/definition/date_trunc.json | 8 +- .../functions/kibana/definition/equals.json | 4 +- .../kibana/definition/greater_than.json | 4 +- .../definition/greater_than_or_equal.json | 4 +- .../kibana/definition/less_than.json | 4 +- .../kibana/definition/less_than_or_equal.json | 4 +- .../esql/functions/kibana/definition/max.json | 4 +- .../esql/functions/kibana/definition/min.json | 4 +- .../kibana/definition/mv_append.json | 6 +- .../functions/kibana/definition/mv_count.json | 2 +- .../kibana/definition/mv_dedupe.json | 4 +- .../functions/kibana/definition/mv_first.json | 4 +- .../functions/kibana/definition/mv_last.json | 4 +- .../functions/kibana/definition/mv_max.json | 4 +- .../functions/kibana/definition/mv_min.json | 4 +- .../functions/kibana/definition/mv_slice.json | 4 +- .../functions/kibana/definition/mv_sort.json | 4 +- .../kibana/definition/not_equals.json | 4 +- .../esql/functions/kibana/definition/now.json | 2 +- .../esql/functions/kibana/definition/sub.json | 16 +- .../kibana/definition/to_datetime.json | 18 +-- .../kibana/definition/to_double.json | 2 +- .../kibana/definition/to_integer.json | 2 +- .../functions/kibana/definition/to_long.json | 2 +- .../kibana/definition/to_string.json | 2 +- .../kibana/definition/to_unsigned_long.json | 2 +- .../esql/functions/kibana/definition/top.json | 4 +- .../functions/kibana/definition/values.json | 4 +- .../esql/functions/kibana/docs/to_datetime.md | 2 +- .../esql/functions/parameters/bucket.asciidoc | 2 +- .../esql/functions/types/add.asciidoc | 8 +- .../esql/functions/types/bucket.asciidoc | 22 +-- .../esql/functions/types/case.asciidoc | 2 +- .../esql/functions/types/coalesce.asciidoc | 2 +- .../esql/functions/types/count.asciidoc | 2 +- .../functions/types/count_distinct.asciidoc | 8 +- .../esql/functions/types/date_diff.asciidoc | 4 +- .../functions/types/date_extract.asciidoc | 4 +- .../esql/functions/types/date_format.asciidoc | 4 +- .../esql/functions/types/date_parse.asciidoc | 8 +- .../esql/functions/types/date_trunc.asciidoc | 4 +- .../esql/functions/types/equals.asciidoc | 2 +- .../functions/types/greater_than.asciidoc | 2 +- .../types/greater_than_or_equal.asciidoc | 2 +- .../esql/functions/types/less_than.asciidoc | 2 +- .../types/less_than_or_equal.asciidoc | 2 +- .../esql/functions/types/max.asciidoc | 2 +- .../esql/functions/types/min.asciidoc | 2 +- .../esql/functions/types/mv_append.asciidoc | 2 +- .../esql/functions/types/mv_count.asciidoc | 2 +- .../esql/functions/types/mv_dedupe.asciidoc | 2 +- .../esql/functions/types/mv_first.asciidoc | 2 +- .../esql/functions/types/mv_last.asciidoc | 2 +- .../esql/functions/types/mv_max.asciidoc | 2 +- .../esql/functions/types/mv_min.asciidoc | 2 +- .../esql/functions/types/mv_slice.asciidoc | 2 +- .../esql/functions/types/mv_sort.asciidoc | 2 +- .../esql/functions/types/not_equals.asciidoc | 2 +- .../esql/functions/types/now.asciidoc | 2 +- .../esql/functions/types/sub.asciidoc | 4 +- .../esql/functions/types/to_datetime.asciidoc | 14 +- .../esql/functions/types/to_double.asciidoc | 2 +- .../esql/functions/types/to_integer.asciidoc | 2 +- .../esql/functions/types/to_long.asciidoc | 2 +- .../esql/functions/types/to_string.asciidoc | 2 +- .../functions/types/to_unsigned_long.asciidoc | 2 +- .../esql/functions/types/top.asciidoc | 2 +- .../esql/functions/types/values.asciidoc | 2 +- .../xpack/esql/core/type/DataType.java | 8 + .../function/scalar/convert/ToDatetime.java | 2 +- .../function/AbstractFunctionTestCase.java | 22 ++- 82 files changed, 265 insertions(+), 259 deletions(-) diff --git a/docs/reference/esql/functions/description/to_datetime.asciidoc b/docs/reference/esql/functions/description/to_datetime.asciidoc index ee6866da9ee34..91cbfa0b5fe1e 100644 --- a/docs/reference/esql/functions/description/to_datetime.asciidoc +++ b/docs/reference/esql/functions/description/to_datetime.asciidoc @@ -4,4 +4,4 @@ Converts an input value to a date value. A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use <>. -NOTE: Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date istruncated, not rounded. +NOTE: Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded. diff --git a/docs/reference/esql/functions/kibana/definition/add.json b/docs/reference/esql/functions/kibana/definition/add.json index e20299821facb..0932a76966560 100644 --- a/docs/reference/esql/functions/kibana/definition/add.json +++ b/docs/reference/esql/functions/kibana/definition/add.json @@ -8,7 +8,7 @@ "params" : [ { "name" : "lhs", - "type" : "date_period", + "type" : "date", "optional" : false, "description" : "A numeric value or a date time value." }, @@ -20,61 +20,61 @@ } ], "variadic" : false, - "returnType" : "date_period" + "returnType" : "date" }, { "params" : [ { "name" : "lhs", - "type" : "date_period", + "type" : "date", "optional" : false, "description" : "A numeric value or a date time value." }, { "name" : "rhs", - "type" : "datetime", + "type" : "time_duration", "optional" : false, "description" : "A numeric value or a date time value." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date_period", "optional" : false, "description" : "A numeric value or a date time value." }, { "name" : "rhs", - "type" : "date_period", + "type" : "date", "optional" : false, "description" : "A numeric value or a date time value." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date_period", "optional" : false, "description" : "A numeric value or a date time value." }, { "name" : "rhs", - "type" : "time_duration", + "type" : "date_period", "optional" : false, "description" : "A numeric value or a date time value." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date_period" }, { "params" : [ @@ -248,13 +248,13 @@ }, { "name" : "rhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "A numeric value or a date time value." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json index 14bd74c1c20f3..94214a3a4f047 100644 --- a/docs/reference/esql/functions/kibana/definition/bucket.json +++ b/docs/reference/esql/functions/kibana/definition/bucket.json @@ -8,7 +8,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -16,17 +16,17 @@ "name" : "buckets", "type" : "date_period", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -34,29 +34,29 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", - "type" : "datetime", + "type" : "date", "optional" : true, "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, { "name" : "to", - "type" : "datetime", + "type" : "date", "optional" : true, "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -64,11 +64,11 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", - "type" : "datetime", + "type" : "date", "optional" : true, "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, @@ -80,13 +80,13 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -94,11 +94,11 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", - "type" : "datetime", + "type" : "date", "optional" : true, "description" : "Start of the range. Can be a number, a date or a date expressed as a string." }, @@ -110,13 +110,13 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -124,7 +124,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -134,19 +134,19 @@ }, { "name" : "to", - "type" : "datetime", + "type" : "date", "optional" : true, "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -154,7 +154,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -170,13 +170,13 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -184,7 +184,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -200,13 +200,13 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -214,7 +214,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -224,19 +224,19 @@ }, { "name" : "to", - "type" : "datetime", + "type" : "date", "optional" : true, "description" : "End of the range. Can be a number, a date or a date expressed as a string." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -244,7 +244,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -260,13 +260,13 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -274,7 +274,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -290,13 +290,13 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Numeric or date expression from which to derive buckets." }, @@ -304,11 +304,11 @@ "name" : "buckets", "type" : "time_duration", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -322,7 +322,7 @@ "name" : "buckets", "type" : "double", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, @@ -340,7 +340,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, @@ -358,7 +358,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -388,7 +388,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -418,7 +418,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -448,7 +448,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -478,7 +478,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -508,7 +508,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -538,7 +538,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -568,7 +568,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -598,7 +598,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -628,7 +628,7 @@ "name" : "buckets", "type" : "long", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, @@ -646,7 +646,7 @@ "name" : "buckets", "type" : "double", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, @@ -664,7 +664,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, @@ -682,7 +682,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -712,7 +712,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -742,7 +742,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -772,7 +772,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -802,7 +802,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -832,7 +832,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -862,7 +862,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -892,7 +892,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -922,7 +922,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -952,7 +952,7 @@ "name" : "buckets", "type" : "long", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, @@ -970,7 +970,7 @@ "name" : "buckets", "type" : "double", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, @@ -988,7 +988,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, @@ -1006,7 +1006,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1036,7 +1036,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1066,7 +1066,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1096,7 +1096,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1126,7 +1126,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1156,7 +1156,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1186,7 +1186,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1216,7 +1216,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1246,7 +1246,7 @@ "name" : "buckets", "type" : "integer", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." }, { "name" : "from", @@ -1276,7 +1276,7 @@ "name" : "buckets", "type" : "long", "optional" : false, - "description" : "Target number of buckets." + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." } ], "variadic" : false, diff --git a/docs/reference/esql/functions/kibana/definition/case.json b/docs/reference/esql/functions/kibana/definition/case.json index 5959eed62d37b..27705cd3897f9 100644 --- a/docs/reference/esql/functions/kibana/definition/case.json +++ b/docs/reference/esql/functions/kibana/definition/case.json @@ -50,13 +50,13 @@ }, { "name" : "trueValue", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." } ], "variadic" : true, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/coalesce.json b/docs/reference/esql/functions/kibana/definition/coalesce.json index f00f471e63ecc..2459a4d51bb2d 100644 --- a/docs/reference/esql/functions/kibana/definition/coalesce.json +++ b/docs/reference/esql/functions/kibana/definition/coalesce.json @@ -74,19 +74,19 @@ "params" : [ { "name" : "first", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Expression to evaluate." }, { "name" : "rest", - "type" : "datetime", + "type" : "date", "optional" : true, "description" : "Other expression to evaluate." } ], "variadic" : true, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/count.json b/docs/reference/esql/functions/kibana/definition/count.json index e05ebc6789816..2a15fb3bdd335 100644 --- a/docs/reference/esql/functions/kibana/definition/count.json +++ b/docs/reference/esql/functions/kibana/definition/count.json @@ -32,7 +32,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : true, "description" : "Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows)." } diff --git a/docs/reference/esql/functions/kibana/definition/count_distinct.json b/docs/reference/esql/functions/kibana/definition/count_distinct.json index 801bd26f7d022..f6a148783ba42 100644 --- a/docs/reference/esql/functions/kibana/definition/count_distinct.json +++ b/docs/reference/esql/functions/kibana/definition/count_distinct.json @@ -74,7 +74,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Column or literal for which to count the number of distinct values." } @@ -86,7 +86,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Column or literal for which to count the number of distinct values." }, @@ -104,7 +104,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Column or literal for which to count the number of distinct values." }, @@ -122,7 +122,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Column or literal for which to count the number of distinct values." }, diff --git a/docs/reference/esql/functions/kibana/definition/date_diff.json b/docs/reference/esql/functions/kibana/definition/date_diff.json index 7995d3c6d32b6..d6589f041075d 100644 --- a/docs/reference/esql/functions/kibana/definition/date_diff.json +++ b/docs/reference/esql/functions/kibana/definition/date_diff.json @@ -14,13 +14,13 @@ }, { "name" : "startTimestamp", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "A string representing a start timestamp" }, { "name" : "endTimestamp", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "A string representing an end timestamp" } @@ -38,13 +38,13 @@ }, { "name" : "startTimestamp", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "A string representing a start timestamp" }, { "name" : "endTimestamp", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "A string representing an end timestamp" } diff --git a/docs/reference/esql/functions/kibana/definition/date_extract.json b/docs/reference/esql/functions/kibana/definition/date_extract.json index 75cedcc191b50..557f0e0a47e54 100644 --- a/docs/reference/esql/functions/kibana/definition/date_extract.json +++ b/docs/reference/esql/functions/kibana/definition/date_extract.json @@ -14,7 +14,7 @@ }, { "name" : "date", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Date expression. If `null`, the function returns `null`." } @@ -32,7 +32,7 @@ }, { "name" : "date", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Date expression. If `null`, the function returns `null`." } diff --git a/docs/reference/esql/functions/kibana/definition/date_format.json b/docs/reference/esql/functions/kibana/definition/date_format.json index 5e8587c046d70..7bd01d7f4ef31 100644 --- a/docs/reference/esql/functions/kibana/definition/date_format.json +++ b/docs/reference/esql/functions/kibana/definition/date_format.json @@ -14,7 +14,7 @@ }, { "name" : "date", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Date expression. If `null`, the function returns `null`." } @@ -32,7 +32,7 @@ }, { "name" : "date", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Date expression. If `null`, the function returns `null`." } diff --git a/docs/reference/esql/functions/kibana/definition/date_parse.json b/docs/reference/esql/functions/kibana/definition/date_parse.json index 890179143bef8..9400340750c2a 100644 --- a/docs/reference/esql/functions/kibana/definition/date_parse.json +++ b/docs/reference/esql/functions/kibana/definition/date_parse.json @@ -20,7 +20,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -38,7 +38,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -56,7 +56,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -74,7 +74,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" } ], "examples" : [ diff --git a/docs/reference/esql/functions/kibana/definition/date_trunc.json b/docs/reference/esql/functions/kibana/definition/date_trunc.json index 3d8658c496529..bd3f362d1670b 100644 --- a/docs/reference/esql/functions/kibana/definition/date_trunc.json +++ b/docs/reference/esql/functions/kibana/definition/date_trunc.json @@ -14,13 +14,13 @@ }, { "name" : "date", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Date expression" } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -32,13 +32,13 @@ }, { "name" : "date", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Date expression" } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" } ], "examples" : [ diff --git a/docs/reference/esql/functions/kibana/definition/equals.json b/docs/reference/esql/functions/kibana/definition/equals.json index 8d0525ac3e91e..eca80ccdbf657 100644 --- a/docs/reference/esql/functions/kibana/definition/equals.json +++ b/docs/reference/esql/functions/kibana/definition/equals.json @@ -63,13 +63,13 @@ "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." }, { "name" : "rhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." } diff --git a/docs/reference/esql/functions/kibana/definition/greater_than.json b/docs/reference/esql/functions/kibana/definition/greater_than.json index 9083e114bfe9d..7831b0f41cd9d 100644 --- a/docs/reference/esql/functions/kibana/definition/greater_than.json +++ b/docs/reference/esql/functions/kibana/definition/greater_than.json @@ -9,13 +9,13 @@ "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." }, { "name" : "rhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." } diff --git a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json index 75888ab25399f..b6a40a838c393 100644 --- a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json +++ b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json @@ -9,13 +9,13 @@ "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." }, { "name" : "rhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." } diff --git a/docs/reference/esql/functions/kibana/definition/less_than.json b/docs/reference/esql/functions/kibana/definition/less_than.json index 30c6c9eab0442..bf6b9c5c08774 100644 --- a/docs/reference/esql/functions/kibana/definition/less_than.json +++ b/docs/reference/esql/functions/kibana/definition/less_than.json @@ -9,13 +9,13 @@ "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." }, { "name" : "rhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." } diff --git a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json index 64f9c463748d1..4e57161887141 100644 --- a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json +++ b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json @@ -9,13 +9,13 @@ "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." }, { "name" : "rhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." } diff --git a/docs/reference/esql/functions/kibana/definition/max.json b/docs/reference/esql/functions/kibana/definition/max.json index 725b42763816d..b13d367d37345 100644 --- a/docs/reference/esql/functions/kibana/definition/max.json +++ b/docs/reference/esql/functions/kibana/definition/max.json @@ -20,13 +20,13 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "" } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/min.json b/docs/reference/esql/functions/kibana/definition/min.json index 68dfdd6cfd8c0..338ed10d67b2e 100644 --- a/docs/reference/esql/functions/kibana/definition/min.json +++ b/docs/reference/esql/functions/kibana/definition/min.json @@ -20,13 +20,13 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "" } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_append.json b/docs/reference/esql/functions/kibana/definition/mv_append.json index 8ee4e7297cc3a..3365226141f8f 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_append.json +++ b/docs/reference/esql/functions/kibana/definition/mv_append.json @@ -62,19 +62,19 @@ "params" : [ { "name" : "field1", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "" }, { "name" : "field2", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "" } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_count.json b/docs/reference/esql/functions/kibana/definition/mv_count.json index d414e5b957495..f125327314f4e 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_count.json +++ b/docs/reference/esql/functions/kibana/definition/mv_count.json @@ -44,7 +44,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Multivalue expression." } diff --git a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json index 7ab287bc94d34..7d66e3dcc0b9b 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json +++ b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json @@ -45,13 +45,13 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Multivalue expression." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_first.json b/docs/reference/esql/functions/kibana/definition/mv_first.json index e3141e800e4ad..de6e642068517 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_first.json +++ b/docs/reference/esql/functions/kibana/definition/mv_first.json @@ -44,13 +44,13 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Multivalue expression." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_last.json b/docs/reference/esql/functions/kibana/definition/mv_last.json index e55d66dbf8b93..ea1293e7acfec 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_last.json +++ b/docs/reference/esql/functions/kibana/definition/mv_last.json @@ -44,13 +44,13 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Multivalue expression." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_max.json b/docs/reference/esql/functions/kibana/definition/mv_max.json index 0783f6d6d5cbc..eb25369f78f77 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_max.json +++ b/docs/reference/esql/functions/kibana/definition/mv_max.json @@ -20,13 +20,13 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Multivalue expression." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_min.json b/docs/reference/esql/functions/kibana/definition/mv_min.json index cc23df386356e..87ad94338492e 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_min.json +++ b/docs/reference/esql/functions/kibana/definition/mv_min.json @@ -20,13 +20,13 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Multivalue expression." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_slice.json b/docs/reference/esql/functions/kibana/definition/mv_slice.json index 30d0e1179dc89..ff52467b7d84a 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_slice.json +++ b/docs/reference/esql/functions/kibana/definition/mv_slice.json @@ -80,7 +80,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Multivalue expression. If `null`, the function returns `null`." }, @@ -98,7 +98,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_sort.json b/docs/reference/esql/functions/kibana/definition/mv_sort.json index 28b4c9e8d6fea..d2bbd2c0fdbf4 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_sort.json +++ b/docs/reference/esql/functions/kibana/definition/mv_sort.json @@ -26,7 +26,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Multivalue expression. If `null`, the function returns `null`." }, @@ -38,7 +38,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/not_equals.json b/docs/reference/esql/functions/kibana/definition/not_equals.json index 41863f7496a25..4b4d22a5abef4 100644 --- a/docs/reference/esql/functions/kibana/definition/not_equals.json +++ b/docs/reference/esql/functions/kibana/definition/not_equals.json @@ -63,13 +63,13 @@ "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." }, { "name" : "rhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "An expression." } diff --git a/docs/reference/esql/functions/kibana/definition/now.json b/docs/reference/esql/functions/kibana/definition/now.json index 9cdb4945afa2e..1a2fc3a1dc42a 100644 --- a/docs/reference/esql/functions/kibana/definition/now.json +++ b/docs/reference/esql/functions/kibana/definition/now.json @@ -6,7 +6,7 @@ "signatures" : [ { "params" : [ ], - "returnType" : "datetime" + "returnType" : "date" } ], "examples" : [ diff --git a/docs/reference/esql/functions/kibana/definition/sub.json b/docs/reference/esql/functions/kibana/definition/sub.json index 413b0e73f89d0..37e3852865e7f 100644 --- a/docs/reference/esql/functions/kibana/definition/sub.json +++ b/docs/reference/esql/functions/kibana/definition/sub.json @@ -8,7 +8,7 @@ "params" : [ { "name" : "lhs", - "type" : "date_period", + "type" : "date", "optional" : false, "description" : "A numeric value or a date time value." }, @@ -20,43 +20,43 @@ } ], "variadic" : false, - "returnType" : "date_period" + "returnType" : "date" }, { "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "A numeric value or a date time value." }, { "name" : "rhs", - "type" : "date_period", + "type" : "time_duration", "optional" : false, "description" : "A numeric value or a date time value." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ { "name" : "lhs", - "type" : "datetime", + "type" : "date_period", "optional" : false, "description" : "A numeric value or a date time value." }, { "name" : "rhs", - "type" : "time_duration", + "type" : "date_period", "optional" : false, "description" : "A numeric value or a date time value." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date_period" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/to_datetime.json b/docs/reference/esql/functions/kibana/definition/to_datetime.json index 778d151c40151..032e8e1cbda34 100644 --- a/docs/reference/esql/functions/kibana/definition/to_datetime.json +++ b/docs/reference/esql/functions/kibana/definition/to_datetime.json @@ -3,19 +3,19 @@ "type" : "eval", "name" : "to_datetime", "description" : "Converts an input value to a date value.\nA string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`.\nTo convert dates in other formats, use <>.", - "note" : "Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date istruncated, not rounded.", + "note" : "Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded.", "signatures" : [ { "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Input value. The input can be a single- or multi-valued column or an expression." } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -27,7 +27,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -39,7 +39,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -51,7 +51,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -63,7 +63,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -75,7 +75,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ @@ -87,7 +87,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" } ], "examples" : [ diff --git a/docs/reference/esql/functions/kibana/definition/to_double.json b/docs/reference/esql/functions/kibana/definition/to_double.json index f4e414068db61..ae7e4832bfb3c 100644 --- a/docs/reference/esql/functions/kibana/definition/to_double.json +++ b/docs/reference/esql/functions/kibana/definition/to_double.json @@ -56,7 +56,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Input value. The input can be a single- or multi-valued column or an expression." } diff --git a/docs/reference/esql/functions/kibana/definition/to_integer.json b/docs/reference/esql/functions/kibana/definition/to_integer.json index 2776d8b29c412..5150d12936711 100644 --- a/docs/reference/esql/functions/kibana/definition/to_integer.json +++ b/docs/reference/esql/functions/kibana/definition/to_integer.json @@ -32,7 +32,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Input value. The input can be a single- or multi-valued column or an expression." } diff --git a/docs/reference/esql/functions/kibana/definition/to_long.json b/docs/reference/esql/functions/kibana/definition/to_long.json index e3218eba9642a..5fd4bce34e7e0 100644 --- a/docs/reference/esql/functions/kibana/definition/to_long.json +++ b/docs/reference/esql/functions/kibana/definition/to_long.json @@ -44,7 +44,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Input value. The input can be a single- or multi-valued column or an expression." } diff --git a/docs/reference/esql/functions/kibana/definition/to_string.json b/docs/reference/esql/functions/kibana/definition/to_string.json index ef03cc06ea636..ea94171834908 100644 --- a/docs/reference/esql/functions/kibana/definition/to_string.json +++ b/docs/reference/esql/functions/kibana/definition/to_string.json @@ -44,7 +44,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Input value. The input can be a single- or multi-valued column or an expression." } diff --git a/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json b/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json index d9cba641573fb..5521241224d61 100644 --- a/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json +++ b/docs/reference/esql/functions/kibana/definition/to_unsigned_long.json @@ -20,7 +20,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "Input value. The input can be a single- or multi-valued column or an expression." } diff --git a/docs/reference/esql/functions/kibana/definition/top.json b/docs/reference/esql/functions/kibana/definition/top.json index 4db3aed40a88d..c688bf5ea77c8 100644 --- a/docs/reference/esql/functions/kibana/definition/top.json +++ b/docs/reference/esql/functions/kibana/definition/top.json @@ -32,7 +32,7 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "The field to collect the top values for." }, @@ -50,7 +50,7 @@ } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/values.json b/docs/reference/esql/functions/kibana/definition/values.json index 3e0036c4d25b6..d9f37cd1ac83d 100644 --- a/docs/reference/esql/functions/kibana/definition/values.json +++ b/docs/reference/esql/functions/kibana/definition/values.json @@ -20,13 +20,13 @@ "params" : [ { "name" : "field", - "type" : "datetime", + "type" : "date", "optional" : false, "description" : "" } ], "variadic" : false, - "returnType" : "datetime" + "returnType" : "date" }, { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/to_datetime.md b/docs/reference/esql/functions/kibana/docs/to_datetime.md index 613381615421a..c194dfd17871a 100644 --- a/docs/reference/esql/functions/kibana/docs/to_datetime.md +++ b/docs/reference/esql/functions/kibana/docs/to_datetime.md @@ -11,4 +11,4 @@ To convert dates in other formats, use <>. ROW string = ["1953-09-02T00:00:00.000Z", "1964-06-02T00:00:00.000Z", "1964-06-02 00:00:00"] | EVAL datetime = TO_DATETIME(string) ``` -Note: Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date istruncated, not rounded. +Note: Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is truncated, not rounded. diff --git a/docs/reference/esql/functions/parameters/bucket.asciidoc b/docs/reference/esql/functions/parameters/bucket.asciidoc index 342ea560aaa0b..09c720d6095f3 100644 --- a/docs/reference/esql/functions/parameters/bucket.asciidoc +++ b/docs/reference/esql/functions/parameters/bucket.asciidoc @@ -6,7 +6,7 @@ Numeric or date expression from which to derive buckets. `buckets`:: -Target number of buckets. +Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted. `from`:: Start of the range. Can be a number, a date or a date expressed as a string. diff --git a/docs/reference/esql/functions/types/add.asciidoc b/docs/reference/esql/functions/types/add.asciidoc index a0215a803d4e3..54d1aec463c1a 100644 --- a/docs/reference/esql/functions/types/add.asciidoc +++ b/docs/reference/esql/functions/types/add.asciidoc @@ -5,10 +5,10 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== lhs | rhs | result +date | date_period | date +date | time_duration | date +date_period | date | date date_period | date_period | date_period -date_period | datetime | datetime -datetime | date_period | datetime -datetime | time_duration | datetime double | double | double double | integer | double double | long | double @@ -18,7 +18,7 @@ integer | long | long long | double | double long | integer | long long | long | long -time_duration | datetime | datetime +time_duration | date | date time_duration | time_duration | time_duration unsigned_long | unsigned_long | unsigned_long |=== diff --git a/docs/reference/esql/functions/types/bucket.asciidoc b/docs/reference/esql/functions/types/bucket.asciidoc index 1cbfad14ca379..172e84b6f7860 100644 --- a/docs/reference/esql/functions/types/bucket.asciidoc +++ b/docs/reference/esql/functions/types/bucket.asciidoc @@ -5,17 +5,17 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== field | buckets | from | to | result -datetime | date_period | | | datetime -datetime | integer | datetime | datetime | datetime -datetime | integer | datetime | keyword | datetime -datetime | integer | datetime | text | datetime -datetime | integer | keyword | datetime | datetime -datetime | integer | keyword | keyword | datetime -datetime | integer | keyword | text | datetime -datetime | integer | text | datetime | datetime -datetime | integer | text | keyword | datetime -datetime | integer | text | text | datetime -datetime | time_duration | | | datetime +date | date_period | | | date +date | integer | date | date | date +date | integer | date | keyword | date +date | integer | date | text | date +date | integer | keyword | date | date +date | integer | keyword | keyword | date +date | integer | keyword | text | date +date | integer | text | date | date +date | integer | text | keyword | date +date | integer | text | text | date +date | time_duration | | | date double | double | | | double double | integer | double | double | double double | integer | double | integer | double diff --git a/docs/reference/esql/functions/types/case.asciidoc b/docs/reference/esql/functions/types/case.asciidoc index 85e4193b5bf2f..f6c8cfe9361d1 100644 --- a/docs/reference/esql/functions/types/case.asciidoc +++ b/docs/reference/esql/functions/types/case.asciidoc @@ -7,7 +7,7 @@ condition | trueValue | result boolean | boolean | boolean boolean | cartesian_point | cartesian_point -boolean | datetime | datetime +boolean | date | date boolean | double | double boolean | geo_point | geo_point boolean | integer | integer diff --git a/docs/reference/esql/functions/types/coalesce.asciidoc b/docs/reference/esql/functions/types/coalesce.asciidoc index 841d836f6837e..368a12db0dca4 100644 --- a/docs/reference/esql/functions/types/coalesce.asciidoc +++ b/docs/reference/esql/functions/types/coalesce.asciidoc @@ -9,7 +9,7 @@ boolean | boolean | boolean boolean | | boolean cartesian_point | cartesian_point | cartesian_point cartesian_shape | cartesian_shape | cartesian_shape -datetime | datetime | datetime +date | date | date geo_point | geo_point | geo_point geo_shape | geo_shape | geo_shape integer | integer | integer diff --git a/docs/reference/esql/functions/types/count.asciidoc b/docs/reference/esql/functions/types/count.asciidoc index 70e79d4899605..959c94c1ec358 100644 --- a/docs/reference/esql/functions/types/count.asciidoc +++ b/docs/reference/esql/functions/types/count.asciidoc @@ -7,7 +7,7 @@ field | result boolean | long cartesian_point | long -datetime | long +date | long double | long geo_point | long integer | long diff --git a/docs/reference/esql/functions/types/count_distinct.asciidoc b/docs/reference/esql/functions/types/count_distinct.asciidoc index 4b201d45732f1..c365c8814573c 100644 --- a/docs/reference/esql/functions/types/count_distinct.asciidoc +++ b/docs/reference/esql/functions/types/count_distinct.asciidoc @@ -9,10 +9,10 @@ boolean | integer | long boolean | long | long boolean | unsigned_long | long boolean | | long -datetime | integer | long -datetime | long | long -datetime | unsigned_long | long -datetime | | long +date | integer | long +date | long | long +date | unsigned_long | long +date | | long double | integer | long double | long | long double | unsigned_long | long diff --git a/docs/reference/esql/functions/types/date_diff.asciidoc b/docs/reference/esql/functions/types/date_diff.asciidoc index 98adcef51e75c..b0a4818f412ac 100644 --- a/docs/reference/esql/functions/types/date_diff.asciidoc +++ b/docs/reference/esql/functions/types/date_diff.asciidoc @@ -5,6 +5,6 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== unit | startTimestamp | endTimestamp | result -keyword | datetime | datetime | integer -text | datetime | datetime | integer +keyword | date | date | integer +text | date | date | integer |=== diff --git a/docs/reference/esql/functions/types/date_extract.asciidoc b/docs/reference/esql/functions/types/date_extract.asciidoc index 43702ef0671a7..ec9bf70c221cc 100644 --- a/docs/reference/esql/functions/types/date_extract.asciidoc +++ b/docs/reference/esql/functions/types/date_extract.asciidoc @@ -5,6 +5,6 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== datePart | date | result -keyword | datetime | long -text | datetime | long +keyword | date | long +text | date | long |=== diff --git a/docs/reference/esql/functions/types/date_format.asciidoc b/docs/reference/esql/functions/types/date_format.asciidoc index a76f38653b9b8..b2e97dfa8835a 100644 --- a/docs/reference/esql/functions/types/date_format.asciidoc +++ b/docs/reference/esql/functions/types/date_format.asciidoc @@ -5,6 +5,6 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== dateFormat | date | result -keyword | datetime | keyword -text | datetime | keyword +keyword | date | keyword +text | date | keyword |=== diff --git a/docs/reference/esql/functions/types/date_parse.asciidoc b/docs/reference/esql/functions/types/date_parse.asciidoc index 314d02eb06271..f3eab18309dd8 100644 --- a/docs/reference/esql/functions/types/date_parse.asciidoc +++ b/docs/reference/esql/functions/types/date_parse.asciidoc @@ -5,8 +5,8 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== datePattern | dateString | result -keyword | keyword | datetime -keyword | text | datetime -text | keyword | datetime -text | text | datetime +keyword | keyword | date +keyword | text | date +text | keyword | date +text | text | date |=== diff --git a/docs/reference/esql/functions/types/date_trunc.asciidoc b/docs/reference/esql/functions/types/date_trunc.asciidoc index 8df45cfef54a8..aa7dee99c6c44 100644 --- a/docs/reference/esql/functions/types/date_trunc.asciidoc +++ b/docs/reference/esql/functions/types/date_trunc.asciidoc @@ -5,6 +5,6 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== interval | date | result -date_period | datetime | datetime -time_duration | datetime | datetime +date_period | date | date +time_duration | date | date |=== diff --git a/docs/reference/esql/functions/types/equals.asciidoc b/docs/reference/esql/functions/types/equals.asciidoc index 497c9319fedb3..ad0e46ef4b8da 100644 --- a/docs/reference/esql/functions/types/equals.asciidoc +++ b/docs/reference/esql/functions/types/equals.asciidoc @@ -8,7 +8,7 @@ lhs | rhs | result boolean | boolean | boolean cartesian_point | cartesian_point | boolean cartesian_shape | cartesian_shape | boolean -datetime | datetime | boolean +date | date | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/greater_than.asciidoc b/docs/reference/esql/functions/types/greater_than.asciidoc index 771daf1a953b2..c506328126a94 100644 --- a/docs/reference/esql/functions/types/greater_than.asciidoc +++ b/docs/reference/esql/functions/types/greater_than.asciidoc @@ -5,7 +5,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== lhs | rhs | result -datetime | datetime | boolean +date | date | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc index 771daf1a953b2..c506328126a94 100644 --- a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc +++ b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc @@ -5,7 +5,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== lhs | rhs | result -datetime | datetime | boolean +date | date | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/less_than.asciidoc b/docs/reference/esql/functions/types/less_than.asciidoc index 771daf1a953b2..c506328126a94 100644 --- a/docs/reference/esql/functions/types/less_than.asciidoc +++ b/docs/reference/esql/functions/types/less_than.asciidoc @@ -5,7 +5,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== lhs | rhs | result -datetime | datetime | boolean +date | date | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc index 771daf1a953b2..c506328126a94 100644 --- a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc +++ b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc @@ -5,7 +5,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== lhs | rhs | result -datetime | datetime | boolean +date | date | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/max.asciidoc b/docs/reference/esql/functions/types/max.asciidoc index 705745d76dbab..35ce5811e0cd0 100644 --- a/docs/reference/esql/functions/types/max.asciidoc +++ b/docs/reference/esql/functions/types/max.asciidoc @@ -6,7 +6,7 @@ |=== field | result boolean | boolean -datetime | datetime +date | date double | double integer | integer ip | ip diff --git a/docs/reference/esql/functions/types/min.asciidoc b/docs/reference/esql/functions/types/min.asciidoc index 705745d76dbab..35ce5811e0cd0 100644 --- a/docs/reference/esql/functions/types/min.asciidoc +++ b/docs/reference/esql/functions/types/min.asciidoc @@ -6,7 +6,7 @@ |=== field | result boolean | boolean -datetime | datetime +date | date double | double integer | integer ip | ip diff --git a/docs/reference/esql/functions/types/mv_append.asciidoc b/docs/reference/esql/functions/types/mv_append.asciidoc index 49dcef6dc8860..a1894e429ae82 100644 --- a/docs/reference/esql/functions/types/mv_append.asciidoc +++ b/docs/reference/esql/functions/types/mv_append.asciidoc @@ -8,7 +8,7 @@ field1 | field2 | result boolean | boolean | boolean cartesian_point | cartesian_point | cartesian_point cartesian_shape | cartesian_shape | cartesian_shape -datetime | datetime | datetime +date | date | date double | double | double geo_point | geo_point | geo_point geo_shape | geo_shape | geo_shape diff --git a/docs/reference/esql/functions/types/mv_count.asciidoc b/docs/reference/esql/functions/types/mv_count.asciidoc index 8af6b76591acb..260c531731f04 100644 --- a/docs/reference/esql/functions/types/mv_count.asciidoc +++ b/docs/reference/esql/functions/types/mv_count.asciidoc @@ -8,7 +8,7 @@ field | result boolean | integer cartesian_point | integer cartesian_shape | integer -datetime | integer +date | integer double | integer geo_point | integer geo_shape | integer diff --git a/docs/reference/esql/functions/types/mv_dedupe.asciidoc b/docs/reference/esql/functions/types/mv_dedupe.asciidoc index a6b78f781f17a..68e546451c8cb 100644 --- a/docs/reference/esql/functions/types/mv_dedupe.asciidoc +++ b/docs/reference/esql/functions/types/mv_dedupe.asciidoc @@ -8,7 +8,7 @@ field | result boolean | boolean cartesian_point | cartesian_point cartesian_shape | cartesian_shape -datetime | datetime +date | date double | double geo_point | geo_point geo_shape | geo_shape diff --git a/docs/reference/esql/functions/types/mv_first.asciidoc b/docs/reference/esql/functions/types/mv_first.asciidoc index e077c57971a4a..35633544d99a0 100644 --- a/docs/reference/esql/functions/types/mv_first.asciidoc +++ b/docs/reference/esql/functions/types/mv_first.asciidoc @@ -8,7 +8,7 @@ field | result boolean | boolean cartesian_point | cartesian_point cartesian_shape | cartesian_shape -datetime | datetime +date | date double | double geo_point | geo_point geo_shape | geo_shape diff --git a/docs/reference/esql/functions/types/mv_last.asciidoc b/docs/reference/esql/functions/types/mv_last.asciidoc index e077c57971a4a..35633544d99a0 100644 --- a/docs/reference/esql/functions/types/mv_last.asciidoc +++ b/docs/reference/esql/functions/types/mv_last.asciidoc @@ -8,7 +8,7 @@ field | result boolean | boolean cartesian_point | cartesian_point cartesian_shape | cartesian_shape -datetime | datetime +date | date double | double geo_point | geo_point geo_shape | geo_shape diff --git a/docs/reference/esql/functions/types/mv_max.asciidoc b/docs/reference/esql/functions/types/mv_max.asciidoc index 4e5f0a5e0ae89..8ea36aebbad37 100644 --- a/docs/reference/esql/functions/types/mv_max.asciidoc +++ b/docs/reference/esql/functions/types/mv_max.asciidoc @@ -6,7 +6,7 @@ |=== field | result boolean | boolean -datetime | datetime +date | date double | double integer | integer ip | ip diff --git a/docs/reference/esql/functions/types/mv_min.asciidoc b/docs/reference/esql/functions/types/mv_min.asciidoc index 4e5f0a5e0ae89..8ea36aebbad37 100644 --- a/docs/reference/esql/functions/types/mv_min.asciidoc +++ b/docs/reference/esql/functions/types/mv_min.asciidoc @@ -6,7 +6,7 @@ |=== field | result boolean | boolean -datetime | datetime +date | date double | double integer | integer ip | ip diff --git a/docs/reference/esql/functions/types/mv_slice.asciidoc b/docs/reference/esql/functions/types/mv_slice.asciidoc index 568de10f53d32..0a9dc073370c7 100644 --- a/docs/reference/esql/functions/types/mv_slice.asciidoc +++ b/docs/reference/esql/functions/types/mv_slice.asciidoc @@ -8,7 +8,7 @@ field | start | end | result boolean | integer | integer | boolean cartesian_point | integer | integer | cartesian_point cartesian_shape | integer | integer | cartesian_shape -datetime | integer | integer | datetime +date | integer | integer | date double | integer | integer | double geo_point | integer | integer | geo_point geo_shape | integer | integer | geo_shape diff --git a/docs/reference/esql/functions/types/mv_sort.asciidoc b/docs/reference/esql/functions/types/mv_sort.asciidoc index 24925ca8a6587..93965187482ac 100644 --- a/docs/reference/esql/functions/types/mv_sort.asciidoc +++ b/docs/reference/esql/functions/types/mv_sort.asciidoc @@ -6,7 +6,7 @@ |=== field | order | result boolean | keyword | boolean -datetime | keyword | datetime +date | keyword | date double | keyword | double integer | keyword | integer ip | keyword | ip diff --git a/docs/reference/esql/functions/types/not_equals.asciidoc b/docs/reference/esql/functions/types/not_equals.asciidoc index 497c9319fedb3..ad0e46ef4b8da 100644 --- a/docs/reference/esql/functions/types/not_equals.asciidoc +++ b/docs/reference/esql/functions/types/not_equals.asciidoc @@ -8,7 +8,7 @@ lhs | rhs | result boolean | boolean | boolean cartesian_point | cartesian_point | boolean cartesian_shape | cartesian_shape | boolean -datetime | datetime | boolean +date | date | boolean double | double | boolean double | integer | boolean double | long | boolean diff --git a/docs/reference/esql/functions/types/now.asciidoc b/docs/reference/esql/functions/types/now.asciidoc index 5737d98f2f7db..b474ab1042050 100644 --- a/docs/reference/esql/functions/types/now.asciidoc +++ b/docs/reference/esql/functions/types/now.asciidoc @@ -5,5 +5,5 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== result -datetime +date |=== diff --git a/docs/reference/esql/functions/types/sub.asciidoc b/docs/reference/esql/functions/types/sub.asciidoc index d309f651705f0..c3ded301ebe68 100644 --- a/docs/reference/esql/functions/types/sub.asciidoc +++ b/docs/reference/esql/functions/types/sub.asciidoc @@ -5,9 +5,9 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== lhs | rhs | result +date | date_period | date +date | time_duration | date date_period | date_period | date_period -datetime | date_period | datetime -datetime | time_duration | datetime double | double | double double | integer | double double | long | double diff --git a/docs/reference/esql/functions/types/to_datetime.asciidoc b/docs/reference/esql/functions/types/to_datetime.asciidoc index 52c4cebb661cf..80c986efca794 100644 --- a/docs/reference/esql/functions/types/to_datetime.asciidoc +++ b/docs/reference/esql/functions/types/to_datetime.asciidoc @@ -5,11 +5,11 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== field | result -datetime | datetime -double | datetime -integer | datetime -keyword | datetime -long | datetime -text | datetime -unsigned_long | datetime +date | date +double | date +integer | date +keyword | date +long | date +text | date +unsigned_long | date |=== diff --git a/docs/reference/esql/functions/types/to_double.asciidoc b/docs/reference/esql/functions/types/to_double.asciidoc index cff686c7bc4ca..d5f5833cd7249 100644 --- a/docs/reference/esql/functions/types/to_double.asciidoc +++ b/docs/reference/esql/functions/types/to_double.asciidoc @@ -9,7 +9,7 @@ boolean | double counter_double | double counter_integer | double counter_long | double -datetime | double +date | double double | double integer | double keyword | double diff --git a/docs/reference/esql/functions/types/to_integer.asciidoc b/docs/reference/esql/functions/types/to_integer.asciidoc index 974f3c9c82d88..d67f8f07affd9 100644 --- a/docs/reference/esql/functions/types/to_integer.asciidoc +++ b/docs/reference/esql/functions/types/to_integer.asciidoc @@ -7,7 +7,7 @@ field | result boolean | integer counter_integer | integer -datetime | integer +date | integer double | integer integer | integer keyword | integer diff --git a/docs/reference/esql/functions/types/to_long.asciidoc b/docs/reference/esql/functions/types/to_long.asciidoc index b3959c5444e34..a07990cb1cfbf 100644 --- a/docs/reference/esql/functions/types/to_long.asciidoc +++ b/docs/reference/esql/functions/types/to_long.asciidoc @@ -8,7 +8,7 @@ field | result boolean | long counter_integer | long counter_long | long -datetime | long +date | long double | long integer | long keyword | long diff --git a/docs/reference/esql/functions/types/to_string.asciidoc b/docs/reference/esql/functions/types/to_string.asciidoc index f14cfbb39929f..26a5b31a2a589 100644 --- a/docs/reference/esql/functions/types/to_string.asciidoc +++ b/docs/reference/esql/functions/types/to_string.asciidoc @@ -8,7 +8,7 @@ field | result boolean | keyword cartesian_point | keyword cartesian_shape | keyword -datetime | keyword +date | keyword double | keyword geo_point | keyword geo_shape | keyword diff --git a/docs/reference/esql/functions/types/to_unsigned_long.asciidoc b/docs/reference/esql/functions/types/to_unsigned_long.asciidoc index a271e1a19321d..87b21f3948dad 100644 --- a/docs/reference/esql/functions/types/to_unsigned_long.asciidoc +++ b/docs/reference/esql/functions/types/to_unsigned_long.asciidoc @@ -6,7 +6,7 @@ |=== field | result boolean | unsigned_long -datetime | unsigned_long +date | unsigned_long double | unsigned_long integer | unsigned_long keyword | unsigned_long diff --git a/docs/reference/esql/functions/types/top.asciidoc b/docs/reference/esql/functions/types/top.asciidoc index ff71b2d153e3a..0eb329c10b9ed 100644 --- a/docs/reference/esql/functions/types/top.asciidoc +++ b/docs/reference/esql/functions/types/top.asciidoc @@ -6,7 +6,7 @@ |=== field | limit | order | result boolean | integer | keyword | boolean -datetime | integer | keyword | datetime +date | integer | keyword | date double | integer | keyword | double integer | integer | keyword | integer ip | integer | keyword | ip diff --git a/docs/reference/esql/functions/types/values.asciidoc b/docs/reference/esql/functions/types/values.asciidoc index 705745d76dbab..35ce5811e0cd0 100644 --- a/docs/reference/esql/functions/types/values.asciidoc +++ b/docs/reference/esql/functions/types/values.asciidoc @@ -6,7 +6,7 @@ |=== field | result boolean | boolean -datetime | datetime +date | date double | double integer | integer ip | ip diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 065ada06bfa1e..979368c300e00 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -447,6 +447,14 @@ public String esType() { return esType; } + /** + * Return the Elasticsearch field name of this type if there is one, + * otherwise return the ESQL specific name. + */ + public String esNameIfPossible() { + return esType != null ? esType : typeName; + } + /** * The name we give to types on the response. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java index 2c86dfbac12ce..c66ba7f87a1c5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java @@ -58,7 +58,7 @@ public class ToDatetime extends AbstractConvertFunction { Converts an input value to a date value. A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use <>.""", - note = "Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is" + note = "Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is " + "truncated, not rounded.", examples = { @Example(file = "date", tag = "to_datetime-str", explanation = """ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index cece2badb2955..efb078cbe80e0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -88,7 +88,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -708,13 +707,12 @@ public static void testFunctionInfo() { for (int i = 0; i < args.size(); i++) { typesFromSignature.add(new HashSet<>()); } - Function typeName = dt -> dt.esType() != null ? dt.esType() : dt.typeName(); for (Map.Entry, DataType> entry : signatures().entrySet()) { List types = entry.getKey(); for (int i = 0; i < args.size() && i < types.size(); i++) { - typesFromSignature.get(i).add(typeName.apply(types.get(i))); + typesFromSignature.get(i).add(types.get(i).esNameIfPossible()); } - returnFromSignature.add(typeName.apply(entry.getValue())); + returnFromSignature.add(entry.getValue().esNameIfPossible()); } for (int i = 0; i < args.size(); i++) { @@ -871,15 +869,15 @@ private static void renderTypes(List argNames) throws IOException { } StringBuilder b = new StringBuilder(); for (DataType arg : sig.getKey()) { - b.append(arg.typeName()).append(" | "); + b.append(arg.esNameIfPossible()).append(" | "); } b.append("| ".repeat(argNames.size() - sig.getKey().size())); - b.append(sig.getValue().typeName()); + b.append(sig.getValue().esNameIfPossible()); table.add(b.toString()); } Collections.sort(table); if (table.isEmpty()) { - table.add(signatures.values().iterator().next().typeName()); + table.add(signatures.values().iterator().next().esNameIfPossible()); } String rendered = DOCS_WARNING + """ @@ -1085,7 +1083,7 @@ private static void renderKibanaFunctionDefinition( builder.startArray("params"); builder.endArray(); // There should only be one return type so just use that as the example - builder.field("returnType", signatures().values().iterator().next().typeName()); + builder.field("returnType", signatures().values().iterator().next().esNameIfPossible()); builder.endObject(); } else { int minArgCount = (int) args.stream().filter(a -> false == a.optional()).count(); @@ -1106,14 +1104,14 @@ private static void renderKibanaFunctionDefinition( EsqlFunctionRegistry.ArgSignature arg = args.get(i); builder.startObject(); builder.field("name", arg.name()); - builder.field("type", sig.getKey().get(i).typeName()); + builder.field("type", sig.getKey().get(i).esNameIfPossible()); builder.field("optional", arg.optional()); builder.field("description", arg.description()); builder.endObject(); } builder.endArray(); builder.field("variadic", variadic); - builder.field("returnType", sig.getValue().typeName()); + builder.field("returnType", sig.getValue().esNameIfPossible()); builder.endObject(); } } @@ -1149,12 +1147,12 @@ public int compare(Map.Entry, DataType> lhs, Map.Entry Date: Tue, 20 Aug 2024 12:52:30 -0400 Subject: [PATCH 13/32] Unmute the test, it had already been fixed but wasn't unmuted (#111800) --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index c2e0d48c31a20..b166fb137bc48 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -38,9 +38,6 @@ tests: - class: "org.elasticsearch.xpack.deprecation.DeprecationHttpIT" issue: "https://github.com/elastic/elasticsearch/issues/108628" method: "testDeprecatedSettingsReturnWarnings" -- class: "org.elasticsearch.xpack.inference.InferenceCrudIT" - issue: "https://github.com/elastic/elasticsearch/issues/109391" - method: "testDeleteEndpointWhileReferencedByPipeline" - class: "org.elasticsearch.xpack.test.rest.XPackRestIT" issue: "https://github.com/elastic/elasticsearch/issues/109687" method: "test {p0=sql/translate/Translate SQL}" From a9fa443402a5e2ef86f577b6eea202eed9c8925c Mon Sep 17 00:00:00 2001 From: Volodymyr Krasnikov <129072588+volodk85@users.noreply.github.com> Date: Tue, 20 Aug 2024 10:59:35 -0700 Subject: [PATCH 14/32] Avoid unsafe futures in SharedBlobCacheService (#111526) --- .../blobcache/shared/SharedBlobCacheService.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java index 8ca62a3b95023..584e551f1cf6b 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java @@ -15,7 +15,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.RefCountingRunnable; -import org.elasticsearch.action.support.UnsafePlainActionFuture; import org.elasticsearch.blobcache.BlobCacheMetrics; import org.elasticsearch.blobcache.BlobCacheUtils; import org.elasticsearch.blobcache.common.ByteRange; @@ -39,7 +38,6 @@ import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.monitor.fs.FsProbe; import org.elasticsearch.node.NodeRoleSettings; -import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -1162,9 +1160,7 @@ private int readSingleRegion( RangeMissingHandler writer, int region ) throws InterruptedException, ExecutionException { - final PlainActionFuture readFuture = new UnsafePlainActionFuture<>( - BlobStoreRepository.STATELESS_SHARD_PREWARMING_THREAD_NAME - ); + final PlainActionFuture readFuture = new PlainActionFuture<>(); final CacheFileRegion fileRegion = get(cacheKey, length, region); final long regionStart = getRegionStart(region); fileRegion.populateAndRead( @@ -1186,9 +1182,7 @@ private int readMultiRegions( int startRegion, int endRegion ) throws InterruptedException, ExecutionException { - final PlainActionFuture readsComplete = new UnsafePlainActionFuture<>( - BlobStoreRepository.STATELESS_SHARD_PREWARMING_THREAD_NAME - ); + final PlainActionFuture readsComplete = new PlainActionFuture<>(); final AtomicInteger bytesRead = new AtomicInteger(); try (var listeners = new RefCountingListener(1, readsComplete)) { for (int region = startRegion; region <= endRegion; region++) { From 0f8ce788ad8901a5d457de72ff9164c68860484e Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Tue, 20 Aug 2024 14:34:44 -0400 Subject: [PATCH 15/32] [ML] Migrate Inference to ChunkedToXContent (#111655) In preperation of streaming Inference responses, we are migrating RestInferenceAction and corresponding result objects from ToXContent to ChunkedToXContent. RestInferenceAction will now use the built in ChunkedRestResponseBodyPart and send a single item before closing the stream. --- docs/changelog/111655.yaml | 5 ++ .../inference/InferenceServiceResults.java | 4 +- .../inference/action/InferenceAction.java | 20 +++--- .../results/ChatCompletionResults.java | 12 ++-- .../results/ErrorChunkedInferenceResults.java | 8 +-- ...nferenceChunkedSparseEmbeddingResults.java | 10 +-- ...erenceChunkedTextEmbeddingByteResults.java | 12 ++-- ...renceChunkedTextEmbeddingFloatResults.java | 10 +-- .../InferenceTextEmbeddingByteResults.java | 12 ++-- .../InferenceTextEmbeddingFloatResults.java | 12 ++-- .../inference/results/RankedDocsResults.java | 12 ++-- .../results/SparseEmbeddingResults.java | 14 ++-- .../results/RankedDocsResultsTests.java | 4 +- ...stractChunkedBWCSerializationTestCase.java | 67 +++++++++++++++++++ .../inference/rest/RestInferenceAction.java | 4 +- 15 files changed, 129 insertions(+), 77 deletions(-) create mode 100644 docs/changelog/111655.yaml create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java diff --git a/docs/changelog/111655.yaml b/docs/changelog/111655.yaml new file mode 100644 index 0000000000000..077714d15a712 --- /dev/null +++ b/docs/changelog/111655.yaml @@ -0,0 +1,5 @@ +pr: 111655 +summary: Migrate Inference to `ChunkedToXContent` +area: Machine Learning +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java index 62166115820f5..f8330404c1538 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java @@ -9,12 +9,12 @@ package org.elasticsearch.inference; import org.elasticsearch.common.io.stream.NamedWriteable; -import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import java.util.List; import java.util.Map; -public interface InferenceServiceResults extends NamedWriteable, ToXContentFragment { +public interface InferenceServiceResults extends NamedWriteable, ChunkedToXContent { /** * Transform the result to match the format required for the TransportCoordinatedInferenceAction. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java index 53e404b48dc2e..7ecb5aef4ce8d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/InferenceAction.java @@ -14,8 +14,11 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; @@ -24,8 +27,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.inference.results.LegacyTextEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; @@ -34,6 +36,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -318,7 +321,7 @@ public String toString() { } } - public static class Response extends ActionResponse implements ToXContentObject { + public static class Response extends ActionResponse implements ChunkedToXContentObject { private final InferenceServiceResults results; @@ -398,11 +401,12 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - results.toXContent(builder, params); - builder.endObject(); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat( + ChunkedToXContentHelper.startObject(), + results.toXContentChunked(params), + ChunkedToXContentHelper.endObject() + ); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java index bbd4d026f0d55..902c69cef558e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChatCompletionResults.java @@ -10,12 +10,15 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -46,13 +49,8 @@ public ChatCompletionResults(StreamInput in) throws IOException { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(COMPLETION); - for (Result result : results) { - result.toXContent(builder, params); - } - builder.endArray(); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.array(COMPLETION, results.iterator()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java index 376b8763a5eb9..18f88a8ff022a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ErrorChunkedInferenceResults.java @@ -11,10 +11,11 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContent; -import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.Iterator; @@ -89,9 +90,8 @@ public String toString() { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(NAME, exception.getMessage()); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.field(NAME, exception.getMessage()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java index f1265873ad6dd..187b186fcd91d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.xcontent.ToXContent; @@ -77,13 +78,8 @@ public List getChunkedResults() { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(FIELD_NAME); - for (MlChunkedTextExpansionResults.ChunkedResult chunk : chunkedResults) { - chunk.toXContent(builder, params); - } - builder.endArray(); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.array(FIELD_NAME, chunkedResults.iterator()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java index b78bce8c5c2cd..cc245c40c51e3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java @@ -12,8 +12,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -61,14 +63,8 @@ public InferenceChunkedTextEmbeddingByteResults(StreamInput in) throws IOExcepti } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - // TODO add isTruncated flag - builder.startArray(FIELD_NAME); - for (var embedding : chunks) { - embedding.toXContent(builder, params); - } - builder.endArray(); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.array(FIELD_NAME, chunks.iterator()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java index 9fead334dcbc0..4b4d77cd3f043 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.xcontent.ToXContent; @@ -74,14 +75,9 @@ public static InferenceChunkedTextEmbeddingFloatResults ofMlResults(MlChunkedTex } @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + public Iterator toXContentChunked(ToXContent.Params params) { // TODO add isTruncated flag - builder.startArray(FIELD_NAME); - for (var embedding : chunks) { - embedding.toXContent(builder, params); - } - builder.endArray(); - return builder; + return ChunkedToXContentHelper.array(FIELD_NAME, chunks.iterator()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java index 8d94083bf3241..16dca7b04d526 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java @@ -13,8 +13,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; @@ -22,6 +24,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -58,13 +61,8 @@ public int getFirstEmbeddingSize() { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(TEXT_EMBEDDING_BYTES); - for (InferenceByteEmbedding embedding : embeddings) { - embedding.toXContent(builder, params); - } - builder.endArray(); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.array(TEXT_EMBEDDING_BYTES, embeddings.iterator()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java index 1822e3af28c2d..9f9bdfec7cfae 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java @@ -14,10 +14,12 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; @@ -25,6 +27,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -99,13 +102,8 @@ public int getFirstEmbeddingSize() { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(TEXT_EMBEDDING); - for (InferenceFloatEmbedding embedding : embeddings) { - embedding.toXContent(builder, params); - } - builder.endArray(); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.array(TEXT_EMBEDDING, embeddings.iterator()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java index 9196a57c868ba..6ebf15bf34937 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResults.java @@ -11,17 +11,20 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.TaskType; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -172,13 +175,8 @@ public List getRankedDocs() { } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(RERANK); - for (RankedDoc rankedDoc : rankedDocs) { - rankedDoc.toXContent(builder, params); - } - builder.endArray(); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.array(RERANK, rankedDocs.iterator()); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java index 1db6dcc802d00..dd8229c604ecb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java @@ -12,10 +12,12 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; @@ -23,6 +25,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -68,15 +71,8 @@ public static SparseEmbeddingResults of(List results } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray(SPARSE_EMBEDDING); - - for (Embedding embedding : embeddings) { - embedding.toXContent(builder, params); - } - - builder.endArray(); - return builder; + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.array(SPARSE_EMBEDDING, embeddings.iterator()); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java index 603531f0aedf9..b84aaa2bcfc1b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/results/RankedDocsResultsTests.java @@ -10,7 +10,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; +import org.elasticsearch.xpack.core.ml.AbstractChunkedBWCSerializationTestCase; import java.io.IOException; import java.util.ArrayList; @@ -18,7 +18,7 @@ import static org.elasticsearch.TransportVersions.ML_RERANK_DOC_OPTIONAL; -public class RankedDocsResultsTests extends AbstractBWCSerializationTestCase { +public class RankedDocsResultsTests extends AbstractChunkedBWCSerializationTestCase { @Override protected Writeable.Reader instanceReader() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java new file mode 100644 index 0000000000000..a23ce2c107fe3 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.test.AbstractChunkedSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase.DEFAULT_BWC_VERSIONS; + +public abstract class AbstractChunkedBWCSerializationTestCase extends + AbstractChunkedSerializingTestCase { + + /** + * Returns the expected instance if serialized from the given version. + */ + protected abstract T mutateInstanceForVersion(T instance, TransportVersion version); + + /** + * The bwc versions to test serialization against + */ + protected List bwcVersions() { + return DEFAULT_BWC_VERSIONS; + } + + /** + * Test serialization and deserialization of the test instance across versions + */ + public final void testBwcSerialization() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + T testInstance = createTestInstance(); + for (TransportVersion bwcVersion : bwcVersions()) { + assertBwcSerialization(testInstance, bwcVersion); + } + } + } + + /** + * Assert that instances copied at a particular version are equal. The version is useful + * for sanity checking the backwards compatibility of the wire. It isn't a substitute for + * real backwards compatibility tests but it is *so* much faster. + */ + protected final void assertBwcSerialization(T testInstance, TransportVersion version) throws IOException { + T deserializedInstance = copyWriteable(testInstance, getNamedWriteableRegistry(), instanceReader(), version); + assertOnBWCObject(deserializedInstance, mutateInstanceForVersion(testInstance, version), version); + } + + /** + * @param bwcSerializedObject The object deserialized from the previous version + * @param testInstance The original test instance + * @param version The version which serialized + */ + protected void assertOnBWCObject(T bwcSerializedObject, T testInstance, TransportVersion version) { + assertNotSame(version.toString(), bwcSerializedObject, testInstance); + assertEquals(version.toString(), bwcSerializedObject, testInstance); + assertEquals(version.toString(), bwcSerializedObject.hashCode(), testInstance.hashCode()); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java index f4bbcbebf0340..f5c30d0a94c54 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestInferenceAction.java @@ -13,7 +13,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.rest.action.RestChunkedToXContentListener; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import java.io.IOException; @@ -59,6 +59,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient InferenceAction.Request.DEFAULT_TIMEOUT ); requestBuilder.setInferenceTimeout(inferTimeout); - return channel -> client.execute(InferenceAction.INSTANCE, requestBuilder.build(), new RestToXContentListener<>(channel)); + return channel -> client.execute(InferenceAction.INSTANCE, requestBuilder.build(), new RestChunkedToXContentListener<>(channel)); } } From 94c48ac589b52a46868801a9d06ae48bb175f3c5 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Tue, 20 Aug 2024 14:35:39 -0400 Subject: [PATCH 16/32] [ML] Remove HttpClientContext (#111914) No requests are making use of this parameter, and it is clashing with Bedrock. We will bring it back in a later change as part of the Http Request portion of the Sender. --- ...AmazonBedrockExecuteOnlyRequestSender.java | 2 -- .../external/http/retry/RequestSender.java | 2 -- .../http/retry/RetryingHttpSender.java | 10 ++++-- ...onBedrockChatCompletionRequestManager.java | 3 +- ...AmazonBedrockEmbeddingsRequestManager.java | 3 +- .../sender/ExecutableInferenceRequest.java | 3 +- .../http/retry/RetryingHttpSenderTests.java | 34 ++++++++----------- .../sender/RequestExecutorServiceTests.java | 18 +++++----- .../http/sender/RequestManagerTests.java | 2 -- 9 files changed, 35 insertions(+), 42 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecuteOnlyRequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecuteOnlyRequestSender.java index a08acab655936..0826d990a80a5 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecuteOnlyRequestSender.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/amazonbedrock/AmazonBedrockExecuteOnlyRequestSender.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.amazonbedrock; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; @@ -45,7 +44,6 @@ public AmazonBedrockExecuteOnlyRequestSender(AmazonBedrockClientCache clientCach public void send( Logger logger, Request request, - HttpClientContext context, Supplier hasRequestTimedOutFunction, ResponseHandler responseHandler, ActionListener listener diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RequestSender.java index 8244e5ad29e95..8e55b0988de6a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RequestSender.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RequestSender.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.http.retry; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.InferenceServiceResults; @@ -19,7 +18,6 @@ public interface RequestSender { void send( Logger logger, Request request, - HttpClientContext context, Supplier hasRequestTimedOutFunction, ResponseHandler responseHandler, ActionListener listener diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java index dd45501564e4e..263bdea5ce368 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSender.java @@ -189,12 +189,18 @@ public boolean shouldRetry(Exception e) { public void send( Logger logger, Request request, - HttpClientContext context, Supplier hasRequestTimedOutFunction, ResponseHandler responseHandler, ActionListener listener ) { - InternalRetrier retrier = new InternalRetrier(logger, request, context, hasRequestTimedOutFunction, responseHandler, listener); + var retrier = new InternalRetrier( + logger, + request, + HttpClientContext.create(), + hasRequestTimedOutFunction, + responseHandler, + listener + ); retrier.run(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java index 8642a19b26a7d..1c6bb58717942 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockChatCompletionRequestManager.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.http.sender; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; @@ -52,7 +51,7 @@ public void execute( var responseHandler = new AmazonBedrockChatCompletionResponseHandler(); try { - requestSender.send(logger, request, HttpClientContext.create(), hasRequestCompletedFunction, responseHandler, listener); + requestSender.send(logger, request, hasRequestCompletedFunction, responseHandler, listener); } catch (Exception e) { var errorMessage = Strings.format( "Failed to send [completion] request from inference entity id [%s]", diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockEmbeddingsRequestManager.java index 2f94cdf342938..34aacbf67af6f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockEmbeddingsRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AmazonBedrockEmbeddingsRequestManager.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.http.sender; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; @@ -61,7 +60,7 @@ public void execute( var responseHandler = new AmazonBedrockEmbeddingsResponseHandler(); var request = new AmazonBedrockEmbeddingsRequest(truncator, truncatedInput, embeddingsModel, requestEntity, timeout); try { - requestSender.send(logger, request, HttpClientContext.create(), hasRequestCompletedFunction, responseHandler, listener); + requestSender.send(logger, request, hasRequestCompletedFunction, responseHandler, listener); } catch (Exception e) { var errorMessage = Strings.format( "Failed to send [text_embedding] request from inference entity id [%s]", diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java index 214eba4ee3485..241466422e47b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableInferenceRequest.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.http.sender; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; @@ -33,7 +32,7 @@ public void run() { var inferenceEntityId = request.createHttpRequest().inferenceEntityId(); try { - requestSender.send(logger, request, HttpClientContext.create(), hasFinished, responseHandler, listener); + requestSender.send(logger, request, hasFinished, responseHandler, listener); } catch (Exception e) { var errorMessage = Strings.format("Failed to send request from inference entity id [%s]", inferenceEntityId); logger.warn(errorMessage, e); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java index c2842a1278a49..f70ab43908827 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/retry/RetryingHttpSenderTests.java @@ -10,7 +10,6 @@ import org.apache.http.ConnectionClosedException; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; @@ -80,7 +79,7 @@ public void testSend_CallsSenderAgain_AfterValidateResponseThrowsAnException() t var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); assertThat(listener.actionGet(TIMEOUT), is(inferenceResults)); verify(httpClient, times(2)).send(any(), any(), any()); @@ -111,7 +110,7 @@ public void testSend_CallsSenderAgain_WhenAFailureStatusCodeIsReturned() throws var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); assertThat(listener.actionGet(TIMEOUT), is(inferenceResults)); verify(httpClient, times(2)).send(any(), any(), any()); @@ -139,7 +138,7 @@ public void testSend_CallsSenderAgain_WhenParsingFailsOnce() throws IOException var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); assertThat(listener.actionGet(TIMEOUT), is(inferenceResults)); verify(httpClient, times(2)).send(any(), any(), any()); @@ -167,7 +166,7 @@ public void testSend_DoesNotCallSenderAgain_WhenParsingFailsWithNonRetryableExce var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 0); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 0); var thrownException = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT)); assertThat(thrownException.getMessage(), is("failed")); @@ -202,7 +201,7 @@ public void testSend_CallsSenderAgain_WhenHttpResultListenerCallsOnFailureOnce() var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); assertThat(listener.actionGet(TIMEOUT), is(inferenceResults)); verify(httpClient, times(2)).send(any(), any(), any()); @@ -235,7 +234,7 @@ public void testSend_CallsSenderAgain_WhenHttpResultListenerCallsOnFailureOnce_W var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); assertThat(listener.actionGet(TIMEOUT), is(inferenceResults)); verify(httpClient, times(2)).send(any(), any(), any()); @@ -268,7 +267,7 @@ public void testSend_CallsSenderAgain_WhenHttpResultListenerCallsOnFailureOnceWi var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); assertThat(listener.actionGet(TIMEOUT), is(inferenceResults)); verify(httpClient, times(2)).send(any(), any(), any()); @@ -295,7 +294,7 @@ public void testSend_ReturnsFailure_WhenHttpResultListenerCallsOnFailureOnceWith var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 0); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 0); var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat(thrownException.getMessage(), is("Invalid host [null], please check that the URL is correct.")); @@ -317,10 +316,7 @@ public void testSend_ReturnsElasticsearchExceptionFailure_WhenTheHttpClientThrow var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks( - () -> retrier.send(mock(Logger.class), mockRequest("id"), HttpClientContext.create(), () -> false, handler, listener), - 0 - ); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest("id"), () -> false, handler, listener), 0); var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); assertThat(thrownException.getMessage(), is("Http client failed to send request from inference entity id [id]")); @@ -354,7 +350,7 @@ public void testSend_ReturnsFailure_WhenValidateResponseThrowsAnException_AfterO var retrier = createRetrier(sender); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); var thrownException = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT)); assertThat(thrownException.getMessage(), is("failed again")); @@ -391,7 +387,7 @@ public void testSend_ReturnsFailure_WhenValidateResponseThrowsAnElasticsearchExc var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); var thrownException = expectThrows(RetryException.class, () -> listener.actionGet(TIMEOUT)); assertThat(thrownException.getMessage(), is("failed again")); @@ -423,7 +419,7 @@ public void testSend_ReturnsFailure_WhenHttpResultsListenerCallsOnFailure_AfterO var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 1); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 1); var thrownException = expectThrows(RetryException.class, () -> listener.actionGet(TIMEOUT)); assertThat(thrownException.getMessage(), is("failed again")); @@ -449,7 +445,7 @@ public void testSend_ReturnsFailure_WhenHttpResultsListenerCallsOnFailure_WithNo var retrier = createRetrier(httpClient); var listener = new PlainActionFuture(); - executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener), 0); + executeTasks(() -> retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener), 0); var thrownException = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT)); assertThat(thrownException.getMessage(), is("failed")); @@ -484,7 +480,7 @@ public void testSend_DoesNotRetryIndefinitely() throws IOException { ); var listener = new PlainActionFuture(); - retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener); + retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener); // Assert that the retrying sender stopped after max retires even though the exception is retryable var thrownException = expectThrows(UncategorizedExecutionException.class, () -> listener.actionGet(TIMEOUT)); @@ -524,7 +520,7 @@ public void testSend_DoesNotRetryIndefinitely_WithAlwaysRetryingResponseHandler( ); var listener = new PlainActionFuture(); - retrier.send(mock(Logger.class), mockRequest(), HttpClientContext.create(), () -> false, handler, listener); + retrier.send(mock(Logger.class), mockRequest(), () -> false, handler, listener); // Assert that the retrying sender stopped after max retires var thrownException = expectThrows(UncategorizedExecutionException.class, () -> listener.actionGet(TIMEOUT)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java index 762a3a74184a4..e09e4968571e5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java @@ -123,7 +123,7 @@ public void testIsTerminated_AfterStopFromSeparateThread() { waitToShutdown.countDown(); waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); return Void.TYPE; - }).when(requestSender).send(any(), any(), any(), any(), any(), any()); + }).when(requestSender).send(any(), any(), any(), any(), any()); var service = createRequestExecutorService(null, requestSender); @@ -203,7 +203,7 @@ public void testTaskThrowsError_CallsOnFailure() { doAnswer(invocation -> { service.shutdown(); throw new IllegalArgumentException("failed"); - }).when(requestSender).send(any(), any(), any(), any(), any(), any()); + }).when(requestSender).send(any(), any(), any(), any(), any()); PlainActionFuture listener = new PlainActionFuture<>(); @@ -270,13 +270,13 @@ public void testExecute_PreservesThreadContext() throws InterruptedException, Ex assertNull(serviceThreadContext.getHeader(headerKey)); @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[5]; + ActionListener listener = invocation.getArgument(4, ActionListener.class); listener.onResponse(null); waitToShutdown.countDown(); waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); return Void.TYPE; - }).when(requestSender).send(any(), any(), any(), any(), any(), any()); + }).when(requestSender).send(any(), any(), any(), any(), any()); var finishedOnResponse = new CountDownLatch(1); ActionListener listener = new ActionListener<>() { @@ -422,7 +422,7 @@ public void testChangingCapacity_SetsCapacityToTwo() throws ExecutionException, waitToShutdown.countDown(); waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); return Void.TYPE; - }).when(requestSender).send(any(), any(), any(), any(), any(), any()); + }).when(requestSender).send(any(), any(), any(), any(), any()); Future executorTermination = submitShutdownRequest(waitToShutdown, waitToReturnFromSend, service); @@ -467,7 +467,7 @@ public void testChangingCapacity_DoesNotRejectsOverflowTasks_BecauseOfQueueFull( waitToShutdown.countDown(); waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); return Void.TYPE; - }).when(requestSender).send(any(), any(), any(), any(), any(), any()); + }).when(requestSender).send(any(), any(), any(), any(), any()); Future executorTermination = submitShutdownRequest(waitToShutdown, waitToReturnFromSend, service); @@ -528,7 +528,7 @@ public void testChangingCapacity_ToZero_SetsQueueCapacityToUnbounded() throws IO waitToShutdown.countDown(); waitToReturnFromSend.await(TIMEOUT.getSeconds(), TimeUnit.SECONDS); return Void.TYPE; - }).when(requestSender).send(any(), any(), any(), any(), any(), any()); + }).when(requestSender).send(any(), any(), any(), any(), any()); Future executorTermination = submitShutdownRequest(waitToShutdown, waitToReturnFromSend, service); @@ -598,11 +598,11 @@ public void testDoesNotExecuteTask_WhenCannotReserveTokens_AndThenCanReserve_And doAnswer(invocation -> { service.shutdown(); return Void.TYPE; - }).when(requestSender).send(any(), any(), any(), any(), any(), any()); + }).when(requestSender).send(any(), any(), any(), any(), any()); service.start(); - verify(requestSender, times(1)).send(any(), any(), any(), any(), any(), any()); + verify(requestSender, times(1)).send(any(), any(), any(), any(), any()); } public void testRemovesRateLimitGroup_AfterStaleDuration() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java index 8b7c01ae133cf..d8a1f2c4227e4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManagerTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.external.http.sender; -import org.apache.http.client.protocol.HttpClientContext; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.InferenceServiceResults; @@ -47,7 +46,6 @@ public static RequestManager createMock(RequestSender requestSender, String infe requestSender.send( mock(Logger.class), RequestTests.mockRequest(inferenceEntityId), - HttpClientContext.create(), () -> false, mock(ResponseHandler.class), listener From 8841c61ee347994b9afd4ff0af57f56a06ea52d3 Mon Sep 17 00:00:00 2001 From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com> Date: Tue, 20 Aug 2024 14:44:56 -0400 Subject: [PATCH 17/32] EsqlCapabilities for casting string to version (#112032) --- muted-tests.yml | 3 --- .../qa/testFixtures/src/main/resources/comparison.csv-spec | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index b166fb137bc48..f59b1540aaf16 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -131,9 +131,6 @@ tests: - class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT method: testSnapshotRestore {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/111799 -- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT - method: test {comparison.RangeVersion SYNC} - issue: https://github.com/elastic/elasticsearch/issues/111814 - class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT method: "test {p0=esql/26_aggs_bucket/friendlier BUCKET interval hourly: #110916}" issue: https://github.com/elastic/elasticsearch/issues/111901 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec index ef07f1dae3c1a..e0b921947e16d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec @@ -181,6 +181,7 @@ emp_no:integer |first_name:keyword ; rangeVersion +required_capability: string_literal_auto_casting_extended from apps | where version > "2" and version < "4" | keep id, version From fe5fa0575b82d7ce2c4570335d3f4a4d364bec27 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 21 Aug 2024 05:16:20 +1000 Subject: [PATCH 18/32] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {mv_percentile.FromIndex SYNC} #112036 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index f59b1540aaf16..33a65c8a90e23 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -178,6 +178,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT method: testScaledFloat issue: https://github.com/elastic/elasticsearch/issues/112003 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {mv_percentile.FromIndex SYNC} + issue: https://github.com/elastic/elasticsearch/issues/112036 # Examples: # From 94aa6ce1b262d6a41d51c4351249de8f3d714e76 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 21 Aug 2024 05:16:33 +1000 Subject: [PATCH 19/32] Mute org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT test {mv_percentile.FromIndex ASYNC} #112037 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 33a65c8a90e23..049ec985c7456 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -181,6 +181,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {mv_percentile.FromIndex SYNC} issue: https://github.com/elastic/elasticsearch/issues/112036 +- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT + method: test {mv_percentile.FromIndex ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/112037 # Examples: # From 29f9ad9a64e359362e6a40ebd05453c4bb15fc57 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 20 Aug 2024 12:45:20 -0700 Subject: [PATCH 20/32] Allow for negative epoch seconds (#111938) The change to allow nanoseconds in ZonedDateTime split the epoch seconds from the nanosecond subelement. However, the epoch seconds were then written as a vlong, when in fact they could be negative if the date is before epoch. This commit changes the format to use zlong instead, which supports negatives. closes #111923 --- .../org/elasticsearch/TransportVersions.java | 3 ++- .../common/io/stream/StreamInput.java | 7 +++-- .../common/io/stream/StreamOutput.java | 9 +++++-- .../common/io/stream/AbstractStreamTests.java | 27 ++++++++++++------- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 3bece535aab0f..fad57b3d6c854 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -189,10 +189,11 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_ORIGINAL_INDICES = def(8_719_00_0); public static final TransportVersion ML_INFERENCE_EIS_INTEGRATION_ADDED = def(8_720_00_0); public static final TransportVersion INGEST_PIPELINE_EXCEPTION_ADDED = def(8_721_00_0); - public static final TransportVersion ZDT_NANOS_SUPPORT = def(8_722_00_0); + public static final TransportVersion ZDT_NANOS_SUPPORT_BROKEN = def(8_722_00_0); public static final TransportVersion REMOVE_GLOBAL_RETENTION_FROM_TEMPLATES = def(8_723_00_0); public static final TransportVersion RANDOM_RERANKER_RETRIEVER = def(8_724_00_0); public static final TransportVersion ESQL_PROFILE_SLEEPS = def(8_725_00_0); + public static final TransportVersion ZDT_NANOS_SUPPORT = def(8_726_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 8de49ded03a4e..c4c18cfd376ad 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -903,8 +903,11 @@ public final Instant readOptionalInstant() throws IOException { private ZonedDateTime readZonedDateTime() throws IOException { final String timeZoneId = readString(); final Instant instant; - if (getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT)) { - instant = Instant.ofEpochSecond(readVLong(), readInt()); + if (getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT_BROKEN)) { + // epoch seconds can be negative, but it was incorrectly first written as vlong + boolean zlong = getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT); + long seconds = zlong ? readZLong() : readVLong(); + instant = Instant.ofEpochSecond(seconds, readInt()); } else { instant = Instant.ofEpochMilli(readLong()); } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 9d5b9a107ee6a..c65ae2e3463d4 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -767,8 +767,13 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep final ZonedDateTime zonedDateTime = (ZonedDateTime) v; o.writeString(zonedDateTime.getZone().getId()); Instant instant = zonedDateTime.toInstant(); - if (o.getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT)) { - o.writeVLong(instant.getEpochSecond()); + if (o.getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT_BROKEN)) { + // epoch seconds can be negative, but it was incorrectly first written as vlong + if (o.getTransportVersion().onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT)) { + o.writeZLong(instant.getEpochSecond()); + } else { + o.writeVLong(instant.getEpochSecond()); + } o.writeInt(instant.getNano()); } else { o.writeLong(instant.toEpochMilli()); diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java index b4aa58ae13f7b..8451d2fd64b9c 100644 --- a/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java +++ b/server/src/test/java/org/elasticsearch/common/io/stream/AbstractStreamTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -53,6 +52,8 @@ import static java.time.Instant.ofEpochSecond; import static java.time.ZonedDateTime.ofInstant; +import static org.elasticsearch.TransportVersions.ZDT_NANOS_SUPPORT; +import static org.elasticsearch.TransportVersions.ZDT_NANOS_SUPPORT_BROKEN; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; @@ -726,11 +727,15 @@ public void testReadAfterReachingEndOfStream() throws IOException { } public void testZonedDateTimeSerialization() throws IOException { - checkZonedDateTimeSerialization(TransportVersions.ZDT_NANOS_SUPPORT); + checkZonedDateTimeSerialization(ZDT_NANOS_SUPPORT); + } + + public void testZonedDateTimeMillisBwcSerializationV1() throws IOException { + checkZonedDateTimeSerialization(TransportVersionUtils.getPreviousVersion(ZDT_NANOS_SUPPORT_BROKEN)); } public void testZonedDateTimeMillisBwcSerialization() throws IOException { - checkZonedDateTimeSerialization(TransportVersionUtils.getPreviousVersion(TransportVersions.ZDT_NANOS_SUPPORT)); + checkZonedDateTimeSerialization(TransportVersionUtils.getPreviousVersion(ZDT_NANOS_SUPPORT)); } public void checkZonedDateTimeSerialization(TransportVersion tv) throws IOException { @@ -738,14 +743,18 @@ public void checkZonedDateTimeSerialization(TransportVersion tv) throws IOExcept assertGenericRoundtrip(ofInstant(ofEpochSecond(1), randomZone()), tv); // just want to test a large number that will use 5+ bytes long maxEpochSecond = Integer.MAX_VALUE; + long minEpochSecond = tv.between(ZDT_NANOS_SUPPORT_BROKEN, ZDT_NANOS_SUPPORT) ? 0 : Integer.MIN_VALUE; assertGenericRoundtrip(ofInstant(ofEpochSecond(maxEpochSecond), randomZone()), tv); - assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(0, maxEpochSecond)), randomZone()), tv); - assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(0, maxEpochSecond), 1_000_000), randomZone()), tv); - assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(0, maxEpochSecond), 999_000_000), randomZone()), tv); - if (tv.onOrAfter(TransportVersions.ZDT_NANOS_SUPPORT)) { - assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(0, maxEpochSecond), 999_999_999), randomZone()), tv); + assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond)), randomZone()), tv); + assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 1_000_000), randomZone()), tv); + assertGenericRoundtrip(ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 999_000_000), randomZone()), tv); + if (tv.onOrAfter(ZDT_NANOS_SUPPORT)) { + assertGenericRoundtrip( + ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), 999_999_999), randomZone()), + tv + ); assertGenericRoundtrip( - ofInstant(ofEpochSecond(randomLongBetween(0, maxEpochSecond), randomIntBetween(0, 999_999_999)), randomZone()), + ofInstant(ofEpochSecond(randomLongBetween(minEpochSecond, maxEpochSecond), randomIntBetween(0, 999_999_999)), randomZone()), tv ); } From 3a122a7d71a74346d8fd2d78130c305b0f8266bb Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 21 Aug 2024 06:24:58 +1000 Subject: [PATCH 21/32] Mute org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT testForceSleepsProfile {SYNC} #112039 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 049ec985c7456..e206d7229083a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -184,6 +184,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT method: test {mv_percentile.FromIndex ASYNC} issue: https://github.com/elastic/elasticsearch/issues/112037 +- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT + method: testForceSleepsProfile {SYNC} + issue: https://github.com/elastic/elasticsearch/issues/112039 # Examples: # From c1019d4c5d05c5f8b0b7b7dfd57ff50a64013e23 Mon Sep 17 00:00:00 2001 From: Stef Nestor <26751266+stefnestor@users.noreply.github.com> Date: Tue, 20 Aug 2024 14:58:18 -0600 Subject: [PATCH 22/32] (Doc+) Link API doc to parent object - part1 (#111951) * (Doc+) Link API to parent Doc part1 --------- Co-authored-by: shainaraskas Co-authored-by: shainaraskas <58563081+shainaraskas@users.noreply.github.com> --- docs/reference/autoscaling/apis/autoscaling-apis.asciidoc | 2 +- .../autoscaling/apis/delete-autoscaling-policy.asciidoc | 2 +- .../autoscaling/apis/get-autoscaling-capacity.asciidoc | 2 +- .../autoscaling/apis/get-autoscaling-policy.asciidoc | 2 +- .../autoscaling/apis/put-autoscaling-policy.asciidoc | 2 +- docs/reference/autoscaling/deciders/fixed-decider.asciidoc | 2 +- .../autoscaling/deciders/frozen-existence-decider.asciidoc | 2 +- .../autoscaling/deciders/frozen-shards-decider.asciidoc | 2 +- .../autoscaling/deciders/frozen-storage-decider.asciidoc | 2 +- .../autoscaling/deciders/machine-learning-decider.asciidoc | 2 +- .../autoscaling/deciders/proactive-storage-decider.asciidoc | 2 +- .../autoscaling/deciders/reactive-storage-decider.asciidoc | 2 +- docs/reference/autoscaling/index.asciidoc | 2 +- .../apis/delete-analytics-collection.asciidoc | 2 +- docs/reference/behavioral-analytics/apis/index.asciidoc | 2 +- .../apis/list-analytics-collection.asciidoc | 2 +- .../apis/post-analytics-collection-event.asciidoc | 2 +- .../apis/put-analytics-collection.asciidoc | 2 +- .../ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc | 2 +- .../ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc | 2 +- .../ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc | 2 +- .../ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc | 2 +- .../ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc | 2 +- docs/reference/ccr/apis/ccr-apis.asciidoc | 2 +- docs/reference/ccr/apis/follow/get-follow-info.asciidoc | 2 +- docs/reference/ccr/apis/follow/get-follow-stats.asciidoc | 2 +- docs/reference/ccr/apis/follow/post-forget-follower.asciidoc | 2 +- docs/reference/ccr/apis/follow/post-pause-follow.asciidoc | 2 +- docs/reference/ccr/apis/follow/post-resume-follow.asciidoc | 2 +- docs/reference/ccr/apis/follow/post-unfollow.asciidoc | 2 +- docs/reference/ccr/apis/follow/put-follow.asciidoc | 2 +- docs/reference/ccr/apis/get-ccr-stats.asciidoc | 2 +- docs/reference/cluster/allocation-explain.asciidoc | 2 +- docs/reference/cluster/delete-desired-balance.asciidoc | 2 +- docs/reference/cluster/get-desired-balance.asciidoc | 2 +- .../data-streams/change-mappings-and-settings.asciidoc | 2 +- docs/reference/data-streams/downsampling-manual.asciidoc | 4 ++-- .../data-streams/lifecycle/apis/delete-lifecycle.asciidoc | 2 +- .../data-streams/lifecycle/apis/explain-lifecycle.asciidoc | 2 +- .../data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc | 2 +- .../data-streams/lifecycle/apis/get-lifecycle.asciidoc | 2 +- .../data-streams/lifecycle/apis/put-lifecycle.asciidoc | 2 +- .../tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc | 4 ++-- docs/reference/data-streams/modify-data-streams-api.asciidoc | 2 +- docs/reference/data-streams/promote-data-stream-api.asciidoc | 2 +- docs/reference/data-streams/tsds-reindex.asciidoc | 2 +- docs/reference/eql/eql-apis.asciidoc | 2 +- docs/reference/esql/esql-apis.asciidoc | 2 +- docs/reference/esql/esql-async-query-delete-api.asciidoc | 2 +- docs/reference/ilm/apis/delete-lifecycle.asciidoc | 2 +- docs/reference/ilm/apis/explain.asciidoc | 2 +- docs/reference/ilm/apis/get-lifecycle.asciidoc | 2 +- docs/reference/ilm/apis/get-status.asciidoc | 2 +- docs/reference/ilm/apis/move-to-step.asciidoc | 2 +- docs/reference/ilm/apis/put-lifecycle.asciidoc | 2 +- docs/reference/ilm/apis/remove-policy-from-index.asciidoc | 2 +- docs/reference/ilm/apis/retry-policy.asciidoc | 2 +- docs/reference/ilm/apis/start.asciidoc | 2 +- docs/reference/ilm/apis/stop.asciidoc | 2 +- docs/reference/ilm/error-handling.asciidoc | 2 +- docs/reference/ilm/ilm-index-lifecycle.asciidoc | 2 +- 61 files changed, 63 insertions(+), 63 deletions(-) diff --git a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc index 090eda5ef5436..e4da2c45ee978 100644 --- a/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc +++ b/docs/reference/autoscaling/apis/autoscaling-apis.asciidoc @@ -4,7 +4,7 @@ NOTE: {cloud-only} -You can use the following APIs to perform autoscaling operations. +You can use the following APIs to perform {cloud}/ec-autoscaling.html[autoscaling operations]. [discrete] [[autoscaling-api-top-level]] diff --git a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc index 608b7bd7cb903..190428485a003 100644 --- a/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/delete-autoscaling-policy.asciidoc @@ -7,7 +7,7 @@ NOTE: {cloud-only} -Delete autoscaling policy. +Delete {cloud}/ec-autoscaling.html[autoscaling] policy. [[autoscaling-delete-autoscaling-policy-request]] ==== {api-request-title} diff --git a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc index 05724b9c48b6e..d635d8c8f7bd0 100644 --- a/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc +++ b/docs/reference/autoscaling/apis/get-autoscaling-capacity.asciidoc @@ -7,7 +7,7 @@ NOTE: {cloud-only} -Get autoscaling capacity. +Get {cloud}/ec-autoscaling.html[autoscaling] capacity. [[autoscaling-get-autoscaling-capacity-request]] ==== {api-request-title} diff --git a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc index ad00d69d1aeb2..973eedcb361c9 100644 --- a/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/get-autoscaling-policy.asciidoc @@ -7,7 +7,7 @@ NOTE: {cloud-only} -Get autoscaling policy. +Get {cloud}/ec-autoscaling.html[autoscaling] policy. [[autoscaling-get-autoscaling-policy-request]] ==== {api-request-title} diff --git a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc index ff79def51ebb9..e564f83411eb4 100644 --- a/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc +++ b/docs/reference/autoscaling/apis/put-autoscaling-policy.asciidoc @@ -7,7 +7,7 @@ NOTE: {cloud-only} -Creates or updates an autoscaling policy. +Creates or updates an {cloud}/ec-autoscaling.html[autoscaling] policy. [[autoscaling-put-autoscaling-policy-request]] ==== {api-request-title} diff --git a/docs/reference/autoscaling/deciders/fixed-decider.asciidoc b/docs/reference/autoscaling/deciders/fixed-decider.asciidoc index c46d1dffe2cc8..5a8b009d9f063 100644 --- a/docs/reference/autoscaling/deciders/fixed-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/fixed-decider.asciidoc @@ -6,7 +6,7 @@ experimental[] [WARNING] The fixed decider is intended for testing only. Do not use this decider in production. -The `fixed` decider responds with a fixed required capacity. It is not enabled +The {cloud}/ec-autoscaling.html[autoscaling] `fixed` decider responds with a fixed required capacity. It is not enabled by default but can be enabled for any policy by explicitly configuring it. ==== Configuration settings diff --git a/docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc b/docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc index 832cf330053aa..0fc9ad444a213 100644 --- a/docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/frozen-existence-decider.asciidoc @@ -2,7 +2,7 @@ [[autoscaling-frozen-existence-decider]] === Frozen existence decider -The frozen existence decider (`frozen_existence`) ensures that once the first +The {cloud}/ec-autoscaling.html[autoscaling] frozen existence decider (`frozen_existence`) ensures that once the first index enters the frozen ILM phase, the frozen tier is scaled into existence. The frozen existence decider is enabled for all policies governing frozen data diff --git a/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc b/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc index ab11da04c8642..1977f95797ef0 100644 --- a/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/frozen-shards-decider.asciidoc @@ -2,7 +2,7 @@ [[autoscaling-frozen-shards-decider]] === Frozen shards decider -The frozen shards decider (`frozen_shards`) calculates the memory required to search +The {cloud}/ec-autoscaling.html[autoscaling] frozen shards decider (`frozen_shards`) calculates the memory required to search the current set of partially mounted indices in the frozen tier. Based on a required memory amount per shard, it calculates the necessary memory in the frozen tier. diff --git a/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc b/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc index 5a10f31f1365b..3a8e7cdb518b3 100644 --- a/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/frozen-storage-decider.asciidoc @@ -2,7 +2,7 @@ [[autoscaling-frozen-storage-decider]] === Frozen storage decider -The frozen storage decider (`frozen_storage`) calculates the local storage +The {cloud}/ec-autoscaling.html[autoscaling] frozen storage decider (`frozen_storage`) calculates the local storage required to search the current set of partially mounted indices based on a percentage of the total data set size of such indices. It signals that additional storage capacity is necessary when existing capacity is less than the diff --git a/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc b/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc index 26ced6ad7bb26..5432d96a47edb 100644 --- a/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/machine-learning-decider.asciidoc @@ -2,7 +2,7 @@ [[autoscaling-machine-learning-decider]] === Machine learning decider -The {ml} decider (`ml`) calculates the memory and CPU requirements to run {ml} +The {cloud}/ec-autoscaling.html[autoscaling] {ml} decider (`ml`) calculates the memory and CPU requirements to run {ml} jobs and trained models. The {ml} decider is enabled for policies governing `ml` nodes. diff --git a/docs/reference/autoscaling/deciders/proactive-storage-decider.asciidoc b/docs/reference/autoscaling/deciders/proactive-storage-decider.asciidoc index 763f1de96f6b9..33c989f3b12eb 100644 --- a/docs/reference/autoscaling/deciders/proactive-storage-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/proactive-storage-decider.asciidoc @@ -2,7 +2,7 @@ [[autoscaling-proactive-storage-decider]] === Proactive storage decider -The proactive storage decider (`proactive_storage`) calculates the storage required to contain +The {cloud}/ec-autoscaling.html[autoscaling] proactive storage decider (`proactive_storage`) calculates the storage required to contain the current data set plus an estimated amount of expected additional data. The proactive storage decider is enabled for all policies governing nodes with the `data_hot` role. diff --git a/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc b/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc index 50897178a88de..7c38df75169fd 100644 --- a/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc +++ b/docs/reference/autoscaling/deciders/reactive-storage-decider.asciidoc @@ -2,7 +2,7 @@ [[autoscaling-reactive-storage-decider]] === Reactive storage decider -The reactive storage decider (`reactive_storage`) calculates the storage required to contain +The {cloud}/ec-autoscaling.html[autoscaling] reactive storage decider (`reactive_storage`) calculates the storage required to contain the current data set. It signals that additional storage capacity is necessary when existing capacity has been exceeded (reactively). diff --git a/docs/reference/autoscaling/index.asciidoc b/docs/reference/autoscaling/index.asciidoc index fbf1a9536973e..e70c464889419 100644 --- a/docs/reference/autoscaling/index.asciidoc +++ b/docs/reference/autoscaling/index.asciidoc @@ -4,7 +4,7 @@ NOTE: {cloud-only} -The autoscaling feature enables an operator to configure tiers of nodes that +The {cloud}/ec-autoscaling.html[autoscaling] feature enables an operator to configure tiers of nodes that self-monitor whether or not they need to scale based on an operator-defined policy. Then, via the autoscaling API, an Elasticsearch cluster can report whether or not it needs additional resources to meet the policy. For example, an diff --git a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc index 9b15bcca3fc85..a6894a933b460 100644 --- a/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/delete-analytics-collection.asciidoc @@ -17,7 +17,7 @@ PUT _application/analytics/my_analytics_collection //// -Removes an Analytics Collection and its associated data stream. +Removes a <> Collection and its associated data stream. [[delete-analytics-collection-request]] ==== {api-request-title} diff --git a/docs/reference/behavioral-analytics/apis/index.asciidoc b/docs/reference/behavioral-analytics/apis/index.asciidoc index 042b50259b1bb..692d3374f89f5 100644 --- a/docs/reference/behavioral-analytics/apis/index.asciidoc +++ b/docs/reference/behavioral-analytics/apis/index.asciidoc @@ -9,7 +9,7 @@ beta::[] --- -Use the following APIs to manage tasks and resources related to Behavioral Analytics: +Use the following APIs to manage tasks and resources related to <>: * <> * <> diff --git a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc index 8d2491ff8a6ee..14511a1258278 100644 --- a/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/list-analytics-collection.asciidoc @@ -24,7 +24,7 @@ DELETE _application/analytics/my_analytics_collection2 // TEARDOWN //// -Returns information about Analytics Collections. +Returns information about <> Collections. [[list-analytics-collection-request]] ==== {api-request-title} diff --git a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc index 84d9cb5351799..f82717e22ed34 100644 --- a/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc +++ b/docs/reference/behavioral-analytics/apis/post-analytics-collection-event.asciidoc @@ -22,7 +22,7 @@ DELETE _application/analytics/my_analytics_collection // TEARDOWN //// -Post an event to an Analytics Collection. +Post an event to a <> Collection. [[post-analytics-collection-event-request]] ==== {api-request-title} diff --git a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc index 48273fb3906c4..cbbab2ae3e26c 100644 --- a/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc +++ b/docs/reference/behavioral-analytics/apis/put-analytics-collection.asciidoc @@ -16,7 +16,7 @@ DELETE _application/analytics/my_analytics_collection // TEARDOWN //// -Creates an Analytics Collection. +Creates a <> Collection. [[put-analytics-collection-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc index 1c72fb8742b93..b510163bab50b 100644 --- a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc @@ -5,7 +5,7 @@ Delete auto-follow pattern ++++ -Delete auto-follow patterns. +Delete {ccr} <>. [[ccr-delete-auto-follow-pattern-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc index 46ef288b05088..a2969e993ddfb 100644 --- a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc @@ -5,7 +5,7 @@ Get auto-follow pattern ++++ -Get auto-follow patterns. +Get {ccr} <>. [[ccr-get-auto-follow-pattern-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc index 1e64ab813e2ad..c5ae5a7b4af9d 100644 --- a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc @@ -5,7 +5,7 @@ Pause auto-follow pattern ++++ -Pauses an auto-follow pattern. +Pauses a {ccr} <>. [[ccr-pause-auto-follow-pattern-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc index d08997068f705..6769f21ca5cef 100644 --- a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc @@ -5,7 +5,7 @@ Create auto-follow pattern ++++ -Creates an auto-follow pattern. +Creates a {ccr} <>. [[ccr-put-auto-follow-pattern-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc index 04da9b4a35ba0..a580bb3838f9b 100644 --- a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc @@ -5,7 +5,7 @@ Resume auto-follow pattern ++++ -Resumes an auto-follow pattern. +Resumes a {ccr} <>. [[ccr-resume-auto-follow-pattern-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/ccr-apis.asciidoc b/docs/reference/ccr/apis/ccr-apis.asciidoc index 0c9f033639eda..ae94e1931af85 100644 --- a/docs/reference/ccr/apis/ccr-apis.asciidoc +++ b/docs/reference/ccr/apis/ccr-apis.asciidoc @@ -2,7 +2,7 @@ [[ccr-apis]] == {ccr-cap} APIs -You can use the following APIs to perform {ccr} operations. +You can use the following APIs to perform <> operations. [discrete] [[ccr-api-top-level]] diff --git a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc index 68fd6e210f884..6c049d9c92b59 100644 --- a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc @@ -5,7 +5,7 @@ Get follower info ++++ -Retrieves information about all follower indices. +Retrieves information about all <> follower indices. [[ccr-get-follow-info-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc index 72224cc7f51f4..4892f86b3523d 100644 --- a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc @@ -5,7 +5,7 @@ Get follower stats ++++ -Get follower stats. +Get <> follower stats. [[ccr-get-follow-stats-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc b/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc index ea7e8640056bf..1917c08d6640d 100644 --- a/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc +++ b/docs/reference/ccr/apis/follow/post-forget-follower.asciidoc @@ -5,7 +5,7 @@ Forget follower ++++ -Removes the follower retention leases from the leader. +Removes the <> follower retention leases from the leader. [[ccr-post-forget-follower-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc index a4ab69aba8d84..6d4730d10efe6 100644 --- a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc @@ -5,7 +5,7 @@ Pause follower ++++ -Pauses a follower index. +Pauses a <> follower index. [[ccr-post-pause-follow-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc index 47ba51a3fb8a0..b023a8cb5cb70 100644 --- a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc @@ -5,7 +5,7 @@ Resume follower ++++ -Resumes a follower index. +Resumes a <> follower index. [[ccr-post-resume-follow-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc index b96777b455d3b..dab11ef9e7a54 100644 --- a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc @@ -5,7 +5,7 @@ Unfollow ++++ -Converts a follower index to a regular index. +Converts a <> follower index to a regular index. [[ccr-post-unfollow-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc index eb83e2a13dcf1..b7ae9ac987474 100644 --- a/docs/reference/ccr/apis/follow/put-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc @@ -5,7 +5,7 @@ Create follower ++++ -Creates a follower index. +Creates a <> follower index. [[ccr-put-follow-request]] ==== {api-request-title} diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc index 128df5e47c777..92e6bae0bdce8 100644 --- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc +++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc @@ -6,7 +6,7 @@ Get {ccr-init} stats ++++ -Get {ccr} stats. +Get <> stats. [[ccr-get-stats-request]] ==== {api-request-title} diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc index 0b0fde6546c29..809c9d74f1450 100644 --- a/docs/reference/cluster/allocation-explain.asciidoc +++ b/docs/reference/cluster/allocation-explain.asciidoc @@ -4,7 +4,7 @@ Cluster allocation explain ++++ -Provides an explanation for a shard's current allocation. +Provides an explanation for a shard's current <>. [source,console] ---- diff --git a/docs/reference/cluster/delete-desired-balance.asciidoc b/docs/reference/cluster/delete-desired-balance.asciidoc index f81dcab011da4..c67834269e505 100644 --- a/docs/reference/cluster/delete-desired-balance.asciidoc +++ b/docs/reference/cluster/delete-desired-balance.asciidoc @@ -6,7 +6,7 @@ NOTE: {cloud-only} -Discards the current desired balance and computes a new desired balance starting from the current allocation of shards. +Discards the current <> and computes a new desired balance starting from the current allocation of shards. This can sometimes help {es} find a desired balance which needs fewer shard movements to achieve, especially if the cluster has experienced changes so substantial that the current desired balance is no longer optimal without {es} having detected that the current desired balance will take more shard movements to achieve than needed. However, this API diff --git a/docs/reference/cluster/get-desired-balance.asciidoc b/docs/reference/cluster/get-desired-balance.asciidoc index 3fd87dcfedc4f..74afdaa52daf1 100644 --- a/docs/reference/cluster/get-desired-balance.asciidoc +++ b/docs/reference/cluster/get-desired-balance.asciidoc @@ -8,7 +8,7 @@ NOTE: {cloud-only} Exposes: -* the desired balance computation and reconciliation stats +* the <> computation and reconciliation stats * balancing stats such as distribution of shards, disk and ingest forecasts across nodes and data tiers (based on the current cluster state) * routing table with each shard current and desired location diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index 076b315558b60..1290f289e5bbd 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -5,7 +5,7 @@ [[data-streams-change-mappings-and-settings]] === Change mappings and settings for a data stream -Each data stream has a <> has a <>. Mappings and index settings from this template are applied to new backing indices created for the stream. This includes the stream's first backing index, which is auto-generated when the stream is created. diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc index 771a08d97d949..44ae77d072034 100644 --- a/docs/reference/data-streams/downsampling-manual.asciidoc +++ b/docs/reference/data-streams/downsampling-manual.asciidoc @@ -14,7 +14,7 @@ DELETE _ingest/pipeline/my-timestamp-pipeline // TEARDOWN //// -The recommended way to downsample a time series data stream (TSDS) is +The recommended way to <> a <> is <>. However, if you're not using ILM, you can downsample a TSDS manually. This guide shows you how, using typical Kubernetes cluster monitoring data. @@ -32,7 +32,7 @@ To test out manual downsampling, follow these steps: ==== Prerequisites * Refer to the <>. -* It is not possible to downsample a data stream directly, nor +* It is not possible to downsample a <> directly, nor multiple indices at once. It's only possible to downsample one time series index (TSDS backing index). * In order to downsample an index, it needs to be read-only. For a TSDS write diff --git a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc index f20c949c2fbc8..315f7fa85e45f 100644 --- a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc @@ -4,7 +4,7 @@ Delete Data Stream Lifecycle ++++ -Deletes the lifecycle from a set of data streams. +Deletes the <> from a set of data streams. [[delete-lifecycle-api-prereqs]] ==== {api-prereq-title} diff --git a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc index 7968bb78939e8..2b15886ebe192 100644 --- a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc @@ -4,7 +4,7 @@ Explain Data Stream Lifecycle ++++ -Retrieves the current data stream lifecycle status for one or more data stream backing indices. +Retrieves the current <> status for one or more data stream backing indices. [[explain-lifecycle-api-prereqs]] ==== {api-prereq-title} diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc index a99fa19d9db8d..f48fa1eb52daa 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc @@ -4,7 +4,7 @@ Get Data Stream Lifecycle ++++ -Gets stats about the execution of data stream lifecycle. +Gets stats about the execution of <>. [[get-lifecycle-stats-api-prereqs]] ==== {api-prereq-title} diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index 331285af395b6..c83572a4e0795 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -4,7 +4,7 @@ Get Data Stream Lifecycle ++++ -Gets the lifecycle of a set of data streams. +Gets the <> of a set of <>. [[get-lifecycle-api-prereqs]] ==== {api-prereq-title} diff --git a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc index 7d33a5b5f880c..c60c105e818ab 100644 --- a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc @@ -4,7 +4,7 @@ Put Data Stream Lifecycle ++++ -Configures the data stream lifecycle for the targeted data streams. +Configures the data stream <> for the targeted <>. [[put-lifecycle-api-prereqs]] ==== {api-prereq-title} diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index 5b2e2a1ec70a2..8d959d8f4ad84 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -2,8 +2,8 @@ [[tutorial-migrate-data-stream-from-ilm-to-dsl]] === Tutorial: Migrate ILM managed data stream to data stream lifecycle -In this tutorial we'll look at migrating an existing data stream from Index Lifecycle Management ({ilm-init}) to -data stream lifecycle. The existing {ilm-init} managed backing indices will continue +In this tutorial we'll look at migrating an existing data stream from <> to +<>. The existing {ilm-init} managed backing indices will continue to be managed by {ilm-init} until they age out and get deleted by {ilm-init}; however, the new backing indices will be managed by data stream lifecycle. This way, a data stream is gradually migrated away from being managed by {ilm-init} to diff --git a/docs/reference/data-streams/modify-data-streams-api.asciidoc b/docs/reference/data-streams/modify-data-streams-api.asciidoc index f05e76e67c32f..2da869083df22 100644 --- a/docs/reference/data-streams/modify-data-streams-api.asciidoc +++ b/docs/reference/data-streams/modify-data-streams-api.asciidoc @@ -4,7 +4,7 @@ Modify data streams ++++ -Performs one or more data stream modification actions in a single atomic +Performs one or more <> modification actions in a single atomic operation. [source,console] diff --git a/docs/reference/data-streams/promote-data-stream-api.asciidoc b/docs/reference/data-streams/promote-data-stream-api.asciidoc index 281e9b549abcb..111c7a2256f8a 100644 --- a/docs/reference/data-streams/promote-data-stream-api.asciidoc +++ b/docs/reference/data-streams/promote-data-stream-api.asciidoc @@ -5,7 +5,7 @@ Promote data stream ++++ -The purpose of the promote data stream api is to turn +The purpose of the promote <> API is to turn a data stream that is replicated by CCR into a regular data stream. diff --git a/docs/reference/data-streams/tsds-reindex.asciidoc b/docs/reference/data-streams/tsds-reindex.asciidoc index ea4ba16df5c4a..9d6594db4e779 100644 --- a/docs/reference/data-streams/tsds-reindex.asciidoc +++ b/docs/reference/data-streams/tsds-reindex.asciidoc @@ -9,7 +9,7 @@ [[tsds-reindex-intro]] ==== Introduction -With reindexing, you can copy documents from an old time-series data stream (TSDS) to a new one. Data streams support +With reindexing, you can copy documents from an old <> to a new one. Data streams support reindexing in general, with a few <>. Still, time-series data streams introduce additional challenges due to tight control on the accepted timestamp range for each backing index they contain. Direct use of the reindex API would likely error out due to attempting to insert documents with timestamps that are diff --git a/docs/reference/eql/eql-apis.asciidoc b/docs/reference/eql/eql-apis.asciidoc index d3f591ccfe6c1..e8cc2b21492ae 100644 --- a/docs/reference/eql/eql-apis.asciidoc +++ b/docs/reference/eql/eql-apis.asciidoc @@ -1,7 +1,7 @@ [[eql-apis]] == EQL APIs -Event Query Language (EQL) is a query language for event-based time series data, +<> is a query language for event-based time series data, such as logs, metrics, and traces. For an overview of EQL and related tutorials, see <>. diff --git a/docs/reference/esql/esql-apis.asciidoc b/docs/reference/esql/esql-apis.asciidoc index 686a71506bc14..8586cd1ae6bce 100644 --- a/docs/reference/esql/esql-apis.asciidoc +++ b/docs/reference/esql/esql-apis.asciidoc @@ -1,7 +1,7 @@ [[esql-apis]] == {esql} APIs -The {es} Query Language ({esql}) provides a powerful way to filter, transform, +The <> provides a powerful way to filter, transform, and analyze data stored in {es}, and in the future in other runtimes. For an overview of {esql} and related tutorials, see <>. diff --git a/docs/reference/esql/esql-async-query-delete-api.asciidoc b/docs/reference/esql/esql-async-query-delete-api.asciidoc index 90f8c06b9124a..5cad566f7f9c0 100644 --- a/docs/reference/esql/esql-async-query-delete-api.asciidoc +++ b/docs/reference/esql/esql-async-query-delete-api.asciidoc @@ -4,7 +4,7 @@ {esql} async query delete API ++++ -The {esql} async query delete API is used to manually delete an async query +The <> async query delete API is used to manually delete an async query by ID. If the query is still running, the query will be cancelled. Otherwise, the stored results are deleted. diff --git a/docs/reference/ilm/apis/delete-lifecycle.asciidoc b/docs/reference/ilm/apis/delete-lifecycle.asciidoc index 632cb982b3968..fc9a35e4ef570 100644 --- a/docs/reference/ilm/apis/delete-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/delete-lifecycle.asciidoc @@ -5,7 +5,7 @@ Delete policy ++++ -Deletes an index lifecycle policy. +Deletes an index <> policy. [[ilm-delete-lifecycle-request]] ==== {api-request-title} diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc index 348a9e7f99e78..a1ddde8c9f2d9 100644 --- a/docs/reference/ilm/apis/explain.asciidoc +++ b/docs/reference/ilm/apis/explain.asciidoc @@ -5,7 +5,7 @@ Explain lifecycle ++++ -Retrieves the current lifecycle status for one or more indices. For data +Retrieves the current <> status for one or more indices. For data streams, the API retrieves the current lifecycle status for the stream's backing indices. diff --git a/docs/reference/ilm/apis/get-lifecycle.asciidoc b/docs/reference/ilm/apis/get-lifecycle.asciidoc index 7443610065487..b4e07389a9fb7 100644 --- a/docs/reference/ilm/apis/get-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/get-lifecycle.asciidoc @@ -5,7 +5,7 @@ Get policy ++++ -Retrieves a lifecycle policy. +Retrieves a <> policy. [[ilm-get-lifecycle-request]] ==== {api-request-title} diff --git a/docs/reference/ilm/apis/get-status.asciidoc b/docs/reference/ilm/apis/get-status.asciidoc index 7e9e963f6f369..f2ab8d65ec9a1 100644 --- a/docs/reference/ilm/apis/get-status.asciidoc +++ b/docs/reference/ilm/apis/get-status.asciidoc @@ -7,7 +7,7 @@ Get {ilm} status ++++ -Retrieves the current {ilm} ({ilm-init}) status. +Retrieves the current <> ({ilm-init}) status. You can start or stop {ilm-init} with the <> and <> APIs. diff --git a/docs/reference/ilm/apis/move-to-step.asciidoc b/docs/reference/ilm/apis/move-to-step.asciidoc index 19cc9f7088867..f3441fa997cff 100644 --- a/docs/reference/ilm/apis/move-to-step.asciidoc +++ b/docs/reference/ilm/apis/move-to-step.asciidoc @@ -5,7 +5,7 @@ Move to step ++++ -Triggers execution of a specific step in the lifecycle policy. +Triggers execution of a specific step in the <> policy. [[ilm-move-to-step-request]] ==== {api-request-title} diff --git a/docs/reference/ilm/apis/put-lifecycle.asciidoc b/docs/reference/ilm/apis/put-lifecycle.asciidoc index ffd59a14d8c25..390f6b1bb4d15 100644 --- a/docs/reference/ilm/apis/put-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/put-lifecycle.asciidoc @@ -5,7 +5,7 @@ Create or update lifecycle policy ++++ -Creates or updates lifecycle policy. See <> for +Creates or updates <> policy. See <> for definitions of policy components. [[ilm-put-lifecycle-request]] diff --git a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc index 711eccc298df1..107cab4d5aa19 100644 --- a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc +++ b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc @@ -5,7 +5,7 @@ Remove policy ++++ -Removes assigned lifecycle policies from an index or a data stream's backing +Removes assigned <> policies from an index or a data stream's backing indices. [[ilm-remove-policy-request]] diff --git a/docs/reference/ilm/apis/retry-policy.asciidoc b/docs/reference/ilm/apis/retry-policy.asciidoc index cb2587fbb151b..8f01f15e0c3ad 100644 --- a/docs/reference/ilm/apis/retry-policy.asciidoc +++ b/docs/reference/ilm/apis/retry-policy.asciidoc @@ -5,7 +5,7 @@ Retry policy ++++ -Retry executing the policy for an index that is in the ERROR step. +Retry executing the <> policy for an index that is in the ERROR step. [[ilm-retry-policy-request]] ==== {api-request-title} diff --git a/docs/reference/ilm/apis/start.asciidoc b/docs/reference/ilm/apis/start.asciidoc index 32db585c6b14c..c38b3d9ca8831 100644 --- a/docs/reference/ilm/apis/start.asciidoc +++ b/docs/reference/ilm/apis/start.asciidoc @@ -7,7 +7,7 @@ Start {ilm} ++++ -Start the {ilm} ({ilm-init}) plugin. +Start the <> ({ilm-init}) plugin. [[ilm-start-request]] ==== {api-request-title} diff --git a/docs/reference/ilm/apis/stop.asciidoc b/docs/reference/ilm/apis/stop.asciidoc index 1e9cfb94d0b1f..a6100d794c2d3 100644 --- a/docs/reference/ilm/apis/stop.asciidoc +++ b/docs/reference/ilm/apis/stop.asciidoc @@ -7,7 +7,7 @@ Stop {ilm} ++++ -Stop the {ilm} ({ilm-init}) plugin. +Stop the <> ({ilm-init}) plugin. [[ilm-stop-request]] ==== {api-request-title} diff --git a/docs/reference/ilm/error-handling.asciidoc b/docs/reference/ilm/error-handling.asciidoc index d922fa6687823..f810afc6c2b5f 100644 --- a/docs/reference/ilm/error-handling.asciidoc +++ b/docs/reference/ilm/error-handling.asciidoc @@ -2,7 +2,7 @@ [[index-lifecycle-error-handling]] == Troubleshooting {ilm} errors -When {ilm-init} executes a lifecycle policy, it's possible for errors to occur +When <> executes a lifecycle policy, it's possible for errors to occur while performing the necessary index operations for a step. When this happens, {ilm-init} moves the index to an `ERROR` step. If {ilm-init} cannot resolve the error automatically, execution is halted diff --git a/docs/reference/ilm/ilm-index-lifecycle.asciidoc b/docs/reference/ilm/ilm-index-lifecycle.asciidoc index acf59645dae13..040e02742f5e7 100644 --- a/docs/reference/ilm/ilm-index-lifecycle.asciidoc +++ b/docs/reference/ilm/ilm-index-lifecycle.asciidoc @@ -5,7 +5,7 @@ Index lifecycle ++++ -{ilm-init} defines five index lifecycle _phases_: +<> defines five index lifecycle _phases_: * **Hot**: The index is actively being updated and queried. * **Warm**: The index is no longer being updated but is still being queried. From 29e922c902c85a214993d01f35120a7f5ba9d8f4 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 20 Aug 2024 23:57:38 +0200 Subject: [PATCH 23/32] Remove search worker pool (#111099) No more need for this pool, now that Lucene can safely execute on the current pool. --- docs/reference/modules/threadpool.asciidoc | 8 +----- .../elasticsearch/node/NodeConstruction.java | 13 +++++++++ .../search/DefaultSearchContext.java | 3 +- .../elasticsearch/search/SearchService.java | 20 ++++++++----- .../support/TimeSeriesIndexSearcher.java | 28 ++----------------- .../search/internal/ContextIndexSearcher.java | 2 +- .../elasticsearch/threadpool/ThreadPool.java | 6 ---- .../search/SearchServiceTests.java | 8 +++--- .../search/dfs/DfsPhaseTests.java | 2 +- .../snapshots/SnapshotResiliencyTests.java | 2 -- .../threadpool/ThreadPoolTests.java | 21 -------------- .../aggregations/AggregatorTestCase.java | 2 +- .../ConcurrentSearchSingleNodeTests.java | 27 ++++++------------ .../ConcurrentSearchTestPluginTests.java | 27 ++++++------------ .../input/MetadataCachingIndexInput.java | 1 - 15 files changed, 54 insertions(+), 116 deletions(-) diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc index ed4becbfbb6d0..9e6e5fb80f999 100644 --- a/docs/reference/modules/threadpool.asciidoc +++ b/docs/reference/modules/threadpool.asciidoc @@ -13,16 +13,10 @@ There are several thread pools, but the important ones include: [[search-threadpool]] `search`:: - For coordination of count/search operations at the shard level whose computation - is offloaded to the search_worker thread pool. Used also by fetch and other search + For count/search operations at the shard level. Used also by fetch and other search related operations Thread pool type is `fixed` with a size of `int((`<>`pass:[ * ]3) / 2) + 1`, and queue_size of `1000`. -`search_worker`:: - For the heavy workload of count/search operations that may be executed concurrently - across segments within the same shard when possible. Thread pool type is `fixed` - with a size of `int((`<>`pass:[ * ]3) / 2) + 1`, and unbounded queue_size . - [[search-throttled]]`search_throttled`:: For count/search/suggest/get operations on `search_throttled indices`. Thread pool type is `fixed` with a size of `1`, and queue_size of `100`. diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index a4db9a0a0e149..2599972665033 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -81,6 +81,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; @@ -497,6 +498,7 @@ private SettingsModule validateSettings(Settings envSettings, Settings settings, for (final ExecutorBuilder builder : threadPool.builders()) { additionalSettings.addAll(builder.getRegisteredSettings()); } + addBwcSearchWorkerSettings(additionalSettings); SettingsExtension.load().forEach(e -> additionalSettings.addAll(e.getSettings())); // this is as early as we can validate settings at this point. we already pass them to ThreadPool @@ -527,6 +529,17 @@ private SettingsModule validateSettings(Settings envSettings, Settings settings, return settingsModule; } + @UpdateForV9 + private static void addBwcSearchWorkerSettings(List> additionalSettings) { + // TODO remove the below settings, they are unused and only here to enable BwC for deployments that still use them + additionalSettings.add( + Setting.intSetting("thread_pool.search_worker.queue_size", 0, Setting.Property.NodeScope, Setting.Property.DeprecatedWarning) + ); + additionalSettings.add( + Setting.intSetting("thread_pool.search_worker.size", 0, Setting.Property.NodeScope, Setting.Property.DeprecatedWarning) + ); + } + private SearchModule createSearchModule(Settings settings, ThreadPool threadPool, TelemetryProvider telemetryProvider) { IndexSearcher.setMaxClauseCount(SearchUtils.calculateMaxClauseValue(threadPool)); return new SearchModule(settings, pluginsService.filterPlugins(SearchPlugin.class).toList(), telemetryProvider); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index dc92cfd11fce3..203834648eb67 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -186,7 +186,7 @@ final class DefaultSearchContext extends SearchContext { enableQueryPhaseParallelCollection, field -> getFieldCardinality(field, readerContext.indexService(), engineSearcher.getDirectoryReader()) ); - if (executor == null) { + if (maximumNumberOfSlices <= 1) { this.searcher = new ContextIndexSearcher( engineSearcher.getIndexReader(), engineSearcher.getSimilarity(), @@ -290,6 +290,7 @@ static int determineMaximumNumberOfSlices( ToLongFunction fieldCardinality ) { return executor instanceof ThreadPoolExecutor tpe + && tpe.getQueue().isEmpty() && isParallelCollectionSupportedForResults(resultsType, request.source(), fieldCardinality, enableQueryPhaseParallelCollection) ? tpe.getMaximumPoolSize() : 1; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 67d5d6337d77c..26c3bf6ceeffe 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -143,7 +143,6 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -229,7 +228,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv "search.worker_threads_enabled", true, Property.NodeScope, - Property.Dynamic + Property.Dynamic, + Property.DeprecatedWarning ); public static final Setting QUERY_PHASE_PARALLEL_COLLECTION_ENABLED = Setting.boolSetting( @@ -282,7 +282,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final FetchPhase fetchPhase; private final RankFeatureShardPhase rankFeatureShardPhase; - private volatile boolean enableSearchWorkerThreads; + private volatile Executor searchExecutor; private volatile boolean enableQueryPhaseParallelCollection; private volatile long defaultKeepAlive; @@ -376,7 +376,10 @@ public SearchService( clusterService.getClusterSettings() .addSettingsUpdateConsumer(ENABLE_REWRITE_AGGS_TO_FILTER_BY_FILTER, this::setEnableRewriteAggsToFilterByFilter); - enableSearchWorkerThreads = SEARCH_WORKER_THREADS_ENABLED.get(settings); + if (SEARCH_WORKER_THREADS_ENABLED.get(settings)) { + searchExecutor = threadPool.executor(Names.SEARCH); + } + clusterService.getClusterSettings().addSettingsUpdateConsumer(SEARCH_WORKER_THREADS_ENABLED, this::setEnableSearchWorkerThreads); enableQueryPhaseParallelCollection = QUERY_PHASE_PARALLEL_COLLECTION_ENABLED.get(settings); @@ -385,7 +388,11 @@ public SearchService( } private void setEnableSearchWorkerThreads(boolean enableSearchWorkerThreads) { - this.enableSearchWorkerThreads = enableSearchWorkerThreads; + if (enableSearchWorkerThreads) { + searchExecutor = threadPool.executor(Names.SEARCH); + } else { + searchExecutor = null; + } } private void setEnableQueryPhaseParallelCollection(boolean enableQueryPhaseParallelCollection) { @@ -1126,7 +1133,6 @@ private DefaultSearchContext createSearchContext( reader.indexShard().shardId(), request.getClusterAlias() ); - ExecutorService executor = this.enableSearchWorkerThreads ? threadPool.executor(Names.SEARCH_WORKER) : null; searchContext = new DefaultSearchContext( reader, request, @@ -1135,7 +1141,7 @@ private DefaultSearchContext createSearchContext( timeout, fetchPhase, lowLevelCancellation, - executor, + searchExecutor, resultsType, enableQueryPhaseParallelCollection, minimumDocsPerSlice diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java index 21138f46e974e..c3faf0e1900dc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java @@ -21,7 +21,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.ThreadInterruptedException; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.lucene.search.function.MinScoreScorer; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; @@ -37,9 +36,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.FutureTask; -import java.util.concurrent.RunnableFuture; import java.util.function.IntSupplier; import static org.elasticsearch.index.IndexSortConfig.TIME_SERIES_SORT; @@ -93,28 +89,8 @@ public void setMinimumScore(Float minimumScore) { public void search(Query query, BucketCollector bucketCollector) throws IOException { query = searcher.rewrite(query); Weight weight = searcher.createWeight(query, bucketCollector.scoreMode(), 1); - if (searcher.getExecutor() == null) { - search(bucketCollector, weight); - bucketCollector.postCollection(); - return; - } - // offload to the search worker thread pool whenever possible. It will be null only when search.worker_threads_enabled is false - RunnableFuture task = new FutureTask<>(() -> { - search(bucketCollector, weight); - bucketCollector.postCollection(); - return null; - }); - searcher.getExecutor().execute(task); - try { - task.get(); - } catch (InterruptedException e) { - throw new ThreadInterruptedException(e); - } catch (ExecutionException e) { - if (e.getCause() instanceof RuntimeException runtimeException) { - throw runtimeException; - } - throw new RuntimeException(e.getCause()); - } + search(bucketCollector, weight); + bucketCollector.postCollection(); } private void search(BucketCollector bucketCollector, Weight weight) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 00c451fa42b2c..ee6b4fc7bad5b 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -355,7 +355,7 @@ private T search(Weight weight, CollectorManager /** * Similar to the lucene implementation, with the following changes made: - * 1) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search worker threads + * 1) postCollection is performed after each segment is collected. This is needed for aggregations, performed by search threads * so it can be parallelized. Also, it needs to happen in the same thread where doc_values are read, as it consumes them and Lucene * does not allow consuming them from a different thread. * 2) handles the ES TimeExceededException diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 29ab3ec7e0848..2877e0b46a390 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -88,7 +88,6 @@ public static class Names { public static final String ANALYZE = "analyze"; public static final String WRITE = "write"; public static final String SEARCH = "search"; - public static final String SEARCH_WORKER = "search_worker"; public static final String SEARCH_COORDINATION = "search_coordination"; public static final String AUTO_COMPLETE = "auto_complete"; public static final String SEARCH_THROTTLED = "search_throttled"; @@ -158,7 +157,6 @@ public static ThreadPoolType fromType(String type) { entry(Names.ANALYZE, ThreadPoolType.FIXED), entry(Names.WRITE, ThreadPoolType.FIXED), entry(Names.SEARCH, ThreadPoolType.FIXED), - entry(Names.SEARCH_WORKER, ThreadPoolType.FIXED), entry(Names.SEARCH_COORDINATION, ThreadPoolType.FIXED), entry(Names.AUTO_COMPLETE, ThreadPoolType.FIXED), entry(Names.MANAGEMENT, ThreadPoolType.SCALING), @@ -267,10 +265,6 @@ public ThreadPool(final Settings settings, MeterRegistry meterRegistry, final Ex new TaskTrackingConfig(true, searchAutoscalingEWMA) ) ); - builders.put( - Names.SEARCH_WORKER, - new FixedExecutorBuilder(settings, Names.SEARCH_WORKER, searchOrGetThreadPoolSize, -1, TaskTrackingConfig.DEFAULT) - ); builders.put( Names.SEARCH_COORDINATION, new FixedExecutorBuilder( diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index 701f093746b9f..3b8ff74e92653 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -2747,7 +2747,7 @@ public void testEnableSearchWorkerThreads() throws IOException { */ public void testSlicingBehaviourForParallelCollection() throws Exception { IndexService indexService = createIndex("index", Settings.EMPTY); - ThreadPoolExecutor executor = (ThreadPoolExecutor) indexService.getThreadPool().executor(ThreadPool.Names.SEARCH_WORKER); + ThreadPoolExecutor executor = (ThreadPoolExecutor) indexService.getThreadPool().executor(ThreadPool.Names.SEARCH); final int configuredMaxPoolSize = 10; executor.setMaximumPoolSize(configuredMaxPoolSize); // We set this explicitly to be independent of CPU cores. int numDocs = randomIntBetween(50, 100); @@ -2837,7 +2837,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { { try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.FETCH, true)) { ContextIndexSearcher searcher = searchContext.searcher(); - assertNotNull(searcher.getExecutor()); + assertNull(searcher.getExecutor()); final long priorExecutorTaskCount = executor.getCompletedTaskCount(); searcher.search(termQuery, new TotalHitCountCollectorManager()); assertBusy( @@ -2853,7 +2853,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { { try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.NONE, true)) { ContextIndexSearcher searcher = searchContext.searcher(); - assertNotNull(searcher.getExecutor()); + assertNull(searcher.getExecutor()); final long priorExecutorTaskCount = executor.getCompletedTaskCount(); searcher.search(termQuery, new TotalHitCountCollectorManager()); assertBusy( @@ -2876,7 +2876,7 @@ public void testSlicingBehaviourForParallelCollection() throws Exception { { try (SearchContext searchContext = service.createContext(readerContext, request, task, ResultsType.QUERY, true)) { ContextIndexSearcher searcher = searchContext.searcher(); - assertNotNull(searcher.getExecutor()); + assertNull(searcher.getExecutor()); final long priorExecutorTaskCount = executor.getCompletedTaskCount(); searcher.search(termQuery, new TotalHitCountCollectorManager()); assertBusy( diff --git a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java index a0f37bcbb7fb1..a2b5671944405 100644 --- a/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/dfs/DfsPhaseTests.java @@ -39,7 +39,7 @@ public class DfsPhaseTests extends ESTestCase { @Before public final void init() { threadPool = new TestThreadPool(DfsPhaseTests.class.getName()); - threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH_WORKER); + threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH); } @After diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index b54a786e05c9d..04a7f2f538fdc 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1844,8 +1844,6 @@ private Environment createEnvironment(String nodeName) { Settings.builder() .put(NODE_NAME_SETTING.getKey(), nodeName) .put(PATH_HOME_SETTING.getKey(), tempDir.resolve(nodeName).toAbsolutePath()) - // test uses the same executor service for all thread pools, search worker would need to be a different one - .put(SearchService.SEARCH_WORKER_THREADS_ENABLED.getKey(), false) .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo").toAbsolutePath()) .putList( ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), diff --git a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java index b19f058d2c6c6..395ae07765016 100644 --- a/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java +++ b/server/src/test/java/org/elasticsearch/threadpool/ThreadPoolTests.java @@ -24,8 +24,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedTransferQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.util.concurrent.EsExecutors.TaskTrackingConfig.DEFAULT; @@ -370,25 +368,6 @@ public void testWriteThreadPoolUsesTaskExecutionTimeTrackingEsThreadPoolExecutor } } - public void testSearchWorkedThreadPool() { - final int allocatedProcessors = randomIntBetween(1, EsExecutors.allocatedProcessors(Settings.EMPTY)); - final ThreadPool threadPool = new TestThreadPool( - "test", - Settings.builder().put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), allocatedProcessors).build() - ); - try { - ExecutorService executor = threadPool.executor(ThreadPool.Names.SEARCH_WORKER); - assertThat(executor, instanceOf(ThreadPoolExecutor.class)); - ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) executor; - int expectedPoolSize = allocatedProcessors * 3 / 2 + 1; - assertEquals(expectedPoolSize, threadPoolExecutor.getCorePoolSize()); - assertEquals(expectedPoolSize, threadPoolExecutor.getMaximumPoolSize()); - assertThat(threadPoolExecutor.getQueue(), instanceOf(LinkedTransferQueue.class)); - } finally { - assertTrue(terminate(threadPool)); - } - } - public void testScheduledOneShotRejection() { final var name = "fixed-bounded"; final var threadPool = new TestThreadPool( diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index f3fc4479a21a4..6ca513516d90e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -211,7 +211,7 @@ public abstract class AggregatorTestCase extends ESTestCase { @Before public final void initPlugins() { threadPool = new TestThreadPool(AggregatorTestCase.class.getName()); - threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH_WORKER); + threadPoolExecutor = (ThreadPoolExecutor) threadPool.executor(ThreadPool.Names.SEARCH); List plugins = new ArrayList<>(getSearchPlugins()); plugins.add(new AggCardinalityUpperBoundPlugin()); SearchModule searchModule = new SearchModule(Settings.EMPTY, plugins); diff --git a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java index 5bb393ff70e83..00f08b1fa8eca 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchSingleNodeTests.java @@ -8,33 +8,22 @@ package org.elasticsearch.search.internal; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESSingleNodeTestCase; -import java.io.IOException; - public class ConcurrentSearchSingleNodeTests extends ESSingleNodeTestCase { private final boolean concurrentSearch = randomBoolean(); - public void testConcurrentSearch() throws IOException { + public void testConcurrentSearch() { client().admin().indices().prepareCreate("index").get(); - IndicesService indicesService = getInstanceFromNode(IndicesService.class); - IndexService indexService = indicesService.iterator().next(); - IndexShard shard = indexService.getShard(0); - SearchService searchService = getInstanceFromNode(SearchService.class); - ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shard.shardId(), 0L, AliasFilter.EMPTY); - try (SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, TimeValue.MINUS_ONE)) { - ContextIndexSearcher searcher = searchContext.searcher(); - if (concurrentSearch) { - assertEquals(1, searcher.getMinimumDocsPerSlice()); - } else { - assertEquals(50_000, searcher.getMinimumDocsPerSlice()); - } + ClusterService clusterService = getInstanceFromNode(ClusterService.class); + int minDocsPerSlice = SearchService.MINIMUM_DOCS_PER_SLICE.get(clusterService.getSettings()); + if (concurrentSearch) { + assertEquals(1, minDocsPerSlice); + } else { + assertEquals(50_000, minDocsPerSlice); } } diff --git a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java index 29da297ce292e..75d23b3baeabf 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/internal/ConcurrentSearchTestPluginTests.java @@ -8,34 +8,23 @@ package org.elasticsearch.search.internal; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.search.SearchService; import org.elasticsearch.test.ESIntegTestCase; -import java.io.IOException; - @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1) public class ConcurrentSearchTestPluginTests extends ESIntegTestCase { private final boolean concurrentSearch = randomBoolean(); - public void testConcurrentSearch() throws IOException { + public void testConcurrentSearch() { client().admin().indices().prepareCreate("index").get(); - IndicesService indicesService = internalCluster().getDataNodeInstance(IndicesService.class); - IndexService indexService = indicesService.iterator().next(); - IndexShard shard = indexService.getShard(0); - SearchService searchService = internalCluster().getDataNodeInstance(SearchService.class); - ShardSearchRequest shardSearchRequest = new ShardSearchRequest(shard.shardId(), 0L, AliasFilter.EMPTY); - try (SearchContext searchContext = searchService.createSearchContext(shardSearchRequest, TimeValue.MINUS_ONE)) { - ContextIndexSearcher searcher = searchContext.searcher(); - if (concurrentSearch) { - assertEquals(1, searcher.getMinimumDocsPerSlice()); - } else { - assertEquals(50_000, searcher.getMinimumDocsPerSlice()); - } + ClusterService clusterService = internalCluster().getDataNodeInstance(ClusterService.class); + int minDocsPerSlice = SearchService.MINIMUM_DOCS_PER_SLICE.get(clusterService.getSettings()); + if (concurrentSearch) { + assertEquals(1, minDocsPerSlice); + } else { + assertEquals(50_000, minDocsPerSlice); } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java index 8c978c3445526..9875ab03088aa 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java @@ -221,7 +221,6 @@ public static boolean assertCurrentThreadMayAccessBlobStore() { ThreadPool.Names.SNAPSHOT, ThreadPool.Names.GENERIC, ThreadPool.Names.SEARCH, - ThreadPool.Names.SEARCH_WORKER, ThreadPool.Names.SEARCH_THROTTLED, // Cache asynchronous fetching runs on a dedicated thread pool. From 3b8c9ad5dd2467298cedfad7caa6977b3a81cbda Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 21 Aug 2024 09:51:26 +0700 Subject: [PATCH 24/32] Pin default codec to index mode. (#111997) By default, the 'default' codec will be used as default, in case of logsdb index mode best_compression codec will be used as default. --- .../org/elasticsearch/index/IndexMode.java | 10 +++++++ .../codec/zstd/Zstd814StoredFieldsFormat.java | 4 +++ .../index/engine/EngineConfig.java | 8 ++++-- ...cTests.java => CodecIntegrationTests.java} | 27 ++++++++++++++++++- 4 files changed, 46 insertions(+), 3 deletions(-) rename server/src/test/java/org/elasticsearch/index/codec/{LegacyCodecTests.java => CodecIntegrationTests.java} (51%) diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 49eb6d84f0b1e..b137cfe27a514 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentDimensions; @@ -297,6 +298,11 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { public boolean isSyntheticSourceEnabled() { return true; } + + @Override + public String getDefaultCodec() { + return CodecService.BEST_COMPRESSION_CODEC; + } }; private static void validateTimeSeriesSettings(Map, Object> settings) { @@ -466,6 +472,10 @@ public String getName() { */ public abstract boolean isSyntheticSourceEnabled(); + public String getDefaultCodec() { + return CodecService.DEFAULT_CODEC; + } + /** * Parse a string into an {@link IndexMode}. */ diff --git a/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java index b827bb6436f07..840b37611374a 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java @@ -78,6 +78,10 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo return super.fieldsWriter(directory, si, context); } + public Mode getMode() { + return mode; + } + private static class ZstdCompressionMode extends CompressionMode { private final int level; diff --git a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 079d6479a63e4..317adcc67cf59 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.unit.MemorySizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecProvider; import org.elasticsearch.index.codec.CodecService; @@ -96,7 +97,10 @@ public Supplier retentionLeasesSupplier() { * This setting is also settable on the node and the index level, it's commonly used in hot/cold node archs where index is likely * allocated on both `kind` of nodes. */ - public static final Setting INDEX_CODEC_SETTING = new Setting<>("index.codec", "default", s -> { + public static final Setting INDEX_CODEC_SETTING = new Setting<>("index.codec", settings -> { + IndexMode indexMode = IndexSettings.MODE.get(settings); + return indexMode.getDefaultCodec(); + }, s -> { switch (s) { case CodecService.DEFAULT_CODEC: case CodecService.LEGACY_DEFAULT_CODEC: @@ -181,7 +185,7 @@ public EngineConfig( this.similarity = similarity; this.codecProvider = codecProvider; this.eventListener = eventListener; - codecName = indexSettings.getValue(INDEX_CODEC_SETTING); + this.codecName = indexSettings.getValue(INDEX_CODEC_SETTING); this.mapperService = mapperService; // We need to make the indexing buffer for this shard at least as large // as the amount of memory that is available for all engines on the diff --git a/server/src/test/java/org/elasticsearch/index/codec/LegacyCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecIntegrationTests.java similarity index 51% rename from server/src/test/java/org/elasticsearch/index/codec/LegacyCodecTests.java rename to server/src/test/java/org/elasticsearch/index/codec/CodecIntegrationTests.java index dbe83af1a0cfb..05b9cf42e6236 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/LegacyCodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecIntegrationTests.java @@ -9,11 +9,12 @@ package org.elasticsearch.index.codec; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat; import org.elasticsearch.test.ESSingleNodeTestCase; import static org.hamcrest.Matchers.equalTo; -public class LegacyCodecTests extends ESSingleNodeTestCase { +public class CodecIntegrationTests extends ESSingleNodeTestCase { public void testCanConfigureLegacySettings() { assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()); @@ -26,4 +27,28 @@ public void testCanConfigureLegacySettings() { codec = client().admin().indices().prepareGetSettings("index2").execute().actionGet().getSetting("index2", "index.codec"); assertThat(codec, equalTo("legacy_best_compression")); } + + public void testDefaultCodecLogsdb() { + assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()); + + var indexService = createIndex("index1", Settings.builder().put("index.mode", "logsdb").build()); + var storedFieldsFormat = (Zstd814StoredFieldsFormat) indexService.getShard(0) + .getEngineOrNull() + .config() + .getCodec() + .storedFieldsFormat(); + assertThat(storedFieldsFormat.getMode(), equalTo(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION)); + } + + public void testDefaultCodec() { + assumeTrue("Only when zstd_stored_fields feature flag is enabled", CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()); + + var indexService = createIndex("index1"); + var storedFieldsFormat = (Zstd814StoredFieldsFormat) indexService.getShard(0) + .getEngineOrNull() + .config() + .getCodec() + .storedFieldsFormat(); + assertThat(storedFieldsFormat.getMode(), equalTo(Zstd814StoredFieldsFormat.Mode.BEST_SPEED)); + } } From 3153bd0c63e667b6328b9e96b9fdd2876c2b1188 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:32:43 +1000 Subject: [PATCH 25/32] Mute org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT testForceSleepsProfile {ASYNC} #112049 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e206d7229083a..011de13677436 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -187,6 +187,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT method: testForceSleepsProfile {SYNC} issue: https://github.com/elastic/elasticsearch/issues/112039 +- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT + method: testForceSleepsProfile {ASYNC} + issue: https://github.com/elastic/elasticsearch/issues/112049 # Examples: # From 5b7d98568f04cefee91d5e5e7bf377ea971aa8a6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 21 Aug 2024 06:11:22 +0000 Subject: [PATCH 26/32] [Automated] Update Lucene snapshot to 9.12.0-snapshot-25253a1a016 --- gradle/verification-metadata.xml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index de1db00b952b7..2cf3a529a50c7 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2818,127 +2818,127 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + From a294265bfd2b2e1097eea24c3a51dc6aacd11085 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:34:08 +0300 Subject: [PATCH 27/32] Re-enable xpack yaml tests (#112031) --- muted-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 011de13677436..5a92196e4e51b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -173,8 +173,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/111923 - class: org.elasticsearch.xpack.sql.qa.security.JdbcCsvSpecIT issue: https://github.com/elastic/elasticsearch/issues/111923 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - issue: https://github.com/elastic/elasticsearch/issues/111944 - class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT method: testScaledFloat issue: https://github.com/elastic/elasticsearch/issues/112003 From a121fcb791705d5b1215665e0525260dee6212de Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Wed, 21 Aug 2024 08:41:07 +0200 Subject: [PATCH 28/32] Fix validation expception message if adaptive allocations is behind feature flag (#111970) --- .../ElasticsearchInternalServiceSettings.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index 1acf19c5373b7..8de791325a6df 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -16,6 +16,7 @@ import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsFeatureFlag; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.inference.services.ServiceUtils; @@ -87,12 +88,18 @@ protected static ElasticsearchInternalServiceSettings.Builder fromMap( String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); if (numAllocations == null && adaptiveAllocationsSettings == null) { - validationException.addValidationError( - ServiceUtils.missingOneOfSettingsErrorMsg( - List.of(NUM_ALLOCATIONS, ADAPTIVE_ALLOCATIONS), - ModelConfigurations.SERVICE_SETTINGS - ) - ); + if (AdaptiveAllocationsFeatureFlag.isEnabled()) { + validationException.addValidationError( + ServiceUtils.missingOneOfSettingsErrorMsg( + List.of(NUM_ALLOCATIONS, ADAPTIVE_ALLOCATIONS), + ModelConfigurations.SERVICE_SETTINGS + ) + ); + } else { + validationException.addValidationError( + ServiceUtils.missingSettingErrorMsg(NUM_ALLOCATIONS, ModelConfigurations.SERVICE_SETTINGS) + ); + } } // if an error occurred while parsing, we'll set these to an invalid value, so we don't accidentally get a From 0d38528e0e6f01010ba4a6d11d84ff7a5ddf32f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:11:48 +0200 Subject: [PATCH 29/32] Fix testLicenseTombstoneWithUsedTrialFromXContext (#112051) Relates: https://github.com/elastic/elasticsearch/issues/103093 --- .../license/LicensesMetadataSerializationTests.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java index be43705984435..e2218dfab1f1c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesMetadataSerializationTests.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.license; -import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; @@ -121,13 +120,12 @@ public void testLicenseTombstoneFromXContext() throws Exception { assertThat(metadataFromXContent.getLicense(), equalTo(LicensesMetadata.LICENSE_TOMBSTONE)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103093") public void testLicenseTombstoneWithUsedTrialFromXContext() throws Exception { final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject("licenses"); builder.nullField("license"); - builder.field("trial_license", Version.CURRENT.toString()); + builder.field("trial_license", TrialLicenseVersion.CURRENT); builder.endObject(); builder.endObject(); LicensesMetadata metadataFromXContent = getLicensesMetadataFromXContent(createParser(builder)); From 2681bb867d970048553ead7a38e995bf88616e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Wed, 21 Aug 2024 10:18:21 +0200 Subject: [PATCH 30/32] ESQL: Added exception catching to aggregators (#111829) Add the `warnExceptions` capability to aggregators. Same as for evaluators. This requires: - Having a "Warnings-like" object in the compute module. I created a new class for that, temporarily - Added new "FaillibleState" objects, that hold a "failed" boolean/bitset to mark failed groups - Catching exceptions in all combine methods receiving a single group (Not catching in evaluateFinal/Intermediate, nor in combines that receive the full state) - Shortcircuiting operations if the group is in "failed" state - Having a third intermediate state: `values, seen, failed` Extracted from https://github.com/elastic/elasticsearch/pull/111639. Check it to see a use case --- .../elasticsearch/compute/ann/Aggregator.java | 6 + .../compute/ann/GroupingAggregator.java | 6 + x-pack/plugin/esql/compute/build.gradle | 52 ++++++ ...AggregatorFunctionSupplierImplementer.java | 56 ++++-- .../compute/gen/AggregatorImplementer.java | 159 ++++++++++++----- .../compute/gen/AggregatorProcessor.java | 17 +- .../compute/gen/Annotations.java | 45 +++++ .../compute/gen/EvaluatorProcessor.java | 35 +--- .../gen/GroupingAggregatorImplementer.java | 132 ++++++++++---- .../org/elasticsearch/compute/gen/Types.java | 7 + .../BooleanFallibleArrayState.java | 125 +++++++++++++ .../aggregation/BooleanFallibleState.java | 62 +++++++ .../compute/aggregation/BooleanState.java | 4 - .../aggregation/DoubleFallibleArrayState.java | 124 +++++++++++++ .../aggregation/DoubleFallibleState.java | 62 +++++++ .../compute/aggregation/DoubleState.java | 4 - .../aggregation/FloatFallibleArrayState.java | 124 +++++++++++++ .../aggregation/FloatFallibleState.java | 62 +++++++ .../compute/aggregation/FloatState.java | 4 - .../aggregation/IntFallibleArrayState.java | 124 +++++++++++++ .../compute/aggregation/IntFallibleState.java | 62 +++++++ .../compute/aggregation/IntState.java | 4 - .../aggregation/LongFallibleArrayState.java | 130 ++++++++++++++ .../aggregation/LongFallibleState.java | 62 +++++++ .../compute/aggregation/LongState.java | 4 - .../aggregation/AbstractArrayState.java | 7 + .../AbstractFallibleArrayState.java | 48 +++++ .../aggregation/CountAggregatorFunction.java | 2 +- .../compute/aggregation/Warnings.java | 74 ++++++++ .../aggregation/X-FallibleArrayState.java.st | 166 ++++++++++++++++++ .../aggregation/X-FallibleState.java.st | 62 +++++++ .../compute/aggregation/X-State.java.st | 8 - 32 files changed, 1690 insertions(+), 149 deletions(-) create mode 100644 x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Annotations.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleArrayState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleArrayState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleState.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractFallibleArrayState.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/Warnings.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st diff --git a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java index 69db6a1310c9e..444dbcc1b9e58 100644 --- a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java +++ b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java @@ -57,4 +57,10 @@ IntermediateState[] value() default {}; + /** + * Exceptions thrown by the `combine*(...)` methods to catch and convert + * into a warning and turn into a null value. + */ + Class[] warnExceptions() default {}; + } diff --git a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java index 0216ea07e5c7c..8d81b60e20e4d 100644 --- a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java +++ b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java @@ -22,6 +22,12 @@ IntermediateState[] value() default {}; + /** + * Exceptions thrown by the `combine*(...)` methods to catch and convert + * into a warning and turn into a null value. + */ + Class[] warnExceptions() default {}; + /** * If {@code true} then the @timestamp LongVector will be appended to the input blocks of the aggregation function. */ diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle index ccf93a277a50d..971bfd39c231f 100644 --- a/x-pack/plugin/esql/compute/build.gradle +++ b/x-pack/plugin/esql/compute/build.gradle @@ -446,6 +446,32 @@ tasks.named('stringTemplates').configure { it.inputFile = stateInputFile it.outputFile = "org/elasticsearch/compute/aggregation/DoubleState.java" } + File fallibleStateInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st") + template { + it.properties = booleanProperties + it.inputFile = fallibleStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/BooleanFallibleState.java" + } + template { + it.properties = intProperties + it.inputFile = fallibleStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/IntFallibleState.java" + } + template { + it.properties = longProperties + it.inputFile = fallibleStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/LongFallibleState.java" + } + template { + it.properties = floatProperties + it.inputFile = fallibleStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/FloatFallibleState.java" + } + template { + it.properties = doubleProperties + it.inputFile = fallibleStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/DoubleFallibleState.java" + } // block lookups File lookupInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-Lookup.java.st") template { @@ -504,6 +530,32 @@ tasks.named('stringTemplates').configure { it.inputFile = arrayStateInputFile it.outputFile = "org/elasticsearch/compute/aggregation/FloatArrayState.java" } + File fallibleArrayStateInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st") + template { + it.properties = booleanProperties + it.inputFile = fallibleArrayStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java" + } + template { + it.properties = intProperties + it.inputFile = fallibleArrayStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/IntFallibleArrayState.java" + } + template { + it.properties = longProperties + it.inputFile = fallibleArrayStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/LongFallibleArrayState.java" + } + template { + it.properties = doubleProperties + it.inputFile = fallibleArrayStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java" + } + template { + it.properties = floatProperties + it.inputFile = fallibleArrayStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java" + } File valuesAggregatorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st") template { it.properties = intProperties diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorFunctionSupplierImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorFunctionSupplierImplementer.java index 3f031db2978f9..f11ccbced6fbe 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorFunctionSupplierImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorFunctionSupplierImplementer.java @@ -10,6 +10,7 @@ import com.squareup.javapoet.ClassName; import com.squareup.javapoet.JavaFile; import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.TypeName; import com.squareup.javapoet.TypeSpec; import org.elasticsearch.compute.ann.Aggregator; @@ -31,6 +32,7 @@ import static org.elasticsearch.compute.gen.Types.AGGREGATOR_FUNCTION_SUPPLIER; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.LIST_INTEGER; +import static org.elasticsearch.compute.gen.Types.STRING; /** * Implements "AggregationFunctionSupplier" from a class annotated with both @@ -40,6 +42,7 @@ public class AggregatorFunctionSupplierImplementer { private final TypeElement declarationType; private final AggregatorImplementer aggregatorImplementer; private final GroupingAggregatorImplementer groupingAggregatorImplementer; + private final boolean hasWarnings; private final List createParameters; private final ClassName implementation; @@ -47,11 +50,13 @@ public AggregatorFunctionSupplierImplementer( Elements elements, TypeElement declarationType, AggregatorImplementer aggregatorImplementer, - GroupingAggregatorImplementer groupingAggregatorImplementer + GroupingAggregatorImplementer groupingAggregatorImplementer, + boolean hasWarnings ) { this.declarationType = declarationType; this.aggregatorImplementer = aggregatorImplementer; this.groupingAggregatorImplementer = groupingAggregatorImplementer; + this.hasWarnings = hasWarnings; Set createParameters = new LinkedHashSet<>(); if (aggregatorImplementer != null) { @@ -86,6 +91,11 @@ private TypeSpec type() { builder.addModifiers(Modifier.PUBLIC, Modifier.FINAL); builder.addSuperinterface(AGGREGATOR_FUNCTION_SUPPLIER); + if (hasWarnings) { + builder.addField(TypeName.INT, "warningsLineNumber"); + builder.addField(TypeName.INT, "warningsColumnNumber"); + builder.addField(STRING, "warningsSourceText"); + } createParameters.stream().forEach(p -> p.declareField(builder)); builder.addMethod(ctor()); if (aggregatorImplementer != null) { @@ -100,6 +110,14 @@ private TypeSpec type() { private MethodSpec ctor() { MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC); + if (hasWarnings) { + builder.addParameter(TypeName.INT, "warningsLineNumber"); + builder.addParameter(TypeName.INT, "warningsColumnNumber"); + builder.addParameter(STRING, "warningsSourceText"); + builder.addStatement("this.warningsLineNumber = warningsLineNumber"); + builder.addStatement("this.warningsColumnNumber = warningsColumnNumber"); + builder.addStatement("this.warningsSourceText = warningsSourceText"); + } createParameters.stream().forEach(p -> p.buildCtor(builder)); return builder.build(); } @@ -114,30 +132,48 @@ private MethodSpec unsupportedNonGroupingAggregator() { } private MethodSpec aggregator() { - MethodSpec.Builder builder = MethodSpec.methodBuilder("aggregator") - .addParameter(DRIVER_CONTEXT, "driverContext") - .returns(aggregatorImplementer.implementation()); + MethodSpec.Builder builder = MethodSpec.methodBuilder("aggregator"); builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC); + builder.addParameter(DRIVER_CONTEXT, "driverContext"); + builder.returns(aggregatorImplementer.implementation()); + + if (hasWarnings) { + builder.addStatement( + "var warnings = Warnings.createWarnings(driverContext.warningsMode(), " + + "warningsLineNumber, warningsColumnNumber, warningsSourceText)" + ); + } + builder.addStatement( "return $T.create($L)", aggregatorImplementer.implementation(), - Stream.concat(Stream.of("driverContext, channels"), aggregatorImplementer.createParameters().stream().map(Parameter::name)) - .collect(Collectors.joining(", ")) + Stream.concat( + Stream.concat(hasWarnings ? Stream.of("warnings") : Stream.of(), Stream.of("driverContext, channels")), + aggregatorImplementer.createParameters().stream().map(Parameter::name) + ).collect(Collectors.joining(", ")) ); return builder.build(); } private MethodSpec groupingAggregator() { - MethodSpec.Builder builder = MethodSpec.methodBuilder("groupingAggregator") - .addParameter(DRIVER_CONTEXT, "driverContext") - .returns(groupingAggregatorImplementer.implementation()); + MethodSpec.Builder builder = MethodSpec.methodBuilder("groupingAggregator"); builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC); + builder.addParameter(DRIVER_CONTEXT, "driverContext"); + builder.returns(groupingAggregatorImplementer.implementation()); + + if (hasWarnings) { + builder.addStatement( + "var warnings = Warnings.createWarnings(driverContext.warningsMode(), " + + "warningsLineNumber, warningsColumnNumber, warningsSourceText)" + ); + } + builder.addStatement( "return $T.create($L)", groupingAggregatorImplementer.implementation(), Stream.concat( - Stream.of("channels, driverContext"), + Stream.concat(hasWarnings ? Stream.of("warnings") : Stream.of(), Stream.of("channels, driverContext")), groupingAggregatorImplementer.createParameters().stream().map(Parameter::name) ).collect(Collectors.joining(", ")) ); diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index 914724905541d..67ce0cf709704 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -21,10 +21,15 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.Modifier; import javax.lang.model.element.TypeElement; +import javax.lang.model.type.TypeMirror; import javax.lang.model.util.Elements; import static java.util.stream.Collectors.joining; @@ -40,6 +45,7 @@ import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.BYTES_REF_BLOCK; import static org.elasticsearch.compute.gen.Types.BYTES_REF_VECTOR; +import static org.elasticsearch.compute.gen.Types.COMPUTE_WARNINGS; import static org.elasticsearch.compute.gen.Types.DOUBLE_BLOCK; import static org.elasticsearch.compute.gen.Types.DOUBLE_VECTOR; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; @@ -68,6 +74,7 @@ */ public class AggregatorImplementer { private final TypeElement declarationType; + private final List warnExceptions; private final ExecutableElement init; private final ExecutableElement combine; private final ExecutableElement combineValueCount; @@ -76,18 +83,28 @@ public class AggregatorImplementer { private final ClassName implementation; private final TypeName stateType; private final boolean stateTypeHasSeen; + private final boolean stateTypeHasFailed; private final boolean valuesIsBytesRef; private final List intermediateState; private final List createParameters; - public AggregatorImplementer(Elements elements, TypeElement declarationType, IntermediateState[] interStateAnno) { + public AggregatorImplementer( + Elements elements, + TypeElement declarationType, + IntermediateState[] interStateAnno, + List warnExceptions + ) { this.declarationType = declarationType; + this.warnExceptions = warnExceptions; this.init = findRequiredMethod(declarationType, new String[] { "init", "initSingle" }, e -> true); this.stateType = choseStateType(); - stateTypeHasSeen = elements.getAllMembers(elements.getTypeElement(stateType.toString())) + this.stateTypeHasSeen = elements.getAllMembers(elements.getTypeElement(stateType.toString())) .stream() .anyMatch(e -> e.toString().equals("seen()")); + this.stateTypeHasFailed = elements.getAllMembers(elements.getTypeElement(stateType.toString())) + .stream() + .anyMatch(e -> e.toString().equals("failed()")); this.combine = findRequiredMethod(declarationType, new String[] { "combine" }, e -> { if (e.getParameters().size() == 0) { @@ -126,7 +143,10 @@ private TypeName choseStateType() { if (false == initReturn.isPrimitive()) { return initReturn; } - return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "State"); + if (warnExceptions.isEmpty()) { + return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "State"); + } + return ClassName.get("org.elasticsearch.compute.aggregation", firstUpper(initReturn.toString()) + "FallibleState"); } static String valueType(ExecutableElement init, ExecutableElement combine) { @@ -202,6 +222,11 @@ private TypeSpec type() { .initializer(initInterState()) .build() ); + + if (warnExceptions.isEmpty() == false) { + builder.addField(COMPUTE_WARNINGS, "warnings", Modifier.PRIVATE, Modifier.FINAL); + } + builder.addField(DRIVER_CONTEXT, "driverContext", Modifier.PRIVATE, Modifier.FINAL); builder.addField(stateType, "state", Modifier.PRIVATE, Modifier.FINAL); builder.addField(LIST_INTEGER, "channels", Modifier.PRIVATE, Modifier.FINAL); @@ -228,17 +253,26 @@ private TypeSpec type() { private MethodSpec create() { MethodSpec.Builder builder = MethodSpec.methodBuilder("create"); builder.addModifiers(Modifier.PUBLIC, Modifier.STATIC).returns(implementation); + if (warnExceptions.isEmpty() == false) { + builder.addParameter(COMPUTE_WARNINGS, "warnings"); + } builder.addParameter(DRIVER_CONTEXT, "driverContext"); builder.addParameter(LIST_INTEGER, "channels"); for (Parameter p : createParameters) { builder.addParameter(p.type(), p.name()); } if (createParameters.isEmpty()) { - builder.addStatement("return new $T(driverContext, channels, $L)", implementation, callInit()); + builder.addStatement( + "return new $T($LdriverContext, channels, $L)", + implementation, + warnExceptions.isEmpty() ? "" : "warnings, ", + callInit() + ); } else { builder.addStatement( - "return new $T(driverContext, channels, $L, $L)", + "return new $T($LdriverContext, channels, $L, $L)", implementation, + warnExceptions.isEmpty() ? "" : "warnings, ", callInit(), createParameters.stream().map(p -> p.name()).collect(joining(", ")) ); @@ -275,16 +309,22 @@ private CodeBlock initInterState() { private MethodSpec ctor() { MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC); + if (warnExceptions.isEmpty() == false) { + builder.addParameter(COMPUTE_WARNINGS, "warnings"); + } builder.addParameter(DRIVER_CONTEXT, "driverContext"); builder.addParameter(LIST_INTEGER, "channels"); builder.addParameter(stateType, "state"); + + if (warnExceptions.isEmpty() == false) { + builder.addStatement("this.warnings = warnings"); + } builder.addStatement("this.driverContext = driverContext"); builder.addStatement("this.channels = channels"); builder.addStatement("this.state = state"); for (Parameter p : createParameters()) { - builder.addParameter(p.type(), p.name()); - builder.addStatement("this.$N = $N", p.name(), p.name()); + p.buildCtor(builder); } return builder.build(); } @@ -306,6 +346,11 @@ private MethodSpec intermediateBlockCount() { private MethodSpec addRawInput() { MethodSpec.Builder builder = MethodSpec.methodBuilder("addRawInput"); builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC).addParameter(PAGE, "page"); + if (stateTypeHasFailed) { + builder.beginControlFlow("if (state.failed())"); + builder.addStatement("return"); + builder.endControlFlow(); + } builder.addStatement("$T block = page.getBlock(channels.get(0))", valueBlockType(init, combine)); builder.addStatement("$T vector = block.asVector()", valueVectorType(init, combine)); builder.beginControlFlow("if (vector != null)").addStatement("addRawVector(vector)"); @@ -366,20 +411,27 @@ private MethodSpec addRawBlock() { } private void combineRawInput(MethodSpec.Builder builder, String blockVariable) { + TypeName returnType = TypeName.get(combine.getReturnType()); + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("try"); + } if (valuesIsBytesRef) { combineRawInputForBytesRef(builder, blockVariable); - return; - } - TypeName returnType = TypeName.get(combine.getReturnType()); - if (returnType.isPrimitive()) { + } else if (returnType.isPrimitive()) { combineRawInputForPrimitive(returnType, builder, blockVariable); - return; - } - if (returnType == TypeName.VOID) { + } else if (returnType == TypeName.VOID) { combineRawInputForVoid(builder, blockVariable); - return; + } else { + throw new IllegalArgumentException("combine must return void or a primitive"); + } + if (warnExceptions.isEmpty() == false) { + String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; + builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); + builder.addStatement("warnings.registerException(e)"); + builder.addStatement("state.failed(true)"); + builder.addStatement("return"); + builder.endControlFlow(); } - throw new IllegalArgumentException("combine must return void or a primitive"); } private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder builder, String blockVariable) { @@ -423,16 +475,37 @@ private MethodSpec addIntermediateInput() { } builder.addStatement("$T.combineIntermediate(state, " + intermediateStateRowAccess() + ")", declarationType); } else if (hasPrimitiveState()) { - assert intermediateState.size() == 2; - assert intermediateState.get(1).name().equals("seen"); - builder.beginControlFlow("if (seen.getBoolean(0))"); - { - var state = intermediateState.get(0); - var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))"; - builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod()); - builder.addStatement("state.seen(true)"); + if (warnExceptions.isEmpty()) { + assert intermediateState.size() == 2; + assert intermediateState.get(1).name().equals("seen"); + builder.beginControlFlow("if (seen.getBoolean(0))"); + } else { + assert intermediateState.size() == 3; + assert intermediateState.get(1).name().equals("seen"); + assert intermediateState.get(2).name().equals("failed"); + builder.beginControlFlow("if (failed.getBoolean(0))"); + { + builder.addStatement("state.failed(true)"); + builder.addStatement("state.seen(true)"); + } + builder.nextControlFlow("else if (seen.getBoolean(0))"); + } + + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("try"); + } + var state = intermediateState.get(0); + var s = "state.$L($T.combine(state.$L(), " + state.name() + "." + vectorAccessorName(state.elementType()) + "(0)))"; + builder.addStatement(s, primitiveStateMethod(), declarationType, primitiveStateMethod()); + builder.addStatement("state.seen(true)"); + if (warnExceptions.isEmpty() == false) { + String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; + builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); + builder.addStatement("warnings.registerException(e)"); + builder.addStatement("state.failed(true)"); builder.endControlFlow(); } + builder.endControlFlow(); } else { throw new IllegalArgumentException("Don't know how to combine intermediate input. Define combineIntermediate"); } @@ -445,15 +518,15 @@ String intermediateStateRowAccess() { private String primitiveStateMethod() { switch (stateType.toString()) { - case "org.elasticsearch.compute.aggregation.BooleanState": + case "org.elasticsearch.compute.aggregation.BooleanState", "org.elasticsearch.compute.aggregation.BooleanFallibleState": return "booleanValue"; - case "org.elasticsearch.compute.aggregation.IntState": + case "org.elasticsearch.compute.aggregation.IntState", "org.elasticsearch.compute.aggregation.IntFallibleState": return "intValue"; - case "org.elasticsearch.compute.aggregation.LongState": + case "org.elasticsearch.compute.aggregation.LongState", "org.elasticsearch.compute.aggregation.LongFallibleState": return "longValue"; - case "org.elasticsearch.compute.aggregation.DoubleState": + case "org.elasticsearch.compute.aggregation.DoubleState", "org.elasticsearch.compute.aggregation.DoubleFallibleState": return "doubleValue"; - case "org.elasticsearch.compute.aggregation.FloatState": + case "org.elasticsearch.compute.aggregation.FloatState", "org.elasticsearch.compute.aggregation.FloatFallibleState": return "floatValue"; default: throw new IllegalArgumentException( @@ -480,8 +553,11 @@ private MethodSpec evaluateFinal() { .addParameter(BLOCK_ARRAY, "blocks") .addParameter(TypeName.INT, "offset") .addParameter(DRIVER_CONTEXT, "driverContext"); - if (stateTypeHasSeen) { - builder.beginControlFlow("if (state.seen() == false)"); + if (stateTypeHasSeen || stateTypeHasFailed) { + var condition = Stream.of(stateTypeHasSeen ? "state.seen() == false" : null, stateTypeHasFailed ? "state.failed()" : null) + .filter(Objects::nonNull) + .collect(joining(" || ")); + builder.beginControlFlow("if ($L)", condition); builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1)", BLOCK); builder.addStatement("return"); builder.endControlFlow(); @@ -496,19 +572,19 @@ private MethodSpec evaluateFinal() { private void primitiveStateToResult(MethodSpec.Builder builder) { switch (stateType.toString()) { - case "org.elasticsearch.compute.aggregation.BooleanState": + case "org.elasticsearch.compute.aggregation.BooleanState", "org.elasticsearch.compute.aggregation.BooleanFallibleState": builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantBooleanBlockWith(state.booleanValue(), 1)"); return; - case "org.elasticsearch.compute.aggregation.IntState": + case "org.elasticsearch.compute.aggregation.IntState", "org.elasticsearch.compute.aggregation.IntFallibleState": builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantIntBlockWith(state.intValue(), 1)"); return; - case "org.elasticsearch.compute.aggregation.LongState": + case "org.elasticsearch.compute.aggregation.LongState", "org.elasticsearch.compute.aggregation.LongFallibleState": builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantLongBlockWith(state.longValue(), 1)"); return; - case "org.elasticsearch.compute.aggregation.DoubleState": + case "org.elasticsearch.compute.aggregation.DoubleState", "org.elasticsearch.compute.aggregation.DoubleFallibleState": builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantDoubleBlockWith(state.doubleValue(), 1)"); return; - case "org.elasticsearch.compute.aggregation.FloatState": + case "org.elasticsearch.compute.aggregation.FloatState", "org.elasticsearch.compute.aggregation.FloatFallibleState": builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantFloatBlockWith(state.floatValue(), 1)"); return; default: @@ -534,13 +610,12 @@ private MethodSpec close() { return builder.build(); } + private static final Pattern PRIMITIVE_STATE_PATTERN = Pattern.compile( + "org.elasticsearch.compute.aggregation.(Boolean|Int|Long|Double|Float)(Fallible)?State" + ); + private boolean hasPrimitiveState() { - return switch (stateType.toString()) { - case "org.elasticsearch.compute.aggregation.BooleanState", "org.elasticsearch.compute.aggregation.IntState", - "org.elasticsearch.compute.aggregation.LongState", "org.elasticsearch.compute.aggregation.DoubleState", - "org.elasticsearch.compute.aggregation.FloatState" -> true; - default -> false; - }; + return PRIMITIVE_STATE_PATTERN.matcher(stateType.toString()).matches(); } record IntermediateStateDesc(String name, String elementType, boolean block) { diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java index d07b24047b7e2..4b1f946a1d176 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java @@ -80,9 +80,14 @@ public boolean process(Set set, RoundEnvironment roundEnv } for (TypeElement aggClass : annotatedClasses) { AggregatorImplementer implementer = null; + var warnExceptionsTypes = Annotations.listAttributeValues( + aggClass, + Set.of(Aggregator.class, GroupingAggregator.class), + "warnExceptions" + ); if (aggClass.getAnnotation(Aggregator.class) != null) { IntermediateState[] intermediateState = aggClass.getAnnotation(Aggregator.class).value(); - implementer = new AggregatorImplementer(env.getElementUtils(), aggClass, intermediateState); + implementer = new AggregatorImplementer(env.getElementUtils(), aggClass, intermediateState, warnExceptionsTypes); write(aggClass, "aggregator", implementer.sourceFile(), env); } GroupingAggregatorImplementer groupingAggregatorImplementer = null; @@ -96,6 +101,7 @@ public boolean process(Set set, RoundEnvironment roundEnv env.getElementUtils(), aggClass, intermediateState, + warnExceptionsTypes, includeTimestamps ); write(aggClass, "grouping aggregator", groupingAggregatorImplementer.sourceFile(), env); @@ -104,8 +110,13 @@ public boolean process(Set set, RoundEnvironment roundEnv write( aggClass, "aggregator function supplier", - new AggregatorFunctionSupplierImplementer(env.getElementUtils(), aggClass, implementer, groupingAggregatorImplementer) - .sourceFile(), + new AggregatorFunctionSupplierImplementer( + env.getElementUtils(), + aggClass, + implementer, + groupingAggregatorImplementer, + warnExceptionsTypes.isEmpty() == false + ).sourceFile(), env ); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Annotations.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Annotations.java new file mode 100644 index 0000000000000..d3892f7d2a40b --- /dev/null +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Annotations.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import javax.lang.model.element.AnnotationValue; +import javax.lang.model.element.Element; +import javax.lang.model.type.TypeMirror; + +public class Annotations { + private Annotations() {} + + /** + * Returns the values of the requested attribute, from all the matching annotations on the given element. + * + * @param element the element to inspect + * @param annotations the annotations to look for + * @param attributeName the attribute to extract + */ + public static List listAttributeValues(Element element, Set> annotations, String attributeName) { + List result = new ArrayList<>(); + for (var mirror : element.getAnnotationMirrors()) { + String annotationType = mirror.getAnnotationType().toString(); + if (annotations.stream().anyMatch(a -> a.getName().equals(annotationType))) { + for (var e : mirror.getElementValues().entrySet()) { + if (false == e.getKey().getSimpleName().toString().equals(attributeName)) { + continue; + } + for (var v : (List) e.getValue().getValue()) { + result.add((TypeMirror) ((AnnotationValue) v).getValue()); + } + } + } + } + return result; + } +} diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java index ea3ee938298de..09012c7b3a48a 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java @@ -11,7 +11,6 @@ import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.MvEvaluator; -import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -21,11 +20,9 @@ import javax.annotation.processing.RoundEnvironment; import javax.lang.model.SourceVersion; import javax.lang.model.element.AnnotationMirror; -import javax.lang.model.element.AnnotationValue; import javax.lang.model.element.Element; import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.TypeElement; -import javax.lang.model.type.TypeMirror; import javax.tools.Diagnostic; /** @@ -69,6 +66,11 @@ public Iterable getCompletions( public boolean process(Set set, RoundEnvironment roundEnvironment) { for (TypeElement ann : set) { for (Element evaluatorMethod : roundEnvironment.getElementsAnnotatedWith(ann)) { + var warnExceptionsTypes = Annotations.listAttributeValues( + evaluatorMethod, + Set.of(Evaluator.class, MvEvaluator.class, ConvertEvaluator.class), + "warnExceptions" + ); Evaluator evaluatorAnn = evaluatorMethod.getAnnotation(Evaluator.class); if (evaluatorAnn != null) { try { @@ -80,7 +82,7 @@ public boolean process(Set set, RoundEnvironment roundEnv env.getTypeUtils(), (ExecutableElement) evaluatorMethod, evaluatorAnn.extraName(), - warnExceptions(evaluatorMethod) + warnExceptionsTypes ).sourceFile(), env ); @@ -102,7 +104,7 @@ public boolean process(Set set, RoundEnvironment roundEnv mvEvaluatorAnn.finish(), mvEvaluatorAnn.single(), mvEvaluatorAnn.ascending(), - warnExceptions(evaluatorMethod) + warnExceptionsTypes ).sourceFile(), env ); @@ -121,7 +123,7 @@ public boolean process(Set set, RoundEnvironment roundEnv env.getElementUtils(), (ExecutableElement) evaluatorMethod, convertEvaluatorAnn.extraName(), - warnExceptions(evaluatorMethod) + warnExceptionsTypes ).sourceFile(), env ); @@ -134,25 +136,4 @@ public boolean process(Set set, RoundEnvironment roundEnv } return true; } - - private static List warnExceptions(Element evaluatorMethod) { - List result = new ArrayList<>(); - for (var mirror : evaluatorMethod.getAnnotationMirrors()) { - String annotationType = mirror.getAnnotationType().toString(); - if (annotationType.equals(Evaluator.class.getName()) - || annotationType.equals(MvEvaluator.class.getName()) - || annotationType.equals(ConvertEvaluator.class.getName())) { - - for (var e : mirror.getElementValues().entrySet()) { - if (false == e.getKey().getSimpleName().toString().equals("warnExceptions")) { - continue; - } - for (var v : (List) e.getValue().getValue()) { - result.add((TypeMirror) ((AnnotationValue) v).getValue()); - } - } - } - } - return result; - } } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index 79df41f304c06..0c4aeca996a19 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -22,11 +22,13 @@ import java.util.List; import java.util.Locale; import java.util.function.Consumer; +import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.Modifier; import javax.lang.model.element.TypeElement; +import javax.lang.model.type.TypeMirror; import javax.lang.model.util.Elements; import static java.util.stream.Collectors.joining; @@ -38,6 +40,7 @@ import static org.elasticsearch.compute.gen.Types.BIG_ARRAYS; import static org.elasticsearch.compute.gen.Types.BLOCK_ARRAY; import static org.elasticsearch.compute.gen.Types.BYTES_REF; +import static org.elasticsearch.compute.gen.Types.COMPUTE_WARNINGS; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.ELEMENT_TYPE; import static org.elasticsearch.compute.gen.Types.GROUPING_AGGREGATOR_FUNCTION; @@ -63,6 +66,7 @@ */ public class GroupingAggregatorImplementer { private final TypeElement declarationType; + private final List warnExceptions; private final ExecutableElement init; private final ExecutableElement combine; private final ExecutableElement combineStates; @@ -79,9 +83,11 @@ public GroupingAggregatorImplementer( Elements elements, TypeElement declarationType, IntermediateState[] interStateAnno, + List warnExceptions, boolean includeTimestampVector ) { this.declarationType = declarationType; + this.warnExceptions = warnExceptions; this.init = findRequiredMethod(declarationType, new String[] { "init", "initGrouping" }, e -> true); this.stateType = choseStateType(); @@ -129,7 +135,10 @@ private TypeName choseStateType() { } String head = initReturn.toString().substring(0, 1).toUpperCase(Locale.ROOT); String tail = initReturn.toString().substring(1); - return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "ArrayState"); + if (warnExceptions.isEmpty()) { + return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "ArrayState"); + } + return ClassName.get("org.elasticsearch.compute.aggregation", head + tail + "FallibleArrayState"); } public JavaFile sourceFile() { @@ -154,6 +163,9 @@ private TypeSpec type() { .build() ); builder.addField(stateType, "state", Modifier.PRIVATE, Modifier.FINAL); + if (warnExceptions.isEmpty() == false) { + builder.addField(COMPUTE_WARNINGS, "warnings", Modifier.PRIVATE, Modifier.FINAL); + } builder.addField(LIST_INTEGER, "channels", Modifier.PRIVATE, Modifier.FINAL); builder.addField(DRIVER_CONTEXT, "driverContext", Modifier.PRIVATE, Modifier.FINAL); @@ -182,17 +194,26 @@ private TypeSpec type() { private MethodSpec create() { MethodSpec.Builder builder = MethodSpec.methodBuilder("create"); builder.addModifiers(Modifier.PUBLIC, Modifier.STATIC).returns(implementation); + if (warnExceptions.isEmpty() == false) { + builder.addParameter(COMPUTE_WARNINGS, "warnings"); + } builder.addParameter(LIST_INTEGER, "channels"); builder.addParameter(DRIVER_CONTEXT, "driverContext"); for (Parameter p : createParameters) { builder.addParameter(p.type(), p.name()); } if (createParameters.isEmpty()) { - builder.addStatement("return new $T(channels, $L, driverContext)", implementation, callInit()); + builder.addStatement( + "return new $T($Lchannels, $L, driverContext)", + implementation, + warnExceptions.isEmpty() ? "" : "warnings, ", + callInit() + ); } else { builder.addStatement( - "return new $T(channels, $L, driverContext, $L)", + "return new $T($Lchannels, $L, driverContext, $L)", implementation, + warnExceptions.isEmpty() ? "" : "warnings, ", callInit(), createParameters.stream().map(p -> p.name()).collect(joining(", ")) ); @@ -235,9 +256,15 @@ private CodeBlock initInterState() { private MethodSpec ctor() { MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC); + if (warnExceptions.isEmpty() == false) { + builder.addParameter(COMPUTE_WARNINGS, "warnings"); + } builder.addParameter(LIST_INTEGER, "channels"); builder.addParameter(stateType, "state"); builder.addParameter(DRIVER_CONTEXT, "driverContext"); + if (warnExceptions.isEmpty() == false) { + builder.addStatement("this.warnings = warnings"); + } builder.addStatement("this.channels = channels"); builder.addStatement("this.state = state"); builder.addStatement("this.driverContext = driverContext"); @@ -349,6 +376,12 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { builder.addStatement("int groupId = Math.toIntExact(groups.getInt(groupPosition))"); } + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("if (state.hasFailed(groupId))"); + builder.addStatement("continue"); + builder.endControlFlow(); + } + if (valuesIsBlock) { builder.beginControlFlow("if (values.isNull(groupPosition + positionOffset))"); builder.addStatement("continue"); @@ -371,31 +404,35 @@ private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { } private void combineRawInput(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { - if (valuesIsBytesRef) { - combineRawInputForBytesRef(builder, blockVariable, offsetVariable); - return; - } - if (includeTimestampVector) { - combineRawInputWithTimestamp(builder, offsetVariable); - return; - } TypeName valueType = TypeName.get(combine.getParameters().get(combine.getParameters().size() - 1).asType()); - if (valueType.isPrimitive() == false) { - throw new IllegalArgumentException("second parameter to combine must be a primitive"); - } String secondParameterGetter = "get" + valueType.toString().substring(0, 1).toUpperCase(Locale.ROOT) + valueType.toString().substring(1); TypeName returnType = TypeName.get(combine.getReturnType()); - if (returnType.isPrimitive()) { - combineRawInputForPrimitive(builder, secondParameterGetter, blockVariable, offsetVariable); - return; + + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("try"); } - if (returnType == TypeName.VOID) { + if (valuesIsBytesRef) { + combineRawInputForBytesRef(builder, blockVariable, offsetVariable); + } else if (includeTimestampVector) { + combineRawInputWithTimestamp(builder, offsetVariable); + } else if (valueType.isPrimitive() == false) { + throw new IllegalArgumentException("second parameter to combine must be a primitive"); + } else if (returnType.isPrimitive()) { + combineRawInputForPrimitive(builder, secondParameterGetter, blockVariable, offsetVariable); + } else if (returnType == TypeName.VOID) { combineRawInputForVoid(builder, secondParameterGetter, blockVariable, offsetVariable); - return; + } else { + throw new IllegalArgumentException("combine must return void or a primitive"); + } + if (warnExceptions.isEmpty() == false) { + String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; + builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); + builder.addStatement("warnings.registerException(e)"); + builder.addStatement("state.setFailed(groupId)"); + builder.endControlFlow(); } - throw new IllegalArgumentException("combine must return void or a primitive"); } private void combineRawInputForPrimitive( @@ -481,20 +518,40 @@ private MethodSpec addIntermediateInput() { { builder.addStatement("int groupId = Math.toIntExact(groups.getInt(groupPosition))"); if (hasPrimitiveState()) { - assert intermediateState.size() == 2; - assert intermediateState.get(1).name().equals("seen"); - builder.beginControlFlow("if (seen.getBoolean(groupPosition + positionOffset))"); - { - var name = intermediateState.get(0).name(); - var m = vectorAccessorName(intermediateState.get(0).elementType()); - builder.addStatement( - "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L(groupPosition + positionOffset)))", - declarationType, - name, - m - ); + if (warnExceptions.isEmpty()) { + assert intermediateState.size() == 2; + assert intermediateState.get(1).name().equals("seen"); + builder.beginControlFlow("if (seen.getBoolean(groupPosition + positionOffset))"); + } else { + assert intermediateState.size() == 3; + assert intermediateState.get(1).name().equals("seen"); + assert intermediateState.get(2).name().equals("failed"); + builder.beginControlFlow("if (failed.getBoolean(groupPosition + positionOffset))"); + { + builder.addStatement("state.setFailed(groupId)"); + } + builder.nextControlFlow("else if (seen.getBoolean(groupPosition + positionOffset))"); + } + + if (warnExceptions.isEmpty() == false) { + builder.beginControlFlow("try"); + } + var name = intermediateState.get(0).name(); + var vectorAccessor = vectorAccessorName(intermediateState.get(0).elementType()); + builder.addStatement( + "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.$L(groupPosition + positionOffset)))", + declarationType, + name, + vectorAccessor + ); + if (warnExceptions.isEmpty() == false) { + String catchPattern = "catch (" + warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | ")) + " e)"; + builder.nextControlFlow(catchPattern, warnExceptions.stream().map(TypeName::get).toArray()); + builder.addStatement("warnings.registerException(e)"); + builder.addStatement("state.setFailed(groupId)"); builder.endControlFlow(); } + builder.endControlFlow(); } else { builder.addStatement("$T.combineIntermediate(state, groupId, " + intermediateStateRowAccess() + ")", declarationType); } @@ -582,12 +639,11 @@ private MethodSpec close() { return builder.build(); } + private static final Pattern PRIMITIVE_STATE_PATTERN = Pattern.compile( + "org.elasticsearch.compute.aggregation.(Boolean|Int|Long|Double|Float)(Fallible)?ArrayState" + ); + private boolean hasPrimitiveState() { - return switch (stateType.toString()) { - case "org.elasticsearch.compute.aggregation.BooleanArrayState", "org.elasticsearch.compute.aggregation.IntArrayState", - "org.elasticsearch.compute.aggregation.LongArrayState", "org.elasticsearch.compute.aggregation.DoubleArrayState", - "org.elasticsearch.compute.aggregation.FloatArrayState" -> true; - default -> false; - }; + return PRIMITIVE_STATE_PATTERN.matcher(stateType.toString()).matches(); } } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index 2b42adc67d71a..096d0b86e6cff 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -27,6 +27,8 @@ public class Types { private static final String OPERATOR_PACKAGE = PACKAGE + ".operator"; private static final String DATA_PACKAGE = PACKAGE + ".data"; + static final TypeName STRING = ClassName.get("java.lang", "String"); + static final TypeName LIST_INTEGER = ParameterizedTypeName.get(ClassName.get(List.class), TypeName.INT.box()); static final ClassName PAGE = ClassName.get(DATA_PACKAGE, "Page"); @@ -128,6 +130,11 @@ public class Types { ); static final ClassName WARNINGS = ClassName.get("org.elasticsearch.xpack.esql.expression.function", "Warnings"); + /** + * Warnings class used in compute module. + * It uses no external dependencies (Like Warnings and Source). + */ + static final ClassName COMPUTE_WARNINGS = ClassName.get("org.elasticsearch.compute.aggregation", "Warnings"); static final ClassName SOURCE = ClassName.get("org.elasticsearch.xpack.esql.core.tree", "Source"); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java new file mode 100644 index 0000000000000..6367fdfb6617e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleArrayState.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of booleans, that also tracks failures. + * It is created in a mode where it won't track + * the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

    + * This class is generated. Do not edit it. + *

    + */ +final class BooleanFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState { + private final boolean init; + + private BitArray values; + private int size; + + BooleanFallibleArrayState(BigArrays bigArrays, boolean init) { + super(bigArrays); + this.values = new BitArray(1, bigArrays); + this.size = 1; + this.values.set(0, init); + this.init = init; + } + + boolean get(int groupId) { + return values.get(groupId); + } + + boolean getOrDefault(int groupId) { + return groupId < size ? values.get(groupId) : init; + } + + void set(int groupId, boolean value) { + ensureCapacity(groupId); + values.set(groupId, value); + trackGroupId(groupId); + } + + Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) { + if (false == trackingGroupIds() && false == anyFailure()) { + try (var builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + builder.appendBoolean(i, values.get(selected.getInt(i))); + } + return builder.build().asBlock(); + } + } + try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group) && !hasFailed(group)) { + builder.appendBoolean(values.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + if (groupId >= size) { + values.fill(size, groupId + 1, init); + size = groupId + 1; + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate( + Block[] blocks, + int offset, + IntVector selected, + org.elasticsearch.compute.operator.DriverContext driverContext + ) { + assert blocks.length >= offset + 3; + try ( + var valuesBuilder = driverContext.blockFactory().newBooleanBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()); + var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (group < size) { + valuesBuilder.appendBoolean(values.get(group)); + } else { + valuesBuilder.appendBoolean(false); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + hasFailedBuilder.appendBoolean(i, hasFailed(group)); + } + blocks[offset + 0] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + blocks[offset + 2] = hasFailedBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleState.java new file mode 100644 index 0000000000000..073f31c390a6f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanFallibleState.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * Aggregator state for a single boolean. + * It stores a third boolean to store if the aggregation failed. + * This class is generated. Do not edit it. + */ +final class BooleanFallibleState implements AggregatorState { + private boolean value; + private boolean seen; + private boolean failed; + + BooleanFallibleState(boolean init) { + this.value = init; + } + + boolean booleanValue() { + return value; + } + + void booleanValue(boolean value) { + this.value = value; + } + + boolean seen() { + return seen; + } + + void seen(boolean seen) { + this.seen = seen; + } + + boolean failed() { + return failed; + } + + void failed(boolean failed) { + this.failed = failed; + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + assert blocks.length >= offset + 3; + blocks[offset + 0] = driverContext.blockFactory().newConstantBooleanBlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1); + } + + @Override + public void close() {} +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanState.java index 7d225c7c06a72..ba4d133dee553 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanState.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/BooleanState.java @@ -18,10 +18,6 @@ final class BooleanState implements AggregatorState { private boolean value; private boolean seen; - BooleanState() { - this(false); - } - BooleanState(boolean init) { this.value = init; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java new file mode 100644 index 0000000000000..dd1d60f7bd246 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleArrayState.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.DoubleArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of doubles, that also tracks failures. + * It is created in a mode where it won't track + * the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

    + * This class is generated. Do not edit it. + *

    + */ +final class DoubleFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState { + private final double init; + + private DoubleArray values; + + DoubleFallibleArrayState(BigArrays bigArrays, double init) { + super(bigArrays); + this.values = bigArrays.newDoubleArray(1, false); + this.values.set(0, init); + this.init = init; + } + + double get(int groupId) { + return values.get(groupId); + } + + double getOrDefault(int groupId) { + return groupId < values.size() ? values.get(groupId) : init; + } + + void set(int groupId, double value) { + ensureCapacity(groupId); + values.set(groupId, value); + trackGroupId(groupId); + } + + Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) { + if (false == trackingGroupIds() && false == anyFailure()) { + try (var builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + builder.appendDouble(i, values.get(selected.getInt(i))); + } + return builder.build().asBlock(); + } + } + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group) && !hasFailed(group)) { + builder.appendDouble(values.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + if (groupId >= values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, groupId + 1); + values.fill(prevSize, values.size(), init); + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate( + Block[] blocks, + int offset, + IntVector selected, + org.elasticsearch.compute.operator.DriverContext driverContext + ) { + assert blocks.length >= offset + 3; + try ( + var valuesBuilder = driverContext.blockFactory().newDoubleBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()); + var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (group < values.size()) { + valuesBuilder.appendDouble(values.get(group)); + } else { + valuesBuilder.appendDouble(0); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + hasFailedBuilder.appendBoolean(i, hasFailed(group)); + } + blocks[offset + 0] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + blocks[offset + 2] = hasFailedBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleState.java new file mode 100644 index 0000000000000..4cdeddec724bf --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleFallibleState.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * Aggregator state for a single double. + * It stores a third boolean to store if the aggregation failed. + * This class is generated. Do not edit it. + */ +final class DoubleFallibleState implements AggregatorState { + private double value; + private boolean seen; + private boolean failed; + + DoubleFallibleState(double init) { + this.value = init; + } + + double doubleValue() { + return value; + } + + void doubleValue(double value) { + this.value = value; + } + + boolean seen() { + return seen; + } + + void seen(boolean seen) { + this.seen = seen; + } + + boolean failed() { + return failed; + } + + void failed(boolean failed) { + this.failed = failed; + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + assert blocks.length >= offset + 3; + blocks[offset + 0] = driverContext.blockFactory().newConstantDoubleBlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1); + } + + @Override + public void close() {} +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleState.java index f1c92c685bcab..90ecc2c1d3c03 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleState.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/DoubleState.java @@ -18,10 +18,6 @@ final class DoubleState implements AggregatorState { private double value; private boolean seen; - DoubleState() { - this(0); - } - DoubleState(double init) { this.value = init; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java new file mode 100644 index 0000000000000..055cf345033c5 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleArrayState.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.FloatArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of floats, that also tracks failures. + * It is created in a mode where it won't track + * the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

    + * This class is generated. Do not edit it. + *

    + */ +final class FloatFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState { + private final float init; + + private FloatArray values; + + FloatFallibleArrayState(BigArrays bigArrays, float init) { + super(bigArrays); + this.values = bigArrays.newFloatArray(1, false); + this.values.set(0, init); + this.init = init; + } + + float get(int groupId) { + return values.get(groupId); + } + + float getOrDefault(int groupId) { + return groupId < values.size() ? values.get(groupId) : init; + } + + void set(int groupId, float value) { + ensureCapacity(groupId); + values.set(groupId, value); + trackGroupId(groupId); + } + + Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) { + if (false == trackingGroupIds() && false == anyFailure()) { + try (var builder = driverContext.blockFactory().newFloatVectorFixedBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + builder.appendFloat(i, values.get(selected.getInt(i))); + } + return builder.build().asBlock(); + } + } + try (FloatBlock.Builder builder = driverContext.blockFactory().newFloatBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group) && !hasFailed(group)) { + builder.appendFloat(values.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + if (groupId >= values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, groupId + 1); + values.fill(prevSize, values.size(), init); + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate( + Block[] blocks, + int offset, + IntVector selected, + org.elasticsearch.compute.operator.DriverContext driverContext + ) { + assert blocks.length >= offset + 3; + try ( + var valuesBuilder = driverContext.blockFactory().newFloatBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()); + var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (group < values.size()) { + valuesBuilder.appendFloat(values.get(group)); + } else { + valuesBuilder.appendFloat(0); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + hasFailedBuilder.appendBoolean(i, hasFailed(group)); + } + blocks[offset + 0] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + blocks[offset + 2] = hasFailedBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleState.java new file mode 100644 index 0000000000000..b050c86258dcd --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatFallibleState.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * Aggregator state for a single float. + * It stores a third boolean to store if the aggregation failed. + * This class is generated. Do not edit it. + */ +final class FloatFallibleState implements AggregatorState { + private float value; + private boolean seen; + private boolean failed; + + FloatFallibleState(float init) { + this.value = init; + } + + float floatValue() { + return value; + } + + void floatValue(float value) { + this.value = value; + } + + boolean seen() { + return seen; + } + + void seen(boolean seen) { + this.seen = seen; + } + + boolean failed() { + return failed; + } + + void failed(boolean failed) { + this.failed = failed; + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + assert blocks.length >= offset + 3; + blocks[offset + 0] = driverContext.blockFactory().newConstantFloatBlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1); + } + + @Override + public void close() {} +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java index 81bdd39e51b6e..6f608271b6e42 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java @@ -18,10 +18,6 @@ final class FloatState implements AggregatorState { private float value; private boolean seen; - FloatState() { - this(0); - } - FloatState(float init) { this.value = init; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleArrayState.java new file mode 100644 index 0000000000000..e45d84720ca1a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleArrayState.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.IntArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of ints, that also tracks failures. + * It is created in a mode where it won't track + * the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

    + * This class is generated. Do not edit it. + *

    + */ +final class IntFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState { + private final int init; + + private IntArray values; + + IntFallibleArrayState(BigArrays bigArrays, int init) { + super(bigArrays); + this.values = bigArrays.newIntArray(1, false); + this.values.set(0, init); + this.init = init; + } + + int get(int groupId) { + return values.get(groupId); + } + + int getOrDefault(int groupId) { + return groupId < values.size() ? values.get(groupId) : init; + } + + void set(int groupId, int value) { + ensureCapacity(groupId); + values.set(groupId, value); + trackGroupId(groupId); + } + + Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) { + if (false == trackingGroupIds() && false == anyFailure()) { + try (var builder = driverContext.blockFactory().newIntVectorFixedBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + builder.appendInt(i, values.get(selected.getInt(i))); + } + return builder.build().asBlock(); + } + } + try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group) && !hasFailed(group)) { + builder.appendInt(values.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + if (groupId >= values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, groupId + 1); + values.fill(prevSize, values.size(), init); + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate( + Block[] blocks, + int offset, + IntVector selected, + org.elasticsearch.compute.operator.DriverContext driverContext + ) { + assert blocks.length >= offset + 3; + try ( + var valuesBuilder = driverContext.blockFactory().newIntBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()); + var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (group < values.size()) { + valuesBuilder.appendInt(values.get(group)); + } else { + valuesBuilder.appendInt(0); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + hasFailedBuilder.appendBoolean(i, hasFailed(group)); + } + blocks[offset + 0] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + blocks[offset + 2] = hasFailedBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleState.java new file mode 100644 index 0000000000000..360f3fdb009e4 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntFallibleState.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * Aggregator state for a single int. + * It stores a third boolean to store if the aggregation failed. + * This class is generated. Do not edit it. + */ +final class IntFallibleState implements AggregatorState { + private int value; + private boolean seen; + private boolean failed; + + IntFallibleState(int init) { + this.value = init; + } + + int intValue() { + return value; + } + + void intValue(int value) { + this.value = value; + } + + boolean seen() { + return seen; + } + + void seen(boolean seen) { + this.seen = seen; + } + + boolean failed() { + return failed; + } + + void failed(boolean failed) { + this.failed = failed; + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + assert blocks.length >= offset + 3; + blocks[offset + 0] = driverContext.blockFactory().newConstantIntBlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1); + } + + @Override + public void close() {} +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntState.java index e7db40eccf9c8..c539c576ef36d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntState.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/IntState.java @@ -18,10 +18,6 @@ final class IntState implements AggregatorState { private int value; private boolean seen; - IntState() { - this(0); - } - IntState(int init) { this.value = init; } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleArrayState.java new file mode 100644 index 0000000000000..cb69579906871 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleArrayState.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of longs, that also tracks failures. + * It is created in a mode where it won't track + * the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

    + * This class is generated. Do not edit it. + *

    + */ +final class LongFallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState { + private final long init; + + private LongArray values; + + LongFallibleArrayState(BigArrays bigArrays, long init) { + super(bigArrays); + this.values = bigArrays.newLongArray(1, false); + this.values.set(0, init); + this.init = init; + } + + long get(int groupId) { + return values.get(groupId); + } + + long getOrDefault(int groupId) { + return groupId < values.size() ? values.get(groupId) : init; + } + + void set(int groupId, long value) { + ensureCapacity(groupId); + values.set(groupId, value); + trackGroupId(groupId); + } + + void increment(int groupId, long value) { + ensureCapacity(groupId); + values.increment(groupId, value); + trackGroupId(groupId); + } + + Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) { + if (false == trackingGroupIds() && false == anyFailure()) { + try (var builder = driverContext.blockFactory().newLongVectorFixedBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + builder.appendLong(i, values.get(selected.getInt(i))); + } + return builder.build().asBlock(); + } + } + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group) && !hasFailed(group)) { + builder.appendLong(values.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + if (groupId >= values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, groupId + 1); + values.fill(prevSize, values.size(), init); + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate( + Block[] blocks, + int offset, + IntVector selected, + org.elasticsearch.compute.operator.DriverContext driverContext + ) { + assert blocks.length >= offset + 3; + try ( + var valuesBuilder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()); + var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (group < values.size()) { + valuesBuilder.appendLong(values.get(group)); + } else { + valuesBuilder.appendLong(0); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + hasFailedBuilder.appendBoolean(i, hasFailed(group)); + } + blocks[offset + 0] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + blocks[offset + 2] = hasFailedBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleState.java new file mode 100644 index 0000000000000..98669ef627d04 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongFallibleState.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * Aggregator state for a single long. + * It stores a third boolean to store if the aggregation failed. + * This class is generated. Do not edit it. + */ +final class LongFallibleState implements AggregatorState { + private long value; + private boolean seen; + private boolean failed; + + LongFallibleState(long init) { + this.value = init; + } + + long longValue() { + return value; + } + + void longValue(long value) { + this.value = value; + } + + boolean seen() { + return seen; + } + + void seen(boolean seen) { + this.seen = seen; + } + + boolean failed() { + return failed; + } + + void failed(boolean failed) { + this.failed = failed; + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + assert blocks.length >= offset + 3; + blocks[offset + 0] = driverContext.blockFactory().newConstantLongBlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1); + } + + @Override + public void close() {} +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongState.java index da78b649782d5..e9d97dcfe7fc1 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongState.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/LongState.java @@ -18,10 +18,6 @@ final class LongState implements AggregatorState { private long value; private boolean seen; - LongState() { - this(0); - } - LongState(long init) { this.value = init; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java index 1573efdd81059..f9962922cc4a7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractArrayState.java @@ -12,6 +12,13 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; +/** + * Base class for array states that track which group ids have been set. + * Most of this class subclasses are autogenerated. + *

    + * Most of this class subclasses are autogenerated. + *

    + */ public class AbstractArrayState implements Releasable { protected final BigArrays bigArrays; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractFallibleArrayState.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractFallibleArrayState.java new file mode 100644 index 0000000000000..d5ad3189e2f9e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/AbstractFallibleArrayState.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.core.Releasables; + +/** + * Base class that extends {@link AbstractArrayState} to add failure tracking. + * That is, when a group id fails, it is marked as failed in the state. + *

    + * Most of this class subclasses are autogenerated. + *

    + */ +public class AbstractFallibleArrayState extends AbstractArrayState { + private BitArray failed; + + public AbstractFallibleArrayState(BigArrays bigArrays) { + super(bigArrays); + } + + final boolean hasFailed(int groupId) { + return failed != null && failed.get(groupId); + } + + protected final boolean anyFailure() { + return failed != null; + } + + protected final void setFailed(int groupId) { + if (failed == null) { + failed = new BitArray(groupId + 1, bigArrays); + } + failed.set(groupId); + } + + @Override + public void close() { + super.close(); + Releasables.close(failed); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountAggregatorFunction.java index 13a4204edfd8f..c32f6f4703a79 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountAggregatorFunction.java @@ -52,7 +52,7 @@ public static List intermediateStateDesc() { private final boolean countAll; public static CountAggregatorFunction create(List inputChannels) { - return new CountAggregatorFunction(inputChannels, new LongState()); + return new CountAggregatorFunction(inputChannels, new LongState(0)); } private CountAggregatorFunction(List channels, LongState state) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/Warnings.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/Warnings.java new file mode 100644 index 0000000000000..eb2255a4e349b --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/Warnings.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.operator.DriverContext; + +import static org.elasticsearch.common.logging.HeaderWarning.addWarning; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; + +/** + * Utilities to collect warnings for running an executor. + */ +public class Warnings { + static final int MAX_ADDED_WARNINGS = 20; + + private final String location; + private final String first; + + private int addedWarnings; + + public static final Warnings NOOP_WARNINGS = new Warnings(-1, -2, "") { + @Override + public void registerException(Exception exception) { + // this space intentionally left blank + } + }; + + /** + * Create a new warnings object based on the given mode + * @param warningsMode The warnings collection strategy to use + * @param lineNumber The line number of the source text. Same as `source.getLineNumber()` + * @param columnNumber The column number of the source text. Same as `source.getColumnNumber()` + * @param sourceText The source text that caused the warning. Same as `source.text()` + * @return A warnings collector object + */ + public static Warnings createWarnings(DriverContext.WarningsMode warningsMode, int lineNumber, int columnNumber, String sourceText) { + switch (warningsMode) { + case COLLECT -> { + return new Warnings(lineNumber, columnNumber, sourceText); + } + case IGNORE -> { + return NOOP_WARNINGS; + } + } + throw new IllegalStateException("Unreachable"); + } + + public Warnings(int lineNumber, int columnNumber, String sourceText) { + location = format("Line {}:{}: ", lineNumber, columnNumber); + first = format( + null, + "{}evaluation of [{}] failed, treating result as null. Only first {} failures recorded.", + location, + sourceText, + MAX_ADDED_WARNINGS + ); + } + + public void registerException(Exception exception) { + if (addedWarnings < MAX_ADDED_WARNINGS) { + if (addedWarnings == 0) { + addWarning(first); + } + // location needs to be added to the exception too, since the headers are deduplicated + addWarning(location + exception.getClass().getName() + ": " + exception.getMessage()); + addedWarnings++; + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st new file mode 100644 index 0000000000000..3c57ab948a79f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleArrayState.java.st @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +$if(boolean)$ +import org.elasticsearch.common.util.BitArray; +$else$ +import org.elasticsearch.common.util.$Type$Array; +$endif$ +import org.elasticsearch.compute.data.Block; +$if(long)$ +import org.elasticsearch.compute.data.IntVector; +$endif$ +import org.elasticsearch.compute.data.$Type$Block; +$if(int)$ +import org.elasticsearch.compute.data.$Type$Vector; +$endif$ +$if(boolean||double||float)$ +import org.elasticsearch.compute.data.IntVector; +$endif$ +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of $type$s, that also tracks failures. + * It is created in a mode where it won't track + * the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

    + * This class is generated. Do not edit it. + *

    + */ +final class $Type$FallibleArrayState extends AbstractFallibleArrayState implements GroupingAggregatorState { + private final $type$ init; + +$if(boolean)$ + private BitArray values; + private int size; + +$else$ + private $Type$Array values; +$endif$ + + $Type$FallibleArrayState(BigArrays bigArrays, $type$ init) { + super(bigArrays); +$if(boolean)$ + this.values = new BitArray(1, bigArrays); + this.size = 1; +$else$ + this.values = bigArrays.new$Type$Array(1, false); +$endif$ + this.values.set(0, init); + this.init = init; + } + + $type$ get(int groupId) { + return values.get(groupId); + } + + $type$ getOrDefault(int groupId) { +$if(boolean)$ + return groupId < size ? values.get(groupId) : init; +$else$ + return groupId < values.size() ? values.get(groupId) : init; +$endif$ + } + + void set(int groupId, $type$ value) { + ensureCapacity(groupId); + values.set(groupId, value); + trackGroupId(groupId); + } + +$if(long)$ + void increment(int groupId, long value) { + ensureCapacity(groupId); + values.increment(groupId, value); + trackGroupId(groupId); + } +$endif$ + + Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) { + if (false == trackingGroupIds() && false == anyFailure()) { + try (var builder = driverContext.blockFactory().new$Type$VectorFixedBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + builder.append$Type$(i, values.get(selected.getInt(i))); + } + return builder.build().asBlock(); + } + } + try ($Type$Block.Builder builder = driverContext.blockFactory().new$Type$BlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group) && !hasFailed(group)) { + builder.append$Type$(values.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { +$if(boolean)$ + if (groupId >= size) { + values.fill(size, groupId + 1, init); + size = groupId + 1; + } +$else$ + if (groupId >= values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, groupId + 1); + values.fill(prevSize, values.size(), init); + } +$endif$ + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate( + Block[] blocks, + int offset, + IntVector selected, + org.elasticsearch.compute.operator.DriverContext driverContext + ) { + assert blocks.length >= offset + 3; + try ( + var valuesBuilder = driverContext.blockFactory().new$Type$BlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()); + var hasFailedBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (group < $if(boolean)$size$else$values.size()$endif$) { + valuesBuilder.append$Type$(values.get(group)); + } else { + valuesBuilder.append$Type$($if(boolean)$false$else$0$endif$); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + hasFailedBuilder.appendBoolean(i, hasFailed(group)); + } + blocks[offset + 0] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + blocks[offset + 2] = hasFailedBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st new file mode 100644 index 0000000000000..27609383e4f61 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-FallibleState.java.st @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * Aggregator state for a single $type$. + * It stores a third boolean to store if the aggregation failed. + * This class is generated. Do not edit it. + */ +final class $Type$FallibleState implements AggregatorState { + private $type$ value; + private boolean seen; + private boolean failed; + + $Type$FallibleState($type$ init) { + this.value = init; + } + + $type$ $type$Value() { + return value; + } + + void $type$Value($type$ value) { + this.value = value; + } + + boolean seen() { + return seen; + } + + void seen(boolean seen) { + this.seen = seen; + } + + boolean failed() { + return failed; + } + + void failed(boolean failed) { + this.failed = failed; + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + assert blocks.length >= offset + 3; + blocks[offset + 0] = driverContext.blockFactory().newConstant$Type$BlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(failed, 1); + } + + @Override + public void close() {} +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st index 2d2d706c9454f..7e0949c86faaa 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-State.java.st @@ -18,14 +18,6 @@ final class $Type$State implements AggregatorState { private $type$ value; private boolean seen; - $Type$State() { -$if(boolean)$ - this(false); -$else$ - this(0); -$endif$ - } - $Type$State($type$ init) { this.value = init; } From 5d5d2e8a6362be73f8ce5e1ec87e1652c7eb5b47 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 21 Aug 2024 11:39:54 +0200 Subject: [PATCH 31/32] [ESQL] Fix cases of geometry collections with one point in CartesianPoint queries (#111193) --- docs/changelog/111193.yaml | 6 ++++++ muted-tests.yml | 3 --- .../xpack/esql/querydsl/query/SpatialRelatesQuery.java | 9 ++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 docs/changelog/111193.yaml diff --git a/docs/changelog/111193.yaml b/docs/changelog/111193.yaml new file mode 100644 index 0000000000000..9e56facb60d3a --- /dev/null +++ b/docs/changelog/111193.yaml @@ -0,0 +1,6 @@ +pr: 111193 +summary: Fix cases of collections with one point +area: Geo +type: bug +issues: + - 110982 diff --git a/muted-tests.yml b/muted-tests.yml index 5a92196e4e51b..145602ad7e1aa 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -74,9 +74,6 @@ tests: - class: org.elasticsearch.xpack.esql.spatial.SpatialPushDownGeoPointIT method: testPushedDownQueriesSingleValue issue: https://github.com/elastic/elasticsearch/issues/111084 -- class: org.elasticsearch.xpack.esql.spatial.SpatialPushDownCartesianPointIT - method: testPushedDownQueriesSingleValue - issue: https://github.com/elastic/elasticsearch/issues/110982 - class: org.elasticsearch.multi_node.GlobalCheckpointSyncActionIT issue: https://github.com/elastic/elasticsearch/issues/111124 - class: org.elasticsearch.cluster.PrevalidateShardPathIT diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java index 7a47b1d38f053..d1e4e12f73868 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.GeoShapeQueryable; import org.elasticsearch.index.mapper.MappedFieldType; @@ -228,10 +227,10 @@ private static org.apache.lucene.search.Query pointShapeQuery( if (geometry == null || geometry.isEmpty()) { throw new QueryShardException(context, "Invalid/empty geometry"); } - if (geometry.type() != ShapeType.POINT && relation == ShapeField.QueryRelation.CONTAINS) { + final XYGeometry[] luceneGeometries = LuceneGeometriesUtils.toXYGeometry(geometry, t -> {}); + if (isPointGeometry(luceneGeometries) == false && relation == ShapeField.QueryRelation.CONTAINS) { return new MatchNoDocsQuery("A point field can never contain a non-point geometry"); } - final XYGeometry[] luceneGeometries = LuceneGeometriesUtils.toXYGeometry(geometry, t -> {}); org.apache.lucene.search.Query intersects = XYPointField.newGeometryQuery(fieldName, luceneGeometries); if (relation == ShapeField.QueryRelation.DISJOINT) { // XYPointField does not support DISJOINT queries, so we build one as EXISTS && !INTERSECTS @@ -250,6 +249,10 @@ private static org.apache.lucene.search.Query pointShapeQuery( return intersects; } + private static boolean isPointGeometry(XYGeometry[] geometries) { + return geometries.length == 1 && geometries[0] instanceof org.apache.lucene.geo.XYPoint; + } + /** * This code is based on the ShapeQueryProcessor.shapeQuery() method */ From f5de9c00c8cba4a5224af30761ebd1590173aa5e Mon Sep 17 00:00:00 2001 From: Stef Nestor <26751266+stefnestor@users.noreply.github.com> Date: Wed, 21 Aug 2024 03:57:09 -0600 Subject: [PATCH 32/32] (Doc+) "min_primary_shard_size" for 10-50GB shards (#111574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 👋🏽 howdy, team! Expands [10-50GB sharding recommendation](https://www.elastic.co/guide/en/elasticsearch/reference/master/size-your-shards.html#shard-size-recommendation) to include ILM's more recent [`min_primary_shard_size`](https://www.elastic.co/guide/en/elasticsearch/reference/master/ilm-rollover.html) option to avoid small shards. --- docs/reference/how-to/size-your-shards.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc index 36aba99adb8c8..5f67014d5bb4a 100644 --- a/docs/reference/how-to/size-your-shards.asciidoc +++ b/docs/reference/how-to/size-your-shards.asciidoc @@ -162,7 +162,8 @@ and smaller shards may be appropriate for {enterprise-search-ref}/index.html[Enterprise Search] and similar use cases. If you use {ilm-init}, set the <>'s -`max_primary_shard_size` threshold to `50gb` to avoid shards larger than 50GB. +`max_primary_shard_size` threshold to `50gb` to avoid shards larger than 50GB +and `min_primary_shard_size` threshold to `10gb` to avoid shards smaller than 10GB. To see the current size of your shards, use the <>.