From d9e6f5452e1bcad67c805570adf033e37865cfdf Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 29 May 2024 10:34:51 +0200 Subject: [PATCH 01/29] [Connector API] Fix bug with with wrong target index for access control sync (#109097) --- docs/changelog/109097.yaml | 6 ++++++ .../sync_job/10_connector_sync_job_post.yml | 21 +++++++++++++++++++ .../application/connector/Connector.java | 5 +++++ .../connector/ConnectorTemplateRegistry.java | 1 + .../syncjob/ConnectorSyncJobIndexService.java | 12 +++++++---- .../ConnectorSyncJobIndexServiceTests.java | 13 ++++++++++++ 6 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/109097.yaml diff --git a/docs/changelog/109097.yaml b/docs/changelog/109097.yaml new file mode 100644 index 0000000000000..a7520f4eaa9be --- /dev/null +++ b/docs/changelog/109097.yaml @@ -0,0 +1,6 @@ +pr: 109097 +summary: "[Connector API] Fix bug with with wrong target index for access control\ + \ sync" +area: Application +type: bug +issues: [] diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml index e703b153dd3bb..8a384eee6bb93 100644 --- a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/10_connector_sync_job_post.yml @@ -275,6 +275,27 @@ setup: - exists: created_at - exists: last_seen +--- +'Create access control sync job - expect prefixed connector index name': + - do: + connector.sync_job_post: + body: + id: test-connector + job_type: access_control + + - set: { id: id } + + - match: { id: $id } + + - do: + connector.sync_job_get: + connector_sync_job_id: $id + + - match: { connector.id: test-connector } + - match: { job_type: access_control } + - match: { connector.index_name: .search-acl-filter-search-test } + + --- 'Create connector sync job with non-existing connector id': - do: diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java index e9361d78ad707..e9447149c7e6c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java @@ -34,6 +34,7 @@ import java.util.Objects; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_PREFIX; /** * Represents a Connector in the Elasticsearch ecosystem. Connectors are used for integrating @@ -269,6 +270,10 @@ public Connector(StreamInput in) throws IOException { } ); + public String getAccessControlIndexName() { + return ACCESS_CONTROL_INDEX_PREFIX + this.indexName; + } + static { PARSER.declareStringOrNull(optionalConstructorArg(), API_KEY_ID_FIELD); PARSER.declareStringOrNull(optionalConstructorArg(), API_KEY_SECRET_ID_FIELD); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java index e4ce4d8181fd8..41976bc6b4272 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java @@ -46,6 +46,7 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { public static final String CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN = ".elastic-connectors-sync-jobs-v1"; public static final String CONNECTOR_SYNC_JOBS_TEMPLATE_NAME = "elastic-connectors-sync-jobs"; + public static final String ACCESS_CONTROL_INDEX_PREFIX = ".search-acl-filter-"; public static final String ACCESS_CONTROL_INDEX_NAME_PATTERN = ".search-acl-filter-*"; public static final String ACCESS_CONTROL_TEMPLATE_NAME = "search-acl-filter"; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index 85f8ef089c6fe..6d81bae3c2e9f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -96,8 +96,9 @@ public void createConnectorSyncJob( ActionListener listener ) { String connectorId = request.getId(); + ConnectorSyncJobType jobType = Objects.requireNonNullElse(request.getJobType(), ConnectorSyncJob.DEFAULT_JOB_TYPE); try { - getSyncJobConnectorInfo(connectorId, listener.delegateFailure((l, connector) -> { + getSyncJobConnectorInfo(connectorId, jobType, listener.delegateFailure((l, connector) -> { if (Strings.isNullOrEmpty(connector.getIndexName())) { l.onFailure( new ElasticsearchStatusException( @@ -125,7 +126,6 @@ public void createConnectorSyncJob( } Instant now = Instant.now(); - ConnectorSyncJobType jobType = Objects.requireNonNullElse(request.getJobType(), ConnectorSyncJob.DEFAULT_JOB_TYPE); ConnectorSyncJobTriggerMethod triggerMethod = Objects.requireNonNullElse( request.getTriggerMethod(), ConnectorSyncJob.DEFAULT_TRIGGER_METHOD @@ -494,7 +494,7 @@ private ConnectorSyncStatus getConnectorSyncJobStatusFromSearchResult(ConnectorS ); } - private void getSyncJobConnectorInfo(String connectorId, ActionListener listener) { + private void getSyncJobConnectorInfo(String connectorId, ConnectorSyncJobType jobType, ActionListener listener) { try { final GetRequest request = new GetRequest(ConnectorIndexService.CONNECTOR_INDEX_NAME, connectorId); @@ -514,11 +514,15 @@ public void onResponse(GetResponse response) { connectorId, XContentType.JSON ); + // Access control syncs write data to a separate index + String targetIndexName = jobType == ConnectorSyncJobType.ACCESS_CONTROL + ? connector.getAccessControlIndexName() + : connector.getIndexName(); // Build the connector representation for sync job final Connector syncJobConnector = new Connector.Builder().setConnectorId(connector.getConnectorId()) .setSyncJobFiltering(transformConnectorFilteringToSyncJobRepresentation(connector.getFiltering())) - .setIndexName(connector.getIndexName()) + .setIndexName(targetIndexName) .setLanguage(connector.getLanguage()) .setPipeline(connector.getPipeline()) .setServiceType(connector.getServiceType()) diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java index 97ebea64eb2d8..5e854bb9d2396 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java @@ -56,6 +56,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.ACCESS_CONTROL_INDEX_PREFIX; import static org.elasticsearch.xpack.application.connector.ConnectorTestUtils.registerSimplifiedConnectorIndexTemplates; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -131,6 +132,18 @@ public void testCreateConnectorSyncJob() throws Exception { assertThat(connectorSyncJob.getDeletedDocumentCount(), equalTo(0L)); } + public void testCreateConnectorSyncJob_WithAccessControlJobType_IndexIsPrefixed() throws Exception { + PostConnectorSyncJobAction.Request createAccessControlJobRequest = ConnectorSyncJobTestUtils + .getRandomPostConnectorSyncJobActionRequest(connectorOneId, ConnectorSyncJobType.ACCESS_CONTROL); + + PostConnectorSyncJobAction.Response createAccessControlJobResponse = awaitPutConnectorSyncJob(createAccessControlJobRequest); + + ConnectorSyncJob connectorSyncJob = awaitGetConnectorSyncJob(createAccessControlJobResponse.getId()); + + assertThat(connectorSyncJob.getJobType(), equalTo(ConnectorSyncJobType.ACCESS_CONTROL)); + assertTrue(connectorSyncJob.getConnector().getIndexName().startsWith(ACCESS_CONTROL_INDEX_PREFIX)); + } + public void testCreateConnectorSyncJob_WithMissingJobType_ExpectDefaultJobTypeToBeSet() throws Exception { PostConnectorSyncJobAction.Request syncJobRequest = new PostConnectorSyncJobAction.Request( connectorOneId, From b84ad26c9b13fe4733fb30fd11aa353fc37e1a85 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 29 May 2024 02:56:43 -0600 Subject: [PATCH 02/29] Use the unsafe future for read single region (#109139) We need to use the unsafe future to block on the same thread pool similar to read multi regions. --- .../blobcache/shared/SharedBlobCacheService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java index c5ef1d7c2bf1d..218f12a97e1a6 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/shared/SharedBlobCacheService.java @@ -1116,7 +1116,9 @@ private int readSingleRegion( RangeMissingHandler writer, int region ) throws InterruptedException, ExecutionException { - final PlainActionFuture readFuture = new PlainActionFuture<>(); + final PlainActionFuture readFuture = new UnsafePlainActionFuture<>( + BlobStoreRepository.STATELESS_SHARD_PREWARMING_THREAD_NAME + ); final CacheFileRegion fileRegion = get(cacheKey, length, region); final long regionStart = getRegionStart(region); fileRegion.populateAndRead( From 1044a0104f6868c52fcc4161f7f973ed3e5a4087 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 29 May 2024 11:48:58 +0200 Subject: [PATCH 03/29] [Connecor API] Add references to tutorial in docs (#109099) --- .../connector/apis/cancel-connector-sync-job-api.asciidoc | 2 ++ docs/reference/connector/apis/check-in-connector-api.asciidoc | 2 ++ .../connector/apis/check-in-connector-sync-job-api.asciidoc | 2 ++ docs/reference/connector/apis/connector-apis.asciidoc | 2 +- docs/reference/connector/apis/create-connector-api.asciidoc | 2 ++ .../connector/apis/create-connector-sync-job-api.asciidoc | 2 ++ docs/reference/connector/apis/delete-connector-api.asciidoc | 2 ++ .../connector/apis/delete-connector-sync-job-api.asciidoc | 2 ++ docs/reference/connector/apis/get-connector-api.asciidoc | 2 ++ .../connector/apis/get-connector-sync-job-api.asciidoc | 2 ++ .../connector/apis/list-connector-sync-jobs-api.asciidoc | 1 + docs/reference/connector/apis/list-connectors-api.asciidoc | 2 ++ .../connector/apis/set-connector-sync-job-error-api.asciidoc | 2 ++ .../connector/apis/set-connector-sync-job-stats-api.asciidoc | 2 ++ .../connector/apis/update-connector-api-key-id-api.asciidoc | 2 ++ .../connector/apis/update-connector-configuration-api.asciidoc | 1 + .../connector/apis/update-connector-error-api.asciidoc | 2 ++ .../connector/apis/update-connector-filtering-api.asciidoc | 2 ++ .../connector/apis/update-connector-index-name-api.asciidoc | 2 ++ .../connector/apis/update-connector-last-sync-api.asciidoc | 2 ++ .../apis/update-connector-name-description-api.asciidoc | 2 ++ .../connector/apis/update-connector-pipeline-api.asciidoc | 2 ++ .../connector/apis/update-connector-scheduling-api.asciidoc | 2 ++ .../connector/apis/update-connector-service-type-api.asciidoc | 2 ++ .../connector/apis/update-connector-status-api.asciidoc | 2 ++ 25 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc index 0519cfdf15984..105d395e42c8b 100644 --- a/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/cancel-connector-sync-job-api.asciidoc @@ -8,6 +8,8 @@ beta::[] Cancels a connector sync job. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[cancel-connector-sync-job-api-request]] ==== {api-request-title} `PUT _connector/_sync_job//_cancel` diff --git a/docs/reference/connector/apis/check-in-connector-api.asciidoc b/docs/reference/connector/apis/check-in-connector-api.asciidoc index e8119028ea24d..8c6b5161a3a72 100644 --- a/docs/reference/connector/apis/check-in-connector-api.asciidoc +++ b/docs/reference/connector/apis/check-in-connector-api.asciidoc @@ -8,6 +8,8 @@ preview::[] Updates the `last_seen` field of a connector with current timestamp. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[check-in-connector-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc index 482c2e8dd4a2f..a052fbb2418cc 100644 --- a/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/check-in-connector-sync-job-api.asciidoc @@ -8,6 +8,8 @@ preview::[] Checks in a connector sync job (updates `last_seen` to the current time). +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[check-in-connector-sync-job-api-request]] ==== {api-request-title} `PUT _connector/_sync_job//_check_in/` diff --git a/docs/reference/connector/apis/connector-apis.asciidoc b/docs/reference/connector/apis/connector-apis.asciidoc index c8b06bb1d0a6e..9d23fcf8d336e 100644 --- a/docs/reference/connector/apis/connector-apis.asciidoc +++ b/docs/reference/connector/apis/connector-apis.asciidoc @@ -3,7 +3,7 @@ beta::[] -The connector and sync jobs APIs provide a convenient way to create and manage Elastic {enterprise-search-ref}/connectors.html[connectors^] and sync jobs in an internal index. +The connector and sync jobs APIs provide a convenient way to create and manage Elastic {enterprise-search-ref}/connectors.html[connectors^] and sync jobs in an internal index. To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. Connectors are {es} integrations that bring content from third-party data sources, which can be deployed on {ecloud} or hosted on your own infrastructure: diff --git a/docs/reference/connector/apis/create-connector-api.asciidoc b/docs/reference/connector/apis/create-connector-api.asciidoc index 2d7944c52f21b..9bd49a3c5ef94 100644 --- a/docs/reference/connector/apis/create-connector-api.asciidoc +++ b/docs/reference/connector/apis/create-connector-api.asciidoc @@ -14,6 +14,8 @@ Connectors are {es} integrations that bring content from third-party data source Find a list of all supported service types in the {enterprise-search-ref}/connectors.html[connectors documentation^]. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [source,console] -------------------------------------------------- PUT _connector/my-connector diff --git a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc index 43a2339c56847..c7cc866930dfb 100644 --- a/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/create-connector-sync-job-api.asciidoc @@ -9,6 +9,8 @@ beta::[] Creates a connector sync job. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [source, console] -------------------------------------------------- POST _connector/_sync_job diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc index d8dfea8c401ce..23acd1b4755b1 100644 --- a/docs/reference/connector/apis/delete-connector-api.asciidoc +++ b/docs/reference/connector/apis/delete-connector-api.asciidoc @@ -11,6 +11,8 @@ This is a destructive action that is not recoverable. Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[delete-connector-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc index 0c3274454845c..7cdabb22f05ee 100644 --- a/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/delete-connector-sync-job-api.asciidoc @@ -9,6 +9,8 @@ beta::[] Removes a connector sync job and its associated data. This is a destructive action that is not recoverable. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[delete-connector-sync-job-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index 7e5186b146c51..4df792c8a0a1a 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -8,6 +8,8 @@ beta::[] Retrieves the details about a connector. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[get-connector-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc index e162fe191375c..fffdada2a2a82 100644 --- a/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-sync-job-api.asciidoc @@ -8,6 +8,8 @@ beta::[] Retrieves the details about a connector sync job. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[get-connector-sync-job-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc index 6d06e7e6b9045..217b29451937d 100644 --- a/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc +++ b/docs/reference/connector/apis/list-connector-sync-jobs-api.asciidoc @@ -9,6 +9,7 @@ beta::[] Returns information about all stored connector sync jobs ordered by their creation date in ascending order. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. [[list-connector-sync-jobs-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc index c24acec5e82ce..c7ea2afd8102f 100644 --- a/docs/reference/connector/apis/list-connectors-api.asciidoc +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -9,6 +9,8 @@ beta::[] Returns information about all created connectors. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[list-connector-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc index 97b5c20f0813c..42203ed8e6103 100644 --- a/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc +++ b/docs/reference/connector/apis/set-connector-sync-job-error-api.asciidoc @@ -8,6 +8,8 @@ preview::[] Sets a connector sync job error. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[set-connector-sync-job-error-api-request]] ==== {api-request-title} `PUT _connector/_sync_job//_error` diff --git a/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc b/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc index 405df07465a28..4dd9cc6e67ab2 100644 --- a/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc +++ b/docs/reference/connector/apis/set-connector-sync-job-stats-api.asciidoc @@ -8,6 +8,8 @@ preview::[] Sets connector sync job stats. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[set-connector-sync-job-stats-api-request]] ==== {api-request-title} `PUT _connector/_sync_job//_stats` diff --git a/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc b/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc index b2b9dd7958191..112ec821df7c9 100644 --- a/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-api-key-id-api.asciidoc @@ -15,6 +15,8 @@ The Connector Secret ID is only required for native connectors. Connector clients do not use this field. See the documentation for {enterprise-search-ref}/native-connectors.html#native-connectors-manage-API-keys-programmatically[managing native connector API keys programmatically^] for more details. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-api-key-id-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc index 51252be4a04c3..e8a710cdacff0 100644 --- a/docs/reference/connector/apis/update-connector-configuration-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-configuration-api.asciidoc @@ -8,6 +8,7 @@ beta::[] Updates a connector's `configuration`, allowing for config value updates within a registered configuration schema. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. [[update-connector-configuration-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-error-api.asciidoc b/docs/reference/connector/apis/update-connector-error-api.asciidoc index 3ec03c6153f4b..67ea6b6d17cf0 100644 --- a/docs/reference/connector/apis/update-connector-error-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-error-api.asciidoc @@ -8,6 +8,8 @@ preview::[] Updates the `error` field of a connector. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-error-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc index 7ae80276d3151..861e72481a59a 100644 --- a/docs/reference/connector/apis/update-connector-filtering-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-filtering-api.asciidoc @@ -10,6 +10,8 @@ Updates the draft `filtering` configuration of a connector and marks the draft v The filtering property is used to configure sync rules (both basic and advanced) for a connector. Learn more in the {enterprise-search-ref}/sync-rules.html[sync rules documentation]. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-filtering-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc index 3f931d8ac5edd..d07007438e09c 100644 --- a/docs/reference/connector/apis/update-connector-index-name-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-index-name-api.asciidoc @@ -8,6 +8,8 @@ beta::[] Updates the `index_name` field of a connector, specifying the index where the data ingested by the connector is stored. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-index-name-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc b/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc index 9326855d3c5d8..918bf4f80a010 100644 --- a/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-last-sync-api.asciidoc @@ -10,6 +10,8 @@ Updates the fields related to the last sync of a connector. This action is used for analytics and monitoring. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-last-sync-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-name-description-api.asciidoc b/docs/reference/connector/apis/update-connector-name-description-api.asciidoc index 9ecf0f5d175fd..7e16874da9fb4 100644 --- a/docs/reference/connector/apis/update-connector-name-description-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-name-description-api.asciidoc @@ -9,6 +9,8 @@ beta::[] Updates the `name` and `description` fields of a connector. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-name-description-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc b/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc index f8f804595eb19..01ed2e39702ea 100644 --- a/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-pipeline-api.asciidoc @@ -10,6 +10,8 @@ Updates the `pipeline` configuration of a connector. When you create a new connector, the configuration of an <> is populated with default settings. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-pipeline-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc index 6d30197dd3390..f932f4c959de2 100644 --- a/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-scheduling-api.asciidoc @@ -8,6 +8,8 @@ beta::[] Updates the `scheduling` configuration of a connector. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-scheduling-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc index fb61a25848a9d..139e9eddf4076 100644 --- a/docs/reference/connector/apis/update-connector-service-type-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-service-type-api.asciidoc @@ -8,6 +8,8 @@ beta::[] Updates the `service_type` of a connector. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-service-type-api-request]] ==== {api-request-title} diff --git a/docs/reference/connector/apis/update-connector-status-api.asciidoc b/docs/reference/connector/apis/update-connector-status-api.asciidoc index d444c82a1bde0..ee9dfcb5f880f 100644 --- a/docs/reference/connector/apis/update-connector-status-api.asciidoc +++ b/docs/reference/connector/apis/update-connector-status-api.asciidoc @@ -8,6 +8,8 @@ preview::[] Updates the `status` of a connector. +To get started with Connector APIs, check out the {enterprise-search-ref}/connectors-tutorial-api.html[tutorial^]. + [[update-connector-status-api-request]] ==== {api-request-title} From 67a8bd219c9de211f904e5c9fcea4502d2637137 Mon Sep 17 00:00:00 2001 From: Artem Prigoda Date: Wed, 29 May 2024 11:58:47 +0200 Subject: [PATCH 04/29] AwaitsFix: https://github.com/elastic/elasticsearch/issues/109154 --- muted-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 4f64492466375..9d78a6714e35e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -35,6 +35,10 @@ tests: - class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT" issue: "https://github.com/elastic/elasticsearch/issues/108993" method: "testTrainedModelInference" +- class: "org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT" + issue: "https://github.com/elastic/elasticsearch/issues/109154" + method: "test {p0=data_stream/200_rollover_failure_store/Lazily roll over a data\ + \ stream's failure store after an ingest failure}" # Examples: # # Mute a single test case in a YAML test suite: From f94c5416e30179a4b6cc0d2f564e96ea0961f800 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 29 May 2024 11:18:24 +0100 Subject: [PATCH 05/29] Fix trappy timeouts in CCR (#109151) Relates #107984 --- .../elasticsearch/xpack/ccr/AutoFollowIT.java | 35 +++++++++---- .../elasticsearch/xpack/ccr/CcrLicenseIT.java | 5 +- .../xpack/ccr/CcrRetentionLeaseIT.java | 19 +++++-- .../xpack/ccr/CloseFollowerIndexIT.java | 2 +- .../elasticsearch/xpack/ccr/FollowInfoIT.java | 32 +++++++----- .../xpack/ccr/FollowStatsIT.java | 24 ++++++--- .../xpack/ccr/IndexFollowingIT.java | 51 +++++++++++++++---- .../xpack/ccr/LocalIndexFollowingIT.java | 10 ++-- .../xpack/ccr/RestartIndexFollowingIT.java | 6 ++- .../ccr/action/AutoFollowCoordinator.java | 3 +- .../ccr/action/TransportPutFollowAction.java | 3 +- .../xpack/ccr/rest/RestCcrStatsAction.java | 3 +- .../RestDeleteAutoFollowPatternAction.java | 4 +- .../xpack/ccr/rest/RestFollowInfoAction.java | 3 +- .../rest/RestGetAutoFollowPatternAction.java | 3 +- .../RestPauseAutoFollowPatternAction.java | 4 +- .../xpack/ccr/rest/RestPauseFollowAction.java | 3 +- .../rest/RestPutAutoFollowPatternAction.java | 5 +- .../xpack/ccr/rest/RestPutFollowAction.java | 4 +- .../RestResumeAutoFollowPatternAction.java | 4 +- .../ccr/rest/RestResumeFollowAction.java | 8 ++- .../xpack/ccr/rest/RestUnfollowAction.java | 8 ++- .../elasticsearch/xpack/CcrIntegTestCase.java | 12 +++-- .../xpack/CcrSingleNodeTestCase.java | 6 +-- ...teAutoFollowPatternActionRequestTests.java | 16 ++++-- .../action/AutoFollowCoordinatorTests.java | 24 +++++++-- .../DeleteAutoFollowPatternRequestTests.java | 2 +- .../ccr/action/FollowInfoRequestTests.java | 2 +- .../GetAutoFollowPatternRequestTests.java | 2 +- .../PutAutoFollowPatternRequestTests.java | 18 +++---- .../action/PutFollowActionRequestTests.java | 8 +-- .../ResumeFollowActionRequestTests.java | 8 +-- ...tActivateAutoFollowPatternActionTests.java | 19 +++++-- ...ortDeleteAutoFollowPatternActionTests.java | 6 +-- ...nsportPutAutoFollowPatternActionTests.java | 6 +-- .../TransportResumeFollowActionTests.java | 2 +- .../ActivateAutoFollowPatternAction.java | 5 +- .../xpack/core/ccr/action/CcrStatsAction.java | 4 +- .../action/DeleteAutoFollowPatternAction.java | 5 +- .../core/ccr/action/FollowInfoAction.java | 5 +- .../action/GetAutoFollowPatternAction.java | 5 +- .../core/ccr/action/PauseFollowAction.java | 5 +- .../action/PutAutoFollowPatternAction.java | 10 ++-- .../core/ccr/action/PutFollowAction.java | 10 ++-- .../core/ccr/action/ResumeFollowAction.java | 10 ++-- .../xpack/core/ccr/action/UnfollowAction.java | 5 +- .../core/ilm/PauseFollowerIndexStep.java | 3 +- .../core/ilm/UnfollowFollowerIndexStep.java | 2 +- .../core/ccr/action/CcrStatsActionTests.java | 8 +-- .../collector/ccr/StatsCollector.java | 2 +- 50 files changed, 290 insertions(+), 159 deletions(-) diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index 8842d9ef35fec..bbd1905374d24 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -273,7 +273,7 @@ public void testAutoFollowParameterAreDelegated() throws Exception { .build(); // Enabling auto following: - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster("leader_cluster"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); // Need to set this, because following an index in the same cluster @@ -314,7 +314,7 @@ public void testAutoFollowParameterAreDelegated() throws Exception { createLeaderIndex("logs-201901", leaderIndexSettings); assertLongBusy(() -> { - FollowInfoAction.Request followInfoRequest = new FollowInfoAction.Request(); + FollowInfoAction.Request followInfoRequest = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); followInfoRequest.setFollowerIndices("copy-logs-201901"); FollowInfoAction.Response followInfoResponse; try { @@ -662,7 +662,10 @@ public void testAutoFollowDatastreamWithClosingFollowerIndex() throws Exception .setSource("foo", "bar", DataStream.TIMESTAMP_FIELD_NAME, randomNonNegativeLong()) .get(); - PutAutoFollowPatternAction.Request followRequest = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request followRequest = new PutAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT + ); followRequest.setName("pattern-1"); followRequest.setRemoteCluster("leader_cluster"); followRequest.setLeaderIndexPatterns(List.of("logs-*")); @@ -727,7 +730,7 @@ private void putAutoFollowPatterns(String name, String[] patterns) { } private void putAutoFollowPatterns(String name, String[] patterns, List exclusionPatterns) { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setName(name); request.setRemoteCluster("leader_cluster"); request.setLeaderIndexPatterns(Arrays.asList(patterns)); @@ -742,7 +745,11 @@ private void putAutoFollowPatterns(String name, String[] patterns, List } private void deleteAutoFollowPattern(final String name) { - DeleteAutoFollowPatternAction.Request request = new DeleteAutoFollowPatternAction.Request(name); + DeleteAutoFollowPatternAction.Request request = new DeleteAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + name + ); if (randomBoolean()) { request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30))); } @@ -750,7 +757,7 @@ private void deleteAutoFollowPattern(final String name) { } private AutoFollowStats getAutoFollowStats() { - CcrStatsAction.Request request = new CcrStatsAction.Request(); + CcrStatsAction.Request request = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT); if (randomBoolean()) { request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30))); } @@ -764,7 +771,12 @@ private void createLeaderIndex(String index, Settings settings) { } private void pauseAutoFollowPattern(final String name) { - ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(name, false); + ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + name, + false + ); if (randomBoolean()) { request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30))); } @@ -772,7 +784,12 @@ private void pauseAutoFollowPattern(final String name) { } private void resumeAutoFollowPattern(final String name) { - ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(name, true); + ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + name, + true + ); if (randomBoolean()) { request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30))); } @@ -780,7 +797,7 @@ private void resumeAutoFollowPattern(final String name) { } private AutoFollowMetadata.AutoFollowPattern getAutoFollowPattern(final String name) { - GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request(); + GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT); request.setName(name); if (randomBoolean()) { request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(10, 20, 30))); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java index 549c5146e6c79..f1febd8aea550 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java @@ -111,7 +111,10 @@ public void onFailure(final Exception e) { public void testThatPutAutoFollowPatternsIsUnavailableWithNonCompliantLicense() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(1); - final PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + final PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT + ); request.setName("name"); request.setRemoteCluster("leader"); request.setLeaderIndexPatterns(Collections.singletonList("*")); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java index 339662c996492..a0b25faea9256 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRetentionLeaseIT.java @@ -472,7 +472,12 @@ public void testUnfollowRemovesRetentionLeases() throws Exception { pauseFollow(followerIndex); assertAcked(followerClient().admin().indices().close(new CloseIndexRequest(followerIndex)).actionGet()); - assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request(followerIndex)).actionGet()); + assertAcked( + followerClient().execute( + UnfollowAction.INSTANCE, + new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, followerIndex) + ).actionGet() + ); final IndicesStatsResponse afterUnfollowStats = leaderClient().admin() .indices() @@ -541,7 +546,10 @@ public void testUnfollowFailsToRemoveRetentionLeases() throws Exception { final ElasticsearchException e = expectThrows( ElasticsearchException.class, - () -> followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request(followerIndex)).actionGet() + () -> followerClient().execute( + UnfollowAction.INSTANCE, + new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, followerIndex) + ).actionGet() ); final ClusterStateResponse followerIndexClusterState = followerClient().admin() @@ -993,7 +1001,12 @@ public void onResponseReceived(final long responseRequestId, final Transport.Res pauseFollow(followerIndex); assertAcked(followerClient().admin().indices().close(new CloseIndexRequest(followerIndex)).actionGet()); - assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request(followerIndex)).actionGet()); + assertAcked( + followerClient().execute( + UnfollowAction.INSTANCE, + new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, followerIndex) + ).actionGet() + ); unfollowLatch.countDown(); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java index a2d45e443e18f..9e84cdac34008 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java @@ -74,7 +74,7 @@ public void testCloseAndReopenFollowerIndex() throws Exception { assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); ensureLeaderYellow("index1"); - PutFollowAction.Request followRequest = new PutFollowAction.Request(); + PutFollowAction.Request followRequest = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); followRequest.setRemoteCluster("leader_cluster"); followRequest.setLeaderIndex("index1"); followRequest.setFollowerIndex("index2"); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowInfoIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowInfoIT.java index efa7fcd0eec53..ab3b5f795d7a2 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowInfoIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowInfoIT.java @@ -39,7 +39,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception { followRequest = getPutFollowRequest("leader2", "follower2"); client().execute(PutFollowAction.INSTANCE, followRequest).get(); - FollowInfoAction.Request request = new FollowInfoAction.Request(); + FollowInfoAction.Request request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndices("follower1"); FollowInfoAction.Response response = client().execute(FollowInfoAction.INSTANCE, request).actionGet(); assertThat(response.getFollowInfos().size(), equalTo(1)); @@ -48,7 +48,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception { assertThat(response.getFollowInfos().get(0).getStatus(), equalTo(Status.ACTIVE)); assertThat(response.getFollowInfos().get(0).getParameters(), notNullValue()); - request = new FollowInfoAction.Request(); + request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndices("follower2"); response = client().execute(FollowInfoAction.INSTANCE, request).actionGet(); assertThat(response.getFollowInfos().size(), equalTo(1)); @@ -57,7 +57,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception { assertThat(response.getFollowInfos().get(0).getStatus(), equalTo(Status.ACTIVE)); assertThat(response.getFollowInfos().get(0).getParameters(), notNullValue()); - request = new FollowInfoAction.Request(); + request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndices("_all"); response = client().execute(FollowInfoAction.INSTANCE, request).actionGet(); response.getFollowInfos().sort(Comparator.comparing(FollowInfoAction.Response.FollowerInfo::getFollowerIndex)); @@ -72,9 +72,11 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception { assertThat(response.getFollowInfos().get(1).getParameters(), notNullValue()); // Pause follower1 index and check the follower info api: - assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet()); + assertAcked( + client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet() + ); - request = new FollowInfoAction.Request(); + request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndices("follower1"); response = client().execute(FollowInfoAction.INSTANCE, request).actionGet(); assertThat(response.getFollowInfos().size(), equalTo(1)); @@ -83,7 +85,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception { assertThat(response.getFollowInfos().get(0).getStatus(), equalTo(Status.PAUSED)); assertThat(response.getFollowInfos().get(0).getParameters(), nullValue()); - request = new FollowInfoAction.Request(); + request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndices("follower2"); response = client().execute(FollowInfoAction.INSTANCE, request).actionGet(); assertThat(response.getFollowInfos().size(), equalTo(1)); @@ -92,7 +94,7 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception { assertThat(response.getFollowInfos().get(0).getStatus(), equalTo(Status.ACTIVE)); assertThat(response.getFollowInfos().get(0).getParameters(), notNullValue()); - request = new FollowInfoAction.Request(); + request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndices("_all"); response = client().execute(FollowInfoAction.INSTANCE, request).actionGet(); response.getFollowInfos().sort(Comparator.comparing(FollowInfoAction.Response.FollowerInfo::getFollowerIndex)); @@ -106,7 +108,9 @@ public void testFollowInfoApiFollowerIndexFiltering() throws Exception { assertThat(response.getFollowInfos().get(1).getStatus(), equalTo(Status.ACTIVE)); assertThat(response.getFollowInfos().get(1).getParameters(), notNullValue()); - assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower2")).actionGet()); + assertAcked( + client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower2")).actionGet() + ); } public void testFollowInfoApiIndexMissing() throws Exception { @@ -122,16 +126,20 @@ public void testFollowInfoApiIndexMissing() throws Exception { followRequest = getPutFollowRequest("leader2", "follower2"); client().execute(PutFollowAction.INSTANCE, followRequest).get(); - FollowInfoAction.Request request1 = new FollowInfoAction.Request(); + FollowInfoAction.Request request1 = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request1.setFollowerIndices("follower3"); expectThrows(IndexNotFoundException.class, () -> client().execute(FollowInfoAction.INSTANCE, request1).actionGet()); - FollowInfoAction.Request request2 = new FollowInfoAction.Request(); + FollowInfoAction.Request request2 = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request2.setFollowerIndices("follower2", "follower3"); expectThrows(IndexNotFoundException.class, () -> client().execute(FollowInfoAction.INSTANCE, request2).actionGet()); - assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet()); - assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower2")).actionGet()); + assertAcked( + client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet() + ); + assertAcked( + client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower2")).actionGet() + ); } } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java index ccf1c72a06178..9fcee30d30b39 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java @@ -100,14 +100,18 @@ public void testFollowStatsApiFollowerIndexFiltering() throws Exception { assertThat(response.getStatsResponses().get(0).status().followerIndex(), equalTo("follower1")); assertThat(response.getStatsResponses().get(1).status().followerIndex(), equalTo("follower2")); - assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet()); - assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower2")).actionGet()); + assertAcked( + client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet() + ); + assertAcked( + client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower2")).actionGet() + ); assertBusy(() -> { - List responseList = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()) - .actionGet() - .getFollowStats() - .getStatsResponses(); + List responseList = client().execute( + CcrStatsAction.INSTANCE, + new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT) + ).actionGet().getFollowStats().getStatsResponses(); assertThat(responseList.size(), equalTo(0)); }); } @@ -139,7 +143,9 @@ public void testFollowStatsApiResourceNotFound() throws Exception { e = expectThrows(ResourceNotFoundException.class, () -> client().execute(FollowStatsAction.INSTANCE, statsRequest).actionGet()); assertThat(e.getMessage(), equalTo("No shard follow tasks for follower indices [follower2]")); - assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet()); + assertAcked( + client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet() + ); } public void testFollowStatsApiWithDeletedFollowerIndex() throws Exception { @@ -202,7 +208,9 @@ public void testFollowStatsApiIncludeShardFollowStatsWithClosedFollowerIndex() t assertThat(response.getStatsResponses().size(), equalTo(1)); assertThat(response.getStatsResponses().get(0).status().followerIndex(), equalTo("follower1")); - assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet()); + assertAcked( + client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower1")).actionGet() + ); } } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 097592a03d5d0..6361b6f89605e 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -435,7 +435,12 @@ public void testDoNotAllowPutMappingToFollower() throws Exception { assertThat(forbiddenException.status(), equalTo(RestStatus.FORBIDDEN)); pauseFollow("index-2"); followerClient().admin().indices().close(new CloseIndexRequest("index-2")).actionGet(); - assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("index-2")).actionGet()); + assertAcked( + followerClient().execute( + UnfollowAction.INSTANCE, + new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index-2") + ).actionGet() + ); followerClient().admin().indices().open(new OpenIndexRequest("index-2")).actionGet(); assertAcked(followerClient().admin().indices().putMapping(putMappingRequest).actionGet()); } @@ -468,7 +473,12 @@ public void testAddAliasAfterUnfollow() throws Exception { followerClient().execute(PutFollowAction.INSTANCE, putFollow("leader", "follower")).get(); pauseFollow("follower"); followerClient().admin().indices().close(new CloseIndexRequest("follower").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet(); - assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("follower")).actionGet()); + assertAcked( + followerClient().execute( + UnfollowAction.INSTANCE, + new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "follower") + ).actionGet() + ); followerClient().admin().indices().open(new OpenIndexRequest("follower").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet(); final IndicesAliasesRequest request = new IndicesAliasesRequest().masterNodeTimeout(TimeValue.MAX_VALUE) .addAliasAction(IndicesAliasesRequest.AliasActions.add().index("follower").alias("follower_alias")); @@ -589,7 +599,7 @@ public void testFollowIndexWithNestedField() throws Exception { } public void testUnfollowNonExistingIndex() { - PauseFollowAction.Request unfollowRequest = new PauseFollowAction.Request("non-existing-index"); + PauseFollowAction.Request unfollowRequest = new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "non-existing-index"); expectThrows(IndexNotFoundException.class, () -> followerClient().execute(PauseFollowAction.INSTANCE, unfollowRequest).actionGet()); } @@ -899,25 +909,33 @@ public void testPauseIndex() throws Exception { ); followerClient().execute(PutFollowAction.INSTANCE, putFollow("leader", "follower")).get(); assertAcked(followerClient().admin().indices().prepareCreate("regular-index").setMasterNodeTimeout(TimeValue.MAX_VALUE)); - assertAcked(followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower")).actionGet()); + assertAcked( + followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower")) + .actionGet() + ); assertThat( expectThrows( IllegalArgumentException.class, - () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower")).actionGet() + () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower")) + .actionGet() ).getMessage(), equalTo("no shard follow tasks for [follower]") ); assertThat( expectThrows( IllegalArgumentException.class, - () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("regular-index")).actionGet() + () -> followerClient().execute( + PauseFollowAction.INSTANCE, + new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "regular-index") + ).actionGet() ).getMessage(), equalTo("index [regular-index] is not a follower index") ); assertThat( expectThrows( IndexNotFoundException.class, - () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("xyz")).actionGet() + () -> followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "xyz")) + .actionGet() ).getMessage(), equalTo("no such index [xyz]") ); @@ -937,7 +955,12 @@ public void testUnfollowIndex() throws Exception { // Turn follow index into a regular index by: pausing shard follow, close index, unfollow index and then open index: pauseFollow("index2"); followerClient().admin().indices().close(new CloseIndexRequest("index2").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet(); - assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("index2")).actionGet()); + assertAcked( + followerClient().execute( + UnfollowAction.INSTANCE, + new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index2") + ).actionGet() + ); followerClient().admin().indices().open(new OpenIndexRequest("index2").masterNodeTimeout(TimeValue.MAX_VALUE)).actionGet(); ensureFollowerGreen("index2"); @@ -960,7 +983,10 @@ public void testUnknownClusterAlias() throws Exception { () -> followerClient().execute(PutFollowAction.INSTANCE, followRequest).actionGet() ); assertThat(e.getMessage(), equalTo("no such remote cluster: [another_cluster]")); - PutAutoFollowPatternAction.Request putAutoFollowRequest = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request putAutoFollowRequest = new PutAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT + ); putAutoFollowRequest.setName("name"); putAutoFollowRequest.setRemoteCluster("another_cluster"); putAutoFollowRequest.setLeaderIndexPatterns(Collections.singletonList("logs-*")); @@ -1447,7 +1473,12 @@ private void runFallBehindTest( followerClient().admin().indices().prepareClose("index2").setMasterNodeTimeout(TimeValue.MAX_VALUE).get(); pauseFollow("index2"); if (randomBoolean()) { - assertAcked(followerClient().execute(UnfollowAction.INSTANCE, new UnfollowAction.Request("index2")).actionGet()); + assertAcked( + followerClient().execute( + UnfollowAction.INSTANCE, + new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index2") + ).actionGet() + ); } final PutFollowAction.Request followRequest2 = putFollow("index1", "index2"); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java index 05fc3b037c795..41fed34ea2106 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java @@ -63,7 +63,7 @@ public void testFollowIndex() throws Exception { assertBusy(() -> assertHitCount(client().prepareSearch("follower"), firstBatchNumDocs + secondBatchNumDocs)); - PauseFollowAction.Request pauseRequest = new PauseFollowAction.Request("follower"); + PauseFollowAction.Request pauseRequest = new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "follower"); client().execute(PauseFollowAction.INSTANCE, pauseRequest); final long thirdBatchNumDocs = randomIntBetween(2, 64); @@ -136,7 +136,7 @@ public void testIndexingMetricsIncremented() throws Exception { } public void testRemoveRemoteConnection() throws Exception { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setName("my_pattern"); request.setRemoteCluster("local"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); @@ -152,7 +152,8 @@ public void testRemoveRemoteConnection() throws Exception { createIndex("logs-20200101", leaderIndexSettings); prepareIndex("logs-20200101").setSource("{}", XContentType.JSON).get(); assertBusy(() -> { - CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet(); + CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT)) + .actionGet(); assertThat( response.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(previousNumberOfSuccessfulFollowedIndices + 1) @@ -171,7 +172,8 @@ public void testRemoveRemoteConnection() throws Exception { // This new document should be replicated to follower index: prepareIndex("logs-20200101").setSource("{}", XContentType.JSON).get(); assertBusy(() -> { - CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet(); + CcrStatsAction.Response response = client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT)) + .actionGet(); assertThat( response.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(previousNumberOfSuccessfulFollowedIndices + 2) diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java index 5c152be35b509..18456b24d4618 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/RestartIndexFollowingIT.java @@ -118,12 +118,14 @@ public void testFollowIndex() throws Exception { }, 30L, TimeUnit.SECONDS); cleanRemoteCluster(); - assertAcked(followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("index2")).actionGet()); + assertAcked( + followerClient().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, "index2")).actionGet() + ); assertAcked(followerClient().admin().indices().prepareClose("index2")); final ActionFuture unfollowFuture = followerClient().execute( UnfollowAction.INSTANCE, - new UnfollowAction.Request("index2") + new UnfollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "index2") ); final ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, unfollowFuture::actionGet); assertThat(elasticsearchException.getMessage(), containsString("no such remote cluster")); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java index a7bf572e9bf73..82af24d2293cc 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java @@ -712,7 +712,8 @@ static PutFollowAction.Request generateRequest( final String leaderIndexName = indexToFollow.getName(); final String followIndexName = getFollowerIndexName(pattern, leaderIndexName); - PutFollowAction.Request request = new PutFollowAction.Request(); + // TODO use longer timeouts here? see https://github.com/elastic/elasticsearch/issues/109150 + PutFollowAction.Request request = new PutFollowAction.Request(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS); request.setRemoteCluster(remoteCluster); request.setLeaderIndex(indexToFollow.getName()); request.setFollowerIndex(followIndexName); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index a0917c1cef815..ed7587556bd28 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -294,10 +294,9 @@ private void initiateFollowing( ) { assert request.waitForActiveShards() != ActiveShardCount.DEFAULT : "PutFollowAction does not support DEFAULT."; FollowParameters parameters = request.getParameters(); - ResumeFollowAction.Request resumeFollowRequest = new ResumeFollowAction.Request(); + ResumeFollowAction.Request resumeFollowRequest = new ResumeFollowAction.Request(request.masterNodeTimeout()); resumeFollowRequest.setFollowerIndex(request.getFollowerIndex()); resumeFollowRequest.setParameters(new FollowParameters(parameters)); - resumeFollowRequest.masterNodeTimeout(request.masterNodeTimeout()); clientWithHeaders.execute( ResumeFollowAction.INSTANCE, resumeFollowRequest, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java index 7c8ebc5a66e80..ff208b0b92f88 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestCcrStatsAction.java @@ -38,11 +38,10 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) { - final CcrStatsAction.Request request = new CcrStatsAction.Request(); + final CcrStatsAction.Request request = new CcrStatsAction.Request(getMasterNodeTimeout(restRequest)); if (restRequest.hasParam("timeout")) { request.setTimeout(restRequest.paramAsTime("timeout", null)); } - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); return channel -> client.execute( CcrStatsAction.INSTANCE, request, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java index 1f96ea6be9dc5..65c00d94635c5 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestDeleteAutoFollowPatternAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ccr.rest; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -32,8 +33,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - Request request = new Request(restRequest.param("name")); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); + final var request = new Request(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, restRequest.param("name")); return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java index e20c34fe38243..12fb8690adf66 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestFollowInfoAction.java @@ -33,8 +33,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) { - final FollowInfoAction.Request request = new FollowInfoAction.Request(); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); + final var request = new FollowInfoAction.Request(getMasterNodeTimeout(restRequest)); request.setFollowerIndices(Strings.splitStringByCommaToArray(restRequest.param("index"))); return channel -> client.execute(FollowInfoAction.INSTANCE, request, new RestRefCountedChunkedToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestGetAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestGetAutoFollowPatternAction.java index 84a8d4f879e02..bb202d74d185a 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestGetAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestGetAutoFollowPatternAction.java @@ -32,9 +32,8 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - Request request = new Request(); + final var request = new Request(getMasterNodeTimeout(restRequest)); request.setName(restRequest.param("name")); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseAutoFollowPatternAction.java index 5a2ba2fe736f7..b27960d5bc4ab 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseAutoFollowPatternAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ccr.rest; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -32,8 +33,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) { - Request request = new Request(restRequest.param("name"), false); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); + final var request = new Request(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, restRequest.param("name"), false); return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseFollowAction.java index 8c0f79f0b2440..a5e967ba7ac5a 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPauseFollowAction.java @@ -32,8 +32,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - Request request = new Request(restRequest.param("index")); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); + final var request = new Request(getMasterNodeTimeout(restRequest), restRequest.param("index")); return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutAutoFollowPatternAction.java index cb42431022501..dedcb941483b0 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutAutoFollowPatternAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ccr.rest; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -40,9 +41,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient private static Request createRequest(RestRequest restRequest) throws IOException { try (XContentParser parser = restRequest.contentOrSourceParamParser()) { - Request request = Request.fromXContent(parser, restRequest.param("name")); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); - return request; + return Request.fromXContent(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, parser, restRequest.param("name")); } } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java index 162431d68fb0f..d2063657c8784 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestPutFollowAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -41,10 +42,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient private static Request createRequest(RestRequest restRequest) throws IOException { try (XContentParser parser = restRequest.contentOrSourceParamParser()) { - final Request request = Request.fromXContent(parser); + final var request = Request.fromXContent(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, parser); request.waitForActiveShards(ActiveShardCount.parseString(restRequest.param("wait_for_active_shards"))); request.setFollowerIndex(restRequest.param("index")); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); return request; } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeAutoFollowPatternAction.java index 3e51386ef1069..13ef9a6d751f7 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeAutoFollowPatternAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ccr.rest; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -32,8 +33,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest restRequest, final NodeClient client) { - Request request = new Request(restRequest.param("name"), true); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); + final var request = new Request(getMasterNodeTimeout(restRequest), TimeValue.THIRTY_SECONDS, restRequest.param("name"), true); return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeFollowAction.java index 86a00ca1ff020..b519d76c841d2 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestResumeFollowAction.java @@ -39,16 +39,14 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient } static Request createRequest(RestRequest restRequest) throws IOException { - Request request; if (restRequest.hasContentOrSourceParam()) { try (XContentParser parser = restRequest.contentOrSourceParamParser()) { - request = Request.fromXContent(parser, restRequest.param("index")); + return Request.fromXContent(getMasterNodeTimeout(restRequest), parser, restRequest.param("index")); } } else { - request = new Request(); + final var request = new Request(getMasterNodeTimeout(restRequest)); request.setFollowerIndex(restRequest.param("index")); + return request; } - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); - return request; } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestUnfollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestUnfollowAction.java index acc6ffb0a67bd..57e69b686b268 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestUnfollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/rest/RestUnfollowAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ccr.rest; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -33,8 +34,11 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - UnfollowAction.Request request = new UnfollowAction.Request(restRequest.param("index")); - request.masterNodeTimeout(getMasterNodeTimeout(restRequest)); + UnfollowAction.Request request = new UnfollowAction.Request( + getMasterNodeTimeout(restRequest), + TimeValue.THIRTY_SECONDS, + restRequest.param("index") + ); return channel -> client.execute(INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 52343be3f2c23..677a82ddafa34 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -484,7 +484,7 @@ protected void ensureEmptyWriteBuffers() throws Exception { protected void pauseFollow(String... indices) throws Exception { for (String index : indices) { - final PauseFollowAction.Request unfollowRequest = new PauseFollowAction.Request(index); + final PauseFollowAction.Request unfollowRequest = new PauseFollowAction.Request(TEST_REQUEST_TIMEOUT, index); assertAcked(followerClient().execute(PauseFollowAction.INSTANCE, unfollowRequest).actionGet()); } ensureNoCcrTasks(); @@ -492,8 +492,10 @@ protected void pauseFollow(String... indices) throws Exception { protected void ensureNoCcrTasks() throws Exception { assertBusy(() -> { - CcrStatsAction.Response statsResponse = followerClient().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()) - .actionGet(); + CcrStatsAction.Response statsResponse = followerClient().execute( + CcrStatsAction.INSTANCE, + new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT) + ).actionGet(); assertThat( "Follow stats not empty: " + Strings.toString(statsResponse.getFollowStats()), statsResponse.getFollowStats().getStatsResponses(), @@ -586,7 +588,7 @@ public static PutFollowAction.Request putFollow(String leaderIndex, String follo } public static PutFollowAction.Request putFollow(String leaderIndex, String followerIndex, ActiveShardCount waitForActiveShards) { - PutFollowAction.Request request = new PutFollowAction.Request(); + PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster("leader_cluster"); request.setLeaderIndex(leaderIndex); request.setFollowerIndex(followerIndex); @@ -602,7 +604,7 @@ public static PutFollowAction.Request putFollow(String leaderIndex, String follo } public static ResumeFollowAction.Request resumeFollow(String followerIndex) { - ResumeFollowAction.Request request = new ResumeFollowAction.Request(); + ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndex(followerIndex); request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(10)); request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10)); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java index efcaee96c008a..6b69c172c0df3 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java @@ -95,11 +95,11 @@ public void removeLocalRemote() throws Exception { } protected AutoFollowStats getAutoFollowStats() { - return client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request()).actionGet().getAutoFollowStats(); + return client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT)).actionGet().getAutoFollowStats(); } protected ResumeFollowAction.Request getResumeFollowRequest(String followerIndex) { - ResumeFollowAction.Request request = new ResumeFollowAction.Request(); + ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndex(followerIndex); request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(1)); request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(1)); @@ -110,7 +110,7 @@ protected ResumeFollowAction.Request getResumeFollowRequest(String followerIndex } protected PutFollowAction.Request getPutFollowRequest(String leaderIndex, String followerIndex) { - PutFollowAction.Request request = new PutFollowAction.Request(); + PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster("local"); request.setLeaderIndex(leaderIndex); request.setFollowerIndex(followerIndex); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java index 03b197b629d02..f9386cf126a24 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ActivateAutoFollowPatternActionRequestTests.java @@ -19,7 +19,12 @@ public class ActivateAutoFollowPatternActionRequestTests extends AbstractWireSer @Override protected ActivateAutoFollowPatternAction.Request createTestInstance() { - return new ActivateAutoFollowPatternAction.Request(randomAlphaOfLength(5), randomBoolean()); + return new ActivateAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomAlphaOfLength(5), + randomBoolean() + ); } @Override @@ -33,12 +38,17 @@ protected Writeable.Reader instanceRead } public void testValidate() { - ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request(null, true); + ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + null, + true + ); ActionRequestValidationException validationException = request.validate(); assertThat(validationException, notNullValue()); assertThat(validationException.getMessage(), containsString("[name] is missing")); - request = new ActivateAutoFollowPatternAction.Request("name", true); + request = new ActivateAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "name", true); validationException = request.validate(); assertThat(validationException, nullValue()); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index f15ca19dd590a..467ef3c68f648 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -460,7 +460,10 @@ public void testAutoFollowerWithPausedActivePatterns() { final ClusterState nextLocalClusterState; if (nextClusterStateVersion == 1) { // cluster state #1 : one pattern is active - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT + ); request.setName("patternLogs"); request.setRemoteCluster(remoteCluster); request.setLeaderIndexPatterns(singletonList("patternLogs-*")); @@ -478,7 +481,10 @@ public void testAutoFollowerWithPausedActivePatterns() { } else if (nextClusterStateVersion == 3) { // cluster state #3 : add a new pattern, two patterns are active - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT + ); request.setName("patternDocs"); request.setRemoteCluster(remoteCluster); request.setLeaderIndexPatterns(singletonList("patternDocs-*")); @@ -496,12 +502,22 @@ public void testAutoFollowerWithPausedActivePatterns() { } else if (nextClusterStateVersion == 5) { // cluster state #5 : first pattern is paused, second pattern is still active - ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request("patternLogs", false); + ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "patternLogs", + false + ); nextLocalClusterState = TransportActivateAutoFollowPatternAction.innerActivate(request, currentLocalState); } else if (nextClusterStateVersion == 6) { // cluster state #5 : second pattern is paused, both patterns are inactive - ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request("patternDocs", false); + ActivateAutoFollowPatternAction.Request request = new ActivateAutoFollowPatternAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "patternDocs", + false + ); nextLocalClusterState = TransportActivateAutoFollowPatternAction.innerActivate(request, currentLocalState); } else { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java index b4a28fd149326..1ca79401ae896 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/DeleteAutoFollowPatternRequestTests.java @@ -19,7 +19,7 @@ protected Writeable.Reader instanceReader @Override protected DeleteAutoFollowPatternAction.Request createTestInstance() { - return new DeleteAutoFollowPatternAction.Request(randomAlphaOfLength(4)); + return new DeleteAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomAlphaOfLength(4)); } @Override diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java index f08ab5aca1aa8..c809572195f94 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoRequestTests.java @@ -19,7 +19,7 @@ protected Writeable.Reader instanceReader() { @Override protected FollowInfoAction.Request createTestInstance() { - FollowInfoAction.Request request = new FollowInfoAction.Request(); + FollowInfoAction.Request request = new FollowInfoAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndices(generateRandomStringArray(4, 4, true, false)); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java index b451e07bd037d..d6b498e0bd3f9 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternRequestTests.java @@ -19,7 +19,7 @@ protected Writeable.Reader instanceReader() @Override protected GetAutoFollowPatternAction.Request createTestInstance() { - GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request(); + GetAutoFollowPatternAction.Request request = new GetAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT); if (randomBoolean()) { request.setName(randomAlphaOfLength(4)); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java index 9e2ca3d8b3013..93d363251da7e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java @@ -26,7 +26,7 @@ public class PutAutoFollowPatternRequestTests extends AbstractXContentSerializin @Override protected PutAutoFollowPatternAction.Request doParseInstance(XContentParser parser) throws IOException { - return PutAutoFollowPatternAction.Request.fromXContent(parser, null); + return PutAutoFollowPatternAction.Request.fromXContent(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, parser, null); } @Override @@ -36,7 +36,7 @@ protected Writeable.Reader instanceReader() @Override protected PutAutoFollowPatternAction.Request createTestInstance() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setName(randomAlphaOfLength(4)); request.setRemoteCluster(randomAlphaOfLength(4)); @@ -62,7 +62,7 @@ protected PutAutoFollowPatternAction.Request mutateInstance(PutAutoFollowPattern protected PutAutoFollowPatternAction.Request createXContextTestInstance(XContentType xContentType) { // follower index parameter is not part of the request body and is provided in the url path. // So this field cannot be used for creating a test instance for xcontent testing. - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster(randomAlphaOfLength(4)); request.setLeaderIndexPatterns(Arrays.asList(generateRandomStringArray(4, 4, false))); if (randomBoolean()) { @@ -78,7 +78,7 @@ protected PutAutoFollowPatternAction.Request createXContextTestInstance(XContent } public void testValidate() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); ActionRequestValidationException validationException = request.validate(); assertThat(validationException, notNullValue()); assertThat(validationException.getMessage(), containsString("[name] is missing")); @@ -118,7 +118,7 @@ public void testValidate() { } public void testValidateName() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster("_alias"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); @@ -128,7 +128,7 @@ public void testValidateName() { } public void testValidateNameComma() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster("_alias"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); @@ -139,7 +139,7 @@ public void testValidateNameComma() { } public void testValidateNameLeadingUnderscore() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster("_alias"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); @@ -150,7 +150,7 @@ public void testValidateNameLeadingUnderscore() { } public void testValidateNameUnderscores() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster("_alias"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); @@ -160,7 +160,7 @@ public void testValidateNameUnderscores() { } public void testValidateNameTooLong() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster("_alias"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java index 171727c3e0bc8..4fa3c0a11acb0 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java @@ -27,7 +27,7 @@ protected Writeable.Reader instanceReader() { @Override protected PutFollowAction.Request createTestInstance() { - PutFollowAction.Request request = new PutFollowAction.Request(); + PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setFollowerIndex(randomAlphaOfLength(4)); request.waitForActiveShards( randomFrom(ActiveShardCount.DEFAULT, ActiveShardCount.NONE, ActiveShardCount.ONE, ActiveShardCount.ALL) @@ -47,7 +47,7 @@ protected PutFollowAction.Request createTestInstance() { protected PutFollowAction.Request createXContextTestInstance(XContentType xContentType) { // follower index parameter and wait for active shards params are not part of the request body and // are provided in the url path. So these fields cannot be used for creating a test instance for xcontent testing. - PutFollowAction.Request request = new PutFollowAction.Request(); + PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setRemoteCluster(randomAlphaOfLength(4)); request.setLeaderIndex(randomAlphaOfLength(4)); request.setSettings( @@ -61,7 +61,7 @@ protected PutFollowAction.Request createXContextTestInstance(XContentType xConte @Override protected PutFollowAction.Request doParseInstance(XContentParser parser) throws IOException { - PutFollowAction.Request request = PutFollowAction.Request.fromXContent(parser); + PutFollowAction.Request request = PutFollowAction.Request.fromXContent(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, parser); request.waitForActiveShards(ActiveShardCount.DEFAULT); request.setFollowerIndex("followerIndex"); return request; @@ -69,7 +69,7 @@ protected PutFollowAction.Request doParseInstance(XContentParser parser) throws @Override protected PutFollowAction.Request mutateInstance(PutFollowAction.Request instance) { - PutFollowAction.Request request = new PutFollowAction.Request(); + PutFollowAction.Request request = new PutFollowAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setFollowerIndex(instance.getFollowerIndex()); request.waitForActiveShards(instance.waitForActiveShards()); request.setRemoteCluster(instance.getRemoteCluster()); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java index 8a835a4d1f715..475c9135d4946 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java @@ -32,7 +32,7 @@ protected Writeable.Reader instanceReader() { @Override protected ResumeFollowAction.Request createTestInstance() { - ResumeFollowAction.Request request = new ResumeFollowAction.Request(); + ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndex(randomAlphaOfLength(4)); generateFollowParameters(request.getParameters()); @@ -48,14 +48,14 @@ protected ResumeFollowAction.Request mutateInstance(ResumeFollowAction.Request i protected ResumeFollowAction.Request createXContextTestInstance(XContentType type) { // follower index parameter is not part of the request body and is provided in the url path. // So this field cannot be used for creating a test instance for xcontent testing. - ResumeFollowAction.Request request = new ResumeFollowAction.Request(); + ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT); generateFollowParameters(request.getParameters()); return request; } @Override protected ResumeFollowAction.Request doParseInstance(XContentParser parser) throws IOException { - return ResumeFollowAction.Request.fromXContent(parser, null); + return ResumeFollowAction.Request.fromXContent(TEST_REQUEST_TIMEOUT, parser, null); } static void generateFollowParameters(FollowParameters followParameters) { @@ -92,7 +92,7 @@ static void generateFollowParameters(FollowParameters followParameters) { } public void testValidate() { - ResumeFollowAction.Request request = new ResumeFollowAction.Request(); + ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndex("index2"); request.getParameters().setMaxRetryDelay(TimeValue.ZERO); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java index 206c71e82c52c..3a6a0d90f60ba 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternActionTests.java @@ -33,7 +33,10 @@ public class TransportActivateAutoFollowPatternActionTests extends ESTestCase { public void testInnerActivateNoAutoFollowMetadata() { Exception e = expectThrows( ResourceNotFoundException.class, - () -> TransportActivateAutoFollowPatternAction.innerActivate(new Request("test", true), ClusterState.EMPTY_STATE) + () -> TransportActivateAutoFollowPatternAction.innerActivate( + new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "test", true), + ClusterState.EMPTY_STATE + ) ); assertThat(e.getMessage(), equalTo("auto-follow pattern [test] is missing")); } @@ -54,7 +57,10 @@ public void testInnerActivateDoesNotExist() { .build(); Exception e = expectThrows( ResourceNotFoundException.class, - () -> TransportActivateAutoFollowPatternAction.innerActivate(new Request("does_not_exist", true), clusterState) + () -> TransportActivateAutoFollowPatternAction.innerActivate( + new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "does_not_exist", true), + clusterState + ) ); assertThat(e.getMessage(), equalTo("auto-follow pattern [does_not_exist] is missing")); } @@ -75,12 +81,17 @@ public void testInnerActivateToggle() { ) .build(); { - Request pauseRequest = new Request("remote_cluster", autoFollowPattern.isActive()); + Request pauseRequest = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "remote_cluster", autoFollowPattern.isActive()); ClusterState updatedState = TransportActivateAutoFollowPatternAction.innerActivate(pauseRequest, clusterState); assertThat(updatedState, sameInstance(clusterState)); } { - Request pauseRequest = new Request("remote_cluster", autoFollowPattern.isActive() == false); + Request pauseRequest = new Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + "remote_cluster", + autoFollowPattern.isActive() == false + ); ClusterState updatedState = TransportActivateAutoFollowPatternAction.innerActivate(pauseRequest, clusterState); assertThat(updatedState, not(sameInstance(clusterState))); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java index 470c15d86dfb1..f94f23c3695bf 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportDeleteAutoFollowPatternActionTests.java @@ -101,7 +101,7 @@ public void testInnerDelete() { ) .build(); - Request request = new Request("name1"); + Request request = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "name1"); AutoFollowMetadata result = TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState) .getMetadata() .custom(AutoFollowMetadata.TYPE); @@ -154,7 +154,7 @@ public void testInnerDeleteDoesNotExist() { ) .build(); - Request request = new Request("name2"); + Request request = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "name2"); Exception e = expectThrows( ResourceNotFoundException.class, () -> TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState) @@ -165,7 +165,7 @@ public void testInnerDeleteDoesNotExist() { public void testInnerDeleteNoAutoFollowMetadata() { ClusterState clusterState = ClusterState.builder(new ClusterName("us_cluster")).metadata(Metadata.builder()).build(); - Request request = new Request("name1"); + Request request = new Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, "name1"); Exception e = expectThrows( ResourceNotFoundException.class, () -> TransportDeleteAutoFollowPatternAction.innerDelete(request, clusterState) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java index cd2bca527bfb9..ea37cdf1aae0c 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java @@ -31,7 +31,7 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { public void testInnerPut() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setName("name1"); request.setRemoteCluster("eu_cluster"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); @@ -73,7 +73,7 @@ public void testInnerPut() { } public void testInnerPut_existingLeaderIndices() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setName("name1"); request.setRemoteCluster("eu_cluster"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); @@ -129,7 +129,7 @@ public void testInnerPut_existingLeaderIndices() { } public void testInnerPut_existingLeaderIndicesAndAutoFollowMetadata() { - PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT); request.setName("name1"); request.setRemoteCluster("eu_cluster"); request.setLeaderIndexPatterns(Arrays.asList("logs-*", "transactions-*")); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 1313e5781f122..7de0d775ba150 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -36,7 +36,7 @@ public class TransportResumeFollowActionTests extends ESTestCase { public static ResumeFollowAction.Request resumeFollow(String followerIndex) { - ResumeFollowAction.Request request = new ResumeFollowAction.Request(); + ResumeFollowAction.Request request = new ResumeFollowAction.Request(TEST_REQUEST_TIMEOUT); request.setFollowerIndex(followerIndex); request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(10)); request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ActivateAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ActivateAutoFollowPatternAction.java index df917b4e97b7d..acab22f620fa0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ActivateAutoFollowPatternAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ActivateAutoFollowPatternAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; import java.util.Objects; @@ -33,8 +34,8 @@ public static class Request extends AcknowledgedRequest { private final String name; private final boolean active; - public Request(final String name, final boolean active) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, final String name, final boolean active) { + super(masterNodeTimeout, ackTimeout); this.name = name; this.active = active; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java index b187e5e39dd33..a0ecc5dd566c2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsAction.java @@ -45,8 +45,8 @@ public Request(StreamInput in) throws IOException { } } - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); + public Request(TimeValue masterNodeTimeout) { + super(masterNodeTimeout); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java index e38a1cfd4a2cb..cdacb184744b2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/DeleteAutoFollowPatternAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; import java.util.Objects; @@ -31,8 +32,8 @@ public static class Request extends AcknowledgedRequest { private final String name; - public Request(String name) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String name) { + super(masterNodeTimeout, ackTimeout); this.name = name; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java index d979a4cf44b9b..821559e16fa38 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; @@ -41,8 +42,8 @@ public static class Request extends MasterNodeReadRequest { private String[] followerIndices; - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); + public Request(TimeValue masterNodeTimeout) { + super(masterNodeTimeout); } public String[] getFollowerIndices() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java index bd6ab5bb5af44..5f08a4c253005 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/GetAutoFollowPatternAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; @@ -34,8 +35,8 @@ public static class Request extends MasterNodeReadRequest { private String name; - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); + public Request(TimeValue masterNodeTimeout) { + super(masterNodeTimeout); } public Request(StreamInput in) throws IOException { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java index c6905b2d06a34..54687001fbbbb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PauseFollowAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; import java.util.Objects; @@ -30,8 +31,8 @@ public static class Request extends MasterNodeRequest { private final String followIndex; - public Request(String followIndex) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); + public Request(TimeValue masterNodeTimeout, String followIndex) { + super(masterNodeTimeout); this.followIndex = Objects.requireNonNull(followIndex, "followIndex"); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java index 333171d864c4f..dcee7274632eb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -62,9 +63,10 @@ public static class Request extends AcknowledgedRequest implements ToXC FollowParameters.initParser(PARSER); } - public static Request fromXContent(XContentParser parser, String name) throws IOException { + public static Request fromXContent(TimeValue masterNodeTimeout, TimeValue ackTimeout, XContentParser parser, String name) + throws IOException { PutAutoFollowPatternParameters parameters = PARSER.parse(parser, null); - Request request = new Request(); + Request request = new Request(masterNodeTimeout, ackTimeout); request.setName(name); request.setRemoteCluster(parameters.remoteCluster); request.setLeaderIndexPatterns(parameters.leaderIndexPatterns); @@ -85,8 +87,8 @@ public static Request fromXContent(XContentParser parser, String name) throws IO private FollowParameters parameters = new FollowParameters(); private List leaderIndexExclusionPatterns = Collections.emptyList(); - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout) { + super(masterNodeTimeout, ackTimeout); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java index db1e84aca9cda..82941c440484d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; @@ -64,10 +65,11 @@ public static final class Request extends AcknowledgedRequest implement FollowParameters.initParser(PARSER); } - public static Request fromXContent(final XContentParser parser) throws IOException { + public static Request fromXContent(TimeValue masterNodeTimeout, TimeValue ackTimeout, final XContentParser parser) + throws IOException { PutFollowParameters parameters = PARSER.parse(parser, null); - Request request = new Request(); + Request request = new Request(masterNodeTimeout, ackTimeout); request.setRemoteCluster(parameters.remoteCluster); request.setLeaderIndex(parameters.leaderIndex); request.setDataStreamName(parameters.dataStreamName); @@ -85,8 +87,8 @@ public static Request fromXContent(final XContentParser parser) throws IOExcepti private FollowParameters parameters = new FollowParameters(); private ActiveShardCount waitForActiveShards = ActiveShardCount.NONE; - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout) { + super(masterNodeTimeout, ackTimeout); } public String getFollowerIndex() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java index 12ddea8d99578..04c77dbbad952 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -43,9 +44,10 @@ public static class Request extends MasterNodeRequest implements ToXCon FollowParameters.initParser(PARSER); } - public static Request fromXContent(final XContentParser parser, final String followerIndex) throws IOException { + public static Request fromXContent(TimeValue masterNodeTimeout, final XContentParser parser, final String followerIndex) + throws IOException { FollowParameters parameters = PARSER.parse(parser, null); - Request request = new Request(); + Request request = new Request(masterNodeTimeout); request.setFollowerIndex(followerIndex); request.setParameters(parameters); return request; @@ -54,8 +56,8 @@ public static Request fromXContent(final XContentParser parser, final String fol private String followerIndex; private FollowParameters parameters = new FollowParameters(); - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); + public Request(TimeValue masterNodeTimeout) { + super(masterNodeTimeout); } public String getFollowerIndex() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java index 9a5f011f39a1b..1647b67a0e6ec 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/UnfollowAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; @@ -33,8 +34,8 @@ public static class Request extends AcknowledgedRequest implements Indi private final String followerIndex; - public Request(String followerIndex) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); + public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String followerIndex) { + super(masterNodeTimeout, ackTimeout); this.followerIndex = followerIndex; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java index 5cecd0eb924f5..d900f168be06c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/PauseFollowerIndexStep.java @@ -52,8 +52,7 @@ void innerPerformAction(String followerIndex, ClusterState currentClusterState, return; } - PauseFollowAction.Request request = new PauseFollowAction.Request(followerIndex); - request.masterNodeTimeout(TimeValue.MAX_VALUE); + PauseFollowAction.Request request = new PauseFollowAction.Request(TimeValue.MAX_VALUE, followerIndex); getClient().execute(PauseFollowAction.INSTANCE, request, listener.delegateFailureAndWrap((l, r) -> { if (r.isAcknowledged() == false) { throw new ElasticsearchException("pause follow request failed to be acknowledged"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStep.java index 8b40a23cc3c44..6e07d4e6ac823 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/UnfollowFollowerIndexStep.java @@ -33,7 +33,7 @@ public boolean isRetryable() { @Override void innerPerformAction(String followerIndex, ClusterState currentClusterState, ActionListener listener) { - UnfollowAction.Request request = new UnfollowAction.Request(followerIndex).masterNodeTimeout(TimeValue.MAX_VALUE); + final var request = new UnfollowAction.Request(TimeValue.MAX_VALUE, TimeValue.MAX_VALUE, followerIndex); getClient().execute(UnfollowAction.INSTANCE, request, ActionListener.wrap(r -> { if (r.isAcknowledged() == false) { throw new ElasticsearchException("unfollow request failed to be acknowledged"); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java index fab9f8de5339f..7c1e71139cb0c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ccr/action/CcrStatsActionTests.java @@ -24,7 +24,7 @@ protected Writeable.Reader instanceReader() { @Override protected CcrStatsAction.Request createTestInstance() { - var request = new CcrStatsAction.Request(); + var request = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT); request.setTimeout(TimeValue.timeValueSeconds(randomFrom(1, 5, 10, 15))); request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(1, 5, 10, 15))); return request; @@ -34,13 +34,13 @@ protected CcrStatsAction.Request createTestInstance() { protected CcrStatsAction.Request mutateInstance(CcrStatsAction.Request instance) throws IOException { return switch (randomInt(1)) { case 0 -> { - var mutatedInstance = new CcrStatsAction.Request(); + var mutatedInstance = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT); mutatedInstance.setTimeout(instance.getTimeout()); mutatedInstance.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30))); yield mutatedInstance; } case 1 -> { - var mutatedInstance = new CcrStatsAction.Request(); + var mutatedInstance = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT); mutatedInstance.setTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30))); mutatedInstance.masterNodeTimeout(instance.masterNodeTimeout()); yield mutatedInstance; @@ -51,7 +51,7 @@ protected CcrStatsAction.Request mutateInstance(CcrStatsAction.Request instance) public void testSerializationBwc() throws IOException { // In previous version `timeout` is not set - var request = new CcrStatsAction.Request(); + var request = new CcrStatsAction.Request(TEST_REQUEST_TIMEOUT); if (randomBoolean()) { request.masterNodeTimeout(TimeValue.timeValueSeconds(randomFrom(20, 25, 30))); } diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java index 9691756285f76..514540a85eb3b 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/ccr/StatsCollector.java @@ -78,7 +78,7 @@ protected Collection doCollect(final MonitoringDoc.Node node, fin final long timestamp = timestamp(); final String clusterUuid = clusterUuid(clusterState); - final CcrStatsAction.Request request = new CcrStatsAction.Request(); + final CcrStatsAction.Request request = new CcrStatsAction.Request(getCollectionTimeout()); final CcrStatsAction.Response response = client.execute(CcrStatsAction.INSTANCE, request).actionGet(getCollectionTimeout()); final AutoFollowStatsMonitoringDoc autoFollowStatsDoc = new AutoFollowStatsMonitoringDoc( From bdaaa9b838e5529f32de2261fec68102d6436769 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 29 May 2024 07:30:05 -0400 Subject: [PATCH 06/29] ESQL: Drop more "nested" query support (#109130) This was all inherited from QL, but ESQL doesn't support `nested` fields yet. And, when it does, it's not going to support nested fields in the same way as QL. And, finally, none of these methods for nested support weren't called by ESQL at all. So they were only used by a few tests. --- .../esql/core/querydsl/query/BoolQuery.java | 31 ---- .../esql/core/querydsl/query/ExistsQuery.java | 2 +- .../core/querydsl/query/GeoDistanceQuery.java | 2 +- .../esql/core/querydsl/query/LeafQuery.java | 33 ---- .../esql/core/querydsl/query/MatchAll.java | 2 +- .../esql/core/querydsl/query/MatchQuery.java | 2 +- .../core/querydsl/query/MultiMatchQuery.java | 2 +- .../esql/core/querydsl/query/NestedQuery.java | 164 ------------------ .../esql/core/querydsl/query/NotQuery.java | 20 --- .../esql/core/querydsl/query/PrefixQuery.java | 2 +- .../xpack/esql/core/querydsl/query/Query.java | 22 --- .../core/querydsl/query/QueryStringQuery.java | 2 +- .../esql/core/querydsl/query/RangeQuery.java | 2 +- .../esql/core/querydsl/query/RegexQuery.java | 2 +- .../esql/core/querydsl/query/TermQuery.java | 2 +- .../esql/core/querydsl/query/TermsQuery.java | 2 +- .../core/querydsl/query/WildcardQuery.java | 2 +- .../core/querydsl/query/BoolQueryTests.java | 90 ++-------- .../core/querydsl/query/LeafQueryTests.java | 23 +-- .../core/querydsl/query/NestedQueryTests.java | 143 --------------- .../esql/querydsl/query/SingleValueQuery.java | 16 -- .../querydsl/query/SpatialRelatesQuery.java | 16 -- 22 files changed, 27 insertions(+), 555 deletions(-) delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQuery.java delete mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQuery.java delete mode 100644 x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQueryTests.java diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java index 3c0723b62d711..dbd75c93ee0e7 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQuery.java @@ -8,12 +8,10 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.util.CollectionUtils; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -44,35 +42,6 @@ public BoolQuery(Source source, boolean isAnd, List queries) { this.queries = queries; } - @Override - public boolean containsNestedField(String path, String field) { - for (Query query : queries) { - if (query.containsNestedField(path, field)) { - return true; - } - } - return false; - } - - @Override - public Query addNestedField(String path, String field, String format, boolean hasDocValues) { - boolean unchanged = true; - List rewritten = new ArrayList<>(queries.size()); - for (Query query : queries) { - var rewrittenQuery = query.addNestedField(path, field, format, hasDocValues); - unchanged &= rewrittenQuery == query; - rewritten.add(rewrittenQuery); - } - return unchanged ? this : new BoolQuery(source(), isAnd, rewritten); - } - - @Override - public void enrichNestedSort(NestedSortBuilder sort) { - for (Query query : queries) { - query.enrichNestedSort(sort); - } - } - @Override public QueryBuilder asBuilder() { BoolQueryBuilder boolQuery = boolQuery(); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java index 7633f7f3f1e9c..be585232cf8d6 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/ExistsQuery.java @@ -11,7 +11,7 @@ import static org.elasticsearch.index.query.QueryBuilders.existsQuery; -public class ExistsQuery extends LeafQuery { +public class ExistsQuery extends Query { private final String name; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java index f3c8829c2159b..f7843cec0c88c 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/GeoDistanceQuery.java @@ -13,7 +13,7 @@ import java.util.Objects; -public class GeoDistanceQuery extends LeafQuery { +public class GeoDistanceQuery extends Query { private final String field; private final double lat; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQuery.java deleted file mode 100644 index 786114a52c7d5..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQuery.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.querydsl.query; - -import org.elasticsearch.search.sort.NestedSortBuilder; -import org.elasticsearch.xpack.esql.core.tree.Source; - -abstract class LeafQuery extends Query { - LeafQuery(Source source) { - super(source); - } - - @Override - public final boolean containsNestedField(String path, String field) { - // No leaf queries are nested - return false; - } - - @Override - public Query addNestedField(String path, String field, String format, boolean hasDocValues) { - // No leaf queries are nested - return this; - } - - @Override - public void enrichNestedSort(NestedSortBuilder sort) { - // No leaf queries are nested - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java index 13f624a283b16..6415a69e2201d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchAll.java @@ -11,7 +11,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -public class MatchAll extends LeafQuery { +public class MatchAll extends Query { public MatchAll(Source source) { super(source); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java index 8c9934af9dfa6..3b7948d37cfad 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MatchQuery.java @@ -22,7 +22,7 @@ import static java.util.Map.entry; -public class MatchQuery extends LeafQuery { +public class MatchQuery extends Query { private static final Map> BUILDER_APPLIERS; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java index 8bede4f16d00e..71e3cb9fd494a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/MultiMatchQuery.java @@ -21,7 +21,7 @@ import static java.util.Map.entry; -public class MultiMatchQuery extends LeafQuery { +public class MultiMatchQuery extends Query { private static final Map> BUILDER_APPLIERS; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQuery.java deleted file mode 100644 index 65c953cdf818d..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQuery.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.querydsl.query; - -import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.fetch.StoredFieldsContext; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.search.sort.NestedSortBuilder; -import org.elasticsearch.xpack.esql.core.tree.Source; - -import java.util.AbstractMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; -import static java.util.Collections.unmodifiableMap; -import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; - -/** - * A query to a nested document. - */ -public class NestedQuery extends Query { - private static long COUNTER = 0; - // TODO: make this configurable - private static final int MAX_INNER_HITS = 99; - private static final List NO_STORED_FIELD = singletonList(StoredFieldsContext._NONE_); - - private final String path; - private final Map> fields; // field -> (useDocValues, format) - private final Query child; - - public NestedQuery(Source source, String path, Query child) { - this(source, path, emptyMap(), child); - } - - public NestedQuery(Source source, String path, Map> fields, Query child) { - super(source); - if (path == null) { - throw new IllegalArgumentException("path is required"); - } - if (fields == null) { - throw new IllegalArgumentException("fields is required"); - } - if (child == null) { - throw new IllegalArgumentException("child is required"); - } - this.path = path; - this.fields = fields; - this.child = child; - } - - @Override - public boolean containsNestedField(String otherPath, String field) { - boolean iContainThisField = this.path.equals(otherPath) && fields.containsKey(field); - boolean myChildContainsThisField = child.containsNestedField(otherPath, field); - return iContainThisField || myChildContainsThisField; - } - - @Override - public Query addNestedField(String otherPath, String field, String format, boolean hasDocValues) { - if (false == this.path.equals(otherPath)) { - // I'm not at the right path so let my child query have a crack at it - Query rewrittenChild = child.addNestedField(otherPath, field, format, hasDocValues); - if (rewrittenChild == child) { - return this; - } - return new NestedQuery(source(), otherPath, fields, rewrittenChild); - } - if (fields.containsKey(field)) { - // I already have the field, no rewriting needed - return this; - } - Map> newFields = Maps.newMapWithExpectedSize(fields.size() + 1); - newFields.putAll(fields); - newFields.put(field, new AbstractMap.SimpleImmutableEntry<>(hasDocValues, format)); - return new NestedQuery(source(), otherPath, unmodifiableMap(newFields), child); - } - - @Override - public void enrichNestedSort(NestedSortBuilder sort) { - child.enrichNestedSort(sort); - if (false == sort.getPath().equals(path)) { - return; - } - - // TODO: Add all filters in nested sorting when https://github.com/elastic/elasticsearch/issues/33079 is implemented - // Adding multiple filters to sort sections makes sense for nested queries where multiple conditions belong to the same - // nested query. The current functionality creates one nested query for each condition involving a nested field. - QueryBuilder childAsBuilder = child.asBuilder(); - if (sort.getFilter() != null && false == sort.getFilter().equals(childAsBuilder)) { - // throw new SqlIllegalArgumentException("nested query should have been grouped in one place"); - return; - } - sort.setFilter(childAsBuilder); - } - - @Override - public QueryBuilder asBuilder() { - // disable score - NestedQueryBuilder query = nestedQuery(path, child.asBuilder(), ScoreMode.None); - - if (fields.isEmpty() == false) { - InnerHitBuilder ihb = new InnerHitBuilder(); - ihb.setSize(0); - ihb.setSize(MAX_INNER_HITS); - ihb.setName(path + "_" + COUNTER++); - - for (Map.Entry> entry : fields.entrySet()) { - if (entry.getValue().getKey()) { - ihb.addFetchField(entry.getKey(), entry.getValue().getValue()); - } else { - ihb.addFetchField(entry.getKey()); - } - } - ihb.setFetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE); - ihb.setStoredFieldNames(NO_STORED_FIELD); - - query.innerHit(ihb); - } - - return query; - } - - String path() { - return path; - } - - Map> fields() { - return fields; - } - - Query child() { - return child; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), path, fields, child); - } - - @Override - public boolean equals(Object obj) { - if (false == super.equals(obj)) { - return false; - } - NestedQuery other = (NestedQuery) obj; - return path.equals(other.path) && fields.equals(other.fields) && child.equals(other.child); - } - - @Override - protected String innerToString() { - return path + "." + fields + "[" + child + "]"; - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java index 4b87bbed80d71..4e36a4ee9f053 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/NotQuery.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.core.querydsl.query; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.xpack.esql.core.tree.Source; import java.util.Objects; @@ -32,25 +31,6 @@ public Query child() { return child; } - @Override - public boolean containsNestedField(String path, String field) { - return child.containsNestedField(path, field); - } - - @Override - public Query addNestedField(String path, String field, String format, boolean hasDocValues) { - Query rewrittenChild = child.addNestedField(path, field, format, hasDocValues); - if (child == rewrittenChild) { - return this; - } - return new NotQuery(source(), child); - } - - @Override - public void enrichNestedSort(NestedSortBuilder sort) { - child.enrichNestedSort(sort); - } - @Override public QueryBuilder asBuilder() { return boolQuery().mustNot(child.asBuilder()); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java index 65dd014e72b70..1d98ff53be2f2 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/PrefixQuery.java @@ -13,7 +13,7 @@ import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; -public class PrefixQuery extends LeafQuery { +public class PrefixQuery extends Query { private final String field, query; private final boolean caseInsensitive; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java index 19db6aa79846a..f3154eb6cd377 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/Query.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.core.querydsl.query; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.xpack.esql.core.tree.Source; /** @@ -45,27 +44,6 @@ public Source source() { return source; } - /** - * Does this query contain a particular nested field? - */ - public abstract boolean containsNestedField(String path, String field); - - /** - * Rewrite this query to one that contains the specified nested field. - *

- * Used to make sure that we fetch nested fields even if they aren't - * explicitly part of the query. - * @return a new query if we could add the nested field, the same query - * instance otherwise - */ - public abstract Query addNestedField(String path, String field, String format, boolean hasDocValues); - - /** - * Attach the one and only one matching nested query's filter to this - * sort. - */ - public abstract void enrichNestedSort(NestedSortBuilder sort); - /** * Convert to an Elasticsearch {@link QueryBuilder} all set up to execute * the query. diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java index 32cd661cab579..8ac90e6314174 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/QueryStringQuery.java @@ -24,7 +24,7 @@ import static java.util.Map.entry; -public class QueryStringQuery extends LeafQuery { +public class QueryStringQuery extends Query { // TODO: it'd be great if these could be constants instead of Strings, needs a core change to make the fields public first private static final Map> BUILDER_APPLIERS = Map.ofEntries( diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java index 745e24fe52fc7..2d66ee86d0f61 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RangeQuery.java @@ -16,7 +16,7 @@ import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -public class RangeQuery extends LeafQuery { +public class RangeQuery extends Query { private final String field; private final Object lower, upper; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java index 9117c32723d76..a8e48de654196 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/RegexQuery.java @@ -13,7 +13,7 @@ import static org.elasticsearch.index.query.QueryBuilders.regexpQuery; -public class RegexQuery extends LeafQuery { +public class RegexQuery extends Query { private final String field, regex; private final boolean caseInsensitive; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java index b195be3741ad1..240f9f581b27e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermQuery.java @@ -14,7 +14,7 @@ import static org.elasticsearch.index.query.QueryBuilders.termQuery; -public class TermQuery extends LeafQuery { +public class TermQuery extends Query { private final String term; private final Object value; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java index a35a41ef09aa4..5b0920929853a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TermsQuery.java @@ -14,7 +14,7 @@ import static org.elasticsearch.index.query.QueryBuilders.termsQuery; -public class TermsQuery extends LeafQuery { +public class TermsQuery extends Query { private final String term; private final Set values; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java index be2c771ced815..9266f2b43d081 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java @@ -14,7 +14,7 @@ import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; -public class WildcardQuery extends LeafQuery { +public class WildcardQuery extends Query { private final String field, query; private final boolean caseInsensitive; diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java index c2f517aba4e1b..1c9d6bc54aebf 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/BoolQueryTests.java @@ -6,18 +6,17 @@ */ package org.elasticsearch.xpack.esql.core.querydsl.query; -import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.SourceTests; import org.elasticsearch.xpack.esql.core.util.StringUtils; -import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Function; +import java.util.function.Supplier; -import static java.util.Collections.singletonMap; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -25,12 +24,16 @@ public class BoolQueryTests extends ESTestCase { static BoolQuery randomBoolQuery(int depth) { - return new BoolQuery( - SourceTests.randomSource(), - randomBoolean(), - NestedQueryTests.randomQuery(depth), - NestedQueryTests.randomQuery(depth) - ); + return new BoolQuery(SourceTests.randomSource(), randomBoolean(), randomQuery(depth), randomQuery(depth)); + } + + static Query randomQuery(int depth) { + List> options = new ArrayList<>(); + options.add(MatchQueryTests::randomMatchQuery); + if (depth > 0) { + options.add(() -> BoolQueryTests.randomBoolQuery(depth - 1)); + } + return randomFrom(options).get(); } public void testEqualsAndHashCode() { @@ -45,77 +48,12 @@ private static BoolQuery mutate(BoolQuery query) { List> options = Arrays.asList( q -> new BoolQuery(SourceTests.mutate(q.source()), q.isAnd(), left(q), right(q)), q -> new BoolQuery(q.source(), false == q.isAnd(), left(q), right(q)), - q -> new BoolQuery(q.source(), q.isAnd(), randomValueOtherThan(left(q), () -> NestedQueryTests.randomQuery(5)), right(q)), - q -> new BoolQuery(q.source(), q.isAnd(), left(q), randomValueOtherThan(right(q), () -> NestedQueryTests.randomQuery(5))) + q -> new BoolQuery(q.source(), q.isAnd(), randomValueOtherThan(left(q), () -> randomQuery(5)), right(q)), + q -> new BoolQuery(q.source(), q.isAnd(), left(q), randomValueOtherThan(right(q), () -> randomQuery(5))) ); return randomFrom(options).apply(query); } - public void testContainsNestedField() { - assertFalse(boolQueryWithoutNestedChildren().containsNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5))); - - String path = randomAlphaOfLength(5); - String field = randomAlphaOfLength(5); - assertTrue(boolQueryWithNestedChildren(path, field).containsNestedField(path, field)); - } - - public void testAddNestedField() { - Query q = boolQueryWithoutNestedChildren(); - assertSame(q, q.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean())); - - String path = randomAlphaOfLength(5); - String field = randomAlphaOfLength(5); - q = boolQueryWithNestedChildren(path, field); - String newField = randomAlphaOfLength(5); - boolean hasDocValues = randomBoolean(); - Query rewritten = q.addNestedField(path, newField, null, hasDocValues); - assertNotSame(q, rewritten); - assertTrue(rewritten.containsNestedField(path, newField)); - } - - public void testEnrichNestedSort() { - Query q = boolQueryWithoutNestedChildren(); - NestedSortBuilder sort = new NestedSortBuilder(randomAlphaOfLength(5)); - q.enrichNestedSort(sort); - assertNull(sort.getFilter()); - - String path = randomAlphaOfLength(5); - String field = randomAlphaOfLength(5); - q = boolQueryWithNestedChildren(path, field); - sort = new NestedSortBuilder(path); - q.enrichNestedSort(sort); - assertNotNull(sort.getFilter()); - } - - private Query boolQueryWithoutNestedChildren() { - return new BoolQuery( - SourceTests.randomSource(), - randomBoolean(), - new MatchAll(SourceTests.randomSource()), - new MatchAll(SourceTests.randomSource()) - ); - } - - private Query boolQueryWithNestedChildren(String path, String field) { - NestedQuery match = new NestedQuery( - SourceTests.randomSource(), - path, - singletonMap(field, new SimpleImmutableEntry<>(randomBoolean(), null)), - new MatchAll(SourceTests.randomSource()) - ); - Query matchAll = new MatchAll(SourceTests.randomSource()); - Query left; - Query right; - if (randomBoolean()) { - left = match; - right = matchAll; - } else { - left = matchAll; - right = match; - } - return new BoolQuery(SourceTests.randomSource(), randomBoolean(), left, right); - } - public void testToString() { assertEquals( "BoolQuery@1:2[ExistsQuery@1:2[f1] AND ExistsQuery@1:8[f2]]", diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java index 1ac96b6d30e8c..15c49f58572cb 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/LeafQueryTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.core.querydsl.query; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.tree.Location; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -18,7 +17,7 @@ import static org.hamcrest.Matchers.equalTo; public class LeafQueryTests extends ESTestCase { - private static class DummyLeafQuery extends LeafQuery { + private static class DummyLeafQuery extends Query { private DummyLeafQuery(Source source) { super(source); } @@ -47,26 +46,6 @@ private static DummyLeafQuery mutate(DummyLeafQuery query) { return new DummyLeafQuery(SourceTests.mutate(query.source())); } - public void testContainsNestedField() { - Query query = new DummyLeafQuery(SourceTests.randomSource()); - // Leaf queries don't contain nested fields. - assertFalse(query.containsNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5))); - } - - public void testAddNestedField() { - Query query = new DummyLeafQuery(SourceTests.randomSource()); - // Leaf queries don't contain nested fields. - assertSame(query, query.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean())); - } - - public void testEnrichNestedSort() { - Query query = new DummyLeafQuery(SourceTests.randomSource()); - // Leaf queries don't contain nested fields. - NestedSortBuilder sort = new NestedSortBuilder(randomAlphaOfLength(5)); - query.enrichNestedSort(sort); - assertNull(sort.getFilter()); - } - public void testNot() { var q = new LeafQueryTests.DummyLeafQuery(new Source(Location.EMPTY, "test")); assertThat(q.negate(new Source(Location.EMPTY, "not")), equalTo(new NotQuery(new Source(Location.EMPTY, "not"), q))); diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQueryTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQueryTests.java deleted file mode 100644 index 1a36531efec52..0000000000000 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/querydsl/query/NestedQueryTests.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.querydsl.query; - -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.search.sort.NestedSortBuilder; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.tree.SourceTests; -import org.elasticsearch.xpack.esql.core.util.StringUtils; - -import java.util.AbstractMap.SimpleImmutableEntry; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.function.Supplier; - -import static java.util.Collections.singletonMap; -import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; -import static org.hamcrest.Matchers.hasEntry; - -public class NestedQueryTests extends ESTestCase { - static Query randomQuery(int depth) { - List> options = new ArrayList<>(); - options.add(MatchQueryTests::randomMatchQuery); - if (depth > 0) { - options.add(() -> randomNestedQuery(depth - 1)); - options.add(() -> BoolQueryTests.randomBoolQuery(depth - 1)); - } - return randomFrom(options).get(); - } - - static NestedQuery randomNestedQuery(int depth) { - return new NestedQuery(SourceTests.randomSource(), randomAlphaOfLength(5), randomFields(), randomQuery(depth)); - } - - private static Map> randomFields() { - int size = between(0, 5); - Map> fields = Maps.newMapWithExpectedSize(size); - while (fields.size() < size) { - fields.put(randomAlphaOfLength(5), new SimpleImmutableEntry<>(randomBoolean(), null)); - } - return fields; - } - - public void testEqualsAndHashCode() { - checkEqualsAndHashCode(randomNestedQuery(5), NestedQueryTests::copy, NestedQueryTests::mutate); - } - - private static NestedQuery copy(NestedQuery query) { - return new NestedQuery(query.source(), query.path(), query.fields(), query.child()); - } - - private static NestedQuery mutate(NestedQuery query) { - List> options = Arrays.asList( - q -> new NestedQuery(SourceTests.mutate(q.source()), q.path(), q.fields(), q.child()), - q -> new NestedQuery(q.source(), randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)), q.fields(), q.child()), - q -> new NestedQuery(q.source(), q.path(), randomValueOtherThan(q.fields(), NestedQueryTests::randomFields), q.child()), - q -> new NestedQuery(q.source(), q.path(), q.fields(), randomValueOtherThan(q.child(), () -> randomQuery(5))) - ); - return randomFrom(options).apply(query); - } - - public void testContainsNestedField() { - NestedQuery q = randomNestedQuery(0); - for (String field : q.fields().keySet()) { - assertTrue(q.containsNestedField(q.path(), field)); - assertFalse(q.containsNestedField(randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)), field)); - } - assertFalse(q.containsNestedField(q.path(), randomValueOtherThanMany(q.fields()::containsKey, () -> randomAlphaOfLength(5)))); - } - - public void testAddNestedField() { - NestedQuery q = randomNestedQuery(0); - for (String field : q.fields().keySet()) { - // add does nothing if the field is already there - assertSame(q, q.addNestedField(q.path(), field, null, randomBoolean())); - String otherPath = randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)); - // add does nothing if the path doesn't match - assertSame(q, q.addNestedField(otherPath, randomAlphaOfLength(5), null, randomBoolean())); - } - - // if the field isn't in the list then add rewrites to a query with all the old fields and the new one - String newField = randomValueOtherThanMany(q.fields()::containsKey, () -> randomAlphaOfLength(5)); - boolean hasDocValues = randomBoolean(); - NestedQuery added = (NestedQuery) q.addNestedField(q.path(), newField, null, hasDocValues); - assertNotSame(q, added); - assertThat(added.fields(), hasEntry(newField, new SimpleImmutableEntry<>(hasDocValues, null))); - assertTrue(added.containsNestedField(q.path(), newField)); - for (Map.Entry> field : q.fields().entrySet()) { - assertThat(added.fields(), hasEntry(field.getKey(), field.getValue())); - assertTrue(added.containsNestedField(q.path(), field.getKey())); - } - } - - public void testEnrichNestedSort() { - NestedQuery q = randomNestedQuery(0); - - // enrich adds the filter if the path matches - { - NestedSortBuilder sort = new NestedSortBuilder(q.path()); - q.enrichNestedSort(sort); - assertEquals(q.child().asBuilder(), sort.getFilter()); - } - - // but doesn't if it doesn't match - { - NestedSortBuilder sort = new NestedSortBuilder(randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5))); - q.enrichNestedSort(sort); - assertNull(sort.getFilter()); - } - - // enriching with the same query twice is fine - { - NestedSortBuilder sort = new NestedSortBuilder(q.path()); - q.enrichNestedSort(sort); - assertEquals(q.child().asBuilder(), sort.getFilter()); - q.enrichNestedSort(sort); - - // But enriching using another query will keep only the first query - Query originalChildQuery = randomValueOtherThan(q.child(), () -> randomQuery(0)); - NestedQuery other = new NestedQuery(SourceTests.randomSource(), q.path(), q.fields(), originalChildQuery); - other.enrichNestedSort(sort); - assertEquals(other.child().asBuilder(), originalChildQuery.asBuilder()); - } - } - - public void testToString() { - NestedQuery q = new NestedQuery( - new Source(1, 1, StringUtils.EMPTY), - "a.b", - singletonMap("f", new SimpleImmutableEntry<>(true, null)), - new MatchAll(new Source(1, 1, StringUtils.EMPTY)) - ); - assertEquals("NestedQuery@1:2[a.b.{f=true=null}[MatchAll@1:2[]]]", q.toString()); - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 8dd1cbdef1386..6545e892741d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -39,7 +39,6 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -83,21 +82,6 @@ public SingleValueQuery(Query next, String field) { this.field = field; } - @Override - public boolean containsNestedField(String path, String field) { - return next.containsNestedField(path, field); - } - - @Override - public Query addNestedField(String path, String field, String format, boolean hasDocValues) { - return next.addNestedField(path, field, format, hasDocValues); - } - - @Override - public void enrichNestedSort(NestedSortBuilder sort) { - next.enrichNestedSort(sort); - } - @Override public Builder asBuilder() { return new Builder(next.asBuilder(), field, new Stats(), next.source()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java index c7875e79533a7..f02dd63999e27 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java @@ -31,7 +31,6 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.lucene.spatial.CartesianShapeDocValuesQuery; -import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -57,21 +56,6 @@ public SpatialRelatesQuery(Source source, String field, ShapeField.QueryRelation this.dataType = dataType; } - @Override - public boolean containsNestedField(String path, String field) { - return false; - } - - @Override - public Query addNestedField(String path, String field, String format, boolean hasDocValues) { - return null; - } - - @Override - public void enrichNestedSort(NestedSortBuilder sort) { - - } - @Override public QueryBuilder asBuilder() { return EsqlDataTypes.isSpatialGeo(dataType) ? new GeoShapeQueryBuilder() : new CartesianShapeQueryBuilder(); From b6241711ef49deeaa2d550bd34d4838d0414301c Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Wed, 29 May 2024 14:00:19 +0200 Subject: [PATCH 07/29] [DOCS] Update CCS matrix for 8.14 (#109142) --- .../ccs-version-compat-matrix.asciidoc | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc index c1d279d3163e8..4a5efe09ea5a0 100644 --- a/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc +++ b/docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc @@ -1,23 +1,24 @@ -[cols="^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^"] +[cols="^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^,^"] |==== -| 17+^h| Remote cluster version +| 18+^h| Remote cluster version h| Local cluster version - | 6.8 | 7.1–7.16 | 7.17 | 8.0 | 8.1 | 8.2 | 8.3 | 8.4 | 8.5 |8.6 |8.7 |8.8 |8.9 |8.10 |8.11 |8.12 | 8.13 -| 6.8 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} -| 7.1–7.16 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} -| 7.17 | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.0 | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.1 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.2 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.3 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}|{yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.4 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.5 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} |{yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.6 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.7 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.8 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.9 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.10 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.11 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.12 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} -| 8.13 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon}| {yes-icon} | {yes-icon} -|==== \ No newline at end of file + | 6.8 | 7.1–7.16 | 7.17 | 8.0 | 8.1 | 8.2 | 8.3 | 8.4 | 8.5 | 8.6 | 8.7 | 8.8 | 8.9 | 8.10 | 8.11 | 8.12 | 8.13 | 8.14 +| 6.8 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} +| 7.1–7.16 | {yes-icon} | {yes-icon} | {yes-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} +| 7.17 | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.0 | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.1 | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.2 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.3 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.4 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.5 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.6 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.7 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.8 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.9 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.10 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.11 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.12 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.13 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} +| 8.14 | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} +|==== From 849c33edeb638efb59860fe1795dc543fa7073b7 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Wed, 29 May 2024 08:02:05 -0400 Subject: [PATCH 08/29] Add SearchOrgs to changelog schema (#109135) --- .../src/main/resources/changelog-schema.json | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json index 2a0cda7fa33c9..8bdfb8d6c0b9c 100644 --- a/build-tools-internal/src/main/resources/changelog-schema.json +++ b/build-tools-internal/src/main/resources/changelog-schema.json @@ -39,6 +39,8 @@ "EQL", "ES|QL", "Engine", + "Experiences", + "Extract&Transform", "FIPS", "Features", "Geo", @@ -48,6 +50,7 @@ "ILM+SLM", "IdentityProvider", "Indices APIs", + "Inference", "Infra/CLI", "Infra/Circuit Breakers", "Infra/Core", @@ -76,6 +79,7 @@ "Ranking", "Recovery", "Reindex", + "Relevance", "Rollup", "SQL", "Search", @@ -174,7 +178,9 @@ } }, "then": { - "required": ["breaking"] + "required": [ + "breaking" + ] } }, { @@ -186,7 +192,9 @@ } }, "then": { - "required": ["breaking"] + "required": [ + "breaking" + ] } } ], @@ -198,7 +206,9 @@ } }, "then": { - "required": ["deprecation"] + "required": [ + "deprecation" + ] }, "additionalProperties": false }, From 1e58f3a48557e14c791a8f3c7c3d5329c9278cd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Wed, 29 May 2024 14:56:59 +0200 Subject: [PATCH 09/29] [DOCS] Fixes sparse vector query docs. (#109153) --- docs/reference/query-dsl/sparse-vector-query.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/query-dsl/sparse-vector-query.asciidoc b/docs/reference/query-dsl/sparse-vector-query.asciidoc index cb3f855afaa51..9a269ad9712a8 100644 --- a/docs/reference/query-dsl/sparse-vector-query.asciidoc +++ b/docs/reference/query-dsl/sparse-vector-query.asciidoc @@ -36,6 +36,7 @@ GET _search ---- // TEST[skip: Requires inference] +[discrete] === Example request using precomputed vectors [source,console] From 86652bde14e00b330402b2ed5366c0f3fe27300a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 29 May 2024 09:05:00 -0400 Subject: [PATCH 10/29] ESQL: Make `Source` `Writeable` (#109137) We'd love to move all of the serialization from `PlanNamedTypes` into the the types themselves. But serialization for `Source` is in ESQL proper, not ESQL-core with all of the types that need to be serialized. This moves that serialization. It does so by making an interface in ESQL-core that contains the `query` itself which is required to serialize `Source`. This seems like the simplest way to get started moving serialization. The shim *should* be temporary, but it'll take some time to move things around. --- .../xpack/esql/core/tree/Source.java | 66 +++- .../xpack/esql/core/util/PlanStreamInput.java | 26 ++ .../xpack/esql/core/util/SourceUtils.java | 76 +--- .../xpack/esql/io/stream/PlanNamedTypes.java | 324 +++++++++--------- .../xpack/esql/io/stream/PlanStreamInput.java | 17 +- .../esql/io/stream/PlanStreamOutput.java | 12 - .../xpack/ql/util/SourceUtils.java | 93 ----- 7 files changed, 283 insertions(+), 331 deletions(-) create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java delete mode 100644 x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/SourceUtils.java diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java index 23d05bdf99273..2129bd8743d0f 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/tree/Source.java @@ -7,9 +7,17 @@ package org.elasticsearch.xpack.esql.core.tree; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; +import org.elasticsearch.xpack.esql.core.util.StringUtils; + +import java.io.IOException; import java.util.Objects; -public final class Source { +public final class Source implements Writeable { public static final Source EMPTY = new Source(Location.EMPTY, ""); @@ -25,6 +33,31 @@ public Source(Location location, String text) { this.text = text; } + public static Source readFrom(S in) throws IOException { + if (in.readBoolean() == false) { + return EMPTY; + } + int line = in.readInt(); + int column = in.readInt(); + int charPositionInLine = column - 1; + + int length = in.readInt(); + String text = sourceText(in.sourceText(), line, column, length); + return new Source(new Location(line, charPositionInLine), text); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + if (this == EMPTY) { + out.writeBoolean(false); + return; + } + out.writeBoolean(true); + out.writeInt(location.getLineNumber()); + out.writeInt(location.getColumnNumber()); + out.writeInt(text.length()); + } + // TODO: rename to location() public Location source() { return location; @@ -61,4 +94,35 @@ public String toString() { public static Source synthetic(String text) { return new Source(Location.EMPTY, text); } + + private static String sourceText(String query, int line, int column, int length) { + if (line <= 0 || column <= 0 || query.isEmpty()) { + return StringUtils.EMPTY; + } + int offset = textOffset(query, line, column); + if (offset + length > query.length()) { + throw new QlIllegalArgumentException( + "location [@" + line + ":" + column + "] and length [" + length + "] overrun query size [" + query.length() + "]" + ); + } + return query.substring(offset, offset + length); + } + + private static int textOffset(String query, int line, int column) { + int offset = 0; + if (line > 1) { + String[] lines = query.split("\n"); + if (line > lines.length) { + throw new QlIllegalArgumentException( + "line location [" + line + "] higher than max [" + lines.length + "] in query [" + query + "]" + ); + } + for (int i = 0; i < line - 1; i++) { + offset += lines[i].length() + 1; // +1 accounts for the removed \n + } + } + offset += column - 1; // -1 since column is 1-based indexed + return offset; + } + } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java new file mode 100644 index 0000000000000..51898f36d5497 --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/PlanStreamInput.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.util; + +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * Interface for streams that can serialize plan components. This exists so + * ESQL proper can expose streaming capability to ESQL-core. If the world is kind + * and just we'll remove this when we flatten everything from ESQL-core into + * ESQL proper. + */ +public interface PlanStreamInput { + /** + * The query sent by the user to build this plan. This is used to rebuild + * {@link Source} without sending the query over the wire over and over + * and over again. + */ + String sourceText(); + +} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java index cbcb31c7a3b56..6bed7f06fa2f5 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/SourceUtils.java @@ -9,8 +9,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.tree.Location; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -20,74 +18,30 @@ public final class SourceUtils { private SourceUtils() {} + /** + * Write a {@link Source} including the text in it. + * @deprecated replace with {@link Source#writeTo}. + * That's not binary compatible so the replacement is complex. + */ + @Deprecated public static void writeSource(StreamOutput out, Source source) throws IOException { - writeSource(out, source, true); - } - - public static void writeSourceNoText(StreamOutput out, Source source) throws IOException { - writeSource(out, source, false); - } - - public static Source readSource(StreamInput in) throws IOException { - return readSource(in, null); - } - - public static Source readSourceWithText(StreamInput in, String queryText) throws IOException { - return readSource(in, queryText); - } - - private static void writeSource(StreamOutput out, Source source, boolean writeText) throws IOException { out.writeInt(source.source().getLineNumber()); out.writeInt(source.source().getColumnNumber()); - if (writeText) { - out.writeString(source.text()); - } else { - out.writeInt(source.text().length()); - } + out.writeString(source.text()); } - private static Source readSource(StreamInput in, @Nullable String queryText) throws IOException { + /** + * Read a {@link Source} including the text in it. + * @deprecated replace with {@link Source#readFrom(StreamInput)}. + * That's not binary compatible so the replacement is complex. + */ + @Deprecated + public static Source readSource(StreamInput in) throws IOException { int line = in.readInt(); int column = in.readInt(); int charPositionInLine = column - 1; - String text; - if (queryText == null) { - text = in.readString(); - } else { - int length = in.readInt(); - text = sourceText(queryText, line, column, length); - } + String text = in.readString(); return new Source(new Location(line, charPositionInLine), text); } - - private static String sourceText(String query, int line, int column, int length) { - if (line <= 0 || column <= 0 || query.isEmpty()) { - return StringUtils.EMPTY; - } - int offset = textOffset(query, line, column); - if (offset + length > query.length()) { - throw new QlIllegalArgumentException( - "location [@" + line + ":" + column + "] and length [" + length + "] overrun query size [" + query.length() + "]" - ); - } - return query.substring(offset, offset + length); - } - - private static int textOffset(String query, int line, int column) { - int offset = 0; - if (line > 1) { - String[] lines = query.split("\n"); - if (line > lines.length) { - throw new QlIllegalArgumentException( - "line location [" + line + "] higher than max [" + lines.length + "] in query [" + query + "]" - ); - } - for (int i = 0; i < line - 1; i++) { - offset += lines[i].length() + 1; // +1 accounts for the removed \n - } - } - offset += column - 1; // -1 since column is 1-based indexed - return offset; - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index d64d70470c605..e846360258ebe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -458,7 +458,7 @@ public static List namedTypeEntries() { // -- physical plan nodes static AggregateExec readAggregateExec(PlanStreamInput in) throws IOException { return new AggregateExec( - in.readSource(), + Source.readFrom(in), in.readPhysicalPlanNode(), in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)), readNamedExpressions(in), @@ -468,7 +468,7 @@ static AggregateExec readAggregateExec(PlanStreamInput in) throws IOException { } static void writeAggregateExec(PlanStreamOutput out, AggregateExec aggregateExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(aggregateExec.child()); out.writeCollection(aggregateExec.groupings(), writerFromPlanWriter(PlanStreamOutput::writeExpression)); writeNamedExpressions(out, aggregateExec.aggregates()); @@ -477,11 +477,17 @@ static void writeAggregateExec(PlanStreamOutput out, AggregateExec aggregateExec } static DissectExec readDissectExec(PlanStreamInput in) throws IOException { - return new DissectExec(in.readSource(), in.readPhysicalPlanNode(), in.readExpression(), readDissectParser(in), readAttributes(in)); + return new DissectExec( + Source.readFrom(in), + in.readPhysicalPlanNode(), + in.readExpression(), + readDissectParser(in), + readAttributes(in) + ); } static void writeDissectExec(PlanStreamOutput out, DissectExec dissectExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(dissectExec.child()); out.writeExpression(dissectExec.inputExpression()); writeDissectParser(out, dissectExec.parser()); @@ -490,7 +496,7 @@ static void writeDissectExec(PlanStreamOutput out, DissectExec dissectExec) thro static EsQueryExec readEsQueryExec(PlanStreamInput in) throws IOException { return new EsQueryExec( - in.readSource(), + Source.readFrom(in), readEsIndex(in), readIndexMode(in), readAttributes(in), @@ -503,7 +509,7 @@ static EsQueryExec readEsQueryExec(PlanStreamInput in) throws IOException { static void writeEsQueryExec(PlanStreamOutput out, EsQueryExec esQueryExec) throws IOException { assert esQueryExec.children().size() == 0; - out.writeNoSource(); + Source.EMPTY.writeTo(out); writeEsIndex(out, esQueryExec.index()); writeIndexMode(out, esQueryExec.indexMode()); writeAttributes(out, esQueryExec.output()); @@ -515,7 +521,7 @@ static void writeEsQueryExec(PlanStreamOutput out, EsQueryExec esQueryExec) thro static EsSourceExec readEsSourceExec(PlanStreamInput in) throws IOException { return new EsSourceExec( - in.readSource(), + Source.readFrom(in), readEsIndex(in), readAttributes(in), in.readOptionalNamedWriteable(QueryBuilder.class), @@ -524,7 +530,7 @@ static EsSourceExec readEsSourceExec(PlanStreamInput in) throws IOException { } static void writeEsSourceExec(PlanStreamOutput out, EsSourceExec esSourceExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); writeEsIndex(out, esSourceExec.index()); writeAttributes(out, esSourceExec.output()); out.writeOptionalNamedWriteable(esSourceExec.query()); @@ -548,17 +554,17 @@ static void writeIndexMode(StreamOutput out, IndexMode indexMode) throws IOExcep } static EvalExec readEvalExec(PlanStreamInput in) throws IOException { - return new EvalExec(in.readSource(), in.readPhysicalPlanNode(), readAliases(in)); + return new EvalExec(Source.readFrom(in), in.readPhysicalPlanNode(), readAliases(in)); } static void writeEvalExec(PlanStreamOutput out, EvalExec evalExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(evalExec.child()); writeAliases(out, evalExec.fields()); } static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException { - final Source source = in.readSource(); + final Source source = Source.readFrom(in); final PhysicalPlan child = in.readPhysicalPlanNode(); final NamedExpression matchField = in.readNamedExpression(); final String policyName = in.readString(); @@ -593,7 +599,7 @@ static EnrichExec readEnrichExec(PlanStreamInput in) throws IOException { } static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(enrich.child()); out.writeNamedExpression(enrich.matchField()); out.writeString(enrich.policyName()); @@ -616,29 +622,29 @@ static void writeEnrichExec(PlanStreamOutput out, EnrichExec enrich) throws IOEx } static ExchangeExec readExchangeExec(PlanStreamInput in) throws IOException { - return new ExchangeExec(in.readSource(), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode()); + return new ExchangeExec(Source.readFrom(in), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode()); } static void writeExchangeExec(PlanStreamOutput out, ExchangeExec exchangeExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); writeAttributes(out, exchangeExec.output()); out.writeBoolean(exchangeExec.isInBetweenAggs()); out.writePhysicalPlanNode(exchangeExec.child()); } static ExchangeSinkExec readExchangeSinkExec(PlanStreamInput in) throws IOException { - return new ExchangeSinkExec(in.readSource(), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode()); + return new ExchangeSinkExec(Source.readFrom(in), readAttributes(in), in.readBoolean(), in.readPhysicalPlanNode()); } static void writeExchangeSinkExec(PlanStreamOutput out, ExchangeSinkExec exchangeSinkExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); writeAttributes(out, exchangeSinkExec.output()); out.writeBoolean(exchangeSinkExec.isIntermediateAgg()); out.writePhysicalPlanNode(exchangeSinkExec.child()); } static ExchangeSourceExec readExchangeSourceExec(PlanStreamInput in) throws IOException { - return new ExchangeSourceExec(in.readSource(), readAttributes(in), in.readBoolean()); + return new ExchangeSourceExec(Source.readFrom(in), readAttributes(in), in.readBoolean()); } static void writeExchangeSourceExec(PlanStreamOutput out, ExchangeSourceExec exchangeSourceExec) throws IOException { @@ -647,28 +653,28 @@ static void writeExchangeSourceExec(PlanStreamOutput out, ExchangeSourceExec exc } static FieldExtractExec readFieldExtractExec(PlanStreamInput in) throws IOException { - return new FieldExtractExec(in.readSource(), in.readPhysicalPlanNode(), readAttributes(in)); + return new FieldExtractExec(Source.readFrom(in), in.readPhysicalPlanNode(), readAttributes(in)); } static void writeFieldExtractExec(PlanStreamOutput out, FieldExtractExec fieldExtractExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(fieldExtractExec.child()); writeAttributes(out, fieldExtractExec.attributesToExtract()); } static FilterExec readFilterExec(PlanStreamInput in) throws IOException { - return new FilterExec(in.readSource(), in.readPhysicalPlanNode(), in.readExpression()); + return new FilterExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readExpression()); } static void writeFilterExec(PlanStreamOutput out, FilterExec filterExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(filterExec.child()); out.writeExpression(filterExec.condition()); } static FragmentExec readFragmentExec(PlanStreamInput in) throws IOException { return new FragmentExec( - in.readSource(), + Source.readFrom(in), in.readLogicalPlanNode(), in.readOptionalNamedWriteable(QueryBuilder.class), in.readOptionalVInt(), @@ -677,7 +683,7 @@ static FragmentExec readFragmentExec(PlanStreamInput in) throws IOException { } static void writeFragmentExec(PlanStreamOutput out, FragmentExec fragmentExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(fragmentExec.fragment()); out.writeOptionalNamedWriteable(fragmentExec.esFilter()); out.writeOptionalVInt(fragmentExec.estimatedRowSize()); @@ -689,7 +695,7 @@ static void writeFragmentExec(PlanStreamOutput out, FragmentExec fragmentExec) t static GrokExec readGrokExec(PlanStreamInput in) throws IOException { Source source; return new GrokExec( - source = in.readSource(), + source = Source.readFrom(in), in.readPhysicalPlanNode(), in.readExpression(), Grok.pattern(source, in.readString()), @@ -698,7 +704,7 @@ static GrokExec readGrokExec(PlanStreamInput in) throws IOException { } static void writeGrokExec(PlanStreamOutput out, GrokExec grokExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(grokExec.child()); out.writeExpression(grokExec.inputExpression()); out.writeString(grokExec.pattern().pattern()); @@ -706,21 +712,21 @@ static void writeGrokExec(PlanStreamOutput out, GrokExec grokExec) throws IOExce } static LimitExec readLimitExec(PlanStreamInput in) throws IOException { - return new LimitExec(in.readSource(), in.readPhysicalPlanNode(), in.readNamed(Expression.class)); + return new LimitExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readNamed(Expression.class)); } static void writeLimitExec(PlanStreamOutput out, LimitExec limitExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(limitExec.child()); out.writeExpression(limitExec.limit()); } static MvExpandExec readMvExpandExec(PlanStreamInput in) throws IOException { - return new MvExpandExec(in.readSource(), in.readPhysicalPlanNode(), in.readNamedExpression(), in.readAttribute()); + return new MvExpandExec(Source.readFrom(in), in.readPhysicalPlanNode(), in.readNamedExpression(), in.readAttribute()); } static void writeMvExpandExec(PlanStreamOutput out, MvExpandExec mvExpandExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(mvExpandExec.child()); out.writeNamedExpression(mvExpandExec.target()); out.writeAttribute(mvExpandExec.expanded()); @@ -728,52 +734,52 @@ static void writeMvExpandExec(PlanStreamOutput out, MvExpandExec mvExpandExec) t static OrderExec readOrderExec(PlanStreamInput in) throws IOException { return new OrderExec( - in.readSource(), + Source.readFrom(in), in.readPhysicalPlanNode(), in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)) ); } static void writeOrderExec(PlanStreamOutput out, OrderExec orderExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(orderExec.child()); out.writeCollection(orderExec.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder)); } static ProjectExec readProjectExec(PlanStreamInput in) throws IOException { - return new ProjectExec(in.readSource(), in.readPhysicalPlanNode(), readNamedExpressions(in)); + return new ProjectExec(Source.readFrom(in), in.readPhysicalPlanNode(), readNamedExpressions(in)); } static void writeProjectExec(PlanStreamOutput out, ProjectExec projectExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(projectExec.child()); writeNamedExpressions(out, projectExec.projections()); } static RowExec readRowExec(PlanStreamInput in) throws IOException { - return new RowExec(in.readSource(), readAliases(in)); + return new RowExec(Source.readFrom(in), readAliases(in)); } static void writeRowExec(PlanStreamOutput out, RowExec rowExec) throws IOException { assert rowExec.children().size() == 0; - out.writeNoSource(); + Source.EMPTY.writeTo(out); writeAliases(out, rowExec.fields()); } @SuppressWarnings("unchecked") static ShowExec readShowExec(PlanStreamInput in) throws IOException { - return new ShowExec(in.readSource(), readAttributes(in), (List>) in.readGenericValue()); + return new ShowExec(Source.readFrom(in), readAttributes(in), (List>) in.readGenericValue()); } static void writeShowExec(PlanStreamOutput out, ShowExec showExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); writeAttributes(out, showExec.output()); out.writeGenericValue(showExec.values()); } static TopNExec readTopNExec(PlanStreamInput in) throws IOException { return new TopNExec( - in.readSource(), + Source.readFrom(in), in.readPhysicalPlanNode(), in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)), in.readNamed(Expression.class), @@ -782,7 +788,7 @@ static TopNExec readTopNExec(PlanStreamInput in) throws IOException { } static void writeTopNExec(PlanStreamOutput out, TopNExec topNExec) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(topNExec.child()); out.writeCollection(topNExec.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder)); out.writeExpression(topNExec.limit()); @@ -792,7 +798,7 @@ static void writeTopNExec(PlanStreamOutput out, TopNExec topNExec) throws IOExce // -- Logical plan nodes static Aggregate readAggregate(PlanStreamInput in) throws IOException { return new Aggregate( - in.readSource(), + Source.readFrom(in), in.readLogicalPlanNode(), in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)), readNamedExpressions(in) @@ -800,18 +806,18 @@ static Aggregate readAggregate(PlanStreamInput in) throws IOException { } static void writeAggregate(PlanStreamOutput out, Aggregate aggregate) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(aggregate.child()); out.writeCollection(aggregate.groupings(), writerFromPlanWriter(PlanStreamOutput::writeExpression)); writeNamedExpressions(out, aggregate.aggregates()); } static Dissect readDissect(PlanStreamInput in) throws IOException { - return new Dissect(in.readSource(), in.readLogicalPlanNode(), in.readExpression(), readDissectParser(in), readAttributes(in)); + return new Dissect(Source.readFrom(in), in.readLogicalPlanNode(), in.readExpression(), readDissectParser(in), readAttributes(in)); } static void writeDissect(PlanStreamOutput out, Dissect dissect) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(dissect.child()); out.writeExpression(dissect.input()); writeDissectParser(out, dissect.parser()); @@ -819,7 +825,7 @@ static void writeDissect(PlanStreamOutput out, Dissect dissect) throws IOExcepti } static EsRelation readEsRelation(PlanStreamInput in) throws IOException { - Source source = in.readSource(); + Source source = Source.readFrom(in); EsIndex esIndex = readEsIndex(in); List attributes = readAttributes(in); if (supportingEsSourceOptions(in.getTransportVersion())) { @@ -832,7 +838,7 @@ static EsRelation readEsRelation(PlanStreamInput in) throws IOException { static void writeEsRelation(PlanStreamOutput out, EsRelation relation) throws IOException { assert relation.children().size() == 0; - out.writeNoSource(); + Source.EMPTY.writeTo(out); writeEsIndex(out, relation.index()); writeAttributes(out, relation.output()); if (supportingEsSourceOptions(out.getTransportVersion())) { @@ -866,11 +872,11 @@ private static void writeEsSourceOptions(PlanStreamOutput out) throws IOExceptio } static Eval readEval(PlanStreamInput in) throws IOException { - return new Eval(in.readSource(), in.readLogicalPlanNode(), readAliases(in)); + return new Eval(Source.readFrom(in), in.readLogicalPlanNode(), readAliases(in)); } static void writeEval(PlanStreamOutput out, Eval eval) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(eval.child()); writeAliases(out, eval.fields()); } @@ -880,7 +886,7 @@ static Enrich readEnrich(PlanStreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { mode = in.readEnum(Enrich.Mode.class); } - final Source source = in.readSource(); + final Source source = Source.readFrom(in); final LogicalPlan child = in.readLogicalPlanNode(); final Expression policyName = in.readExpression(); final NamedExpression matchField = in.readNamedExpression(); @@ -906,7 +912,7 @@ static void writeEnrich(PlanStreamOutput out, Enrich enrich) throws IOException out.writeEnum(enrich.mode()); } - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(enrich.child()); out.writeExpression(enrich.policyName()); out.writeNamedExpression(enrich.matchField()); @@ -930,21 +936,21 @@ static void writeEnrich(PlanStreamOutput out, Enrich enrich) throws IOException } static EsqlProject readEsqlProject(PlanStreamInput in) throws IOException { - return new EsqlProject(in.readSource(), in.readLogicalPlanNode(), readNamedExpressions(in)); + return new EsqlProject(Source.readFrom(in), in.readLogicalPlanNode(), readNamedExpressions(in)); } static void writeEsqlProject(PlanStreamOutput out, EsqlProject project) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(project.child()); writeNamedExpressions(out, project.projections()); } static Filter readFilter(PlanStreamInput in) throws IOException { - return new Filter(in.readSource(), in.readLogicalPlanNode(), in.readExpression()); + return new Filter(Source.readFrom(in), in.readLogicalPlanNode(), in.readExpression()); } static void writeFilter(PlanStreamOutput out, Filter filter) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(filter.child()); out.writeExpression(filter.condition()); } @@ -952,7 +958,7 @@ static void writeFilter(PlanStreamOutput out, Filter filter) throws IOException static Grok readGrok(PlanStreamInput in) throws IOException { Source source; return new Grok( - source = in.readSource(), + source = Source.readFrom(in), in.readLogicalPlanNode(), in.readExpression(), Grok.pattern(source, in.readString()), @@ -961,7 +967,7 @@ static Grok readGrok(PlanStreamInput in) throws IOException { } static void writeGrok(PlanStreamOutput out, Grok grok) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(grok.child()); out.writeExpression(grok.input()); out.writeString(grok.parser().pattern()); @@ -969,21 +975,21 @@ static void writeGrok(PlanStreamOutput out, Grok grok) throws IOException { } static Limit readLimit(PlanStreamInput in) throws IOException { - return new Limit(in.readSource(), in.readNamed(Expression.class), in.readLogicalPlanNode()); + return new Limit(Source.readFrom(in), in.readNamed(Expression.class), in.readLogicalPlanNode()); } static void writeLimit(PlanStreamOutput out, Limit limit) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(limit.limit()); out.writeLogicalPlanNode(limit.child()); } static MvExpand readMvExpand(PlanStreamInput in) throws IOException { - return new MvExpand(in.readSource(), in.readLogicalPlanNode(), in.readNamedExpression(), in.readAttribute()); + return new MvExpand(Source.readFrom(in), in.readLogicalPlanNode(), in.readNamedExpression(), in.readAttribute()); } static void writeMvExpand(PlanStreamOutput out, MvExpand mvExpand) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(mvExpand.child()); out.writeNamedExpression(mvExpand.target()); out.writeAttribute(mvExpand.expanded()); @@ -991,31 +997,31 @@ static void writeMvExpand(PlanStreamOutput out, MvExpand mvExpand) throws IOExce static OrderBy readOrderBy(PlanStreamInput in) throws IOException { return new OrderBy( - in.readSource(), + Source.readFrom(in), in.readLogicalPlanNode(), in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)) ); } static void writeOrderBy(PlanStreamOutput out, OrderBy order) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(order.child()); out.writeCollection(order.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder)); } static Project readProject(PlanStreamInput in) throws IOException { - return new Project(in.readSource(), in.readLogicalPlanNode(), readNamedExpressions(in)); + return new Project(Source.readFrom(in), in.readLogicalPlanNode(), readNamedExpressions(in)); } static void writeProject(PlanStreamOutput out, Project project) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(project.child()); writeNamedExpressions(out, project.projections()); } static TopN readTopN(PlanStreamInput in) throws IOException { return new TopN( - in.readSource(), + Source.readFrom(in), in.readLogicalPlanNode(), in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)), in.readNamed(Expression.class) @@ -1023,7 +1029,7 @@ static TopN readTopN(PlanStreamInput in) throws IOException { } static void writeTopN(PlanStreamOutput out, TopN topN) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(topN.child()); out.writeCollection(topN.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder)); out.writeExpression(topN.limit()); @@ -1059,7 +1065,7 @@ static void writeAliases(PlanStreamOutput out, List aliases) throws IOExc static FieldAttribute readFieldAttribute(PlanStreamInput in) throws IOException { return new FieldAttribute( - in.readSource(), + Source.readFrom(in), in.readOptionalWithReader(PlanNamedTypes::readFieldAttribute), in.readString(), in.dataTypeFromTypeName(in.readString()), @@ -1072,7 +1078,7 @@ static FieldAttribute readFieldAttribute(PlanStreamInput in) throws IOException } static void writeFieldAttribute(PlanStreamOutput out, FieldAttribute fileAttribute) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeOptionalWriteable(fileAttribute.parent() == null ? null : o -> writeFieldAttribute(out, fileAttribute.parent())); out.writeString(fileAttribute.name()); out.writeString(fileAttribute.dataType().typeName()); @@ -1085,7 +1091,7 @@ static void writeFieldAttribute(PlanStreamOutput out, FieldAttribute fileAttribu static ReferenceAttribute readReferenceAttr(PlanStreamInput in) throws IOException { return new ReferenceAttribute( - in.readSource(), + Source.readFrom(in), in.readString(), in.dataTypeFromTypeName(in.readString()), in.readOptionalString(), @@ -1096,7 +1102,7 @@ static ReferenceAttribute readReferenceAttr(PlanStreamInput in) throws IOExcepti } static void writeReferenceAttr(PlanStreamOutput out, ReferenceAttribute referenceAttribute) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeString(referenceAttribute.name()); out.writeString(referenceAttribute.dataType().typeName()); out.writeOptionalString(referenceAttribute.qualifier()); @@ -1107,7 +1113,7 @@ static void writeReferenceAttr(PlanStreamOutput out, ReferenceAttribute referenc static MetadataAttribute readMetadataAttr(PlanStreamInput in) throws IOException { return new MetadataAttribute( - in.readSource(), + Source.readFrom(in), in.readString(), in.dataTypeFromTypeName(in.readString()), in.readOptionalString(), @@ -1119,7 +1125,7 @@ static MetadataAttribute readMetadataAttr(PlanStreamInput in) throws IOException } static void writeMetadataAttr(PlanStreamOutput out, MetadataAttribute metadataAttribute) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeString(metadataAttribute.name()); out.writeString(metadataAttribute.dataType().typeName()); out.writeOptionalString(metadataAttribute.qualifier()); @@ -1131,7 +1137,7 @@ static void writeMetadataAttr(PlanStreamOutput out, MetadataAttribute metadataAt static UnsupportedAttribute readUnsupportedAttr(PlanStreamInput in) throws IOException { return new UnsupportedAttribute( - in.readSource(), + Source.readFrom(in), in.readString(), readUnsupportedEsField(in), in.readOptionalString(), @@ -1140,7 +1146,7 @@ static UnsupportedAttribute readUnsupportedAttr(PlanStreamInput in) throws IOExc } static void writeUnsupportedAttr(PlanStreamOutput out, UnsupportedAttribute unsupportedAttribute) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeString(unsupportedAttribute.name()); writeUnsupportedEsField(out, unsupportedAttribute.field()); out.writeOptionalString(unsupportedAttribute.hasCustomMessage() ? unsupportedAttribute.unresolvedMessage() : null); @@ -1250,7 +1256,7 @@ static void writeUnsupportedEsField(PlanStreamOutput out, UnsupportedEsField uns // -- BinaryComparison static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) throws IOException { - var source = in.readSource(); + var source = Source.readFrom(in); EsqlBinaryComparison.BinaryComparisonOperation operation = EsqlBinaryComparison.BinaryComparisonOperation.readFromStream(in); var left = in.readExpression(); var right = in.readExpression(); @@ -1260,7 +1266,7 @@ static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) t } static void writeBinComparison(PlanStreamOutput out, EsqlBinaryComparison binaryComparison) throws IOException { - out.writeSource(binaryComparison.source()); + binaryComparison.source().writeTo(out); binaryComparison.getFunctionType().writeTo(out); out.writeExpression(binaryComparison.left()); out.writeExpression(binaryComparison.right()); @@ -1269,14 +1275,14 @@ static void writeBinComparison(PlanStreamOutput out, EsqlBinaryComparison binary // -- InsensitiveEquals static InsensitiveEquals readInsensitiveEquals(PlanStreamInput in, String name) throws IOException { - var source = in.readSource(); + var source = Source.readFrom(in); var left = in.readExpression(); var right = in.readExpression(); return new InsensitiveEquals(source, left, right); } static void writeInsensitiveEquals(PlanStreamOutput out, InsensitiveEquals eq) throws IOException { - out.writeSource(eq.source()); + eq.source().writeTo(out); out.writeExpression(eq.left()); out.writeExpression(eq.right()); } @@ -1284,11 +1290,15 @@ static void writeInsensitiveEquals(PlanStreamOutput out, InsensitiveEquals eq) t // -- InComparison static In readInComparison(PlanStreamInput in) throws IOException { - return new In(in.readSource(), in.readExpression(), in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression))); + return new In( + Source.readFrom(in), + in.readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); } static void writeInComparison(PlanStreamOutput out, In in) throws IOException { - out.writeSource(in.source()); + in.source().writeTo(out); out.writeExpression(in.value()); out.writeCollection(in.list(), writerFromPlanWriter(PlanStreamOutput::writeExpression)); } @@ -1296,21 +1306,21 @@ static void writeInComparison(PlanStreamOutput out, In in) throws IOException { // -- RegexMatch static WildcardLike readWildcardLike(PlanStreamInput in, String name) throws IOException { - return new WildcardLike(in.readSource(), in.readExpression(), new WildcardPattern(in.readString())); + return new WildcardLike(Source.readFrom(in), in.readExpression(), new WildcardPattern(in.readString())); } static void writeWildcardLike(PlanStreamOutput out, WildcardLike like) throws IOException { - out.writeSource(like.source()); + like.source().writeTo(out); out.writeExpression(like.field()); out.writeString(like.pattern().pattern()); } static RLike readRLike(PlanStreamInput in, String name) throws IOException { - return new RLike(in.readSource(), in.readExpression(), new RLikePattern(in.readString())); + return new RLike(Source.readFrom(in), in.readExpression(), new RLikePattern(in.readString())); } static void writeRLike(PlanStreamOutput out, RLike like) throws IOException { - out.writeSource(like.source()); + like.source().writeTo(out); out.writeExpression(like.field()); out.writeString(like.pattern().asJavaRegex()); } @@ -1323,14 +1333,14 @@ static void writeRLike(PlanStreamOutput out, RLike like) throws IOException { ); static BinaryLogic readBinaryLogic(PlanStreamInput in, String name) throws IOException { - var source = in.readSource(); + var source = Source.readFrom(in); var left = in.readExpression(); var right = in.readExpression(); return BINARY_LOGIC_CTRS.get(name).apply(source, left, right); } static void writeBinaryLogic(PlanStreamOutput out, BinaryLogic binaryLogic) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(binaryLogic.left()); out.writeExpression(binaryLogic.right()); } @@ -1385,11 +1395,11 @@ static UnaryScalarFunction readESQLUnaryScalar(PlanStreamInput in, String name) if (ctr == null) { throw new IOException("Constructor for ESQLUnaryScalar not found for name:" + name); } - return ctr.apply(in.readSource(), in.readExpression()); + return ctr.apply(Source.readFrom(in), in.readExpression()); } static void writeESQLUnaryScalar(PlanStreamOutput out, UnaryScalarFunction function) throws IOException { - out.writeSource(function.source()); + function.source().writeTo(out); out.writeExpression(function.field()); } @@ -1404,11 +1414,11 @@ static ScalarFunction readNoArgScalar(PlanStreamInput in, String name) throws IO if (ctr == null) { throw new IOException("Constructor not found:" + name); } - return ctr.apply(in.readSource()); + return ctr.apply(Source.readFrom(in)); } static void writeNoArgScalar(PlanStreamOutput out, ScalarFunction function) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); } static final Map< @@ -1430,32 +1440,32 @@ static org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarF if (ctr == null) { throw new IOException("Constructor for QLUnaryScalar not found for name:" + name); } - return ctr.apply(in.readSource(), in.readExpression()); + return ctr.apply(Source.readFrom(in), in.readExpression()); } static void writeQLUnaryScalar( PlanStreamOutput out, org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction function ) throws IOException { - out.writeSource(function.source()); + function.source().writeTo(out); out.writeExpression(function.field()); } // -- ScalarFunction static Atan2 readAtan2(PlanStreamInput in) throws IOException { - return new Atan2(in.readSource(), in.readExpression(), in.readExpression()); + return new Atan2(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writeAtan2(PlanStreamOutput out, Atan2 atan2) throws IOException { - out.writeSource(atan2.source()); + atan2.source().writeTo(out); out.writeExpression(atan2.y()); out.writeExpression(atan2.x()); } static Bucket readBucket(PlanStreamInput in) throws IOException { return new Bucket( - in.readSource(), + Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class), @@ -1464,7 +1474,7 @@ static Bucket readBucket(PlanStreamInput in) throws IOException { } static void writeBucket(PlanStreamOutput out, Bucket bucket) throws IOException { - out.writeSource(bucket.source()); + bucket.source().writeTo(out); out.writeExpression(bucket.field()); out.writeExpression(bucket.buckets()); out.writeOptionalExpression(bucket.from()); @@ -1481,11 +1491,15 @@ static void writeBucket(PlanStreamOutput out, Bucket bucket) throws IOException static ScalarFunction readVarag(PlanStreamInput in, String name) throws IOException { return VARARG_CTORS.get(name) - .apply(in.readSource(), in.readExpression(), in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression))); + .apply( + Source.readFrom(in), + in.readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); } static void writeVararg(PlanStreamOutput out, ScalarFunction vararg) throws IOException { - out.writeSource(vararg.source()); + vararg.source().writeTo(out); out.writeExpression(vararg.children().get(0)); out.writeCollection( vararg.children().subList(1, vararg.children().size()), @@ -1494,23 +1508,23 @@ static void writeVararg(PlanStreamOutput out, ScalarFunction vararg) throws IOEx } static CountDistinct readCountDistinct(PlanStreamInput in) throws IOException { - return new CountDistinct(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new CountDistinct(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeCountDistinct(PlanStreamOutput out, CountDistinct countDistinct) throws IOException { List fields = countDistinct.children(); assert fields.size() == 1 || fields.size() == 2; - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(fields.get(0)); out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); } static DateDiff readDateDiff(PlanStreamInput in) throws IOException { - return new DateDiff(in.readSource(), in.readExpression(), in.readExpression(), in.readExpression()); + return new DateDiff(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readExpression()); } static void writeDateDiff(PlanStreamOutput out, DateDiff function) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); List fields = function.children(); assert fields.size() == 3; out.writeExpression(fields.get(0)); @@ -1519,11 +1533,11 @@ static void writeDateDiff(PlanStreamOutput out, DateDiff function) throws IOExce } static DateExtract readDateExtract(PlanStreamInput in) throws IOException { - return new DateExtract(in.readSource(), in.readExpression(), in.readExpression(), in.configuration()); + return new DateExtract(Source.readFrom(in), in.readExpression(), in.readExpression(), in.configuration()); } static void writeDateExtract(PlanStreamOutput out, DateExtract function) throws IOException { - out.writeSource(function.source()); + function.source().writeTo(out); List fields = function.children(); assert fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1531,11 +1545,11 @@ static void writeDateExtract(PlanStreamOutput out, DateExtract function) throws } static DateFormat readDateFormat(PlanStreamInput in) throws IOException { - return new DateFormat(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class), in.configuration()); + return new DateFormat(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class), in.configuration()); } static void writeDateFormat(PlanStreamOutput out, DateFormat dateFormat) throws IOException { - out.writeSource(dateFormat.source()); + dateFormat.source().writeTo(out); List fields = dateFormat.children(); assert fields.size() == 1 || fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1543,11 +1557,11 @@ static void writeDateFormat(PlanStreamOutput out, DateFormat dateFormat) throws } static DateParse readDateTimeParse(PlanStreamInput in) throws IOException { - return new DateParse(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new DateParse(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeDateTimeParse(PlanStreamOutput out, DateParse function) throws IOException { - out.writeSource(function.source()); + function.source().writeTo(out); List fields = function.children(); assert fields.size() == 1 || fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1555,11 +1569,11 @@ static void writeDateTimeParse(PlanStreamOutput out, DateParse function) throws } static DateTrunc readDateTrunc(PlanStreamInput in) throws IOException { - return new DateTrunc(in.readSource(), in.readExpression(), in.readExpression()); + return new DateTrunc(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writeDateTrunc(PlanStreamOutput out, DateTrunc dateTrunc) throws IOException { - out.writeSource(dateTrunc.source()); + dateTrunc.source().writeTo(out); List fields = dateTrunc.children(); assert fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1588,51 +1602,51 @@ static void writeSpatialRelatesFunction(PlanStreamOutput out, SpatialRelatesFunc } static Now readNow(PlanStreamInput in) throws IOException { - return new Now(in.readSource(), in.configuration()); + return new Now(Source.readFrom(in), in.configuration()); } static void writeNow(PlanStreamOutput out, Now function) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); } static Round readRound(PlanStreamInput in) throws IOException { - return new Round(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new Round(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeRound(PlanStreamOutput out, Round round) throws IOException { - out.writeSource(round.source()); + round.source().writeTo(out); out.writeExpression(round.field()); out.writeOptionalExpression(round.decimals()); } static Pow readPow(PlanStreamInput in) throws IOException { - return new Pow(in.readSource(), in.readExpression(), in.readExpression()); + return new Pow(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writePow(PlanStreamOutput out, Pow pow) throws IOException { - out.writeSource(pow.source()); + pow.source().writeTo(out); out.writeExpression(pow.base()); out.writeExpression(pow.exponent()); } static Percentile readPercentile(PlanStreamInput in) throws IOException { - return new Percentile(in.readSource(), in.readExpression(), in.readExpression()); + return new Percentile(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writePercentile(PlanStreamOutput out, Percentile percentile) throws IOException { List fields = percentile.children(); assert fields.size() == 2 : "percentile() aggregation must have two arguments"; - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(fields.get(0)); out.writeExpression(fields.get(1)); } static StartsWith readStartsWith(PlanStreamInput in) throws IOException { - return new StartsWith(in.readSource(), in.readExpression(), in.readExpression()); + return new StartsWith(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writeStartsWith(PlanStreamOutput out, StartsWith startsWith) throws IOException { - out.writeSource(startsWith.source()); + startsWith.source().writeTo(out); List fields = startsWith.children(); assert fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1640,23 +1654,23 @@ static void writeStartsWith(PlanStreamOutput out, StartsWith startsWith) throws } static EndsWith readEndsWith(PlanStreamInput in) throws IOException { - return new EndsWith(in.readSource(), in.readExpression(), in.readExpression()); + return new EndsWith(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writeEndsWith(PlanStreamOutput out, EndsWith endsWith) throws IOException { List fields = endsWith.children(); assert fields.size() == 2; - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(fields.get(0)); out.writeExpression(fields.get(1)); } static Substring readSubstring(PlanStreamInput in) throws IOException { - return new Substring(in.readSource(), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new Substring(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeSubstring(PlanStreamOutput out, Substring substring) throws IOException { - out.writeSource(substring.source()); + substring.source().writeTo(out); List fields = substring.children(); assert fields.size() == 2 || fields.size() == 3; out.writeExpression(fields.get(0)); @@ -1665,11 +1679,11 @@ static void writeSubstring(PlanStreamOutput out, Substring substring) throws IOE } static Locate readLocate(PlanStreamInput in) throws IOException { - return new Locate(in.readSource(), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new Locate(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeLocate(PlanStreamOutput out, Locate locate) throws IOException { - out.writeSource(locate.source()); + locate.source().writeTo(out); List fields = locate.children(); assert fields.size() == 2 || fields.size() == 3; out.writeExpression(fields.get(0)); @@ -1706,11 +1720,11 @@ static void writeToUpper(PlanStreamOutput out, ToUpper toUpper) throws IOExcepti } static Left readLeft(PlanStreamInput in) throws IOException { - return new Left(in.readSource(), in.readExpression(), in.readExpression()); + return new Left(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writeLeft(PlanStreamOutput out, Left left) throws IOException { - out.writeSource(left.source()); + left.source().writeTo(out); List fields = left.children(); assert fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1718,11 +1732,11 @@ static void writeLeft(PlanStreamOutput out, Left left) throws IOException { } static Right readRight(PlanStreamInput in) throws IOException { - return new Right(in.readSource(), in.readExpression(), in.readExpression()); + return new Right(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writeRight(PlanStreamOutput out, Right right) throws IOException { - out.writeSource(right.source()); + right.source().writeTo(out); List fields = right.children(); assert fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1730,25 +1744,25 @@ static void writeRight(PlanStreamOutput out, Right right) throws IOException { } static Split readSplit(PlanStreamInput in) throws IOException { - return new Split(in.readSource(), in.readExpression(), in.readExpression()); + return new Split(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writeSplit(PlanStreamOutput out, Split split) throws IOException { - out.writeSource(split.source()); + split.source().writeTo(out); out.writeExpression(split.left()); out.writeExpression(split.right()); } static CIDRMatch readCIDRMatch(PlanStreamInput in) throws IOException { return new CIDRMatch( - in.readSource(), + Source.readFrom(in), in.readExpression(), in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) ); } static void writeCIDRMatch(PlanStreamOutput out, CIDRMatch cidrMatch) throws IOException { - out.writeSource(cidrMatch.source()); + cidrMatch.source().writeTo(out); List children = cidrMatch.children(); assert children.size() > 1; out.writeExpression(children.get(0)); @@ -1766,14 +1780,14 @@ static void writeCIDRMatch(PlanStreamOutput out, CIDRMatch cidrMatch) throws IOE ); static ArithmeticOperation readArithmeticOperation(PlanStreamInput in, String name) throws IOException { - var source = in.readSource(); + var source = Source.readFrom(in); var left = in.readExpression(); var right = in.readExpression(); return ARITHMETIC_CTRS.get(name).apply(source, left, right); } static void writeArithmeticOperation(PlanStreamOutput out, ArithmeticOperation arithmeticOperation) throws IOException { - out.writeSource(arithmeticOperation.source()); + arithmeticOperation.source().writeTo(out); out.writeExpression(arithmeticOperation.left()); out.writeExpression(arithmeticOperation.right()); } @@ -1792,11 +1806,11 @@ static void writeArithmeticOperation(PlanStreamOutput out, ArithmeticOperation a ); static AggregateFunction readAggFunction(PlanStreamInput in, String name) throws IOException { - return AGG_CTRS.get(name).apply(in.readSource(), in.readExpression()); + return AGG_CTRS.get(name).apply(Source.readFrom(in), in.readExpression()); } static void writeAggFunction(PlanStreamOutput out, AggregateFunction aggregateFunction) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(aggregateFunction.field()); } @@ -1814,20 +1828,20 @@ static void writeAggFunction(PlanStreamOutput out, AggregateFunction aggregateFu ); static AbstractMultivalueFunction readMvFunction(PlanStreamInput in, String name) throws IOException { - return MV_CTRS.get(name).apply(in.readSource(), in.readExpression()); + return MV_CTRS.get(name).apply(Source.readFrom(in), in.readExpression()); } static void writeMvFunction(PlanStreamOutput out, AbstractMultivalueFunction fn) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(fn.field()); } static MvConcat readMvConcat(PlanStreamInput in) throws IOException { - return new MvConcat(in.readSource(), in.readExpression(), in.readExpression()); + return new MvConcat(Source.readFrom(in), in.readExpression(), in.readExpression()); } static void writeMvConcat(PlanStreamOutput out, MvConcat fn) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(fn.left()); out.writeExpression(fn.right()); } @@ -1836,7 +1850,7 @@ static void writeMvConcat(PlanStreamOutput out, MvConcat fn) throws IOException static Alias readAlias(PlanStreamInput in) throws IOException { return new Alias( - in.readSource(), + Source.readFrom(in), in.readString(), in.readOptionalString(), in.readNamed(Expression.class), @@ -1846,7 +1860,7 @@ static Alias readAlias(PlanStreamInput in) throws IOException { } static void writeAlias(PlanStreamOutput out, Alias alias) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeString(alias.name()); out.writeOptionalString(alias.qualifier()); out.writeExpression(alias.child()); @@ -1857,14 +1871,14 @@ static void writeAlias(PlanStreamOutput out, Alias alias) throws IOException { // -- Expressions (other) static Literal readLiteral(PlanStreamInput in) throws IOException { - Source source = in.readSource(); + Source source = Source.readFrom(in); Object value = in.readGenericValue(); DataType dataType = in.dataTypeFromTypeName(in.readString()); return new Literal(source, mapToLiteralValue(in, dataType, value), dataType); } static void writeLiteral(PlanStreamOutput out, Literal literal) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeGenericValue(mapFromLiteralValue(out, literal.dataType(), literal.value())); out.writeString(literal.dataType().typeName()); } @@ -1917,7 +1931,7 @@ private static long wkbAsLong(DataType dataType, BytesRef wkb) { static Order readOrder(PlanStreamInput in) throws IOException { return new org.elasticsearch.xpack.esql.expression.Order( - in.readSource(), + Source.readFrom(in), in.readNamed(Expression.class), in.readEnum(Order.OrderDirection.class), in.readEnum(Order.NullsPosition.class) @@ -1925,7 +1939,7 @@ static Order readOrder(PlanStreamInput in) throws IOException { } static void writeOrder(PlanStreamOutput out, Order order) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); out.writeExpression(order.child()); out.writeEnum(order.direction()); out.writeEnum(order.nullsPosition()); @@ -1974,11 +1988,11 @@ static void writeDissectParser(PlanStreamOutput out, Parser dissectParser) throw } static Log readLog(PlanStreamInput in) throws IOException { - return new Log(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new Log(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeLog(PlanStreamOutput out, Log log) throws IOException { - out.writeSource(log.source()); + log.source().writeTo(out); List fields = log.children(); assert fields.size() == 1 || fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1986,11 +2000,11 @@ static void writeLog(PlanStreamOutput out, Log log) throws IOException { } static MvSort readMvSort(PlanStreamInput in) throws IOException { - return new MvSort(in.readSource(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new MvSort(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeMvSort(PlanStreamOutput out, MvSort mvSort) throws IOException { - out.writeSource(mvSort.source()); + mvSort.source().writeTo(out); List fields = mvSort.children(); assert fields.size() == 1 || fields.size() == 2; out.writeExpression(fields.get(0)); @@ -1998,11 +2012,11 @@ static void writeMvSort(PlanStreamOutput out, MvSort mvSort) throws IOException } static MvSlice readMvSlice(PlanStreamInput in) throws IOException { - return new MvSlice(in.readSource(), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new MvSlice(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeMvSlice(PlanStreamOutput out, MvSlice fn) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); List fields = fn.children(); assert fields.size() == 2 || fields.size() == 3; out.writeExpression(fields.get(0)); @@ -2011,11 +2025,11 @@ static void writeMvSlice(PlanStreamOutput out, MvSlice fn) throws IOException { } static MvZip readMvZip(PlanStreamInput in) throws IOException { - return new MvZip(in.readSource(), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); + return new MvZip(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); } static void writeMvZip(PlanStreamOutput out, MvZip fn) throws IOException { - out.writeNoSource(); + Source.EMPTY.writeTo(out); List fields = fn.children(); assert fields.size() == 2 || fields.size() == 3; out.writeExpression(fields.get(0)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java index 3faf85b989c09..c8dc24e4c2208 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java @@ -30,7 +30,6 @@ import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanNamedReader; @@ -48,13 +47,13 @@ import java.util.function.LongFunction; import java.util.function.Supplier; -import static org.elasticsearch.xpack.esql.core.util.SourceUtils.readSourceWithText; - /** * A customized stream input used to deserialize ESQL physical plan fragments. Complements stream * input with methods that read plan nodes, Attributes, Expressions, etc. */ -public final class PlanStreamInput extends NamedWriteableAwareStreamInput { +public final class PlanStreamInput extends NamedWriteableAwareStreamInput + implements + org.elasticsearch.xpack.esql.core.util.PlanStreamInput { /** * A Mapper of stream named id, represented as a primitive long value, to NameId instance. @@ -123,11 +122,6 @@ public PhysicalPlan readOptionalPhysicalPlanNode() throws IOException { return readOptionalNamed(PhysicalPlan.class); } - public Source readSource() throws IOException { - boolean hasSource = readBoolean(); - return hasSource ? readSourceWithText(this, configuration.query()) : Source.EMPTY; - } - public Expression readExpression() throws IOException { return readNamed(Expression.class); } @@ -268,6 +262,11 @@ public Block[] readCachedBlockArray() throws IOException { } } + @Override + public String sourceText() { + return configuration.query(); + } + static void throwOnNullOptionalRead(Class type) throws IOException { final IOException e = new IOException("read optional named returned null which is not allowed, type:" + type); assert false : e; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java index 6cf64de339383..351918699aac4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java @@ -23,7 +23,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; @@ -33,8 +32,6 @@ import java.util.Map; import java.util.function.Function; -import static org.elasticsearch.xpack.esql.core.util.SourceUtils.writeSourceNoText; - /** * A customized stream output used to serialize ESQL physical plan fragments. Complements stream * output with methods that write plan nodes, Attributes, Expressions, etc. @@ -98,15 +95,6 @@ public void writeOptionalPhysicalPlanNode(PhysicalPlan physicalPlan) throws IOEx } } - public void writeSource(Source source) throws IOException { - writeBoolean(true); - writeSourceNoText(this, source); - } - - public void writeNoSource() throws IOException { - writeBoolean(false); - } - public void writeExpression(Expression expression) throws IOException { writeNamed(Expression.class, expression); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/SourceUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/SourceUtils.java deleted file mode 100644 index afba73373df92..0000000000000 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/SourceUtils.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.ql.util; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.xpack.ql.QlIllegalArgumentException; -import org.elasticsearch.xpack.ql.tree.Location; -import org.elasticsearch.xpack.ql.tree.Source; - -import java.io.IOException; - -public final class SourceUtils { - - private SourceUtils() {} - - public static void writeSource(StreamOutput out, Source source) throws IOException { - writeSource(out, source, true); - } - - public static void writeSourceNoText(StreamOutput out, Source source) throws IOException { - writeSource(out, source, false); - } - - public static Source readSource(StreamInput in) throws IOException { - return readSource(in, null); - } - - public static Source readSourceWithText(StreamInput in, String queryText) throws IOException { - return readSource(in, queryText); - } - - private static void writeSource(StreamOutput out, Source source, boolean writeText) throws IOException { - out.writeInt(source.source().getLineNumber()); - out.writeInt(source.source().getColumnNumber()); - if (writeText) { - out.writeString(source.text()); - } else { - out.writeInt(source.text().length()); - } - } - - private static Source readSource(StreamInput in, @Nullable String queryText) throws IOException { - int line = in.readInt(); - int column = in.readInt(); - int charPositionInLine = column - 1; - - String text; - if (queryText == null) { - text = in.readString(); - } else { - int length = in.readInt(); - text = sourceText(queryText, line, column, length); - } - return new Source(new Location(line, charPositionInLine), text); - } - - private static String sourceText(String query, int line, int column, int length) { - if (line <= 0 || column <= 0 || query.isEmpty()) { - return StringUtils.EMPTY; - } - int offset = textOffset(query, line, column); - if (offset + length > query.length()) { - throw new QlIllegalArgumentException( - "location [@" + line + ":" + column + "] and length [" + length + "] overrun query size [" + query.length() + "]" - ); - } - return query.substring(offset, offset + length); - } - - private static int textOffset(String query, int line, int column) { - int offset = 0; - if (line > 1) { - String[] lines = query.split("\n"); - if (line > lines.length) { - throw new QlIllegalArgumentException( - "line location [" + line + "] higher than max [" + lines.length + "] in query [" + query + "]" - ); - } - for (int i = 0; i < line - 1; i++) { - offset += lines[i].length() + 1; // +1 accounts for the removed \n - } - } - offset += column - 1; // -1 since column is 1-based indexed - return offset; - } -} From da9282e3f571e5af834953db679763bde8d1530d Mon Sep 17 00:00:00 2001 From: Henning Andersen <33268011+henningandersen@users.noreply.github.com> Date: Wed, 29 May 2024 16:23:22 +0200 Subject: [PATCH 11/29] Longer timeout for local node startup during CI (#109141) Sometimes CI is slow enough that the 2 min timeout fails. Extending it. Seen in CI failure from #109139 --- .../test/cluster/local/AbstractLocalClusterFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 9bd3403060b2a..0101c76b21f76 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -64,7 +64,7 @@ public abstract class AbstractLocalClusterFactory { private static final Logger LOGGER = LogManager.getLogger(AbstractLocalClusterFactory.class); - private static final Duration NODE_UP_TIMEOUT = Duration.ofMinutes(2); + private static final Duration NODE_UP_TIMEOUT = Duration.ofMinutes(3); private static final Map, DistributionDescriptor> TEST_DISTRIBUTIONS = new ConcurrentHashMap<>(); private static final String TESTS_CLUSTER_MODULES_PATH_SYSPROP = "tests.cluster.modules.path"; private static final String TESTS_CLUSTER_PLUGINS_PATH_SYSPROP = "tests.cluster.plugins.path"; From f16f71e2a2bd025cb2bc63cad0670deb91017818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Wed, 29 May 2024 16:23:45 +0200 Subject: [PATCH 12/29] ESQL: Add ip_prefix function (#109070) Added ESQL function to get the prefix of an IP. It works now with both IPv4 and IPv6. For users planning to use it with mixed IPs, we may need to add a function like "is_ipv4()" first. **About the skipped test:** There's currently a "bug" in the evaluators//functions that return null. Evaluators can't handle them. We'll work on support for that in another PR. It affects other functions, like `substring()`. In this function, however, it only affects in "wrong" cases (Like an invalid prefix), so it has no impact. Fixes https://github.com/elastic/elasticsearch/issues/99064 --- .gitattributes | 5 + docs/changelog/109070.yaml | 6 + .../functions/description/ip_prefix.asciidoc | 5 + .../functions/examples/ip_prefix.asciidoc | 13 ++ .../kibana/definition/ip_prefix.json | 35 ++++ .../esql/functions/kibana/docs/ip_prefix.md | 11 + .../esql/functions/layout/ip_prefix.asciidoc | 15 ++ .../functions/parameters/ip_prefix.asciidoc | 12 ++ .../esql/functions/signature/ip_prefix.svg | 1 + .../esql/functions/types/ip_prefix.asciidoc | 9 + .../xpack/esql/CsvTestUtils.java | 2 +- .../src/main/resources/ip.csv-spec | 105 ++++++++++ .../src/main/resources/meta.csv-spec | 6 +- .../function/scalar/ip/IpPrefixEvaluator.java | 183 +++++++++++++++++ .../scalar/ip/IpPrefixOnlyV4Evaluator.java | 148 ++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 6 + .../function/EsqlFunctionRegistry.java | 2 + .../function/scalar/ip/IpPrefix.java | 191 ++++++++++++++++++ .../xpack/esql/io/stream/PlanNamedTypes.java | 2 + .../function/scalar/ip/IpPrefixTests.java | 116 +++++++++++ 20 files changed, 871 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/109070.yaml create mode 100644 docs/reference/esql/functions/description/ip_prefix.asciidoc create mode 100644 docs/reference/esql/functions/examples/ip_prefix.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/ip_prefix.json create mode 100644 docs/reference/esql/functions/kibana/docs/ip_prefix.md create mode 100644 docs/reference/esql/functions/layout/ip_prefix.asciidoc create mode 100644 docs/reference/esql/functions/parameters/ip_prefix.asciidoc create mode 100644 docs/reference/esql/functions/signature/ip_prefix.svg create mode 100644 docs/reference/esql/functions/types/ip_prefix.asciidoc create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java diff --git a/.gitattributes b/.gitattributes index ebe1a34db3479..6a8de5462ec3f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,8 @@ x-pack/plugin/esql/src/main/antlr/*.tokens linguist-generated=true x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*.interp linguist-generated=true x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java linguist-generated=true x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java linguist-generated=true +x-pack/plugin/esql/src/main/generated/** linguist-generated=true + +# ESQL functions docs are autogenerated. More information at `docs/reference/esql/functions/README.md` +docs/reference/esql/functions/*/** linguist-generated=true + diff --git a/docs/changelog/109070.yaml b/docs/changelog/109070.yaml new file mode 100644 index 0000000000000..8dbc0ec1c6cf2 --- /dev/null +++ b/docs/changelog/109070.yaml @@ -0,0 +1,6 @@ +pr: 109070 +summary: "ESQL: Add `ip_prefix` function" +area: ES|QL +type: feature +issues: + - 99064 diff --git a/docs/reference/esql/functions/description/ip_prefix.asciidoc b/docs/reference/esql/functions/description/ip_prefix.asciidoc new file mode 100644 index 0000000000000..4b7a88486dea2 --- /dev/null +++ b/docs/reference/esql/functions/description/ip_prefix.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Truncates an IP to a given prefix length. diff --git a/docs/reference/esql/functions/examples/ip_prefix.asciidoc b/docs/reference/esql/functions/examples/ip_prefix.asciidoc new file mode 100644 index 0000000000000..19f0ed266afb5 --- /dev/null +++ b/docs/reference/esql/functions/examples/ip_prefix.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/ip.csv-spec[tag=ipPrefix] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/ip.csv-spec[tag=ipPrefix-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/ip_prefix.json b/docs/reference/esql/functions/kibana/definition/ip_prefix.json new file mode 100644 index 0000000000000..00c3cf75a949e --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/ip_prefix.json @@ -0,0 +1,35 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "ip_prefix", + "description" : "Truncates an IP to a given prefix length.", + "signatures" : [ + { + "params" : [ + { + "name" : "ip", + "type" : "ip", + "optional" : false, + "description" : "IP address of type `ip` (both IPv4 and IPv6 are supported)." + }, + { + "name" : "prefixLengthV4", + "type" : "integer", + "optional" : false, + "description" : "Prefix length for IPv4 addresses." + }, + { + "name" : "prefixLengthV6", + "type" : "integer", + "optional" : false, + "description" : "Prefix length for IPv6 addresses." + } + ], + "variadic" : false, + "returnType" : "ip" + } + ], + "examples" : [ + "row ip4 = to_ip(\"1.2.3.4\"), ip6 = to_ip(\"fe80::cae2:65ff:fece:feb9\")\n| eval ip4_prefix = ip_prefix(ip4, 24, 0), ip6_prefix = ip_prefix(ip6, 0, 112);" + ] +} diff --git a/docs/reference/esql/functions/kibana/docs/ip_prefix.md b/docs/reference/esql/functions/kibana/docs/ip_prefix.md new file mode 100644 index 0000000000000..5c0009528bb68 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/ip_prefix.md @@ -0,0 +1,11 @@ + + +### IP_PREFIX +Truncates an IP to a given prefix length. + +``` +row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9") +| eval ip4_prefix = ip_prefix(ip4, 24, 0), ip6_prefix = ip_prefix(ip6, 0, 112); +``` diff --git a/docs/reference/esql/functions/layout/ip_prefix.asciidoc b/docs/reference/esql/functions/layout/ip_prefix.asciidoc new file mode 100644 index 0000000000000..ca51c871daf7f --- /dev/null +++ b/docs/reference/esql/functions/layout/ip_prefix.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-ip_prefix]] +=== `IP_PREFIX` + +*Syntax* + +[.text-center] +image::esql/functions/signature/ip_prefix.svg[Embedded,opts=inline] + +include::../parameters/ip_prefix.asciidoc[] +include::../description/ip_prefix.asciidoc[] +include::../types/ip_prefix.asciidoc[] +include::../examples/ip_prefix.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/ip_prefix.asciidoc b/docs/reference/esql/functions/parameters/ip_prefix.asciidoc new file mode 100644 index 0000000000000..945601c2476e6 --- /dev/null +++ b/docs/reference/esql/functions/parameters/ip_prefix.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`ip`:: +IP address of type `ip` (both IPv4 and IPv6 are supported). + +`prefixLengthV4`:: +Prefix length for IPv4 addresses. + +`prefixLengthV6`:: +Prefix length for IPv6 addresses. diff --git a/docs/reference/esql/functions/signature/ip_prefix.svg b/docs/reference/esql/functions/signature/ip_prefix.svg new file mode 100644 index 0000000000000..4699c23357460 --- /dev/null +++ b/docs/reference/esql/functions/signature/ip_prefix.svg @@ -0,0 +1 @@ +IP_PREFIX(ip,prefixLengthV4,prefixLengthV6) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/ip_prefix.asciidoc b/docs/reference/esql/functions/types/ip_prefix.asciidoc new file mode 100644 index 0000000000000..786d99d45d327 --- /dev/null +++ b/docs/reference/esql/functions/types/ip_prefix.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +ip | prefixLengthV4 | prefixLengthV6 | result +ip | integer | integer | ip +|=== diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index 1927cfd03ac06..7d1c168bd203f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -101,7 +101,7 @@ public static Tuple skipVersionRange(String testName) { Map pairs = extractInstructions(testName); String versionRange = pairs.get("skip"); if (versionRange != null) { - String[] skipVersions = versionRange.split("-"); + String[] skipVersions = versionRange.split("-", Integer.MAX_VALUE); if (skipVersions.length != 2) { throw new IllegalArgumentException("malformed version range : " + versionRange); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec index ae683acbb2c3a..0de64d3e2d9d4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec @@ -485,3 +485,108 @@ beta | 127.0.0.1 beta | 127.0.0.1 beta | 127.0.0.1 ; + +ipPrefix +required_capability: fn_ip_prefix +//tag::ipPrefix[] +row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9") +| eval ip4_prefix = ip_prefix(ip4, 24, 0), ip6_prefix = ip_prefix(ip6, 0, 112); +//end::ipPrefix[] + +//tag::ipPrefix-result[] +ip4:ip | ip6:ip | ip4_prefix:ip | ip6_prefix:ip +1.2.3.4 | fe80::cae2:65ff:fece:feb9 | 1.2.3.0 | fe80::cae2:65ff:fece:0000 +//end::ipPrefix-result[] +; + +ipPrefixCompleteIp +required_capability: fn_ip_prefix +row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9") +| eval ip4_prefix = ip_prefix(ip4, 32, 0), ip6_prefix = ip_prefix(ip6, 0, 128); + +ip4:ip | ip6:ip | ip4_prefix:ip | ip6_prefix:ip +1.2.3.4 | fe80::cae2:65ff:fece:feb9 | 1.2.3.4 | fe80::cae2:65ff:fece:feb9 +; + +ipPrefixZeroBits +required_capability: fn_ip_prefix +row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9") +| eval ip4_prefix = ip_prefix(ip4, 0, 128), ip6_prefix = ip_prefix(ip6, 32, 0); + +ip4:ip | ip6:ip | ip4_prefix:ip | ip6_prefix:ip +1.2.3.4 | fe80::cae2:65ff:fece:feb9 | 0.0.0.0 | ::0 +; + +ipPrefixWithBits +required_capability: fn_ip_prefix +row ip4 = to_ip("1.2.3.255"), ip6 = to_ip("fe80::cae2:65ff:fece:feff") +| eval ip4_prefix = ip_prefix(ip4, 25, 0), ip6_prefix = ip_prefix(ip6, 0, 121); + +ip4:ip | ip6:ip | ip4_prefix:ip | ip6_prefix:ip +1.2.3.255 | fe80::cae2:65ff:fece:feff | 1.2.3.128 | fe80::cae2:65ff:fece:fe80 +; + +ipPrefixLengthFromColumn +required_capability: fn_ip_prefix +from hosts +| where host == "alpha" +| sort card +| eval prefix = ip_prefix(ip0, 24, 128) +| keep card, ip0, prefix; + +card:keyword | ip0:ip | prefix:ip +eth0 | 127.0.0.1 | 127.0.0.0 +eth1 | ::1 | ::0 +; + +ipPrefixLengthFromExpression +required_capability: fn_ip_prefix +row ip4 = to_ip("1.2.3.4"), ip6 = to_ip("fe80::cae2:65ff:fece:feb9"), bits_per_byte = 8 +| eval ip4_length = 3 * bits_per_byte, ip4_prefix = ip_prefix(ip4, ip4_length, 0), ip6_prefix = ip_prefix(ip6, 0, 12 * 10); + +ip4:ip | ip6:ip | bits_per_byte:integer | ip4_length:integer | ip4_prefix:ip | ip6_prefix:ip +1.2.3.4 | fe80::cae2:65ff:fece:feb9 | 8 | 24 | 1.2.3.0 | fe80::cae2:65ff:fece:fe00 +; + +ipPrefixAsGroup +required_capability: fn_ip_prefix +from hosts +| stats count(*) by ip_prefix(ip1, 24, 120) +| sort `ip_prefix(ip1, 24, 120)`; +warning:Line 2:21: evaluation of [ip_prefix(ip1, 24, 120)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:21: java.lang.IllegalArgumentException: single-value function encountered multi-value + +count(*):long | ip_prefix(ip1, 24, 120):ip +2 | ::0 +3 | 127.0.0.0 +1 | 128.0.0.0 +1 | fe80::cae2:65ff:fece:fe00 +1 | fe81::cae2:65ff:fece:fe00 +2 | null +; + +ipPrefixWithWrongLengths +required_capability: fn_ip_prefix +row ip4 = to_ip("1.2.3.4") +| eval a = ip_prefix(ip4, -1, 128), b = ip_prefix(ip4, 32, -1), c = ip_prefix(ip4, 33, 0), d = ip_prefix(ip4, 32, 129); +warning:Line 2:12: evaluation of [ip_prefix(ip4, -1, 128)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:12: java.lang.IllegalArgumentException: Prefix length v4 must be in range [0, 32], found -1 +warning:Line 2:41: evaluation of [ip_prefix(ip4, 32, -1)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:41: java.lang.IllegalArgumentException: Prefix length v6 must be in range [0, 128], found -1 +warning:Line 2:69: evaluation of [ip_prefix(ip4, 33, 0)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:69: java.lang.IllegalArgumentException: Prefix length v4 must be in range [0, 32], found 33 +warning:Line 2:96: evaluation of [ip_prefix(ip4, 32, 129)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:96: java.lang.IllegalArgumentException: Prefix length v6 must be in range [0, 128], found 129 + +ip4:ip | a:ip | b:ip | c:ip | d:ip +1.2.3.4 | null | null | null | null +; + +ipPrefixWithNullArguments +required_capability: fn_ip_prefix +row ip4 = to_ip("1.2.3.4") +| eval a = ip_prefix(null, 32, 128), b = ip_prefix(ip4, null, 128), c = ip_prefix(ip4, 32, null); + +ip4:ip | a:ip | b:ip | c:ip +1.2.3.4 | null | null | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index f68dd15e9c516..eff4cb05bd8c0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -30,6 +30,7 @@ double e() "double|integer|long|unsigned_long floor(number:double|integer|long|unsigned_long)" "keyword from_base64(string:keyword|text)" "boolean|double|integer|ip|keyword|long|text|version greatest(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)" +"ip ip_prefix(ip:ip, prefixLengthV4:integer, prefixLengthV6:integer)" "boolean|double|integer|ip|keyword|long|text|version least(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)" "keyword left(string:keyword|text, length:integer)" "integer length(string:keyword|text)" @@ -144,6 +145,7 @@ ends_with |[str, suffix] |["keyword|text", "keyword|te floor |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. from_base64 |string |"keyword|text" |A base64 string. greatest |first |"boolean|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. +ip_prefix |[ip, prefixLengthV4, prefixLengthV6]|[ip, integer, integer] |[IP address of type `ip` (both IPv4 and IPv6 are supported)., Prefix length for IPv4 addresses., Prefix length for IPv6 addresses.] least |first |"boolean|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. left |[string, length] |["keyword|text", integer] |[The string from which to return a substring., The number of characters to return.] length |string |"keyword|text" |String expression. If `null`, the function returns `null`. @@ -259,6 +261,7 @@ ends_with |Returns a boolean that indicates whether a keyword string ends wi floor |Round a number down to the nearest integer. from_base64 |Decode a base64 string. greatest |Returns the maximum value from multiple columns. This is similar to <> except it is intended to run on multiple columns at once. +ip_prefix |Truncates an IP to a given prefix length. least |Returns the minimum value from multiple columns. This is similar to <> except it is intended to run on multiple columns at once. left |Returns the substring that extracts 'length' chars from 'string' starting from the left. length |Returns the character length of a string. @@ -375,6 +378,7 @@ ends_with |boolean floor |"double|integer|long|unsigned_long" |false |false |false from_base64 |keyword |false |false |false greatest |"boolean|double|integer|ip|keyword|long|text|version" |false |true |false +ip_prefix |ip |[false, false, false] |false |false least |"boolean|double|integer|ip|keyword|long|text|version" |false |true |false left |keyword |[false, false] |false |false length |integer |false |false |false @@ -471,5 +475,5 @@ countFunctions#[skip:-8.14.99, reason:BIN added] meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -106 | 106 | 106 +107 | 107 | 107 ; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java new file mode 100644 index 0000000000000..174df48d5ce62 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixEvaluator.java @@ -0,0 +1,183 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.util.function.Function; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link IpPrefix}. + * This class is generated. Do not edit it. + */ +public final class IpPrefixEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator ip; + + private final EvalOperator.ExpressionEvaluator prefixLengthV4; + + private final EvalOperator.ExpressionEvaluator prefixLengthV6; + + private final BytesRef scratch; + + private final DriverContext driverContext; + + public IpPrefixEvaluator(Source source, EvalOperator.ExpressionEvaluator ip, + EvalOperator.ExpressionEvaluator prefixLengthV4, + EvalOperator.ExpressionEvaluator prefixLengthV6, BytesRef scratch, + DriverContext driverContext) { + this.warnings = new Warnings(source); + this.ip = ip; + this.prefixLengthV4 = prefixLengthV4; + this.prefixLengthV6 = prefixLengthV6; + this.scratch = scratch; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock ipBlock = (BytesRefBlock) ip.eval(page)) { + try (IntBlock prefixLengthV4Block = (IntBlock) prefixLengthV4.eval(page)) { + try (IntBlock prefixLengthV6Block = (IntBlock) prefixLengthV6.eval(page)) { + BytesRefVector ipVector = ipBlock.asVector(); + if (ipVector == null) { + return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block, prefixLengthV6Block); + } + IntVector prefixLengthV4Vector = prefixLengthV4Block.asVector(); + if (prefixLengthV4Vector == null) { + return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block, prefixLengthV6Block); + } + IntVector prefixLengthV6Vector = prefixLengthV6Block.asVector(); + if (prefixLengthV6Vector == null) { + return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block, prefixLengthV6Block); + } + return eval(page.getPositionCount(), ipVector, prefixLengthV4Vector, prefixLengthV6Vector); + } + } + } + } + + public BytesRefBlock eval(int positionCount, BytesRefBlock ipBlock, IntBlock prefixLengthV4Block, + IntBlock prefixLengthV6Block) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef ipScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (ipBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (ipBlock.getValueCount(p) != 1) { + if (ipBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (prefixLengthV4Block.isNull(p)) { + result.appendNull(); + continue position; + } + if (prefixLengthV4Block.getValueCount(p) != 1) { + if (prefixLengthV4Block.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (prefixLengthV6Block.isNull(p)) { + result.appendNull(); + continue position; + } + if (prefixLengthV6Block.getValueCount(p) != 1) { + if (prefixLengthV6Block.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + try { + result.appendBytesRef(IpPrefix.process(ipBlock.getBytesRef(ipBlock.getFirstValueIndex(p), ipScratch), prefixLengthV4Block.getInt(prefixLengthV4Block.getFirstValueIndex(p)), prefixLengthV6Block.getInt(prefixLengthV6Block.getFirstValueIndex(p)), scratch)); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + public BytesRefBlock eval(int positionCount, BytesRefVector ipVector, + IntVector prefixLengthV4Vector, IntVector prefixLengthV6Vector) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef ipScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + try { + result.appendBytesRef(IpPrefix.process(ipVector.getBytesRef(p, ipScratch), prefixLengthV4Vector.getInt(p), prefixLengthV6Vector.getInt(p), scratch)); + } catch (IllegalArgumentException e) { + warnings.registerException(e); + result.appendNull(); + } + } + return result.build(); + } + } + + @Override + public String toString() { + return "IpPrefixEvaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + ", prefixLengthV6=" + prefixLengthV6 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(ip, prefixLengthV4, prefixLengthV6); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory ip; + + private final EvalOperator.ExpressionEvaluator.Factory prefixLengthV4; + + private final EvalOperator.ExpressionEvaluator.Factory prefixLengthV6; + + private final Function scratch; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory ip, + EvalOperator.ExpressionEvaluator.Factory prefixLengthV4, + EvalOperator.ExpressionEvaluator.Factory prefixLengthV6, + Function scratch) { + this.source = source; + this.ip = ip; + this.prefixLengthV4 = prefixLengthV4; + this.prefixLengthV6 = prefixLengthV6; + this.scratch = scratch; + } + + @Override + public IpPrefixEvaluator get(DriverContext context) { + return new IpPrefixEvaluator(source, ip.get(context), prefixLengthV4.get(context), prefixLengthV6.get(context), scratch.apply(context), context); + } + + @Override + public String toString() { + return "IpPrefixEvaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + ", prefixLengthV6=" + prefixLengthV6 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java new file mode 100644 index 0000000000000..a6cb7c7f9b687 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixOnlyV4Evaluator.java @@ -0,0 +1,148 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import java.util.function.Function; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link IpPrefix}. + * This class is generated. Do not edit it. + */ +public final class IpPrefixOnlyV4Evaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator ip; + + private final EvalOperator.ExpressionEvaluator prefixLengthV4; + + private final BytesRef scratch; + + private final DriverContext driverContext; + + public IpPrefixOnlyV4Evaluator(Source source, EvalOperator.ExpressionEvaluator ip, + EvalOperator.ExpressionEvaluator prefixLengthV4, BytesRef scratch, + DriverContext driverContext) { + this.warnings = new Warnings(source); + this.ip = ip; + this.prefixLengthV4 = prefixLengthV4; + this.scratch = scratch; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock ipBlock = (BytesRefBlock) ip.eval(page)) { + try (IntBlock prefixLengthV4Block = (IntBlock) prefixLengthV4.eval(page)) { + BytesRefVector ipVector = ipBlock.asVector(); + if (ipVector == null) { + return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block); + } + IntVector prefixLengthV4Vector = prefixLengthV4Block.asVector(); + if (prefixLengthV4Vector == null) { + return eval(page.getPositionCount(), ipBlock, prefixLengthV4Block); + } + return eval(page.getPositionCount(), ipVector, prefixLengthV4Vector).asBlock(); + } + } + } + + public BytesRefBlock eval(int positionCount, BytesRefBlock ipBlock, + IntBlock prefixLengthV4Block) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef ipScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (ipBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (ipBlock.getValueCount(p) != 1) { + if (ipBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + if (prefixLengthV4Block.isNull(p)) { + result.appendNull(); + continue position; + } + if (prefixLengthV4Block.getValueCount(p) != 1) { + if (prefixLengthV4Block.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBytesRef(IpPrefix.process(ipBlock.getBytesRef(ipBlock.getFirstValueIndex(p), ipScratch), prefixLengthV4Block.getInt(prefixLengthV4Block.getFirstValueIndex(p)), scratch)); + } + return result.build(); + } + } + + public BytesRefVector eval(int positionCount, BytesRefVector ipVector, + IntVector prefixLengthV4Vector) { + try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + BytesRef ipScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendBytesRef(IpPrefix.process(ipVector.getBytesRef(p, ipScratch), prefixLengthV4Vector.getInt(p), scratch)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "IpPrefixOnlyV4Evaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(ip, prefixLengthV4); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory ip; + + private final EvalOperator.ExpressionEvaluator.Factory prefixLengthV4; + + private final Function scratch; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory ip, + EvalOperator.ExpressionEvaluator.Factory prefixLengthV4, + Function scratch) { + this.source = source; + this.ip = ip; + this.prefixLengthV4 = prefixLengthV4; + this.scratch = scratch; + } + + @Override + public IpPrefixOnlyV4Evaluator get(DriverContext context) { + return new IpPrefixOnlyV4Evaluator(source, ip.get(context), prefixLengthV4.get(context), scratch.apply(context), context); + } + + @Override + public String toString() { + return "IpPrefixOnlyV4Evaluator[" + "ip=" + ip + ", prefixLengthV4=" + prefixLengthV4 + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index ca90b9e1e3f2e..e8f136c297ce0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -27,6 +27,11 @@ public class EsqlCapabilities { */ private static final String FN_CBRT = "fn_cbrt"; + /** + * Support for function {@code IP_PREFIX}. + */ + private static final String FN_IP_PREFIX = "fn_ip_prefix"; + /** * Optimization for ST_CENTROID changed some results in cartesian data. #108713 */ @@ -47,6 +52,7 @@ public class EsqlCapabilities { private static Set capabilities() { List caps = new ArrayList<>(); caps.add(FN_CBRT); + caps.add(FN_IP_PREFIX); caps.add(ST_CENTROID_AGG_OPTIMIZED); caps.add(METADATA_IGNORED_FIELD); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 90f28b263f07a..382b02d2f9b35 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -52,6 +52,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; +import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Acos; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Asin; @@ -257,6 +258,7 @@ private FunctionDefinition[][] functions() { new FunctionDefinition[] { def(Coalesce.class, Coalesce::new, "coalesce"), }, // IP new FunctionDefinition[] { def(CIDRMatch.class, CIDRMatch::new, "cidr_match") }, + new FunctionDefinition[] { def(IpPrefix.class, IpPrefix::new, "ip_prefix") }, // conversion functions new FunctionDefinition[] { def(FromBase64.class, FromBase64::new, "from_base64"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java new file mode 100644 index 0000000000000..d271742aef82d --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.ann.Fixed; +import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.function.OptionalArgument; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.DataTypes; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isIPAndExact; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataTypes.INTEGER; + +/** + * Truncates an IP value to a given prefix length. + */ +public class IpPrefix extends EsqlScalarFunction implements OptionalArgument { + // Borrowed from Lucene, rfc4291 prefix + private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 }; + + private final Expression ipField; + private final Expression prefixLengthV4Field; + private final Expression prefixLengthV6Field; + + @FunctionInfo( + returnType = "ip", + description = "Truncates an IP to a given prefix length.", + examples = @Example(file = "ip", tag = "ipPrefix") + ) + public IpPrefix( + Source source, + @Param( + name = "ip", + type = { "ip" }, + description = "IP address of type `ip` (both IPv4 and IPv6 are supported)." + ) Expression ipField, + @Param( + name = "prefixLengthV4", + type = { "integer" }, + description = "Prefix length for IPv4 addresses." + ) Expression prefixLengthV4Field, + @Param( + name = "prefixLengthV6", + type = { "integer" }, + description = "Prefix length for IPv6 addresses." + ) Expression prefixLengthV6Field + ) { + super(source, Arrays.asList(ipField, prefixLengthV4Field, prefixLengthV6Field)); + this.ipField = ipField; + this.prefixLengthV4Field = prefixLengthV4Field; + this.prefixLengthV6Field = prefixLengthV6Field; + } + + public static IpPrefix readFrom(PlanStreamInput in) throws IOException { + return new IpPrefix(in.readSource(), in.readExpression(), in.readExpression(), in.readExpression()); + } + + public static void writeTo(PlanStreamOutput out, IpPrefix ipPrefix) throws IOException { + out.writeSource(ipPrefix.source()); + List fields = ipPrefix.children(); + assert fields.size() == 3; + out.writeExpression(fields.get(0)); + out.writeExpression(fields.get(1)); + out.writeExpression(fields.get(2)); + } + + public Expression ipField() { + return ipField; + } + + public Expression prefixLengthV4Field() { + return prefixLengthV4Field; + } + + public Expression prefixLengthV6Field() { + return prefixLengthV6Field; + } + + @Override + public boolean foldable() { + return Expressions.foldable(children()); + } + + @Override + public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + var ipEvaluatorSupplier = toEvaluator.apply(ipField); + var prefixLengthV4EvaluatorSupplier = toEvaluator.apply(prefixLengthV4Field); + var prefixLengthV6EvaluatorSupplier = toEvaluator.apply(prefixLengthV6Field); + + return new IpPrefixEvaluator.Factory( + source(), + ipEvaluatorSupplier, + prefixLengthV4EvaluatorSupplier, + prefixLengthV6EvaluatorSupplier, + context -> new BytesRef(new byte[16]) + ); + } + + @Evaluator(warnExceptions = IllegalArgumentException.class) + static BytesRef process( + BytesRef ip, + int prefixLengthV4, + int prefixLengthV6, + @Fixed(includeInToString = false, build = true) BytesRef scratch + ) { + if (prefixLengthV4 < 0 || prefixLengthV4 > 32) { + throw new IllegalArgumentException("Prefix length v4 must be in range [0, 32], found " + prefixLengthV4); + } + if (prefixLengthV6 < 0 || prefixLengthV6 > 128) { + throw new IllegalArgumentException("Prefix length v6 must be in range [0, 128], found " + prefixLengthV6); + } + + boolean isIpv4 = Arrays.compareUnsigned(ip.bytes, 0, IPV4_PREFIX.length, IPV4_PREFIX, 0, IPV4_PREFIX.length) == 0; + + if (isIpv4) { + makePrefix(ip, scratch, 12 + prefixLengthV4 / 8, prefixLengthV4 % 8); + } else { + makePrefix(ip, scratch, prefixLengthV6 / 8, prefixLengthV6 % 8); + } + + return scratch; + } + + private static void makePrefix(BytesRef ip, BytesRef scratch, int fullBytes, int remainingBits) { + // Copy the first full bytes + System.arraycopy(ip.bytes, ip.offset, scratch.bytes, 0, fullBytes); + + // Copy the last byte ignoring the trailing bits + if (remainingBits > 0) { + byte lastByteMask = (byte) (0xFF << (8 - remainingBits)); + scratch.bytes[fullBytes] = (byte) (ip.bytes[fullBytes] & lastByteMask); + } + + // Copy the last empty bytes + if (fullBytes < 16) { + Arrays.fill(scratch.bytes, fullBytes + 1, 16, (byte) 0); + } + } + + @Override + public DataType dataType() { + return DataTypes.IP; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + return isIPAndExact(ipField, sourceText(), FIRST).and( + isType(prefixLengthV4Field, dt -> dt == INTEGER, sourceText(), SECOND, "integer") + ).and(isType(prefixLengthV6Field, dt -> dt == INTEGER, sourceText(), THIRD, "integer")); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new IpPrefix(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, IpPrefix::new, ipField, prefixLengthV4Field, prefixLengthV6Field); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index e846360258ebe..67a6a5d4bb98d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -98,6 +98,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; +import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Acos; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Asin; @@ -394,6 +395,7 @@ public static List namedTypeEntries() { of(ScalarFunction.class, DateTrunc.class, PlanNamedTypes::writeDateTrunc, PlanNamedTypes::readDateTrunc), of(ScalarFunction.class, E.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), of(ScalarFunction.class, Greatest.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), + of(ScalarFunction.class, IpPrefix.class, IpPrefix::writeTo, IpPrefix::readFrom), of(ScalarFunction.class, Least.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), of(ScalarFunction.class, Log.class, PlanNamedTypes::writeLog, PlanNamedTypes::readLog), of(ScalarFunction.class, Now.class, PlanNamedTypes::writeNow, PlanNamedTypes::readNow), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java new file mode 100644 index 0000000000000..e46eaea849bb5 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.ip; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataTypes; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; + +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public class IpPrefixTests extends AbstractFunctionTestCase { + public IpPrefixTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + var suppliers = List.of( + // V4 + new TestCaseSupplier( + List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("1.2.3.4"), DataTypes.IP, "ip"), + new TestCaseSupplier.TypedData(24, DataTypes.INTEGER, "prefixLengthV4"), + new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 128), DataTypes.INTEGER, "prefixLengthV6") + ), + "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]", + DataTypes.IP, + equalTo(EsqlDataTypeConverter.stringToIP("1.2.3.0")) + ) + ), + new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), () -> { + var randomIp = randomIp(true); + var randomPrefix = randomIntBetween(0, 32); + var cidrString = InetAddresses.toCidrString(randomIp, randomPrefix); + + var ipParameter = EsqlDataTypeConverter.stringToIP(NetworkAddress.format(randomIp)); + var expectedPrefix = EsqlDataTypeConverter.stringToIP( + NetworkAddress.format(InetAddresses.parseIpRangeFromCidr(cidrString).lowerBound()) + ); + + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(ipParameter, DataTypes.IP, "ip"), + new TestCaseSupplier.TypedData(randomPrefix, DataTypes.INTEGER, "prefixLengthV4"), + new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 128), DataTypes.INTEGER, "prefixLengthV6") + ), + "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]", + DataTypes.IP, + equalTo(expectedPrefix) + ); + }), + + // V6 + new TestCaseSupplier( + List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(EsqlDataTypeConverter.stringToIP("::ff"), DataTypes.IP, "ip"), + new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 32), DataTypes.INTEGER, "prefixLengthV4"), + new TestCaseSupplier.TypedData(127, DataTypes.INTEGER, "prefixLengthV6") + ), + "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]", + DataTypes.IP, + equalTo(EsqlDataTypeConverter.stringToIP("::fe")) + ) + ), + new TestCaseSupplier(List.of(DataTypes.IP, DataTypes.INTEGER, DataTypes.INTEGER), () -> { + var randomIp = randomIp(false); + var randomPrefix = randomIntBetween(0, 128); + var cidrString = InetAddresses.toCidrString(randomIp, randomPrefix); + + var ipParameter = EsqlDataTypeConverter.stringToIP(NetworkAddress.format(randomIp)); + var expectedPrefix = EsqlDataTypeConverter.stringToIP( + NetworkAddress.format(InetAddresses.parseIpRangeFromCidr(cidrString).lowerBound()) + ); + + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(ipParameter, DataTypes.IP, "ip"), + new TestCaseSupplier.TypedData(ESTestCase.randomIntBetween(0, 32), DataTypes.INTEGER, "prefixLengthV4"), + new TestCaseSupplier.TypedData(randomPrefix, DataTypes.INTEGER, "prefixLengthV6") + ), + "IpPrefixEvaluator[ip=Attribute[channel=0], prefixLengthV4=Attribute[channel=1], prefixLengthV6=Attribute[channel=2]]", + DataTypes.IP, + equalTo(expectedPrefix) + ); + }) + ); + + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + } + + @Override + protected Expression build(Source source, List args) { + return new IpPrefix(source, args.get(0), args.get(1), args.size() == 3 ? args.get(2) : null); + } +} From 829070dfea109f73f3a04f42c0e71e3f37e90f53 Mon Sep 17 00:00:00 2001 From: Andrei Dan Date: Wed, 29 May 2024 16:20:44 +0100 Subject: [PATCH 13/29] Install the apm templates only if DSL available (#109166) The APM module installs component and index templates that make use of data stream lifecycle. This makes sure the templates are installed only once the cluster has the data stream lifecycle feature. Follow-up to https://github.com/elastic/elasticsearch/pull/108860 Fixes #106461 --- x-pack/plugin/apm-data/build.gradle | 1 + .../apmdata/APMIndexTemplateRegistry.java | 19 ++++++++++++-- .../xpack/apmdata/APMPlugin.java | 9 ++++++- .../APMIndexTemplateRegistryTests.java | 26 +++++++++++++++++-- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/apm-data/build.gradle b/x-pack/plugin/apm-data/build.gradle index cbd843d227ff4..354be306a0ddd 100644 --- a/x-pack/plugin/apm-data/build.gradle +++ b/x-pack/plugin/apm-data/build.gradle @@ -20,6 +20,7 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation project(path: ':x-pack:plugin:stack') testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(':modules:data-streams') clusterModules project(':modules:data-streams') clusterModules project(':modules:ingest-common') clusterModules project(':modules:ingest-geoip') diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java index 6ec287fe2b980..04b0257f4180a 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -10,12 +10,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -39,12 +42,15 @@ */ public class APMIndexTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(APMIndexTemplateRegistry.class); - + // this node feature is a redefinition of {@link DataStreamFeatures#DATA_STREAM_LIFECYCLE} and it's meant to avoid adding a + // dependency to the data-streams module just for this + public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle"); private final int version; private final Map componentTemplates; private final Map composableIndexTemplates; private final List ingestPipelines; + private final FeatureService featureService; private volatile boolean enabled; @SuppressWarnings("unchecked") @@ -53,7 +59,8 @@ public APMIndexTemplateRegistry( ClusterService clusterService, ThreadPool threadPool, Client client, - NamedXContentRegistry xContentRegistry + NamedXContentRegistry xContentRegistry, + FeatureService featureService ) { super(nodeSettings, clusterService, threadPool, client, xContentRegistry); @@ -78,6 +85,7 @@ public APMIndexTemplateRegistry( Map.Entry> pipelineConfig = map.entrySet().iterator().next(); return loadIngestPipeline(pipelineConfig.getKey(), version, (List) pipelineConfig.getValue().get("dependencies")); }).collect(Collectors.toList()); + this.featureService = featureService; } catch (IOException e) { throw new RuntimeException(e); } @@ -105,6 +113,13 @@ protected String getOrigin() { return ClientHelper.APM_ORIGIN; } + @Override + protected boolean isClusterReady(ClusterChangedEvent event) { + // Ensure current version of the components are installed only after versions that support data stream lifecycle + // due to the use of the feature in all the `@lifecycle` component templates + return featureService.clusterHasFeature(event.state(), DATA_STREAM_LIFECYCLE); + } + @Override protected boolean requiresMasterNode() { return true; diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java index f905c17c04b4c..102b0d38461c3 100644 --- a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java @@ -48,7 +48,14 @@ public Collection createComponents(PluginServices services) { Settings settings = services.environment().settings(); ClusterService clusterService = services.clusterService(); registry.set( - new APMIndexTemplateRegistry(settings, clusterService, services.threadPool(), services.client(), services.xContentRegistry()) + new APMIndexTemplateRegistry( + settings, + clusterService, + services.threadPool(), + services.client(), + services.xContentRegistry(), + services.featureService() + ) ); if (enabled) { APMIndexTemplateRegistry registryInstance = registry.get(); diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java index 8228d7011c9c1..e9f0775836c71 100644 --- a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.apmdata; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -29,6 +30,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamFeatures; import org.elasticsearch.features.FeatureService; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.ingest.PipelineConfiguration; @@ -88,7 +90,7 @@ public void createRegistryAndClient() { threadPool = new TestThreadPool(this.getClass().getName()); client = new VerifyingClient(threadPool); clusterService = ClusterServiceUtils.createClusterService(threadPool, clusterSettings); - FeatureService featureService = new FeatureService(List.of()); + FeatureService featureService = new FeatureService(List.of(new DataStreamFeatures())); stackTemplateRegistryAccessor = new StackTemplateRegistryAccessor( new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY, featureService) ); @@ -98,7 +100,8 @@ public void createRegistryAndClient() { clusterService, threadPool, client, - NamedXContentRegistry.EMPTY + NamedXContentRegistry.EMPTY, + featureService ); apmIndexTemplateRegistry.setEnabled(true); } @@ -355,6 +358,25 @@ public void testIndexTemplateConventions() throws Exception { } } + public void testThatNothingIsInstalledWhenAllNodesAreNotUpdated() { + DiscoveryNode updatedNode = DiscoveryNodeUtils.create("updatedNode"); + DiscoveryNode outdatedNode = DiscoveryNodeUtils.create("outdatedNode", ESTestCase.buildNewFakeTransportAddress(), Version.V_8_10_0); + DiscoveryNodes nodes = DiscoveryNodes.builder() + .localNodeId("updatedNode") + .masterNodeId("updatedNode") + .add(updatedNode) + .add(outdatedNode) + .build(); + + client.setVerifier((a, r, l) -> { + fail("if some cluster mode are not updated to at least v.8.11.0 nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Map.of(), Map.of(), nodes); + apmIndexTemplateRegistry.clusterChanged(event); + } + private Map getIndependentComponentTemplateConfigs() { return apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet().stream().filter(template -> { Settings settings = template.getValue().template().settings(); From 0f8c375bc826e61ed916db457f6a208d148dbd41 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 29 May 2024 11:22:04 -0400 Subject: [PATCH 14/29] Forward port release notes for v8.13.4 (#108527) --- docs/reference/release-notes.asciidoc | 2 ++ docs/reference/release-notes/8.13.4.asciidoc | 22 ++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 docs/reference/release-notes/8.13.4.asciidoc diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 9fb5bd70e4d29..9b1257e9054c9 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. * <> * <> +* <> * <> * <> * <> @@ -68,6 +69,7 @@ This section summarizes the changes in each release. include::release-notes/8.15.0.asciidoc[] include::release-notes/8.14.0.asciidoc[] +include::release-notes/8.13.4.asciidoc[] include::release-notes/8.13.3.asciidoc[] include::release-notes/8.13.2.asciidoc[] include::release-notes/8.13.1.asciidoc[] diff --git a/docs/reference/release-notes/8.13.4.asciidoc b/docs/reference/release-notes/8.13.4.asciidoc new file mode 100644 index 0000000000000..bf3f2f497d8fc --- /dev/null +++ b/docs/reference/release-notes/8.13.4.asciidoc @@ -0,0 +1,22 @@ +[[release-notes-8.13.4]] +== {es} version 8.13.4 + +Also see <>. + +[[bug-8.13.4]] +[float] +=== Bug fixes + +Aggregations:: +* Fix Bucket ordering for partial reduction in date histogram and histogram aggregation {es-pull}108184[#108184] (issue: {es-issue}108181[#108181]) + +ES|QL:: +* Fix `BlockHash` `DirectEncoder` {es-pull}108283[#108283] (issue: {es-issue}108268[#108268]) + +Snapshot/Restore:: +* Ensure necessary security context for s3 bulk deletions {es-pull}108280[#108280] (issue: {es-issue}108049[#108049]) + +TSDB:: +* Fix tsdb codec when doc-values spread in two blocks {es-pull}108276[#108276] + + From aed23cd6af7f836f08caedb9dc321fbed4ea8fad Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 29 May 2024 11:30:30 -0400 Subject: [PATCH 15/29] ESQL: Fix compilation Two PR passing in the night, break each other. --- .../esql/expression/function/scalar/ip/IpPrefix.java | 8 ++++---- .../xpack/esql/io/stream/PlanNamedTypes.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java index d271742aef82d..5c576101668b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java @@ -78,12 +78,12 @@ public IpPrefix( } public static IpPrefix readFrom(PlanStreamInput in) throws IOException { - return new IpPrefix(in.readSource(), in.readExpression(), in.readExpression(), in.readExpression()); + return new IpPrefix(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readExpression()); } - public static void writeTo(PlanStreamOutput out, IpPrefix ipPrefix) throws IOException { - out.writeSource(ipPrefix.source()); - List fields = ipPrefix.children(); + public void writeTo(PlanStreamOutput out) throws IOException { + source().writeTo(out); + List fields = children(); assert fields.size() == 3; out.writeExpression(fields.get(0)); out.writeExpression(fields.get(1)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index 67a6a5d4bb98d..d941b4ef3f40b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -395,7 +395,7 @@ public static List namedTypeEntries() { of(ScalarFunction.class, DateTrunc.class, PlanNamedTypes::writeDateTrunc, PlanNamedTypes::readDateTrunc), of(ScalarFunction.class, E.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), of(ScalarFunction.class, Greatest.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), - of(ScalarFunction.class, IpPrefix.class, IpPrefix::writeTo, IpPrefix::readFrom), + of(ScalarFunction.class, IpPrefix.class, (out, prefix) -> prefix.writeTo(out), IpPrefix::readFrom), of(ScalarFunction.class, Least.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), of(ScalarFunction.class, Log.class, PlanNamedTypes::writeLog, PlanNamedTypes::readLog), of(ScalarFunction.class, Now.class, PlanNamedTypes::writeNow, PlanNamedTypes::readNow), From 6bb687bfb25bbbc8cfac50cf5834b20b03dc575a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 29 May 2024 11:34:43 -0400 Subject: [PATCH 16/29] ESQL: Drop unused wrapper (#109125) This isn't used any more. --- .../planner/EsqlExpressionTranslators.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index d78d2d7f79d80..8873bd770a84e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -53,7 +53,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP; import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG; @@ -376,26 +375,13 @@ public static void checkSpatialRelatesFunction(Expression constantExpression, Sh ); } - /** - * We should normally be using the real `wrapFunctionQuery` above, so we get the benefits of `SingleValueQuery`, - * but at the moment `SingleValueQuery` makes use of `SortDocValues` to determine if the results are single or multi-valued, - * and LeafShapeFieldData does not support `SortedBinaryDocValues getBytesValues()`. - * Skipping this code path entirely is a temporary workaround while separate work is being done to simplify `SingleValueQuery` - * to rather rely on a new method on `LeafFieldData`. This is both for the benefit of the spatial queries, as well as an - * improvement overall. - * TODO: Remove this method and call the parent method once the SingleValueQuery improvements have been made - */ - public static Query wrapFunctionQuery(Supplier querySupplier) { - return querySupplier.get(); - } - public static Query doTranslate(SpatialRelatesFunction bc, TranslatorHandler handler) { if (bc.left().foldable()) { checkSpatialRelatesFunction(bc.left(), bc.queryRelation()); - return wrapFunctionQuery(() -> translate(bc, handler, bc.right(), bc.left())); + return translate(bc, handler, bc.right(), bc.left()); } else { checkSpatialRelatesFunction(bc.right(), bc.queryRelation()); - return wrapFunctionQuery(() -> translate(bc, handler, bc.left(), bc.right())); + return translate(bc, handler, bc.left(), bc.right()); } } From 1582c645bf5e89c4271acec6cb7db886a0ca8fb4 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Wed, 29 May 2024 11:40:46 -0400 Subject: [PATCH 17/29] [ML] Offload request to generic threadpool (#109104) buildTextStructureResponse can take a long time, presumably up to 25s, and is currently blocking the transport thread for that amount of time. Offloading the request to the generic threadpool will at least let the transport thread handle other requests concurrently. Co-authored-by: David Turner --- docs/changelog/109104.yaml | 6 +++ .../transport/TextStructExecutor.java | 47 +++++++++++++++++++ .../TransportFindFieldStructureAction.java | 21 +++++---- .../TransportFindMessageStructureAction.java | 20 +++++--- .../TransportFindStructureAction.java | 26 +++++++--- 5 files changed, 96 insertions(+), 24 deletions(-) create mode 100644 docs/changelog/109104.yaml create mode 100644 x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java diff --git a/docs/changelog/109104.yaml b/docs/changelog/109104.yaml new file mode 100644 index 0000000000000..985cf14bc5952 --- /dev/null +++ b/docs/changelog/109104.yaml @@ -0,0 +1,6 @@ +pr: 109104 +summary: Offload request to generic threadpool +area: Machine Learning +type: bug +issues: + - 109100 diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java new file mode 100644 index 0000000000000..89b21c8dfbcf5 --- /dev/null +++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TextStructExecutor.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.textstructure.transport; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.concurrent.ExecutorService; + +import static org.elasticsearch.common.util.concurrent.EsExecutors.DIRECT_EXECUTOR_SERVICE; + +/** + * workaround for https://github.com/elastic/elasticsearch/issues/97916 + * TODO delete this entire class when we can + */ +public class TextStructExecutor { + private final ThreadPool threadPool; + + @Inject + public TextStructExecutor(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + /** + * when the workaround is removed, change the value in each consuming class's constructor passes to the super constructor from + * DIRECT_EXECUTOR_SERVICE back to threadpool.generic() so that we continue to fork off of the transport thread. + */ + ExecutorService handledTransportActionExecutorService() { + return DIRECT_EXECUTOR_SERVICE; + } + + /** + * when the workaround is removed, change the callers of this function to + * {@link ActionListener#completeWith(ActionListener, CheckedSupplier)}. + */ + void execute(ActionListener listener, CheckedSupplier supplier) { + threadPool.generic().execute(ActionRunnable.supply(listener, supplier)); + } +} diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java index 43a990f6f565b..88e60dc3ffd9f 100644 --- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java +++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindFieldStructureAction.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.textstructure.transport; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -31,6 +32,8 @@ import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC; + public class TransportFindFieldStructureAction extends HandledTransportAction { private final Client client; @@ -58,21 +61,18 @@ protected void doExecute(Task task, FindFieldStructureAction.Request request, Ac .setFetchSource(true) .setQuery(QueryBuilders.existsQuery(request.getField())) .setFetchSource(new String[] { request.getField() }, null) - .execute(ActionListener.wrap(searchResponse -> { - long hitCount = searchResponse.getHits().getHits().length; + .execute(listener.delegateFailureAndWrap((delegate, searchResponse) -> { + var hitCount = searchResponse.getHits().getHits().length; if (hitCount < AbstractFindStructureRequest.MIN_SAMPLE_LINE_COUNT) { - listener.onFailure( + delegate.onFailure( new IllegalArgumentException("Input contained too few lines [" + hitCount + "] to obtain a meaningful sample") ); return; } - List messages = getMessages(searchResponse, request.getField()); - try { - listener.onResponse(buildTextStructureResponse(messages, request)); - } catch (Exception e) { - listener.onFailure(e); - } - }, listener::onFailure)); + var messages = getMessages(searchResponse, request.getField()); + // As matching a regular expression might take a while, we run in a different thread to avoid blocking the network thread. + threadPool.generic().execute(ActionRunnable.supply(delegate, () -> buildTextStructureResponse(messages, request))); + })); } private List getMessages(SearchResponse searchResponse, String field) { @@ -83,6 +83,7 @@ private List getMessages(SearchResponse searchResponse, String field) { private FindStructureResponse buildTextStructureResponse(List messages, FindFieldStructureAction.Request request) throws Exception { + assert ThreadPool.assertCurrentThreadPool(GENERIC); TextStructureFinderManager structureFinderManager = new TextStructureFinderManager(threadPool.scheduler()); TextStructureFinder textStructureFinder = structureFinderManager.findTextStructure( messages, diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java index 79c21b3cea306..d915a7babcbfe 100644 --- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java +++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindMessageStructureAction.java @@ -19,32 +19,38 @@ import org.elasticsearch.xpack.textstructure.structurefinder.TextStructureFinderManager; import org.elasticsearch.xpack.textstructure.structurefinder.TextStructureOverrides; +import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC; + public class TransportFindMessageStructureAction extends HandledTransportAction { private final ThreadPool threadPool; + private final TextStructExecutor executor; @Inject - public TransportFindMessageStructureAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool) { + public TransportFindMessageStructureAction( + TransportService transportService, + ActionFilters actionFilters, + ThreadPool threadPool, + TextStructExecutor executor + ) { super( FindMessageStructureAction.NAME, transportService, actionFilters, FindMessageStructureAction.Request::new, - threadPool.generic() + executor.handledTransportActionExecutorService() ); this.threadPool = threadPool; + this.executor = executor; } @Override protected void doExecute(Task task, FindMessageStructureAction.Request request, ActionListener listener) { - try { - listener.onResponse(buildTextStructureResponse(request)); - } catch (Exception e) { - listener.onFailure(e); - } + executor.execute(listener, () -> buildTextStructureResponse(request)); } private FindStructureResponse buildTextStructureResponse(FindMessageStructureAction.Request request) throws Exception { + assert ThreadPool.assertCurrentThreadPool(GENERIC); TextStructureFinderManager structureFinderManager = new TextStructureFinderManager(threadPool.scheduler()); TextStructureFinder textStructureFinder = structureFinderManager.findTextStructure( request.getMessages(), diff --git a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java index 4257a36bc150a..aa546e1747ba4 100644 --- a/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java +++ b/x-pack/plugin/text-structure/src/main/java/org/elasticsearch/xpack/textstructure/transport/TransportFindStructureAction.java @@ -21,26 +21,38 @@ import java.io.InputStream; +import static org.elasticsearch.threadpool.ThreadPool.Names.GENERIC; + public class TransportFindStructureAction extends HandledTransportAction { private final ThreadPool threadPool; + private final TextStructExecutor executor; @Inject - public TransportFindStructureAction(TransportService transportService, ActionFilters actionFilters, ThreadPool threadPool) { - super(FindStructureAction.NAME, transportService, actionFilters, FindStructureAction.Request::new, threadPool.generic()); + public TransportFindStructureAction( + TransportService transportService, + ActionFilters actionFilters, + ThreadPool threadPool, + TextStructExecutor executor + ) { + super( + FindStructureAction.NAME, + transportService, + actionFilters, + FindStructureAction.Request::new, + executor.handledTransportActionExecutorService() + ); this.threadPool = threadPool; + this.executor = executor; } @Override protected void doExecute(Task task, FindStructureAction.Request request, ActionListener listener) { - try { - listener.onResponse(buildTextStructureResponse(request)); - } catch (Exception e) { - listener.onFailure(e); - } + executor.execute(listener, () -> buildTextStructureResponse(request)); } private FindStructureResponse buildTextStructureResponse(FindStructureAction.Request request) throws Exception { + assert ThreadPool.assertCurrentThreadPool(GENERIC); TextStructureFinderManager structureFinderManager = new TextStructureFinderManager(threadPool.scheduler()); try (InputStream sampleStream = request.getSample().streamInput()) { TextStructureFinder textStructureFinder = structureFinderManager.findTextStructure( From 1413c67d99e25a35dd7f5a17f86281b8f1dca750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Wed, 29 May 2024 17:43:10 +0200 Subject: [PATCH 18/29] [DOCS] Amends inference reference docs and tutorials (#109159) * [DOCS] Fixes inference tutorial widgets. * [DOCS] Adds link to notebooks, rearranges sections in PUT inference API docs. --- .../inference/put-inference.asciidoc | 552 +++++++++--------- .../semantic-search-inference.asciidoc | 12 +- .../infer-api-ingest-pipeline-widget.asciidoc | 2 +- .../infer-api-reindex-widget.asciidoc | 2 +- .../infer-api-search-widget.asciidoc | 2 +- 5 files changed, 294 insertions(+), 276 deletions(-) diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index cac03afd29562..354cee3f6a990 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -8,10 +8,11 @@ Creates an {infer} endpoint to perform an {infer} task. IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure -OpenAI, Google AI Studio or Hugging Face. For built-in models and models uploaded though -Eland, the {infer} APIs offer an alternative way to use and manage trained -models. However, if you do not plan to use the {infer} APIs to use these models -or if you want to use non-NLP models, use the <>. +OpenAI, Google AI Studio or Hugging Face. For built-in models and models +uploaded though Eland, the {infer} APIs offer an alternative way to use and +manage trained models. However, if you do not plan to use the {infer} APIs to +use these models or if you want to use non-NLP models, use the +<>. [discrete] @@ -38,14 +39,14 @@ The create {infer} API enables you to create an {infer} endpoint and configure a The following services are available through the {infer} API: +* Azure AI Studio +* Azure OpenAI * Cohere +* Elasticsearch (for built-in models and models uploaded through Eland) * ELSER +* Google AI Studio * Hugging Face * OpenAI -* Azure OpenAI -* Azure AI Studio -* Elasticsearch (for built-in models and models uploaded through Eland) -* Google AI Studio [discrete] @@ -74,82 +75,34 @@ The type of the {infer} task that the model will perform. Available task types: (Required, string) The type of service supported for the specified task type. Available services: + +* `azureopenai`: specify the `completion` or `text_embedding` task type to use the Azure OpenAI service. +* `azureaistudio`: specify the `completion` or `text_embedding` task type to use the Azure AI Studio service. * `cohere`: specify the `completion`, `text_embedding` or the `rerank` task type to use the Cohere service. +* `elasticsearch`: specify the `text_embedding` task type to use the E5 +built-in model or text embedding models uploaded by Eland. * `elser`: specify the `sparse_embedding` task type to use the ELSER service. +* `googleaistudio`: specify the `completion` task to use the Google AI Studio service. * `hugging_face`: specify the `text_embedding` task type to use the Hugging Face service. * `openai`: specify the `completion` or `text_embedding` task type to use the OpenAI service. -* `azureopenai`: specify the `completion` or `text_embedding` task type to use the Azure OpenAI service. -* `azureaistudio`: specify the `completion` or `text_embedding` task type to use the Azure AI Studio service. -* `elasticsearch`: specify the `text_embedding` task type to use the E5 -built-in model or text embedding models uploaded by Eland. -* `googleaistudio`: specify the `completion` task to use the Google AI Studio service. + `service_settings`:: (Required, object) Settings used to install the {infer} model. These settings are specific to the `service` you specified. + -.`service_settings` for the `cohere` service -[%collapsible%closed] -===== -`api_key`::: -(Required, string) -A valid API key of your Cohere account. You can find your Cohere API keys or you -can create a new one -https://dashboard.cohere.com/api-keys[on the API keys settings page]. - -IMPORTANT: You need to provide the API key only once, during the {infer} model -creation. The <> does not retrieve your API key. After -creating the {infer} model, you cannot change the associated API key. If you -want to use a different API key, delete the {infer} model and recreate it with -the same name and the updated API key. - -`embedding_type`:: -(Optional, string) -Only for `text_embedding`. Specifies the types of embeddings you want to get -back. Defaults to `float`. -Valid values are: - * `byte`: use it for signed int8 embeddings (this is a synonym of `int8`). - * `float`: use it for the default float embeddings. - * `int8`: use it for signed int8 embeddings. - -`model_id`:: -(Optional, string) -The name of the model to use for the {infer} task. -To review the available `rerank` models, refer to the -https://docs.cohere.com/reference/rerank-1[Cohere docs]. - -To review the available `text_embedding` models, refer to the -https://docs.cohere.com/reference/embed[Cohere docs]. The default value for -`text_embedding` is `embed-english-v2.0`. -===== -+ -.`service_settings` for the `elser` service +.`service_settings` for the `azureaistudio` service [%collapsible%closed] ===== -`num_allocations`::: -(Required, integer) -The number of model allocations to create. `num_allocations` must not exceed the -number of available processors per node divided by the `num_threads`. -`num_threads`::: -(Required, integer) -The number of threads to use by each model allocation. `num_threads` must not -exceed the number of available processors per node divided by the number of -allocations. Must be a power of 2. Max allowed value is 32. -===== -+ -.`service_settings` for the `hugging_face` service -[%collapsible%closed] -===== `api_key`::: (Required, string) -A valid access token of your Hugging Face account. You can find your Hugging -Face access tokens or you can create a new one -https://huggingface.co/settings/tokens[on the settings page]. +A valid API key of your Azure AI Studio model deployment. +This key can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account. IMPORTANT: You need to provide the API key only once, during the {infer} model creation. The <> does not retrieve your API key. After @@ -157,43 +110,43 @@ creating the {infer} model, you cannot change the associated API key. If you want to use a different API key, delete the {infer} model and recreate it with the same name and the updated API key. -`url`::: -(Required, string) -The URL endpoint to use for the requests. -===== -+ -.`service_settings` for the `openai` service -[%collapsible%closed] -===== -`api_key`::: +`target`::: (Required, string) -A valid API key of your OpenAI account. You can find your OpenAI API keys in -your OpenAI account under the -https://platform.openai.com/api-keys[API keys section]. - -IMPORTANT: You need to provide the API key only once, during the {infer} model -creation. The <> does not retrieve your API key. After -creating the {infer} model, you cannot change the associated API key. If you -want to use a different API key, delete the {infer} model and recreate it with -the same name and the updated API key. +The target URL of your Azure AI Studio model deployment. +This can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account. -`model_id`::: +`provider`::: (Required, string) -The name of the model to use for the {infer} task. Refer to the -https://platform.openai.com/docs/guides/embeddings/what-are-embeddings[OpenAI documentation] -for the list of available text embedding models. +The model provider for your deployment. +Note that some providers may support only certain task types. +Supported providers include: -`organization_id`::: -(Optional, string) -The unique identifier of your organization. You can find the Organization ID in -your OpenAI account under -https://platform.openai.com/account/organization[**Settings** > **Organizations**]. +* `cohere` - available for `text_embedding` and `completion` task types +* `databricks` - available for `completion` task type only +* `meta` - available for `completion` task type only +* `microsoft_phi` - available for `completion` task type only +* `mistral` - available for `completion` task type only +* `openai` - available for `text_embedding` and `completion` task types -`url`::: -(Optional, string) -The URL endpoint to use for the requests. Can be changed for testing purposes. -Defaults to `https://api.openai.com/v1/embeddings`. +`endpoint_type`::: +(Required, string) +One of `token` or `realtime`. +Specifies the type of endpoint that is used in your model deployment. +There are https://learn.microsoft.com/en-us/azure/ai-studio/concepts/deployments-overview#billing-for-deploying-and-inferencing-llms-in-azure-ai-studio[two endpoint types available] for deployment through Azure AI Studio. +"Pay as you go" endpoints are billed per token. +For these, you must specify `token` for your `endpoint_type`. +For "real-time" endpoints which are billed per hour of usage, specify `realtime`. +`rate_limit`::: +(Optional, object) +By default, the `azureaistudio` service sets the number of requests allowed per minute to `240`. +This helps to minimize the number of rate limit errors returned from Azure AI Studio. +To modify this, set the `requests_per_minute` setting of this object in your service settings: +``` +"rate_limit": { + "requests_per_minute": <> +} +``` ===== + .`service_settings` for the `azureopenai` service @@ -230,14 +183,14 @@ We recommend using the https://learn.microsoft.com/en-us/azure/ai-services/opena ===== + -.`service_settings` for the `azureaistudio` service +.`service_settings` for the `cohere` service [%collapsible%closed] ===== - `api_key`::: (Required, string) -A valid API key of your Azure AI Studio model deployment. -This key can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account. +A valid API key of your Cohere account. You can find your Cohere API keys or you +can create a new one +https://dashboard.cohere.com/api-keys[on the API keys settings page]. IMPORTANT: You need to provide the API key only once, during the {infer} model creation. The <> does not retrieve your API key. After @@ -245,43 +198,61 @@ creating the {infer} model, you cannot change the associated API key. If you want to use a different API key, delete the {infer} model and recreate it with the same name and the updated API key. -`target`::: -(Required, string) -The target URL of your Azure AI Studio model deployment. -This can be found on the overview page for your deployment in the management section of your https://ai.azure.com/[Azure AI Studio] account. +`embedding_type`:: +(Optional, string) +Only for `text_embedding`. Specifies the types of embeddings you want to get +back. Defaults to `float`. +Valid values are: + * `byte`: use it for signed int8 embeddings (this is a synonym of `int8`). + * `float`: use it for the default float embeddings. + * `int8`: use it for signed int8 embeddings. -`provider`::: +`model_id`:: +(Optional, string) +The name of the model to use for the {infer} task. +To review the available `rerank` models, refer to the +https://docs.cohere.com/reference/rerank-1[Cohere docs]. + +To review the available `text_embedding` models, refer to the +https://docs.cohere.com/reference/embed[Cohere docs]. The default value for +`text_embedding` is `embed-english-v2.0`. +===== ++ +.`service_settings` for the `elasticsearch` service +[%collapsible%closed] +===== +`model_id`::: (Required, string) -The model provider for your deployment. -Note that some providers may support only certain task types. -Supported providers include: +The name of the model to use for the {infer} task. It can be the +ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or +a text embedding model already +{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland]. -* `openai` - available for `text_embedding` and `completion` task types -* `mistral` - available for `completion` task type only -* `meta` - available for `completion` task type only -* `microsoft_phi` - available for `completion` task type only -* `cohere` - available for `text_embedding` and `completion` task types -* `databricks` - available for `completion` task type only +`num_allocations`::: +(Required, integer) +The number of model allocations to create. `num_allocations` must not exceed the +number of available processors per node divided by the `num_threads`. -`endpoint_type`::: -(Required, string) -One of `token` or `realtime`. -Specifies the type of endpoint that is used in your model deployment. -There are https://learn.microsoft.com/en-us/azure/ai-studio/concepts/deployments-overview#billing-for-deploying-and-inferencing-llms-in-azure-ai-studio[two endpoint types available] for deployment through Azure AI Studio. -"Pay as you go" endpoints are billed per token. -For these, you must specify `token` for your `endpoint_type`. -For "real-time" endpoints which are billed per hour of usage, specify `realtime`. +`num_threads`::: +(Required, integer) +The number of threads to use by each model allocation. `num_threads` must not +exceed the number of available processors per node divided by the number of +allocations. Must be a power of 2. Max allowed value is 32. +===== ++ +.`service_settings` for the `elser` service +[%collapsible%closed] +===== +`num_allocations`::: +(Required, integer) +The number of model allocations to create. `num_allocations` must not exceed the +number of available processors per node divided by the `num_threads`. -`rate_limit`::: -(Optional, object) -By default, the `azureaistudio` service sets the number of requests allowed per minute to `240`. -This helps to minimize the number of rate limit errors returned from Azure AI Studio. -To modify this, set the `requests_per_minute` setting of this object in your service settings: -``` -"rate_limit": { - "requests_per_minute": <> -} -``` +`num_threads`::: +(Required, integer) +The number of threads to use by each model allocation. `num_threads` must not +exceed the number of available processors per node divided by the number of +allocations. Must be a power of 2. Max allowed value is 32. ===== + .`service_settings` for the `googleiastudio` service @@ -311,26 +282,58 @@ To modify this, set the `requests_per_minute` setting of this object in your ser -- ===== + -.`service_settings` for the `elasticsearch` service +.`service_settings` for the `hugging_face` service +[%collapsible%closed] +===== +`api_key`::: +(Required, string) +A valid access token of your Hugging Face account. You can find your Hugging +Face access tokens or you can create a new one +https://huggingface.co/settings/tokens[on the settings page]. + +IMPORTANT: You need to provide the API key only once, during the {infer} model +creation. The <> does not retrieve your API key. After +creating the {infer} model, you cannot change the associated API key. If you +want to use a different API key, delete the {infer} model and recreate it with +the same name and the updated API key. + +`url`::: +(Required, string) +The URL endpoint to use for the requests. +===== ++ +.`service_settings` for the `openai` service [%collapsible%closed] ===== +`api_key`::: +(Required, string) +A valid API key of your OpenAI account. You can find your OpenAI API keys in +your OpenAI account under the +https://platform.openai.com/api-keys[API keys section]. + +IMPORTANT: You need to provide the API key only once, during the {infer} model +creation. The <> does not retrieve your API key. After +creating the {infer} model, you cannot change the associated API key. If you +want to use a different API key, delete the {infer} model and recreate it with +the same name and the updated API key. + `model_id`::: (Required, string) -The name of the model to use for the {infer} task. It can be the -ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or -a text embedding model already -{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland]. +The name of the model to use for the {infer} task. Refer to the +https://platform.openai.com/docs/guides/embeddings/what-are-embeddings[OpenAI documentation] +for the list of available text embedding models. -`num_allocations`::: -(Required, integer) -The number of model allocations to create. `num_allocations` must not exceed the -number of available processors per node divided by the `num_threads`. +`organization_id`::: +(Optional, string) +The unique identifier of your organization. You can find the Organization ID in +your OpenAI account under +https://platform.openai.com/account/organization[**Settings** > **Organizations**]. + +`url`::: +(Optional, string) +The URL endpoint to use for the requests. Can be changed for testing purposes. +Defaults to `https://api.openai.com/v1/embeddings`. -`num_threads`::: -(Required, integer) -The number of threads to use by each model allocation. `num_threads` must not -exceed the number of available processors per node divided by the number of -allocations. Must be a power of 2. Max allowed value is 32. ===== `task_settings`:: @@ -341,6 +344,18 @@ Settings to configure the {infer} task. These settings are specific to the .`task_settings` for the `completion` task type [%collapsible%closed] ===== +`do_sample`::: +(Optional, float) +For the `azureaistudio` service only. +Instructs the inference process to perform sampling or not. +Has not affect unless `temperature` or `top_p` is specified. + +`max_new_tokens`::: +(Optional, integer) +For the `azureaistudio` service only. +Provides a hint for the maximum number of output tokens to be generated. +Defaults to 64. + `user`::: (Optional, string) For `openai` service only. Specifies the user issuing the request, which can be @@ -358,18 +373,6 @@ For the `azureaistudio` service only. A number in the range of 0.0 to 2.0 that is an alternative value to temperature that causes the model to consider the results of the tokens with nucleus sampling probability. Should not be used if `temperature` is specified. -`do_sample`::: -(Optional, float) -For the `azureaistudio` service only. -Instructs the inference process to perform sampling or not. -Has not affect unless `temperature` or `top_p` is specified. - -`max_new_tokens`::: -(Optional, integer) -For the `azureaistudio` service only. -Provides a hint for the maximum number of output tokens to be generated. -Defaults to 64. - ===== + .`task_settings` for the `rerank` task type @@ -422,6 +425,100 @@ request, which can be used for abuse detection. This section contains example API calls for every service type. +[discrete] +[[inference-example-azureaistudio]] +===== Azure AI Studio service + +The following example shows how to create an {infer} endpoint called +`azure_ai_studio_embeddings` to perform a `text_embedding` task type. +Note that we do not specify a model here, as it is defined already via our Azure AI Studio deployment. + +The list of embeddings models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio model explorer]. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/azure_ai_studio_embeddings +{ + "service": "azureaistudio", + "service_settings": { + "api_key": "", + "target": "", + "provider": "", + "endpoint_type": "" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The next example shows how to create an {infer} endpoint called +`azure_ai_studio_completion` to perform a `completion` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/completion/azure_ai_studio_completion +{ + "service": "azureaistudio", + "service_settings": { + "api_key": "", + "target": "", + "provider": "", + "endpoint_type": "" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The list of chat completion models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=chat-completion[Azure AI Studio model explorer]. + + +[discrete] +[[inference-example-azureopenai]] +===== Azure OpenAI service + +The following example shows how to create an {infer} endpoint called +`azure_openai_embeddings` to perform a `text_embedding` task type. +Note that we do not specify a model here, as it is defined already via our Azure OpenAI deployment. + +The list of embeddings models that you can choose from in your deployment can be found in the https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#embeddings[Azure models documentation]. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/azure_openai_embeddings +{ + "service": "azureopenai", + "service_settings": { + "api_key": "", + "resource_name": "", + "deployment_id": "", + "api_version": "2024-02-01" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The next example shows how to create an {infer} endpoint called +`azure_openai_completion` to perform a `completion` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/completion/azure_openai_completion +{ + "service": "azureopenai", + "service_settings": { + "api_key": "", + "resource_name": "", + "deployment_id": "", + "api_version": "2024-02-01" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The list of chat completion models that you can choose from in your Azure OpenAI deployment can be found at the following places: + +* https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-models[GPT-4 and GPT-4 Turbo models] +* https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-35[GPT-3.5] + [discrete] [[inference-example-cohere]] @@ -471,7 +568,7 @@ https://docs.cohere.com/docs/elasticsearch-and-cohere#rerank-search-results-with [discrete] [[inference-example-e5]] -===== E5 via the elasticsearch service +===== E5 via the `elasticsearch` service The following example shows how to create an {infer} endpoint called `my-e5-model` to perform a `text_embedding` task type. @@ -500,6 +597,8 @@ further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation]. The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type. +Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more +info. [source,console] ------------------------------------------------------------ @@ -533,6 +632,27 @@ Example response: // NOTCONSOLE +[discrete] +[[inference-example-googleaistudio]] +===== Google AI Studio service + +The following example shows how to create an {infer} endpoint called +`google_ai_studio_completion` to perform a `completion` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/completion/google_ai_studio_completion +{ + "service": "googleaistudio", + "service_settings": { + "api_key": "", + "model_id": "" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + + [discrete] [[inference-example-hugging-face]] ===== Hugging Face service @@ -637,115 +757,3 @@ PUT _inference/completion/openai-completion ------------------------------------------------------------ // TEST[skip:TBD] -[discrete] -[[inference-example-azureopenai]] -===== Azure OpenAI service - -The following example shows how to create an {infer} endpoint called -`azure_openai_embeddings` to perform a `text_embedding` task type. -Note that we do not specify a model here, as it is defined already via our Azure OpenAI deployment. - -The list of embeddings models that you can choose from in your deployment can be found in the https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#embeddings[Azure models documentation]. - -[source,console] ------------------------------------------------------------- -PUT _inference/text_embedding/azure_openai_embeddings -{ - "service": "azureopenai", - "service_settings": { - "api_key": "", - "resource_name": "", - "deployment_id": "", - "api_version": "2024-02-01" - } -} ------------------------------------------------------------- -// TEST[skip:TBD] - -The next example shows how to create an {infer} endpoint called -`azure_openai_completion` to perform a `completion` task type. - -[source,console] ------------------------------------------------------------- -PUT _inference/completion/azure_openai_completion -{ - "service": "azureopenai", - "service_settings": { - "api_key": "", - "resource_name": "", - "deployment_id": "", - "api_version": "2024-02-01" - } -} ------------------------------------------------------------- -// TEST[skip:TBD] - -The list of chat completion models that you can choose from in your Azure OpenAI deployment can be found at the following places: - -* https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-4-and-gpt-4-turbo-models[GPT-4 and GPT-4 Turbo models] -* https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#gpt-35[GPT-3.5] - -[discrete] -[[inference-example-azureaistudio]] -===== Azure AI Studio service - -The following example shows how to create an {infer} endpoint called -`azure_ai_studio_embeddings` to perform a `text_embedding` task type. -Note that we do not specify a model here, as it is defined already via our Azure AI Studio deployment. - -The list of embeddings models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio model explorer]. - -[source,console] ------------------------------------------------------------- -PUT _inference/text_embedding/azure_ai_studio_embeddings -{ - "service": "azureaistudio", - "service_settings": { - "api_key": "", - "target": "", - "provider": "", - "endpoint_type": "" - } -} ------------------------------------------------------------- -// TEST[skip:TBD] - -The next example shows how to create an {infer} endpoint called -`azure_ai_studio_completion` to perform a `completion` task type. - -[source,console] ------------------------------------------------------------- -PUT _inference/completion/azure_ai_studio_completion -{ - "service": "azureaistudio", - "service_settings": { - "api_key": "", - "target": "", - "provider": "", - "endpoint_type": "" - } -} ------------------------------------------------------------- -// TEST[skip:TBD] - -The list of chat completion models that you can choose from in your deployment can be found in the https://ai.azure.com/explore/models?selectedTask=chat-completion[Azure AI Studio model explorer]. - -[discrete] -[[inference-example-googleaistudio]] -===== Google AI Studio service - -The following example shows how to create an {infer} endpoint called -`google_ai_studio_completion` to perform a `completion` task type. - -[source,console] ------------------------------------------------------------- -PUT _inference/completion/google_ai_studio_completion -{ - "service": "googleaistudio", - "service_settings": { - "api_key": ">", - "model_id": "" - } -} ------------------------------------------------------------- -// TEST[skip:TBD] diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc index e7e16d74764fa..89464d46744b2 100644 --- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc @@ -114,4 +114,14 @@ provide the query text and the model you have used to create the embeddings. NOTE: If you cancelled the reindexing process, you run the query only a part of the data which affects the quality of your results. -include::{es-ref-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc[] \ No newline at end of file +include::{es-ref-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc[] + + +[discrete] +[[infer-interactive-tutorials]] +==== Interactive tutorials + +You can also find tutorials in an interactive Colab notebook format using the +{es} Python client: +* https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/inference-cohere.ipynb[Cohere {infer} tutorial notebook] +* https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/07-inference.ipynb[OpenAI {infer} tutorial notebook] \ No newline at end of file diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc index ce0f41524c1a0..80f6da2cf602a 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc @@ -8,7 +8,7 @@ Cohere