From 43b2e877e0b937d491745081479e6a1b2edf93cc Mon Sep 17 00:00:00 2001 From: Patrick Doyle <810052+prdoyle@users.noreply.github.com> Date: Wed, 19 Jun 2024 18:10:50 -0400 Subject: [PATCH 01/16] Revert "Move PluginsService to its own internal package (#109872)" (#109946) This reverts commit b9e7965184efb6da6b06de56821db13911803c3a. --- .../script/ScriptScoreBenchmark.java | 2 +- .../resources/checkstyle_suppressions.xml | 2 +- .../plugin-cli/src/main/java/module-info.java | 25 ------------------ .../plugins/cli/InstallPluginAction.java | 2 +- .../plugins/cli/RemovePluginAction.java | 2 +- .../plugins/cli/InstallPluginActionTests.java | 2 +- .../plugins/cli/ListPluginsCommandTests.java | 2 +- .../plugins/cli/RemovePluginActionTests.java | 2 +- .../plugins/cli/SyncPluginsActionTests.java | 2 +- .../datastreams/DataStreamAutoshardingIT.java | 2 +- .../AbstractFeatureMigrationIntegTest.java | 2 +- .../s3/S3BlobStoreRepositoryMetricsTests.java | 2 +- .../s3/S3BlobStoreRepositoryTests.java | 2 +- .../s3/S3RepositoryThirdPartyTests.java | 2 +- .../s3/RepositoryCredentialsTests.java | 2 +- .../bootstrap/SpawnerNoBootstrapTests.java | 2 +- .../http/HealthRestCancellationIT.java | 2 +- .../http/SearchRestCancellationIT.java | 2 +- .../action/IndicesRequestIT.java | 2 +- .../action/admin/ReloadSecureSettingsIT.java | 2 +- .../admin/cluster/tasks/ListTasksIT.java | 2 +- ...tReplicationActionRetryOnClosedNodeIT.java | 2 +- .../coordination/RestHandlerNodesIT.java | 2 +- .../cluster/routing/ShardRoutingRoleIT.java | 2 +- .../indices/recovery/IndexRecoveryIT.java | 2 +- .../indices/recovery/TaskRecoveryIT.java | 2 +- .../metrics/NodeIndexingMetricsIT.java | 2 +- .../IndexFoldersDeletionListenerIT.java | 1 - .../threadpool/SimpleThreadPoolIT.java | 2 +- server/src/main/java/module-info.java | 1 - ...nsportNodesReloadSecureSettingsAction.java | 2 +- .../org/elasticsearch/bootstrap/Security.java | 2 +- .../org/elasticsearch/bootstrap/Spawner.java | 2 +- .../metadata/DataStreamFactoryRetention.java | 2 +- .../common/settings/ClusterSettings.java | 2 +- .../elasticsearch/indices/IndicesService.java | 2 +- .../indices/IndicesServiceBuilder.java | 2 +- .../java/org/elasticsearch/node/Node.java | 2 +- .../elasticsearch/node/NodeConstruction.java | 2 +- .../org/elasticsearch/node/NodeService.java | 2 +- .../node/NodeServiceProvider.java | 2 +- .../plugins/{loading => }/ModuleSupport.java | 2 +- .../plugins/{loading => }/PluginBundle.java | 4 +-- .../{loading => }/PluginIntrospector.java | 23 +--------------- .../PluginLoaderIndirection.java | 2 +- .../plugins/{loading => }/PluginsService.java | 7 +---- .../plugins/{loading => }/PluginsUtils.java | 3 +-- .../StablePluginPlaceHolder.java | 4 +-- .../{loading => }/UberModuleClassLoader.java | 2 +- .../plugins/internal/ReloadAwarePlugin.java | 3 +-- .../scanners/NamedComponentReader.java | 2 +- .../scanners/StablePluginsRegistry.java | 2 +- ...HierarchyCircuitBreakerTelemetryTests.java | 2 +- .../org/elasticsearch/node/NodeTests.java | 2 +- .../plugins/PluginDescriptorTests.java | 2 -- .../PluginIntrospectorTests.java | 23 +--------------- .../{loading => }/PluginsServiceTests.java | 15 ++++------- .../{loading => }/PluginsUtilsTests.java | 3 +-- .../UberModuleClassLoaderTests.java | 2 +- .../internal/ReloadAwarePluginTests.java | 2 +- .../scanners/StablePluginsRegistryTests.java | 2 +- .../SearchTookTimeTelemetryTests.java | 2 +- .../SearchTransportTelemetryTests.java | 2 +- .../snapshots/SnapshotResiliencyTests.java | 2 +- ...ns.PluginsServiceTests$TestExtensionPoint} | 4 +-- .../plugins/{loading => }/dummy-plugin.jar | Bin .../{loading => }/non-extensible-plugin.jar | Bin .../support/CancellableActionTestPlugin.java | 2 +- .../bootstrap/BootstrapForTesting.java | 2 +- .../java/org/elasticsearch/node/MockNode.java | 4 +-- .../{loading => }/MockPluginsService.java | 5 +--- .../plugins/{loading => }/PluginTestUtil.java | 3 +-- .../test/AbstractBuilderTestCase.java | 4 +-- .../AbstractSearchCancellationTestCase.java | 2 +- .../plugins/MockPluginsServiceTests.java | 1 - ...AutoscalingCapacityRestCancellationIT.java | 2 +- .../TransportNodeDeprecationCheckAction.java | 2 +- ...nsportNodeDeprecationCheckActionTests.java | 2 +- .../DownsampleActionSingleNodeTests.java | 2 +- .../AbstractEqlBlockingIntegTestCase.java | 2 +- .../exporter/http/HttpExporterIT.java | 2 +- .../profiling/action/CancellationIT.java | 2 +- ...archableSnapshotsPrewarmingIntegTests.java | 2 +- .../authc/jwt/JwtRealmSingleNodeTests.java | 2 +- .../ProfileCancellationIntegTests.java | 2 +- .../AbstractSqlBlockingIntegTestCase.java | 2 +- .../WriteLoadForecasterIT.java | 2 +- 87 files changed, 87 insertions(+), 180 deletions(-) delete mode 100644 distribution/tools/plugin-cli/src/main/java/module-info.java rename server/src/main/java/org/elasticsearch/plugins/{loading => }/ModuleSupport.java (99%) rename server/src/main/java/org/elasticsearch/plugins/{loading => }/PluginBundle.java (96%) rename server/src/main/java/org/elasticsearch/plugins/{loading => }/PluginIntrospector.java (87%) rename server/src/main/java/org/elasticsearch/plugins/{loading => }/PluginLoaderIndirection.java (95%) rename server/src/main/java/org/elasticsearch/plugins/{loading => }/PluginsService.java (99%) rename server/src/main/java/org/elasticsearch/plugins/{loading => }/PluginsUtils.java (99%) rename server/src/main/java/org/elasticsearch/plugins/{loading => }/StablePluginPlaceHolder.java (85%) rename server/src/main/java/org/elasticsearch/plugins/{loading => }/UberModuleClassLoader.java (99%) rename server/src/test/java/org/elasticsearch/plugins/{loading => }/PluginIntrospectorTests.java (93%) rename server/src/test/java/org/elasticsearch/plugins/{loading => }/PluginsServiceTests.java (98%) rename server/src/test/java/org/elasticsearch/plugins/{loading => }/PluginsUtilsTests.java (99%) rename server/src/test/java/org/elasticsearch/plugins/{loading => }/UberModuleClassLoaderTests.java (99%) rename server/src/test/resources/META-INF/services/{org.elasticsearch.plugins.loading.PluginsServiceTests$TestExtensionPoint => org.elasticsearch.plugins.PluginsServiceTests$TestExtensionPoint} (71%) rename server/src/test/resources/org/elasticsearch/plugins/{loading => }/dummy-plugin.jar (100%) rename server/src/test/resources/org/elasticsearch/plugins/{loading => }/non-extensible-plugin.jar (100%) rename test/framework/src/main/java/org/elasticsearch/plugins/{loading => }/MockPluginsService.java (97%) rename test/framework/src/main/java/org/elasticsearch/plugins/{loading => }/PluginTestUtil.java (96%) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index fb502302be8f0..5a27abe8be2a4 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -33,8 +33,8 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.script.DocReader; import org.elasticsearch.script.DocValuesDocReader; import org.elasticsearch.script.ScoreScript; diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml index de5a8aea34980..fd01993951959 100644 --- a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml +++ b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml @@ -20,7 +20,7 @@ - + diff --git a/distribution/tools/plugin-cli/src/main/java/module-info.java b/distribution/tools/plugin-cli/src/main/java/module-info.java deleted file mode 100644 index 4898125790e37..0000000000000 --- a/distribution/tools/plugin-cli/src/main/java/module-info.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import org.elasticsearch.cli.CliToolProvider; - -module org.elasticsearch.plugins.cli { - requires jopt.simple; - requires org.apache.lucene.core; - requires org.apache.lucene.suggest; - requires org.bouncycastle.fips.core; - requires org.bouncycastle.pg; - requires org.elasticsearch.base; - requires org.elasticsearch.cli; - requires org.elasticsearch.plugin.scanner; - requires org.elasticsearch.server; - requires org.elasticsearch.xcontent; - requires org.objectweb.asm; - - provides CliToolProvider with org.elasticsearch.plugins.cli.PluginCliProvider, org.elasticsearch.plugins.cli.SyncPluginsCliProvider; -} diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java index 9bfe000feb81e..c7bee4a6c172d 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java @@ -41,7 +41,7 @@ import org.elasticsearch.plugin.scanner.NamedComponentScanner; import org.elasticsearch.plugins.Platforms; import org.elasticsearch.plugins.PluginDescriptor; -import org.elasticsearch.plugins.loading.PluginsUtils; +import org.elasticsearch.plugins.PluginsUtils; import org.objectweb.asm.ClassReader; import java.io.BufferedReader; diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java index 6d19641fb372a..4714ef202b258 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java @@ -13,7 +13,7 @@ import org.elasticsearch.cli.UserException; import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; -import org.elasticsearch.plugins.loading.PluginsUtils; +import org.elasticsearch.plugins.PluginsUtils; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java index fe18507398e1c..3dc7af07d4d83 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java @@ -50,7 +50,7 @@ import org.elasticsearch.plugin.scanner.NamedComponentScanner; import org.elasticsearch.plugins.Platforms; import org.elasticsearch.plugins.PluginDescriptor; -import org.elasticsearch.plugins.loading.PluginTestUtil; +import org.elasticsearch.plugins.PluginTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.PosixPermissionsResetter; import org.elasticsearch.test.compiler.InMemoryJavaCompiler; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java index cec6b2c7881b1..b225bc441794a 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java @@ -18,7 +18,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.plugins.loading.PluginTestUtil; +import org.elasticsearch.plugins.PluginTestUtil; import org.junit.Before; import java.io.IOException; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java index 9d9300155e9cb..73e89fc948029 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java @@ -17,7 +17,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.plugins.loading.PluginTestUtil; +import org.elasticsearch.plugins.PluginTestUtil; import org.elasticsearch.test.ESTestCase; import org.junit.Before; diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java index 1e9002a7db392..9802b4039bb7b 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java @@ -14,8 +14,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.plugins.PluginTestUtil; import org.elasticsearch.plugins.cli.SyncPluginsAction.PluginChanges; -import org.elasticsearch.plugins.loading.PluginTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java index 7bc0192b723ce..a4c9a9d3e1c67 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java @@ -51,7 +51,7 @@ import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.telemetry.InstrumentType; import org.elasticsearch.telemetry.Measurement; diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java index f7e07a82e00fb..3bbc8e4b969ee 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java @@ -28,8 +28,8 @@ import org.elasticsearch.indices.AssociatedIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java index d88cf885ef921..f8503bca3ec67 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.s3.S3BlobStore.Operation; diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 3b9eff683dbbf..88f0e01db3e6a 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -38,7 +38,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java index 8ff9fb478c402..5064910723ab6 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -30,7 +30,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java index bd7827be1cca4..9a1d12fab0af5 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryCredentialsTests.java @@ -21,7 +21,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.rest.AbstractRestChannel; import org.elasticsearch.rest.RestRequest; diff --git a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java index 96de064424312..6369e02e1f605 100644 --- a/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java +++ b/qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.plugins.Platforms; -import org.elasticsearch.plugins.loading.PluginTestUtil; +import org.elasticsearch.plugins.PluginTestUtil; import org.elasticsearch.test.GraalVMThreadsFilter; import org.elasticsearch.test.MockLog; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java index 6da08b71a6d58..dc298fc616506 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HealthRestCancellationIT.java @@ -24,7 +24,7 @@ import org.elasticsearch.health.node.HealthInfo; import org.elasticsearch.plugins.HealthPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import java.util.ArrayList; import java.util.Collection; diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java index 462be7eb85283..73dd1525f8a08 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SearchRestCancellationIT.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java index 7969e401d1677..920677e8c4b4a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java @@ -73,7 +73,7 @@ import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java index 9eb603ad894d3..5d4a922ec3e11 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java @@ -23,8 +23,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ReloadablePlugin; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.test.ESIntegTestCase; import org.junit.BeforeClass; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java index 0b0344187c8c6..4a076cb3b6e66 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/tasks/ListTasksIT.java @@ -21,7 +21,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java index 29abeb8badf90..b89cea7dff089 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/support/replication/TransportReplicationActionRetryOnClosedNodeIT.java @@ -27,7 +27,7 @@ import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.ThreadPool; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java index 08ae42c57fff1..5a7f4609a7d0f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RestHandlerNodesIT.java @@ -18,7 +18,7 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.test.ESIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java index 8aacf625e82cf..bb9324dd7d10c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/ShardRoutingRoleIT.java @@ -48,7 +48,7 @@ import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.test.ESIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index ea7df1cacf3e2..4f15b82ca1f16 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -104,7 +104,7 @@ import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java index 833bd3a8c9e3c..16905dc5b4d37 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/TaskRecoveryIT.java @@ -16,7 +16,7 @@ import org.elasticsearch.index.engine.InternalEngine; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.tasks.TaskInfo; import org.elasticsearch.test.ESIntegTestCase; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java index ce0d7419e96e1..7b26cc5edf1bc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java @@ -19,7 +19,7 @@ import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java index 2d1b829b2e2dd..542a4cd2c4c92 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java @@ -21,7 +21,6 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java index a0bc6dd933756..44b6ef1d51ce0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.telemetry.InstrumentType; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 4138036cbcea1..db7e3d40518ba 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -460,5 +460,4 @@ org.elasticsearch.serverless.shardhealth, org.elasticsearch.serverless.apifiltering; exports org.elasticsearch.lucene.spatial; - exports org.elasticsearch.plugins.loading to org.elasticsearch.deprecation, org.elasticsearch.plugins.cli; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index 559bf803ef62a..f906b7d659b7b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -24,8 +24,8 @@ import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ReloadablePlugin; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Security.java b/server/src/main/java/org/elasticsearch/bootstrap/Security.java index 7b05e974f18dc..12edf344c72a2 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -19,7 +19,7 @@ import org.elasticsearch.jdk.JarHell; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; -import org.elasticsearch.plugins.loading.PluginsUtils; +import org.elasticsearch.plugins.PluginsUtils; import org.elasticsearch.secure_sm.SecureSM; import org.elasticsearch.transport.TcpTransport; diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java b/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java index 0910b75563dbc..2d37da1d10245 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Spawner.java @@ -16,7 +16,7 @@ import org.elasticsearch.nativeaccess.NativeAccess; import org.elasticsearch.plugins.Platforms; import org.elasticsearch.plugins.PluginDescriptor; -import org.elasticsearch.plugins.loading.PluginsUtils; +import org.elasticsearch.plugins.PluginsUtils; import java.io.BufferedReader; import java.io.Closeable; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java index 8f4b48aff6be5..5b96f92193e98 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFactoryRetention.java @@ -11,7 +11,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; /** * Holds the factory retention configuration. Factory retention is the global retention configuration meant to be diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 944f0967456e3..d5f770ebb95fc 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -106,7 +106,7 @@ import org.elasticsearch.node.NodeRoleSettings; import org.elasticsearch.persistent.PersistentTasksClusterService; import org.elasticsearch.persistent.decider.EnableAssignmentDecider; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.readiness.ReadinessService; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.BaseRestHandler; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 79e1334211a2d..199bbc54fa3d6 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -133,7 +133,7 @@ import org.elasticsearch.node.Node; import org.elasticsearch.plugins.FieldPredicate; import org.elasticsearch.plugins.IndexStorePlugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java index 0ba62ccc7aa41..d56cf3c2c1e1a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java @@ -29,7 +29,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.IndexStorePlugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.ShardSearchRequest; diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index bbc0a4183c3b6..11eb8760b2dbb 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -68,7 +68,7 @@ import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.MetadataUpgrader; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.readiness.ReadinessService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.reservedstate.service.FileSettingsService; diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 8b6b0a6d9c992..bcf8451e5fe54 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -156,6 +156,7 @@ import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.RecoveryPlannerPlugin; import org.elasticsearch.plugins.ReloadablePlugin; import org.elasticsearch.plugins.RepositoryPlugin; @@ -170,7 +171,6 @@ import org.elasticsearch.plugins.internal.ReloadAwarePlugin; import org.elasticsearch.plugins.internal.RestExtension; import org.elasticsearch.plugins.internal.SettingsExtension; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.readiness.ReadinessService; import org.elasticsearch.repositories.RepositoriesModule; import org.elasticsearch.repositories.RepositoriesService; diff --git a/server/src/main/java/org/elasticsearch/node/NodeService.java b/server/src/main/java/org/elasticsearch/node/NodeService.java index ee99a1bba19e1..059b05091a6ae 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeService.java +++ b/server/src/main/java/org/elasticsearch/node/NodeService.java @@ -31,7 +31,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.monitor.MonitorService; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.aggregations.support.AggregationUsageService; diff --git a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java index 9114f2da6f43c..914dd51d0c6b2 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java +++ b/server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java @@ -26,7 +26,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.recovery.RecoverySettings; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.readiness.ReadinessService; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; diff --git a/server/src/main/java/org/elasticsearch/plugins/loading/ModuleSupport.java b/server/src/main/java/org/elasticsearch/plugins/ModuleSupport.java similarity index 99% rename from server/src/main/java/org/elasticsearch/plugins/loading/ModuleSupport.java rename to server/src/main/java/org/elasticsearch/plugins/ModuleSupport.java index 5072f21ec0552..e5f0004431b0e 100644 --- a/server/src/main/java/org/elasticsearch/plugins/loading/ModuleSupport.java +++ b/server/src/main/java/org/elasticsearch/plugins/ModuleSupport.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.elasticsearch.core.SuppressForbidden; diff --git a/server/src/main/java/org/elasticsearch/plugins/loading/PluginBundle.java b/server/src/main/java/org/elasticsearch/plugins/PluginBundle.java similarity index 96% rename from server/src/main/java/org/elasticsearch/plugins/loading/PluginBundle.java rename to server/src/main/java/org/elasticsearch/plugins/PluginBundle.java index 4c9c25a71d994..154ffce6ba05f 100644 --- a/server/src/main/java/org/elasticsearch/plugins/loading/PluginBundle.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginBundle.java @@ -6,9 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; - -import org.elasticsearch.plugins.PluginDescriptor; +package org.elasticsearch.plugins; import java.io.IOException; import java.net.URL; diff --git a/server/src/main/java/org/elasticsearch/plugins/loading/PluginIntrospector.java b/server/src/main/java/org/elasticsearch/plugins/PluginIntrospector.java similarity index 87% rename from server/src/main/java/org/elasticsearch/plugins/loading/PluginIntrospector.java rename to server/src/main/java/org/elasticsearch/plugins/PluginIntrospector.java index e3fb9e9c2e07d..8b7e4faa1f226 100644 --- a/server/src/main/java/org/elasticsearch/plugins/loading/PluginIntrospector.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginIntrospector.java @@ -6,30 +6,9 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.plugins.ActionPlugin; -import org.elasticsearch.plugins.AnalysisPlugin; -import org.elasticsearch.plugins.CircuitBreakerPlugin; -import org.elasticsearch.plugins.ClusterPlugin; -import org.elasticsearch.plugins.DiscoveryPlugin; -import org.elasticsearch.plugins.EnginePlugin; -import org.elasticsearch.plugins.ExtensiblePlugin; -import org.elasticsearch.plugins.HealthPlugin; -import org.elasticsearch.plugins.IndexStorePlugin; -import org.elasticsearch.plugins.IngestPlugin; -import org.elasticsearch.plugins.MapperPlugin; -import org.elasticsearch.plugins.NetworkPlugin; -import org.elasticsearch.plugins.PersistentTaskPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.RecoveryPlannerPlugin; -import org.elasticsearch.plugins.ReloadablePlugin; -import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.plugins.ShutdownAwarePlugin; -import org.elasticsearch.plugins.SystemIndexPlugin; import java.lang.reflect.Method; import java.lang.reflect.Modifier; diff --git a/server/src/main/java/org/elasticsearch/plugins/loading/PluginLoaderIndirection.java b/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java similarity index 95% rename from server/src/main/java/org/elasticsearch/plugins/loading/PluginLoaderIndirection.java rename to server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java index d0f41eb1cbaf2..d4a703c163025 100644 --- a/server/src/main/java/org/elasticsearch/plugins/loading/PluginLoaderIndirection.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginLoaderIndirection.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.elasticsearch.plugins.loader.ExtendedPluginsClassLoader; diff --git a/server/src/main/java/org/elasticsearch/plugins/loading/PluginsService.java b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java similarity index 99% rename from server/src/main/java/org/elasticsearch/plugins/loading/PluginsService.java rename to server/src/main/java/org/elasticsearch/plugins/PluginsService.java index 2f6b6ed3ab0e1..038b03c5dd93f 100644 --- a/server/src/main/java/org/elasticsearch/plugins/loading/PluginsService.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsService.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -28,11 +28,6 @@ import org.elasticsearch.jdk.JarHell; import org.elasticsearch.jdk.ModuleQualifiedExportsService; import org.elasticsearch.node.ReportingService; -import org.elasticsearch.plugins.ExtensiblePlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.PluginApiInfo; -import org.elasticsearch.plugins.PluginDescriptor; -import org.elasticsearch.plugins.PluginRuntimeInfo; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.plugins.spi.SPIClassIterator; diff --git a/server/src/main/java/org/elasticsearch/plugins/loading/PluginsUtils.java b/server/src/main/java/org/elasticsearch/plugins/PluginsUtils.java similarity index 99% rename from server/src/main/java/org/elasticsearch/plugins/loading/PluginsUtils.java rename to server/src/main/java/org/elasticsearch/plugins/PluginsUtils.java index 8c90e8abc1818..becc5ef05e846 100644 --- a/server/src/main/java/org/elasticsearch/plugins/loading/PluginsUtils.java +++ b/server/src/main/java/org/elasticsearch/plugins/PluginsUtils.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -14,7 +14,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.jdk.JarHell; -import org.elasticsearch.plugins.PluginDescriptor; import java.io.IOException; import java.net.URL; diff --git a/server/src/main/java/org/elasticsearch/plugins/loading/StablePluginPlaceHolder.java b/server/src/main/java/org/elasticsearch/plugins/StablePluginPlaceHolder.java similarity index 85% rename from server/src/main/java/org/elasticsearch/plugins/loading/StablePluginPlaceHolder.java rename to server/src/main/java/org/elasticsearch/plugins/StablePluginPlaceHolder.java index 09bff29977686..c1bc8fcfd12b7 100644 --- a/server/src/main/java/org/elasticsearch/plugins/loading/StablePluginPlaceHolder.java +++ b/server/src/main/java/org/elasticsearch/plugins/StablePluginPlaceHolder.java @@ -6,9 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; - -import org.elasticsearch.plugins.Plugin; +package org.elasticsearch.plugins; class StablePluginPlaceHolder extends Plugin { private final String name; diff --git a/server/src/main/java/org/elasticsearch/plugins/loading/UberModuleClassLoader.java b/server/src/main/java/org/elasticsearch/plugins/UberModuleClassLoader.java similarity index 99% rename from server/src/main/java/org/elasticsearch/plugins/loading/UberModuleClassLoader.java rename to server/src/main/java/org/elasticsearch/plugins/UberModuleClassLoader.java index 6e0d33f7081af..dca3afb2ed745 100644 --- a/server/src/main/java/org/elasticsearch/plugins/loading/UberModuleClassLoader.java +++ b/server/src/main/java/org/elasticsearch/plugins/UberModuleClassLoader.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.elasticsearch.core.SuppressForbidden; diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/ReloadAwarePlugin.java b/server/src/main/java/org/elasticsearch/plugins/internal/ReloadAwarePlugin.java index 3463d3ec64498..71228efe227c5 100644 --- a/server/src/main/java/org/elasticsearch/plugins/internal/ReloadAwarePlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/internal/ReloadAwarePlugin.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.ReloadablePlugin; -import org.elasticsearch.plugins.loading.PluginsService; /** * A plugin that may receive a {@link ReloadablePlugin} in order to @@ -23,7 +22,7 @@ public interface ReloadAwarePlugin { * *

This callback is in the form of an implementation of {@link ReloadablePlugin}, * but the implementation does not need to be a {@link org.elasticsearch.plugins.Plugin}, - * or be registered with {@link PluginsService}. + * or be registered with {@link org.elasticsearch.plugins.PluginsService}. * * @param reloadablePlugin A plugin that this plugin may be able to reload */ diff --git a/server/src/main/java/org/elasticsearch/plugins/scanners/NamedComponentReader.java b/server/src/main/java/org/elasticsearch/plugins/scanners/NamedComponentReader.java index 96ddba9f2a175..ed32bd245977a 100644 --- a/server/src/main/java/org/elasticsearch/plugins/scanners/NamedComponentReader.java +++ b/server/src/main/java/org/elasticsearch/plugins/scanners/NamedComponentReader.java @@ -11,7 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.core.Strings; -import org.elasticsearch.plugins.loading.PluginBundle; +import org.elasticsearch.plugins.PluginBundle; import org.elasticsearch.xcontent.XContentParserConfiguration; import java.io.BufferedInputStream; diff --git a/server/src/main/java/org/elasticsearch/plugins/scanners/StablePluginsRegistry.java b/server/src/main/java/org/elasticsearch/plugins/scanners/StablePluginsRegistry.java index 5c56f040a75a3..6e2780a08251f 100644 --- a/server/src/main/java/org/elasticsearch/plugins/scanners/StablePluginsRegistry.java +++ b/server/src/main/java/org/elasticsearch/plugins/scanners/StablePluginsRegistry.java @@ -8,7 +8,7 @@ package org.elasticsearch.plugins.scanners; -import org.elasticsearch.plugins.loading.PluginBundle; +import org.elasticsearch.plugins.PluginBundle; import java.util.Collection; import java.util.Collections; diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java index 4e94827380a6e..2cbe1202520df 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerTelemetryTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.RecordingInstruments; diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java index dcee2f3cf9a8b..b36cafd694378 100644 --- a/server/src/test/java/org/elasticsearch/node/NodeTests.java +++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java @@ -36,8 +36,8 @@ import org.elasticsearch.plugins.CircuitBreakerPlugin; import org.elasticsearch.plugins.ClusterCoordinationPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsServiceTests; import org.elasticsearch.plugins.RecoveryPlannerPlugin; -import org.elasticsearch.plugins.loading.PluginsServiceTests; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/elasticsearch/plugins/PluginDescriptorTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginDescriptorTests.java index bc14f87f5b580..7ed4d975fe3be 100644 --- a/server/src/test/java/org/elasticsearch/plugins/PluginDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginDescriptorTests.java @@ -12,8 +12,6 @@ import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.plugins.loading.PluginTestUtil; -import org.elasticsearch.plugins.loading.PluginsUtils; import org.elasticsearch.test.ESTestCase; import java.io.IOException; diff --git a/server/src/test/java/org/elasticsearch/plugins/loading/PluginIntrospectorTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/plugins/loading/PluginIntrospectorTests.java rename to server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java index d341ecd72273d..5e80b6d217a55 100644 --- a/server/src/test/java/org/elasticsearch/plugins/loading/PluginIntrospectorTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginIntrospectorTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator; @@ -22,27 +22,6 @@ import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.indices.recovery.plan.ShardSnapshotsService; import org.elasticsearch.ingest.Processor; -import org.elasticsearch.plugins.ActionPlugin; -import org.elasticsearch.plugins.AnalysisPlugin; -import org.elasticsearch.plugins.CircuitBreakerPlugin; -import org.elasticsearch.plugins.ClusterPlugin; -import org.elasticsearch.plugins.DiscoveryPlugin; -import org.elasticsearch.plugins.EnginePlugin; -import org.elasticsearch.plugins.ExtensiblePlugin; -import org.elasticsearch.plugins.HealthPlugin; -import org.elasticsearch.plugins.IndexStorePlugin; -import org.elasticsearch.plugins.IngestPlugin; -import org.elasticsearch.plugins.MapperPlugin; -import org.elasticsearch.plugins.NetworkPlugin; -import org.elasticsearch.plugins.PersistentTaskPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.RecoveryPlannerPlugin; -import org.elasticsearch.plugins.ReloadablePlugin; -import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.plugins.ScriptPlugin; -import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.plugins.ShutdownAwarePlugin; -import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.PrivilegedOperations; import org.elasticsearch.test.compiler.InMemoryJavaCompiler; diff --git a/server/src/test/java/org/elasticsearch/plugins/loading/PluginsServiceTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java similarity index 98% rename from server/src/test/java/org/elasticsearch/plugins/loading/PluginsServiceTests.java rename to server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java index 6e4984e2dcf6d..28ebed88e8f3c 100644 --- a/server/src/test/java/org/elasticsearch/plugins/loading/PluginsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.Constants; @@ -19,11 +19,6 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.jdk.ModuleQualifiedExportsService; import org.elasticsearch.plugin.analysis.CharFilterFactory; -import org.elasticsearch.plugins.ExtensiblePlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.PluginDescriptor; -import org.elasticsearch.plugins.PluginRuntimeInfo; -import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.scanners.PluginInfo; import org.elasticsearch.plugins.spi.BarPlugin; import org.elasticsearch.plugins.spi.BarTestService; @@ -318,18 +313,18 @@ public void testNonExtensibleDep() throws Exception { public void testPassingMandatoryPluginCheck() { PluginsService.checkMandatoryPlugins( - Set.of("org.elasticsearch.plugins.loading.PluginsServiceTests$FakePlugin"), - Set.of("org.elasticsearch.plugins.loading.PluginsServiceTests$FakePlugin") + Set.of("org.elasticsearch.plugins.PluginsServiceTests$FakePlugin"), + Set.of("org.elasticsearch.plugins.PluginsServiceTests$FakePlugin") ); } public void testFailingMandatoryPluginCheck() { IllegalStateException e = expectThrows( IllegalStateException.class, - () -> PluginsService.checkMandatoryPlugins(Set.of(), Set.of("org.elasticsearch.plugins.loading.PluginsServiceTests$FakePlugin")) + () -> PluginsService.checkMandatoryPlugins(Set.of(), Set.of("org.elasticsearch.plugins.PluginsServiceTests$FakePlugin")) ); assertEquals( - "missing mandatory plugins [org.elasticsearch.plugins.loading.PluginsServiceTests$FakePlugin], found plugins []", + "missing mandatory plugins [org.elasticsearch.plugins.PluginsServiceTests$FakePlugin], found plugins []", e.getMessage() ); } diff --git a/server/src/test/java/org/elasticsearch/plugins/loading/PluginsUtilsTests.java b/server/src/test/java/org/elasticsearch/plugins/PluginsUtilsTests.java similarity index 99% rename from server/src/test/java/org/elasticsearch/plugins/loading/PluginsUtilsTests.java rename to server/src/test/java/org/elasticsearch/plugins/PluginsUtilsTests.java index f44de9ac1c704..a7cc74582afdc 100644 --- a/server/src/test/java/org/elasticsearch/plugins/loading/PluginsUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/PluginsUtilsTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.apache.logging.log4j.Level; import org.apache.lucene.tests.util.LuceneTestCase; @@ -14,7 +14,6 @@ import org.elasticsearch.Version; import org.elasticsearch.core.PathUtils; import org.elasticsearch.jdk.JarHell; -import org.elasticsearch.plugins.PluginDescriptor; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; diff --git a/server/src/test/java/org/elasticsearch/plugins/loading/UberModuleClassLoaderTests.java b/server/src/test/java/org/elasticsearch/plugins/UberModuleClassLoaderTests.java similarity index 99% rename from server/src/test/java/org/elasticsearch/plugins/loading/UberModuleClassLoaderTests.java rename to server/src/test/java/org/elasticsearch/plugins/UberModuleClassLoaderTests.java index 6a12009acce41..e3cd11c8f3b68 100644 --- a/server/src/test/java/org/elasticsearch/plugins/loading/UberModuleClassLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/UberModuleClassLoaderTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; diff --git a/server/src/test/java/org/elasticsearch/plugins/internal/ReloadAwarePluginTests.java b/server/src/test/java/org/elasticsearch/plugins/internal/ReloadAwarePluginTests.java index 4553a65944d30..2401ad25193a2 100644 --- a/server/src/test/java/org/elasticsearch/plugins/internal/ReloadAwarePluginTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/internal/ReloadAwarePluginTests.java @@ -11,8 +11,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.MockNode; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ReloadablePlugin; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.netty4.Netty4Plugin; diff --git a/server/src/test/java/org/elasticsearch/plugins/scanners/StablePluginsRegistryTests.java b/server/src/test/java/org/elasticsearch/plugins/scanners/StablePluginsRegistryTests.java index b37da49034f78..276cbdbc1a452 100644 --- a/server/src/test/java/org/elasticsearch/plugins/scanners/StablePluginsRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/plugins/scanners/StablePluginsRegistryTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.plugins.scanners; -import org.elasticsearch.plugins.loading.PluginBundle; +import org.elasticsearch.plugins.PluginBundle; import org.elasticsearch.test.ESTestCase; import org.mockito.Mockito; diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java index 5dad16c5be397..850af7f85f76a 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTookTimeTelemetryTests.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java index 5edb7147ea259..c7fc11e81483f 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/SearchTransportTelemetryTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 3100bdd6db529..f4aa44f143c40 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -162,8 +162,8 @@ import org.elasticsearch.ingest.IngestService; import org.elasticsearch.monitor.StatusInfo; import org.elasticsearch.node.ResponseCollectorService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.internal.DocumentParsingProvider; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; diff --git a/server/src/test/resources/META-INF/services/org.elasticsearch.plugins.loading.PluginsServiceTests$TestExtensionPoint b/server/src/test/resources/META-INF/services/org.elasticsearch.plugins.PluginsServiceTests$TestExtensionPoint similarity index 71% rename from server/src/test/resources/META-INF/services/org.elasticsearch.plugins.loading.PluginsServiceTests$TestExtensionPoint rename to server/src/test/resources/META-INF/services/org.elasticsearch.plugins.PluginsServiceTests$TestExtensionPoint index 20654d4496c23..65ca6fbcac9a5 100644 --- a/server/src/test/resources/META-INF/services/org.elasticsearch.plugins.loading.PluginsServiceTests$TestExtensionPoint +++ b/server/src/test/resources/META-INF/services/org.elasticsearch.plugins.PluginsServiceTests$TestExtensionPoint @@ -6,5 +6,5 @@ # Side Public License, v 1. # -org.elasticsearch.plugins.loading.PluginsServiceTests$TestExtension1 -org.elasticsearch.plugins.loading.PluginsServiceTests$TestExtension2 +org.elasticsearch.plugins.PluginsServiceTests$TestExtension1 +org.elasticsearch.plugins.PluginsServiceTests$TestExtension2 diff --git a/server/src/test/resources/org/elasticsearch/plugins/loading/dummy-plugin.jar b/server/src/test/resources/org/elasticsearch/plugins/dummy-plugin.jar similarity index 100% rename from server/src/test/resources/org/elasticsearch/plugins/loading/dummy-plugin.jar rename to server/src/test/resources/org/elasticsearch/plugins/dummy-plugin.jar diff --git a/server/src/test/resources/org/elasticsearch/plugins/loading/non-extensible-plugin.jar b/server/src/test/resources/org/elasticsearch/plugins/non-extensible-plugin.jar similarity index 100% rename from server/src/test/resources/org/elasticsearch/plugins/loading/non-extensible-plugin.jar rename to server/src/test/resources/org/elasticsearch/plugins/non-extensible-plugin.jar diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/CancellableActionTestPlugin.java b/test/framework/src/main/java/org/elasticsearch/action/support/CancellableActionTestPlugin.java index d46e95aaef328..dad0e3b613efb 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/support/CancellableActionTestPlugin.java +++ b/test/framework/src/main/java/org/elasticsearch/action/support/CancellableActionTestPlugin.java @@ -14,7 +14,7 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskCancelledException; diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java index 1b3ef22db7a86..8ef80c08517de 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java @@ -215,7 +215,7 @@ public boolean implies(ProtectionDomain domain, Permission permission) { static Map getCodebases() { Map codebases = PolicyUtil.getCodebaseJarMap(JarHell.parseClassPath()); // when testing server, the main elasticsearch code is not yet in a jar, so we need to manually add it - addClassCodebase(codebases, "elasticsearch", "org.elasticsearch.plugins.loading.PluginsService"); + addClassCodebase(codebases, "elasticsearch", "org.elasticsearch.plugins.PluginsService"); addClassCodebase(codebases, "elasticsearch-plugin-classloader", "org.elasticsearch.plugins.loader.ExtendedPluginsClassLoader"); addClassCodebase(codebases, "elasticsearch-nio", "org.elasticsearch.nio.ChannelFactory"); addClassCodebase(codebases, "elasticsearch-secure-sm", "org.elasticsearch.secure_sm.SecureSM"); diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index a348be75d0449..520aff77497ba 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -28,9 +28,9 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.plugins.MockPluginsService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.MockPluginsService; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.readiness.MockReadinessService; import org.elasticsearch.readiness.ReadinessService; import org.elasticsearch.script.MockScriptService; diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/loading/MockPluginsService.java b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java similarity index 97% rename from test/framework/src/main/java/org/elasticsearch/plugins/loading/MockPluginsService.java rename to test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java index a393953ca2f99..9d8e7dedcc06d 100644 --- a/test/framework/src/main/java/org/elasticsearch/plugins/loading/MockPluginsService.java +++ b/test/framework/src/main/java/org/elasticsearch/plugins/MockPluginsService.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -16,9 +16,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.env.Environment; import org.elasticsearch.jdk.ModuleQualifiedExportsService; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.PluginDescriptor; -import org.elasticsearch.plugins.PluginRuntimeInfo; import org.elasticsearch.plugins.spi.SPIClassIterator; import java.lang.reflect.Constructor; diff --git a/test/framework/src/main/java/org/elasticsearch/plugins/loading/PluginTestUtil.java b/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java similarity index 96% rename from test/framework/src/main/java/org/elasticsearch/plugins/loading/PluginTestUtil.java rename to test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java index 4418208a10272..7edb46d3e3786 100644 --- a/test/framework/src/main/java/org/elasticsearch/plugins/loading/PluginTestUtil.java +++ b/test/framework/src/main/java/org/elasticsearch/plugins/PluginTestUtil.java @@ -6,10 +6,9 @@ * Side Public License, v 1. */ -package org.elasticsearch.plugins.loading; +package org.elasticsearch.plugins; import org.elasticsearch.Version; -import org.elasticsearch.plugins.PluginDescriptor; import java.io.IOException; import java.io.OutputStream; diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 42332500a83a8..271df2a971fb1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -65,11 +65,11 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.MockPluginsService; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.plugins.loading.MockPluginsService; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptService; diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractSearchCancellationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractSearchCancellationTestCase.java index c4683bacc75c6..5dc707e94bdd7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractSearchCancellationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractSearchCancellationTestCase.java @@ -21,7 +21,7 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.search.SearchService; diff --git a/test/framework/src/test/java/org/elasticsearch/plugins/MockPluginsServiceTests.java b/test/framework/src/test/java/org/elasticsearch/plugins/MockPluginsServiceTests.java index b8f8cecd3615d..055e56db3fb33 100644 --- a/test/framework/src/test/java/org/elasticsearch/plugins/MockPluginsServiceTests.java +++ b/test/framework/src/test/java/org/elasticsearch/plugins/MockPluginsServiceTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.plugins.loading.MockPluginsService; import org.elasticsearch.test.ESTestCase; import org.junit.Before; diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityRestCancellationIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityRestCancellationIT.java index e14ebcd0930be..5b2803c8f4186 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityRestCancellationIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/action/GetAutoscalingCapacityRestCancellationIT.java @@ -16,7 +16,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESIntegTestCase; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java index 19b306868738a..ba72be655a7ff 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckAction.java @@ -26,7 +26,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java index a153b40f730f0..80692efb7474a 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransportNodeDeprecationCheckActionTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 44367b7de51ea..80bb0368a1afc 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -57,7 +57,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchResponseUtils; diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java index efa19266a38fd..414705aff0b79 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AbstractEqlBlockingIntegTestCase.java @@ -22,7 +22,7 @@ import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.search.internal.ReaderContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java index ef4f179bb93a2..5250a1f764e5c 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java @@ -29,7 +29,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.license.TestUtils; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.RestUtils; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.Scope; diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/CancellationIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/CancellationIT.java index dc71c8fd46f79..183ef3786a62d 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/CancellationIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/CancellationIT.java @@ -19,7 +19,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.search.lookup.LeafStoredFieldsLookup; import org.elasticsearch.tasks.CancellableTask; diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java index ece12dd4d1167..42542b63c80d1 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/SearchableSnapshotsPrewarmingIntegTests.java @@ -38,8 +38,8 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.license.LicenseSettings; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.RepositoryPlugin; -import org.elasticsearch.plugins.loading.PluginsService; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.RepositoriesService; diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java index 9aa0201d6b4cc..2ced54a513146 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java @@ -31,7 +31,7 @@ import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.SecuritySettingsSource; import org.elasticsearch.test.SecuritySingleNodeTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java index 5e004b9c048e5..87a5146113f72 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java @@ -24,7 +24,7 @@ import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.search.internal.ReaderContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.tasks.CancellableTask; diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java index a562c379b301d..7ad54901e2d06 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/AbstractSqlBlockingIntegTestCase.java @@ -22,7 +22,7 @@ import org.elasticsearch.license.LicenseSettings; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.search.internal.ReaderContext; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; diff --git a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java index 4b6e31ae7105f..cb93725b320d1 100644 --- a/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java +++ b/x-pack/plugin/write-load-forecaster/src/internalClusterTest/java/org/elasticsearch/xpack/writeloadforecaster/WriteLoadForecasterIT.java @@ -32,7 +32,7 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.plugins.loading.PluginsService; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; From 674fea275b50939dc99b6679c1051bb8f30f2ee9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 20 Jun 2024 12:28:06 +0200 Subject: [PATCH 02/16] Fix classpath for :test:fixtures:krb5kdc-fixture:cacheTestFixtures (#109952) --- test/fixtures/krb5kdc-fixture/build.gradle | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle index c671d58e1e395..c4bb4237a5b6e 100644 --- a/test/fixtures/krb5kdc-fixture/build.gradle +++ b/test/fixtures/krb5kdc-fixture/build.gradle @@ -21,9 +21,6 @@ dockerFixtures { } configurations { - all { - transitive = false - } krb5ConfHdfsFile { canBeConsumed = true canBeResolved = false @@ -36,21 +33,18 @@ configurations { dependencies { testImplementation project(':test:framework') - api "junit:junit:${versions.junit}" api project(':test:fixtures:testcontainer-utils') - api "org.testcontainers:testcontainers:${versions.testcontainer}" - implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + api("org.testcontainers:testcontainers:${versions.testcontainer}") { + transitive = false + } + implementation("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"){ + transitive = false + } implementation "org.slf4j:slf4j-api:${versions.slf4j}" implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}" - runtimeOnly "com.github.docker-java:docker-java-transport:${versions.dockerJava}" - runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}" - runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}" - runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}" - // ensure we have proper logging during when used in tests runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" runtimeOnly "org.hamcrest:hamcrest:${versions.hamcrest}" From 00f4c1320fcb497945ef917bf3c7765047b5b3fb Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 20 Jun 2024 12:33:31 +0200 Subject: [PATCH 03/16] Revert "Fix classpath for :test:fixtures:krb5kdc-fixture:cacheTestFixtures (#109952)" (#109960) This reverts commit 674fea275b50939dc99b6679c1051bb8f30f2ee9. --- test/fixtures/krb5kdc-fixture/build.gradle | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle index c4bb4237a5b6e..c671d58e1e395 100644 --- a/test/fixtures/krb5kdc-fixture/build.gradle +++ b/test/fixtures/krb5kdc-fixture/build.gradle @@ -21,6 +21,9 @@ dockerFixtures { } configurations { + all { + transitive = false + } krb5ConfHdfsFile { canBeConsumed = true canBeResolved = false @@ -33,18 +36,21 @@ configurations { dependencies { testImplementation project(':test:framework') + api "junit:junit:${versions.junit}" api project(':test:fixtures:testcontainer-utils') - api("org.testcontainers:testcontainers:${versions.testcontainer}") { - transitive = false - } - implementation("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"){ - transitive = false - } + api "org.testcontainers:testcontainers:${versions.testcontainer}" + implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" implementation "org.slf4j:slf4j-api:${versions.slf4j}" implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}" + runtimeOnly "com.github.docker-java:docker-java-transport:${versions.dockerJava}" + runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}" + runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}" + runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}" + // ensure we have proper logging during when used in tests runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" runtimeOnly "org.hamcrest:hamcrest:${versions.hamcrest}" From 4a0b026276613cd48895e3b35cb266a2abdcfa12 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 20 Jun 2024 13:01:35 +0200 Subject: [PATCH 04/16] Fix krb5 fixture classpath (#109962) * Fix krb5dc fixture classpath and handling --- test/fixtures/krb5kdc-fixture/build.gradle | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle index c671d58e1e395..733bfd1d4bd29 100644 --- a/test/fixtures/krb5kdc-fixture/build.gradle +++ b/test/fixtures/krb5kdc-fixture/build.gradle @@ -22,7 +22,7 @@ dockerFixtures { configurations { all { - transitive = false + exclude group: 'org.hamcrest', module: 'hamcrest-core' } krb5ConfHdfsFile { canBeConsumed = true @@ -36,21 +36,18 @@ configurations { dependencies { testImplementation project(':test:framework') - api "junit:junit:${versions.junit}" api project(':test:fixtures:testcontainer-utils') - api "org.testcontainers:testcontainers:${versions.testcontainer}" - implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + api("org.testcontainers:testcontainers:${versions.testcontainer}") { + transitive = false + } + implementation("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"){ + transitive = false + } implementation "org.slf4j:slf4j-api:${versions.slf4j}" - implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" + // implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}" - runtimeOnly "com.github.docker-java:docker-java-transport:${versions.dockerJava}" - runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}" - runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}" - runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}" - // ensure we have proper logging during when used in tests runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" runtimeOnly "org.hamcrest:hamcrest:${versions.hamcrest}" From 7b81e0c91a224c84dbd92f752d6e574292d51b6c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 20 Jun 2024 07:25:27 -0400 Subject: [PATCH 05/16] ESQL: Move more function serialization (#109950) This moves yet more of the function serialization code from `PlanNamedTypes` into `NamedWriteable` to line up better with the rest of Elasticsearch. This moves 13 more functions including all "configuration" functions and all date functions and all varags functions. There are 38 left. --- .../function/scalar/UnaryScalarFunction.java | 20 ++- .../expression/predicate/logical/Not.java | 14 ++ .../expression/predicate/nulls/IsNotNull.java | 18 +++ .../expression/predicate/nulls/IsNull.java | 14 ++ .../function/scalar/EsqlScalarFunction.java | 34 +++- .../function/scalar/UnaryScalarFunction.java | 6 + .../function/scalar/conditional/Case.java | 30 ++++ .../function/scalar/conditional/Greatest.java | 30 ++++ .../function/scalar/conditional/Least.java | 30 ++++ .../function/scalar/date/DateDiff.java | 41 +++++ .../function/scalar/date/DateExtract.java | 40 +++++ .../function/scalar/date/DateFormat.java | 40 +++++ .../function/scalar/date/DateParse.java | 31 ++++ .../function/scalar/date/DateTrunc.java | 36 +++++ .../expression/function/scalar/date/Now.java | 21 ++- .../function/scalar/nulls/Coalesce.java | 30 ++++ .../function/scalar/string/Concat.java | 29 ++++ .../function/scalar/string/ToLower.java | 21 +++ .../function/scalar/string/ToUpper.java | 21 +++ .../xpack/esql/io/stream/PlanNamedTypes.java | 146 +----------------- .../AbstractExpressionSerializationTests.java | 4 + .../AbstractVarargsSerializationTests.java | 39 +++++ .../scalar/NotSerializationTests.java | 36 +++++ .../conditional/CaseSerializationTests.java | 28 ++++ .../GreatestSerializationTests.java | 28 ++++ .../conditional/LeastSerializationTests.java | 28 ++++ .../date/DateDiffSerializationTests.java | 52 +++++++ .../date/DateExtractSerializationTests.java | 45 ++++++ .../date/DateFormatSerializationTests.java | 45 ++++++ .../date/DateParseSerializationTests.java | 45 ++++++ .../date/DateTruncSerializationTests.java | 45 ++++++ .../scalar/date/NowSerializationTests.java | 37 +++++ .../scalar/{math => date}/NowTests.java | 3 +- .../nulls/CoalesceSerializationTests.java | 28 ++++ .../nulls/IsNotNullSerializationTests.java | 37 +++++ .../nulls/IsNullSerializationTests.java | 37 +++++ .../string/ConcatSerializationTests.java | 28 ++++ .../string/ToLowerSerializationTests.java | 41 +++++ .../string/ToUpperSerializationTests.java | 41 +++++ 39 files changed, 1143 insertions(+), 156 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractVarargsSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NotSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowSerializationTests.java rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/{math => date}/NowTests.java (97%) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperSerializationTests.java diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java index 2ef0b892138de..e5c2cedfd087b 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java @@ -6,10 +6,15 @@ */ package org.elasticsearch.xpack.esql.core.expression.function.scalar; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; +import java.io.IOException; import java.util.List; import static java.util.Collections.singletonList; @@ -18,16 +23,21 @@ public abstract class UnaryScalarFunction extends ScalarFunction { private final Expression field; - protected UnaryScalarFunction(Source source) { - super(source); - this.field = null; - } - protected UnaryScalarFunction(Source source, Expression field) { super(source, singletonList(field)); this.field = field; } + protected UnaryScalarFunction(StreamInput in) throws IOException { + this(Source.readFrom((StreamInput & PlanStreamInput) in), ((PlanStreamInput) in).readExpression()); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(field); + } + @Override public final UnaryScalarFunction replaceChildren(List newChildren) { return replaceChild(newChildren.get(0)); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java index 31c63393afaea..5f183a1cc26ea 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.logical; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor; @@ -14,15 +16,27 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import java.io.IOException; + import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isBoolean; public class Not extends UnaryScalarFunction implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Not", Not::new); public Not(Source source, Expression child) { super(source, child); } + private Not(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, Not::new, field()); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java index 52375c5db01a1..e365480a6fd79 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.nulls; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; @@ -16,12 +18,28 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import java.io.IOException; + public class IsNotNull extends UnaryScalarFunction implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "IsNotNull", + IsNotNull::new + ); public IsNotNull(Source source, Expression field) { super(source, field); } + private IsNotNull(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, IsNotNull::new, field()); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java index d52eec9114df6..8b6eb5d4404b0 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.nulls; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; @@ -16,12 +18,24 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import java.io.IOException; + public class IsNull extends UnaryScalarFunction implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "IsNull", IsNull::new); public IsNull(Source source, Expression field) { super(source, field); } + private IsNull(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, IsNull::new, field()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java index 4f991af54ecff..17934c1729ad7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java @@ -7,10 +7,25 @@ package org.elasticsearch.xpack.esql.expression.function.scalar; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateDiff; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateFormat; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateParse; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals; import java.util.List; @@ -25,6 +40,24 @@ *

*/ public abstract class EsqlScalarFunction extends ScalarFunction implements EvaluatorMapper { + public static List getNamedWriteables() { + return List.of( + Case.ENTRY, + Coalesce.ENTRY, + Concat.ENTRY, + Greatest.ENTRY, + InsensitiveEquals.ENTRY, + DateExtract.ENTRY, + DateDiff.ENTRY, + DateFormat.ENTRY, + DateParse.ENTRY, + DateTrunc.ENTRY, + Least.ENTRY, + Now.ENTRY, + ToLower.ENTRY, + ToUpper.ENTRY + ); + } protected EsqlScalarFunction(Source source) { super(source); @@ -38,5 +71,4 @@ protected EsqlScalarFunction(Source source, List fields) { public Object fold() { return EvaluatorMapper.super.fold(); } - } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java index 0866f97b67724..eb2e5ab94487f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java @@ -12,6 +12,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromBase64; @@ -76,10 +79,13 @@ public static List getNamedWriteables() { Cosh.ENTRY, Floor.ENTRY, FromBase64.ENTRY, + IsNotNull.ENTRY, + IsNull.ENTRY, Length.ENTRY, Log10.ENTRY, LTrim.ENTRY, Neg.ENTRY, + Not.ENTRY, RTrim.ENTRY, Signum.ENTRY, Sin.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index f98f5c45acd16..50d0e5484756e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.ElementType; @@ -27,8 +30,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -37,8 +43,12 @@ import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; public final class Case extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Case", Case::new); + record Condition(Expression condition, Expression value) {} private final List conditions; @@ -110,6 +120,26 @@ public Case( elseValue = elseValueIsExplicit() ? children().get(children().size() - 1) : new Literal(source, null, NULL); } + private Case(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + private boolean elseValueIsExplicit() { return children().size() % 2 == 1; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java index 8062019b4c51c..580e2f9900208 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -23,17 +26,24 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMax; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.util.List; import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * Returns the maximum value of multiple columns. */ public class Greatest extends EsqlScalarFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Greatest", Greatest::new); + private DataType dataType; @FunctionInfo( @@ -61,6 +71,26 @@ public Greatest( super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } + private Greatest(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { if (dataType == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java index f983e0125a4db..2255fed9d4947 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -23,17 +26,24 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.util.List; import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * Returns the minimum value of multiple columns. */ public class Least extends EsqlScalarFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Least", Least::new); + private DataType dataType; @FunctionInfo( @@ -59,6 +69,26 @@ public Least( super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } + private Least(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { if (dataType == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index 42e20a9a4615e..2a224598253f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -20,7 +23,10 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -47,6 +53,7 @@ * If the second argument (start) is greater than the third argument (end), then negative values are returned. */ public class DateDiff extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "DateDiff", DateDiff::new); public static final ZoneId UTC = ZoneId.of("Z"); @@ -166,6 +173,40 @@ public DateDiff( this.endTimestamp = endTimestamp; } + private DateDiff(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + ((PlanStreamOutput) out).writeExpression(unit); + ((PlanStreamOutput) out).writeExpression(startTimestamp); + ((PlanStreamOutput) out).writeExpression(endTimestamp); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + Expression unit() { + return unit; + } + + Expression startTimestamp() { + return startTimestamp; + } + + Expression endTimestamp() { + return endTimestamp; + } + @Evaluator(extraName = "Constant", warnExceptions = { IllegalArgumentException.class, InvalidArgumentException.class }) static int process(@Fixed Part datePartFieldUnit, long startTimestamp, long endTimestamp) throws IllegalArgumentException { ZonedDateTime zdtStart = ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTimestamp), UTC); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index c28c5e417c152..f3448a2b7c5ff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -22,8 +25,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.time.ZoneId; import java.time.temporal.ChronoField; import java.util.List; @@ -35,6 +41,11 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.chronoToLong; public class DateExtract extends EsqlConfigurationFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "DateExtract", + DateExtract::new + ); private ChronoField chronoField; @@ -69,6 +80,35 @@ public DateExtract( super(source, List.of(chronoFieldExp, field), configuration); } + private DateExtract(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).configuration() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(datePart()); + ((PlanStreamOutput) out).writeExpression(field()); + } + + Expression datePart() { + return children().get(0); + } + + Expression field() { + return children().get(1); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { var fieldEvaluator = toEvaluator.apply(children().get(1)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index bcc5d7cb16050..9a789c2bb6fb2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -22,9 +25,12 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -37,6 +43,11 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; public class DateFormat extends EsqlConfigurationFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "DateFormat", + DateFormat::new + ); private final Expression field; private final Expression format; @@ -59,6 +70,35 @@ Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.S this.format = date != null ? format : null; } + private DateFormat(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readOptionalWriteable(i -> ((PlanStreamInput) i).readExpression()), + ((PlanStreamInput) in).configuration() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeOptionalWriteable(children().size() == 1 ? null : o -> ((PlanStreamOutput) o).writeExpression(children().get(1))); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + Expression field() { + return field; + } + + Expression format() { + return format; + } + @Override public DataType dataType() { return DataType.KEYWORD; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java index d68664afe8418..12ffe092287ed 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -22,8 +25,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.time.ZoneId; import java.util.List; import java.util.function.Function; @@ -38,6 +44,11 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong; public class DateParse extends EsqlScalarFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "DateParse", + DateParse::new + ); private final Expression field; private final Expression format; @@ -64,6 +75,26 @@ public DateParse( this.format = second != null ? first : null; } + private DateParse(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readOptionalWriteable(i -> ((PlanStreamInput) i).readExpression()) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeOptionalWriteable(children().size() == 2 ? o -> ((PlanStreamOutput) out).writeExpression(children().get(1)) : null); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { return DataType.DATETIME; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java index ddd51d281105d..995e525dda9ec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -20,8 +23,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.time.Duration; import java.time.Period; import java.time.ZoneId; @@ -36,6 +42,12 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; public class DateTrunc extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "DateTrunc", + DateTrunc::new + ); + private final Expression interval; private final Expression timestampField; protected static final ZoneId DEFAULT_TZ = ZoneOffset.UTC; @@ -69,6 +81,30 @@ public DateTrunc( this.timestampField = field; } + private DateTrunc(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), ((PlanStreamInput) in).readExpression(), ((PlanStreamInput) in).readExpression()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(interval); + ((PlanStreamOutput) out).writeExpression(timestampField); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + Expression interval() { + return interval; + } + + Expression field() { + return timestampField; + } + @Override protected TypeResolution resolveType() { if (childrenResolved() == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java index fe54cfd186fec..0f401e3de8045 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -18,11 +21,14 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import java.io.IOException; import java.util.List; import java.util.function.Function; public class Now extends EsqlConfigurationFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Now", Now::new); private final long now; @@ -38,13 +44,18 @@ public Now(Source source, Configuration configuration) { this.now = configuration.now() == null ? System.currentTimeMillis() : configuration.now().toInstant().toEpochMilli(); } - private Now(Source source, long now) { - super(source, List.of(), null); - this.now = now; + private Now(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), ((PlanStreamInput) in).configuration()); } - public static Now newInstance(Source source, long now) { - return new Now(source, now); + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + } + + @Override + public String getWriteableName() { + return ENTRY.name; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java index ff7cd83eedbe2..6a02eb4b94f12 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.Page; @@ -27,19 +30,26 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.util.List; import java.util.function.Function; import java.util.stream.IntStream; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * Function returning the first non-null value. */ public class Coalesce extends EsqlScalarFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Coalesce", Coalesce::new); + private DataType dataType; @FunctionInfo( @@ -100,6 +110,26 @@ public Coalesce( super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } + private Coalesce(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { if (dataType == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java index d01edbe7024e8..69464787f9288 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; @@ -22,7 +25,10 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.util.List; import java.util.function.Function; import java.util.stream.Stream; @@ -30,11 +36,14 @@ import static org.elasticsearch.common.unit.ByteSizeUnit.MB; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * Join strings. */ public class Concat extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Concat", Concat::new); static final long MAX_CONCAT_LENGTH = MB.toBytes(1); @@ -51,6 +60,26 @@ public Concat( super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } + private Concat(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { return DataType.KEYWORD; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java index f14df4f56929a..aadb0b3ac7886 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -17,12 +20,15 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; +import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -31,6 +37,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; public class ToLower extends EsqlConfigurationFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "ToLower", ToLower::new); private final Expression field; @@ -52,6 +59,20 @@ public ToLower( this.field = field; } + private ToLower(StreamInput in) throws IOException { + this(Source.EMPTY, ((PlanStreamInput) in).readExpression(), ((PlanStreamInput) in).configuration()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ((PlanStreamOutput) out).writeExpression(field()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { return field.dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java index 6c903b4bfddeb..398fe1c76a49f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -17,12 +20,15 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; +import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -31,6 +37,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; public class ToUpper extends EsqlConfigurationFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "ToUpper", ToUpper::new); private final Expression field; @@ -52,6 +59,20 @@ public ToUpper( this.field = field; } + private ToUpper(StreamInput in) throws IOException { + this(Source.EMPTY, ((PlanStreamInput) in).readExpression(), ((PlanStreamInput) in).configuration()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ((PlanStreamOutput) out).writeExpression(field()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { return field.dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index 831d105a89076..74e8661596e41 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -62,16 +62,8 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; -import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; -import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; -import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateDiff; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateFormat; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateParse; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Atan2; @@ -82,13 +74,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Round; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.AbstractMultivalueFunction; -import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialContains; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialDisjoint; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialIntersects; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWithin; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; @@ -99,13 +89,10 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Dissect.Parser; @@ -246,27 +233,12 @@ public static List namedTypeEntries() { // BinaryLogic of(BinaryLogic.class, And.class, PlanNamedTypes::writeBinaryLogic, PlanNamedTypes::readBinaryLogic), of(BinaryLogic.class, Or.class, PlanNamedTypes::writeBinaryLogic, PlanNamedTypes::readBinaryLogic), - // UnaryScalarFunction - of(QL_UNARY_SCLR_CLS, IsNotNull.class, PlanNamedTypes::writeQLUnaryScalar, PlanNamedTypes::readQLUnaryScalar), - of(QL_UNARY_SCLR_CLS, IsNull.class, PlanNamedTypes::writeQLUnaryScalar, PlanNamedTypes::readQLUnaryScalar), - of(QL_UNARY_SCLR_CLS, Not.class, PlanNamedTypes::writeQLUnaryScalar, PlanNamedTypes::readQLUnaryScalar), // ScalarFunction of(ScalarFunction.class, Atan2.class, PlanNamedTypes::writeAtan2, PlanNamedTypes::readAtan2), - of(ScalarFunction.class, Case.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), of(ScalarFunction.class, CIDRMatch.class, PlanNamedTypes::writeCIDRMatch, PlanNamedTypes::readCIDRMatch), - of(ScalarFunction.class, Coalesce.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), - of(ScalarFunction.class, Concat.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), - of(ScalarFunction.class, DateDiff.class, PlanNamedTypes::writeDateDiff, PlanNamedTypes::readDateDiff), - of(ScalarFunction.class, DateExtract.class, PlanNamedTypes::writeDateExtract, PlanNamedTypes::readDateExtract), - of(ScalarFunction.class, DateFormat.class, PlanNamedTypes::writeDateFormat, PlanNamedTypes::readDateFormat), - of(ScalarFunction.class, DateParse.class, PlanNamedTypes::writeDateTimeParse, PlanNamedTypes::readDateTimeParse), - of(ScalarFunction.class, DateTrunc.class, PlanNamedTypes::writeDateTrunc, PlanNamedTypes::readDateTrunc), of(ScalarFunction.class, E.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), - of(ScalarFunction.class, Greatest.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), of(ScalarFunction.class, IpPrefix.class, (out, prefix) -> prefix.writeTo(out), IpPrefix::readFrom), - of(ScalarFunction.class, Least.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), of(ScalarFunction.class, Log.class, PlanNamedTypes::writeLog, PlanNamedTypes::readLog), - of(ScalarFunction.class, Now.class, PlanNamedTypes::writeNow, PlanNamedTypes::readNow), of(ScalarFunction.class, Pi.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), of(ScalarFunction.class, Round.class, PlanNamedTypes::writeRound, PlanNamedTypes::readRound), of(ScalarFunction.class, Pow.class, PlanNamedTypes::writePow, PlanNamedTypes::readPow), @@ -284,8 +256,6 @@ public static List namedTypeEntries() { of(ScalarFunction.class, Split.class, PlanNamedTypes::writeSplit, PlanNamedTypes::readSplit), of(ScalarFunction.class, Tau.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), of(ScalarFunction.class, Replace.class, PlanNamedTypes::writeReplace, PlanNamedTypes::readReplace), - of(ScalarFunction.class, ToLower.class, PlanNamedTypes::writeToLower, PlanNamedTypes::readToLower), - of(ScalarFunction.class, ToUpper.class, PlanNamedTypes::writeToUpper, PlanNamedTypes::readToUpper), // GroupingFunctions of(GroupingFunction.class, Bucket.class, PlanNamedTypes::writeBucket, PlanNamedTypes::readBucket), // AggregateFunctions @@ -309,10 +279,11 @@ public static List namedTypeEntries() { AbstractMultivalueFunction.getNamedWriteables(), EsqlArithmeticOperation.getNamedWriteables(), EsqlBinaryComparison.getNamedWriteables(), + EsqlScalarFunction.getNamedWriteables(), FullTextPredicate.getNamedWriteables(), NamedExpression.getNamedWriteables(), UnaryScalarFunction.getNamedWriteables(), - List.of(UnsupportedAttribute.ENTRY, InsensitiveEquals.ENTRY, Literal.ENTRY, org.elasticsearch.xpack.esql.expression.Order.ENTRY) + List.of(UnsupportedAttribute.ENTRY, Literal.ENTRY, org.elasticsearch.xpack.esql.expression.Order.ENTRY) )) { for (NamedWriteableRegistry.Entry e : ee) { entries.add(of(Expression.class, e)); @@ -1081,32 +1052,6 @@ static void writeBucket(PlanStreamOutput out, Bucket bucket) throws IOException out.writeOptionalExpression(bucket.to()); } - static final Map, ScalarFunction>> VARARG_CTORS = Map.ofEntries( - entry(name(Case.class), Case::new), - entry(name(Coalesce.class), Coalesce::new), - entry(name(Concat.class), Concat::new), - entry(name(Greatest.class), Greatest::new), - entry(name(Least.class), Least::new) - ); - - static ScalarFunction readVarag(PlanStreamInput in, String name) throws IOException { - return VARARG_CTORS.get(name) - .apply( - Source.readFrom(in), - in.readExpression(), - in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) - ); - } - - static void writeVararg(PlanStreamOutput out, ScalarFunction vararg) throws IOException { - vararg.source().writeTo(out); - out.writeExpression(vararg.children().get(0)); - out.writeCollection( - vararg.children().subList(1, vararg.children().size()), - writerFromPlanWriter(PlanStreamOutput::writeExpression) - ); - } - static CountDistinct readCountDistinct(PlanStreamInput in) throws IOException { return new CountDistinct(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } @@ -1119,67 +1064,6 @@ static void writeCountDistinct(PlanStreamOutput out, CountDistinct countDistinct out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); } - static DateDiff readDateDiff(PlanStreamInput in) throws IOException { - return new DateDiff(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readExpression()); - } - - static void writeDateDiff(PlanStreamOutput out, DateDiff function) throws IOException { - Source.EMPTY.writeTo(out); - List fields = function.children(); - assert fields.size() == 3; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - out.writeExpression(fields.get(2)); - } - - static DateExtract readDateExtract(PlanStreamInput in) throws IOException { - return new DateExtract(Source.readFrom(in), in.readExpression(), in.readExpression(), in.configuration()); - } - - static void writeDateExtract(PlanStreamOutput out, DateExtract function) throws IOException { - function.source().writeTo(out); - List fields = function.children(); - assert fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - } - - static DateFormat readDateFormat(PlanStreamInput in) throws IOException { - return new DateFormat(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class), in.configuration()); - } - - static void writeDateFormat(PlanStreamOutput out, DateFormat dateFormat) throws IOException { - dateFormat.source().writeTo(out); - List fields = dateFormat.children(); - assert fields.size() == 1 || fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); - } - - static DateParse readDateTimeParse(PlanStreamInput in) throws IOException { - return new DateParse(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); - } - - static void writeDateTimeParse(PlanStreamOutput out, DateParse function) throws IOException { - function.source().writeTo(out); - List fields = function.children(); - assert fields.size() == 1 || fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); - } - - static DateTrunc readDateTrunc(PlanStreamInput in) throws IOException { - return new DateTrunc(Source.readFrom(in), in.readExpression(), in.readExpression()); - } - - static void writeDateTrunc(PlanStreamOutput out, DateTrunc dateTrunc) throws IOException { - dateTrunc.source().writeTo(out); - List fields = dateTrunc.children(); - assert fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - } - static SpatialIntersects readIntersects(PlanStreamInput in) throws IOException { return new SpatialIntersects(Source.EMPTY, in.readExpression(), in.readExpression()); } @@ -1201,14 +1085,6 @@ static void writeSpatialRelatesFunction(PlanStreamOutput out, SpatialRelatesFunc out.writeExpression(spatialRelatesFunction.right()); } - static Now readNow(PlanStreamInput in) throws IOException { - return new Now(Source.readFrom(in), in.configuration()); - } - - static void writeNow(PlanStreamOutput out, Now function) throws IOException { - Source.EMPTY.writeTo(out); - } - static Round readRound(PlanStreamInput in) throws IOException { return new Round(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } @@ -1303,22 +1179,6 @@ static void writeReplace(PlanStreamOutput out, Replace replace) throws IOExcepti out.writeExpression(fields.get(2)); } - static ToLower readToLower(PlanStreamInput in) throws IOException { - return new ToLower(Source.EMPTY, in.readExpression(), in.configuration()); - } - - static void writeToLower(PlanStreamOutput out, ToLower toLower) throws IOException { - out.writeExpression(toLower.field()); - } - - static ToUpper readToUpper(PlanStreamInput in) throws IOException { - return new ToUpper(Source.EMPTY, in.readExpression(), in.configuration()); - } - - static void writeToUpper(PlanStreamOutput out, ToUpper toUpper) throws IOException { - out.writeExpression(toUpper.field()); - } - static Left readLeft(PlanStreamInput in) throws IOException { return new Left(Source.readFrom(in), in.readExpression(), in.readExpression()); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java index 9b33af9f0a2e0..33f9cb3123b8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java @@ -78,6 +78,10 @@ protected boolean alwaysEmptySource() { protected abstract List getNamedWriteables(); + public EsqlConfiguration configuration() { + return config; + } + @Override protected final NamedWriteableRegistry getNamedWriteableRegistry() { List entries = new ArrayList<>(NamedExpression.getNamedWriteables()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractVarargsSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractVarargsSerializationTests.java new file mode 100644 index 0000000000000..67195fa99114b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractVarargsSerializationTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.io.IOException; +import java.util.List; + +public abstract class AbstractVarargsSerializationTests extends AbstractExpressionSerializationTests { + protected abstract T create(Source source, Expression first, List rest); + + @Override + protected final T createTestInstance() { + Source source = randomSource(); + Expression first = randomChild(); + List rest = randomList(0, 10, AbstractExpressionSerializationTests::randomChild); + return create(source, first, rest); + } + + @Override + protected final T mutateInstance(T instance) throws IOException { + Source source = instance.source(); + Expression first = instance.children().get(0); + List rest = instance.children().subList(1, instance.children().size()); + if (randomBoolean()) { + first = randomValueOtherThan(first, AbstractExpressionSerializationTests::randomChild); + } else { + rest = randomValueOtherThan(rest, () -> randomList(0, 10, AbstractExpressionSerializationTests::randomChild)); + } + return create(instance.source(), first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NotSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NotSerializationTests.java new file mode 100644 index 0000000000000..61e3690f1633f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NotSerializationTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; +import java.util.List; + +public class NotSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return UnaryScalarFunction.getNamedWriteables(); + } + + @Override + protected Not createTestInstance() { + return new Not(randomSource(), randomChild()); + } + + @Override + protected Not mutateInstance(Not instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new Not(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseSerializationTests.java new file mode 100644 index 0000000000000..69bbf2f76937f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class CaseSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Case create(Source source, Expression first, List rest) { + return new Case(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestSerializationTests.java new file mode 100644 index 0000000000000..43e1fe405911a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class GreatestSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Greatest create(Source source, Expression first, List rest) { + return new Greatest(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastSerializationTests.java new file mode 100644 index 0000000000000..f552713af4dbe --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class LeastSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Least create(Source source, Expression first, List rest) { + return new Least(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffSerializationTests.java new file mode 100644 index 0000000000000..b1dc1b064ae5a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffSerializationTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateDiffSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateDiff createTestInstance() { + Source source = randomSource(); + Expression unit = randomChild(); + Expression startTimestamp = randomChild(); + Expression endTimestamp = randomChild(); + return new DateDiff(source, unit, startTimestamp, endTimestamp); + } + + @Override + protected DateDiff mutateInstance(DateDiff instance) throws IOException { + Source source = instance.source(); + Expression unit = instance.unit(); + Expression startTimestamp = instance.startTimestamp(); + Expression endTimestamp = instance.endTimestamp(); + switch (between(0, 2)) { + case 0 -> unit = randomValueOtherThan(unit, AbstractExpressionSerializationTests::randomChild); + case 1 -> startTimestamp = randomValueOtherThan(startTimestamp, AbstractExpressionSerializationTests::randomChild); + case 2 -> endTimestamp = randomValueOtherThan(endTimestamp, AbstractExpressionSerializationTests::randomChild); + } + return new DateDiff(source, unit, startTimestamp, endTimestamp); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractSerializationTests.java new file mode 100644 index 0000000000000..6e1c061c84f2e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractSerializationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateExtractSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateExtract createTestInstance() { + Source source = randomSource(); + Expression datePart = randomChild(); + Expression field = randomChild(); + return new DateExtract(source, datePart, field, configuration()); + } + + @Override + protected DateExtract mutateInstance(DateExtract instance) throws IOException { + Source source = instance.source(); + Expression datePart = instance.datePart(); + Expression field = instance.field(); + if (randomBoolean()) { + datePart = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } else { + field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } + return new DateExtract(source, datePart, field, configuration()); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatSerializationTests.java new file mode 100644 index 0000000000000..4dff735318558 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatSerializationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateFormatSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateFormat createTestInstance() { + Source source = randomSource(); + Expression field = randomChild(); + Expression format = randomBoolean() ? null : randomChild(); + return new DateFormat(source, field, format, configuration()); + } + + @Override + protected DateFormat mutateInstance(DateFormat instance) throws IOException { + Source source = instance.source(); + Expression field = instance.field(); + Expression format = instance.format(); + if (randomBoolean()) { + field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } else { + format = randomValueOtherThan(format, () -> randomBoolean() ? null : randomChild()); + } + return new DateFormat(source, field, format, configuration()); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java new file mode 100644 index 0000000000000..e816f2c4a20fb --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateParseSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateParse createTestInstance() { + Source source = randomSource(); + Expression first = randomChild(); + Expression second = randomBoolean() ? null : randomChild(); + return new DateParse(source, first, second); + } + + @Override + protected DateParse mutateInstance(DateParse instance) throws IOException { + Source source = instance.source(); + Expression first = instance.children().get(0); + Expression second = instance.children().size() == 1 ? null : instance.children().get(1); + if (randomBoolean()) { + first = randomValueOtherThan(first, AbstractExpressionSerializationTests::randomChild); + } else { + second = randomValueOtherThan(second, () -> randomBoolean() ? null : randomChild()); + } + return new DateParse(source, first, second); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncSerializationTests.java new file mode 100644 index 0000000000000..09d2e06003128 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncSerializationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateTruncSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateTrunc createTestInstance() { + Source source = randomSource(); + Expression interval = randomChild(); + Expression field = randomChild(); + return new DateTrunc(source, interval, field); + } + + @Override + protected DateTrunc mutateInstance(DateTrunc instance) throws IOException { + Source source = instance.source(); + Expression interval = instance.interval(); + Expression field = instance.field(); + if (randomBoolean()) { + interval = randomValueOtherThan(interval, AbstractExpressionSerializationTests::randomChild); + } else { + field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } + return new DateTrunc(source, interval, field); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowSerializationTests.java new file mode 100644 index 0000000000000..3bb8c2f260561 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class NowSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Now createTestInstance() { + return new Now(randomSource(), configuration()); + } + + @Override + protected Now mutateInstance(Now instance) throws IOException { + return null; + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java similarity index 97% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java index 2c1322abf8cda..8edc21db427d2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.expression.function.scalar.math; +package org.elasticsearch.xpack.esql.expression.function.scalar.date; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceSerializationTests.java new file mode 100644 index 0000000000000..7cab0a957b235 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class CoalesceSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Coalesce create(Source source, Expression first, List rest) { + return new Coalesce(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullSerializationTests.java new file mode 100644 index 0000000000000..23545b3627a1a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class IsNotNullSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return UnaryScalarFunction.getNamedWriteables(); + } + + @Override + protected IsNotNull createTestInstance() { + return new IsNotNull(randomSource(), randomChild()); + } + + @Override + protected IsNotNull mutateInstance(IsNotNull instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new IsNotNull(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullSerializationTests.java new file mode 100644 index 0000000000000..354a2129d7ec0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class IsNullSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return UnaryScalarFunction.getNamedWriteables(); + } + + @Override + protected IsNull createTestInstance() { + return new IsNull(randomSource(), randomChild()); + } + + @Override + protected IsNull mutateInstance(IsNull instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new IsNull(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatSerializationTests.java new file mode 100644 index 0000000000000..30f6acffbaf8a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class ConcatSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Concat create(Source source, Expression first, List rest) { + return new Concat(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerSerializationTests.java new file mode 100644 index 0000000000000..f2dbdbd74470a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerSerializationTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class ToLowerSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected ToLower createTestInstance() { + return new ToLower(randomSource(), randomChild(), configuration()); + } + + @Override + protected ToLower mutateInstance(ToLower instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new ToLower(source, child, configuration()); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperSerializationTests.java new file mode 100644 index 0000000000000..e57aedd79fdfd --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperSerializationTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class ToUpperSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected ToUpper createTestInstance() { + return new ToUpper(randomSource(), randomChild(), configuration()); + } + + @Override + protected ToUpper mutateInstance(ToUpper instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new ToUpper(source, child, configuration()); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} From 4eaaf9c4951603b025574c9321bc66f95814455b Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 20 Jun 2024 14:49:48 +0300 Subject: [PATCH 06/16] ESQL: Adds a test and fixes running the CsvTestsDataLoader.main (#109836) --- x-pack/plugin/esql/build.gradle | 4 +++- .../plugin/esql/qa/testFixtures/build.gradle | 13 ++++++++--- .../xpack/esql/CsvTestsDataLoaderTests.java | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 92071543aa27e..dbec0963d1aab 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -28,7 +28,9 @@ dependencies { // Also contains a dummy processor to allow compilation with unused annotations. annotationProcessor project('compute:gen') - testImplementation project('qa:testFixtures') + testImplementation(project('qa:testFixtures')) { + exclude(group:"org.elasticsearch.plugin", module: "esql") + } testImplementation project(':test:framework') testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: xpackModule('enrich')) diff --git a/x-pack/plugin/esql/qa/testFixtures/build.gradle b/x-pack/plugin/esql/qa/testFixtures/build.gradle index 520873a6cb03e..e8a95011100f5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/build.gradle +++ b/x-pack/plugin/esql/qa/testFixtures/build.gradle @@ -1,9 +1,9 @@ apply plugin: 'elasticsearch.java' - +apply plugin: org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin dependencies { implementation project(':x-pack:plugin:esql:compute') - compileOnly project(':x-pack:plugin:esql') + implementation project(':x-pack:plugin:esql') compileOnly project(path: xpackModule('core')) implementation project(":libs:elasticsearch-x-content") implementation project(':client:rest') @@ -11,7 +11,14 @@ dependencies { implementation project(':test:framework') api(testArtifact(project(xpackModule('esql-core')))) implementation project(':server') - api "net.sf.supercsv:super-csv:${versions.supercsv}" + implementation "net.sf.supercsv:super-csv:${versions.supercsv}" +} + +/** + * This is needed for CsvTestsDataLoaderTests to reflect the classpath that CsvTestsDataLoader actually uses when "main" method is executed. + */ +tasks.named("test").configure { + classpath = classpath - (configurations.resolveableCompileOnly - configurations.runtimeClasspath) } /** diff --git a/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java b/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java new file mode 100644 index 0000000000000..5b40e1d03e92f --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql; + +import org.elasticsearch.test.ESTestCase; + +import java.net.ConnectException; + +import static org.hamcrest.Matchers.startsWith; + +public class CsvTestsDataLoaderTests extends ESTestCase { + + public void testCsvTestsDataLoaderExecution() { + ConnectException ce = expectThrows(ConnectException.class, () -> CsvTestsDataLoader.main(new String[] {})); + assertThat(ce.getMessage(), startsWith("Connection refused")); + } +} From a692aeded973ef8b89eae560f85b3b2244169cac Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:21:20 +0200 Subject: [PATCH 07/16] Include component templates in retention validaiton (#109779) We shouldn't disregard a component template's lifecycle configuration if the index template has one during retention validation. --- docs/changelog/109779.yaml | 5 +++ .../MetadataIndexTemplateService.java | 4 +- ...amLifecycleWithRetentionWarningsTests.java | 42 +++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/109779.yaml diff --git a/docs/changelog/109779.yaml b/docs/changelog/109779.yaml new file mode 100644 index 0000000000000..4ccd8d475ec8d --- /dev/null +++ b/docs/changelog/109779.yaml @@ -0,0 +1,5 @@ +pr: 109779 +summary: Include component templates in retention validaiton +area: Data streams +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index affc331c5ab49..6d99874fd2edb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -805,9 +805,7 @@ static void validateLifecycle( ComposableIndexTemplate template, @Nullable DataStreamGlobalRetention globalRetention ) { - DataStreamLifecycle lifecycle = template.template() != null && template.template().lifecycle() != null - ? template.template().lifecycle() - : resolveLifecycle(template, metadata.componentTemplates()); + DataStreamLifecycle lifecycle = resolveLifecycle(template, metadata.componentTemplates()); if (lifecycle != null) { if (template.getDataStreamTemplate() == null) { throw new IllegalArgumentException( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java index 7e338c52a0a17..922a1405bddff 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java @@ -191,6 +191,48 @@ public void testValidateLifecycleIndexTemplateWithWarning() { ); } + /** + * Make sure we still take into account component templates during validation (and not just the index template). + */ + public void testValidateLifecycleComponentTemplateWithWarning() { + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + HeaderWarning.setThreadContext(threadContext); + TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS); + MetadataIndexTemplateService.validateLifecycle( + Metadata.builder() + .componentTemplates( + Map.of( + "component-template", + new ComponentTemplate( + new Template( + null, + null, + null, + new DataStreamLifecycle( + new DataStreamLifecycle.Retention(randomTimeValue(2, 100, TimeUnit.DAYS)), + null, + null + ) + ), + null, + null + ) + ) + ) + .build(), + randomAlphaOfLength(10), + ComposableIndexTemplate.builder() + .template(new Template(null, null, null, DataStreamLifecycle.DEFAULT)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .indexPatterns(List.of(randomAlphaOfLength(10))) + .componentTemplates(List.of("component-template")) + .build(), + new DataStreamGlobalRetention(defaultRetention, null) + ); + Map> responseHeaders = threadContext.getResponseHeaders(); + assertThat(responseHeaders.size(), is(0)); + } + public void testValidateLifecycleInComponentTemplate() throws Exception { IndicesService indicesService = mock(IndicesService.class); IndexService indexService = mock(IndexService.class); From e79ee173b2c593ba15c8e77c0b286ecda762b5d1 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 20 Jun 2024 14:38:20 +0200 Subject: [PATCH 08/16] ESQL: Unmute MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests testMulitvaluedNullGroup (#109942) Already fixed in #100556. --- .../aggregation/GroupingAggregatorFunctionTestCase.java | 2 +- ...soluteDeviationDoubleGroupingAggregatorFunctionTests.java | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java index d10e1bada5580..65cd2585b8182 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java @@ -276,7 +276,7 @@ public final void testMulitvaluedNullGroupsAndValues() { assertSimpleOutput(origInput, results); } - public void testMulitvaluedNullGroup() { + public final void testMulitvaluedNullGroup() { DriverContext driverContext = driverContext(); BlockFactory blockFactory = driverContext.blockFactory(); int end = between(1, 2); // TODO revert diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests.java index 8eba1842d688d..a6ca769036e54 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests.java @@ -76,9 +76,4 @@ static double median(DoubleStream s) { int c = data.length / 2; return data.length % 2 == 0 ? (data[c - 1] + data[c]) / 2 : data[c]; } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101569") - public void testMulitvaluedNullGroup() { - // only here for muting it - } } From 3566ee9a0129394cb23cc40c6af1f19a7fa68193 Mon Sep 17 00:00:00 2001 From: Artem Prigoda Date: Thu, 20 Jun 2024 15:22:41 +0200 Subject: [PATCH 09/16] Track RequestedRangeNotSatisfiedException separately in S3 Metrics (#109657) Due to RCO changes, we started getting a lot of `RequestedRangeNotSatisfiedExceptions` which are expected. We would like track them separately. This change adds two new metrics to track all client errors analogous to other S3 errors. * es.repositories.exceptions.request_range_not_satisfied.total * es.repositories.exceptions.request_range_not_satisfied.histogram In the future, we can add the error code as an attribute to the metrics, so we can adapt it to all client errors. --- docs/changelog/109657.yaml | 5 ++ .../s3/S3BlobStoreRepositoryMetricsTests.java | 68 ++++++++++++++++--- .../repositories/s3/S3BlobStore.java | 18 +++++ .../repositories/RepositoriesMetrics.java | 8 +++ 4 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/109657.yaml diff --git a/docs/changelog/109657.yaml b/docs/changelog/109657.yaml new file mode 100644 index 0000000000000..35b315b7568c9 --- /dev/null +++ b/docs/changelog/109657.yaml @@ -0,0 +1,5 @@ +pr: 109657 +summary: Track `RequestedRangeNotSatisfiedException` separately in S3 Metrics +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java index f8503bca3ec67..640293ecb80b0 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.collect.Iterators; @@ -23,6 +22,7 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; import org.elasticsearch.repositories.s3.S3BlobStore.Operation; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.telemetry.Measurement; @@ -39,6 +39,7 @@ import static org.elasticsearch.repositories.RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM; +import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_OPERATIONS_TOTAL; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_REQUESTS_TOTAL; @@ -47,8 +48,10 @@ import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.elasticsearch.rest.RestStatus.NOT_FOUND; +import static org.elasticsearch.rest.RestStatus.REQUESTED_RANGE_NOT_SATISFIED; import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint") // Need to set up a new cluster for each test because cluster settings use randomized authentication settings @@ -80,22 +83,29 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .build(); } - public void testMetricsWithErrors() throws IOException { - final String repository = createRepository(randomRepositoryName()); - - final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); - final var blobStoreRepository = (BlobStoreRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName) - .repository(repository); - final BlobStore blobStore = blobStoreRepository.blobStore(); - final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + private static TestTelemetryPlugin getPlugin(String dataNodeName) { + var plugin = internalCluster().getInstance(PluginsService.class, dataNodeName) .filterPlugins(TestTelemetryPlugin.class) .findFirst() .orElseThrow(); - plugin.resetMeter(); + return plugin; + } + + private static BlobContainer getBlobContainer(String dataNodeName, String repository) { + final var blobStoreRepository = (BlobStoreRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName) + .repository(repository); + return blobStoreRepository.blobStore().blobContainer(BlobPath.EMPTY.add(randomIdentifier())); + } + + public void testMetricsWithErrors() throws IOException { + final String repository = createRepository(randomRepositoryName()); + + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final TestTelemetryPlugin plugin = getPlugin(dataNodeName); final OperationPurpose purpose = randomFrom(OperationPurpose.values()); - final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.EMPTY.add(randomIdentifier())); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); final String blobName = randomIdentifier(); // Put a blob @@ -132,6 +142,9 @@ public void testMetricsWithErrors() throws IOException { assertThat(getLongHistogramValue(plugin, METRIC_EXCEPTIONS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); + + // Make sure we don't hit the request range not satisfied counters + assertThat(getLongCounterValue(plugin, METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, Operation.GET_OBJECT), equalTo(0L)); } // List retry exhausted @@ -166,6 +179,39 @@ public void testMetricsWithErrors() throws IOException { assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.DELETE_OBJECTS), equalTo(1L)); } + public void testMetricsForRequestRangeNotSatisfied() { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + final TestTelemetryPlugin plugin = getPlugin(dataNodeName); + + final OperationPurpose purpose = randomFrom(OperationPurpose.values()); + final String blobName = randomIdentifier(); + + for (int i = 0; i < randomIntBetween(1, 3); i++) { + final long batch = i + 1; + addErrorStatus(TOO_MANY_REQUESTS, TOO_MANY_REQUESTS, REQUESTED_RANGE_NOT_SATISFIED); + try { + blobContainer.readBlob(purpose, blobName).close(); + } catch (Exception e) { + assertThat(e, instanceOf(RequestedRangeNotSatisfiedException.class)); + } + + assertThat(getLongCounterValue(plugin, METRIC_REQUESTS_TOTAL, Operation.GET_OBJECT), equalTo(3 * batch)); + assertThat(getLongCounterValue(plugin, METRIC_OPERATIONS_TOTAL, Operation.GET_OBJECT), equalTo(batch)); + assertThat(getLongCounterValue(plugin, METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL, Operation.GET_OBJECT), equalTo(batch)); + assertThat(getLongCounterValue(plugin, METRIC_EXCEPTIONS_TOTAL, Operation.GET_OBJECT), equalTo(batch)); + assertThat(getLongHistogramValue(plugin, METRIC_EXCEPTIONS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); + assertThat( + getLongCounterValue(plugin, METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, Operation.GET_OBJECT), + equalTo(batch) + ); + assertThat(getLongCounterValue(plugin, METRIC_THROTTLES_TOTAL, Operation.GET_OBJECT), equalTo(2 * batch)); + assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.GET_OBJECT), equalTo(2 * batch)); + assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); + } + } + private void addErrorStatus(RestStatus... statuses) { errorStatusQueue.addAll(Arrays.asList(statuses)); } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 2aff610dc82e9..5af53364fb765 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -52,6 +52,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.rest.RestStatus.REQUESTED_RANGE_NOT_SATISFIED; class S3BlobStore implements BlobStore { @@ -177,6 +178,23 @@ public final void collectMetrics(Request request, Response response) { .map(List::size) .orElse(0); + if (exceptionCount > 0) { + final List statusCodes = Objects.requireNonNullElse( + awsRequestMetrics.getProperty(AWSRequestMetrics.Field.StatusCode), + List.of() + ); + // REQUESTED_RANGE_NOT_SATISFIED errors are expected errors due to RCO + // TODO Add more expected client error codes? + final long amountOfRequestRangeNotSatisfiedErrors = statusCodes.stream() + .filter(e -> (Integer) e == REQUESTED_RANGE_NOT_SATISFIED.getStatus()) + .count(); + if (amountOfRequestRangeNotSatisfiedErrors > 0) { + s3RepositoriesMetrics.common() + .requestRangeNotSatisfiedExceptionCounter() + .incrementBy(amountOfRequestRangeNotSatisfiedErrors, attributes); + } + } + s3RepositoriesMetrics.common().operationCounter().incrementBy(1, attributes); if (numberOfAwsErrors == requestCount) { s3RepositoriesMetrics.common().unsuccessfulOperationCounter().incrementBy(1, attributes); diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java index 50aa7881cd2b6..85f06580cee79 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java @@ -16,6 +16,7 @@ public record RepositoriesMetrics( MeterRegistry meterRegistry, LongCounter requestCounter, LongCounter exceptionCounter, + LongCounter requestRangeNotSatisfiedExceptionCounter, LongCounter throttleCounter, LongCounter operationCounter, LongCounter unsuccessfulOperationCounter, @@ -28,6 +29,8 @@ public record RepositoriesMetrics( public static final String METRIC_REQUESTS_TOTAL = "es.repositories.requests.total"; public static final String METRIC_EXCEPTIONS_TOTAL = "es.repositories.exceptions.total"; + public static final String METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL = + "es.repositories.exceptions.request_range_not_satisfied.total"; public static final String METRIC_THROTTLES_TOTAL = "es.repositories.throttles.total"; public static final String METRIC_OPERATIONS_TOTAL = "es.repositories.operations.total"; public static final String METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL = "es.repositories.operations.unsuccessful.total"; @@ -40,6 +43,11 @@ public RepositoriesMetrics(MeterRegistry meterRegistry) { meterRegistry, meterRegistry.registerLongCounter(METRIC_REQUESTS_TOTAL, "repository request counter", "unit"), meterRegistry.registerLongCounter(METRIC_EXCEPTIONS_TOTAL, "repository request exception counter", "unit"), + meterRegistry.registerLongCounter( + METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, + "repository request RequestedRangeNotSatisfiedException counter", + "unit" + ), meterRegistry.registerLongCounter(METRIC_THROTTLES_TOTAL, "repository request throttle counter", "unit"), meterRegistry.registerLongCounter(METRIC_OPERATIONS_TOTAL, "repository operation counter", "unit"), meterRegistry.registerLongCounter(METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL, "repository unsuccessful operation counter", "unit"), From bd13814b87f2a6361fea8e21499da3c0bd1f4c1c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 20 Jun 2024 10:31:38 -0400 Subject: [PATCH 10/16] [ci] Bump disk size for agents to 250GB (#109975) --- .buildkite/pipelines/dra-workflow.yml | 1 + .buildkite/pipelines/intake.template.yml | 8 ++++ .buildkite/pipelines/intake.yml | 8 ++++ .../lucene-snapshot/build-snapshot.yml | 1 + .../pipelines/lucene-snapshot/run-tests.yml | 8 ++++ .../periodic-packaging.bwc.template.yml | 1 + .buildkite/pipelines/periodic-packaging.yml | 34 ++++++++++++++ .../pipelines/periodic-platform-support.yml | 1 + .../pipelines/periodic.bwc.template.yml | 1 + .buildkite/pipelines/periodic.template.yml | 13 +++++ .buildkite/pipelines/periodic.yml | 47 +++++++++++++++++++ .../pull-request/build-benchmark.yml | 1 + .../pipelines/pull-request/bwc-snapshots.yml | 1 + .../pipelines/pull-request/cloud-deploy.yml | 1 + .../pipelines/pull-request/docs-check.yml | 1 + .../pull-request/eql-correctness.yml | 1 + .../pull-request/example-plugins.yml | 1 + .../pipelines/pull-request/full-bwc.yml | 1 + .../pull-request/packaging-upgrade-tests.yml | 1 + .../pipelines/pull-request/part-1-fips.yml | 1 + .buildkite/pipelines/pull-request/part-1.yml | 1 + .../pipelines/pull-request/part-2-fips.yml | 1 + .buildkite/pipelines/pull-request/part-2.yml | 1 + .../pipelines/pull-request/part-3-fips.yml | 1 + .buildkite/pipelines/pull-request/part-3.yml | 1 + .../pipelines/pull-request/part-4-fips.yml | 1 + .buildkite/pipelines/pull-request/part-4.yml | 1 + .../pipelines/pull-request/part-5-fips.yml | 1 + .buildkite/pipelines/pull-request/part-5.yml | 1 + .../pipelines/pull-request/precommit.yml | 1 + .../pull-request/rest-compatibility.yml | 1 + .../pull-request/validate-changelogs.yml | 1 + 32 files changed, 144 insertions(+) diff --git a/.buildkite/pipelines/dra-workflow.yml b/.buildkite/pipelines/dra-workflow.yml index e7bf19816356f..32a2b7d22134a 100644 --- a/.buildkite/pipelines/dra-workflow.yml +++ b/.buildkite/pipelines/dra-workflow.yml @@ -7,6 +7,7 @@ steps: image: family/elasticsearch-ubuntu-2204 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait # The hadoop build depends on the ES artifact # So let's trigger the hadoop build any time we build a new staging artifact diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml index f530f237113a9..1a513971b2c10 100644 --- a/.buildkite/pipelines/intake.template.yml +++ b/.buildkite/pipelines/intake.template.yml @@ -7,6 +7,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -16,6 +17,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -24,6 +26,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -32,6 +35,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -40,6 +44,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -48,6 +53,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -61,6 +67,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -71,6 +78,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index b33fc98ccb01b..4124d4e550d11 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -8,6 +8,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -17,6 +18,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -25,6 +27,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -33,6 +36,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -41,6 +45,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -49,6 +54,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -62,6 +68,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -72,6 +79,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow diff --git a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml index 8cf2a8aacbece..1f69b8faa7ab4 100644 --- a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml +++ b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml @@ -15,6 +15,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - trigger: "elasticsearch-lucene-snapshot-tests" build: diff --git a/.buildkite/pipelines/lucene-snapshot/run-tests.yml b/.buildkite/pipelines/lucene-snapshot/run-tests.yml index c76c54a56494e..49c3396488d82 100644 --- a/.buildkite/pipelines/lucene-snapshot/run-tests.yml +++ b/.buildkite/pipelines/lucene-snapshot/run-tests.yml @@ -7,6 +7,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait: null - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -16,6 +17,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -24,6 +26,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -32,6 +35,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -40,6 +44,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -48,6 +53,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -64,6 +70,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -74,3 +81,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/periodic-packaging.bwc.template.yml b/.buildkite/pipelines/periodic-packaging.bwc.template.yml index b06bc80d3535d..8a6fa2553b204 100644 --- a/.buildkite/pipelines/periodic-packaging.bwc.template.yml +++ b/.buildkite/pipelines/periodic-packaging.bwc.template.yml @@ -11,5 +11,6 @@ image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 406331dda881b..4217fc91bf0fd 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -46,6 +46,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.0.1 @@ -62,6 +63,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.1.1 @@ -78,6 +80,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.2.1 @@ -94,6 +97,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.3.2 @@ -110,6 +114,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.4.2 @@ -126,6 +131,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.5.2 @@ -142,6 +148,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.6.2 @@ -158,6 +165,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.7.1 @@ -174,6 +182,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.8.1 @@ -190,6 +199,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.9.3 @@ -206,6 +216,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.10.2 @@ -222,6 +233,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.11.2 @@ -238,6 +250,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.12.1 @@ -254,6 +267,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.13.4 @@ -270,6 +284,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.14.2 @@ -286,6 +301,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.15.2 @@ -302,6 +318,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.16.3 @@ -318,6 +335,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.17.23 @@ -334,6 +352,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.0.1 @@ -350,6 +369,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.1.3 @@ -366,6 +386,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.2.3 @@ -382,6 +403,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.3.3 @@ -398,6 +420,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.4.3 @@ -414,6 +437,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.5.3 @@ -430,6 +454,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.6.2 @@ -446,6 +471,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.7.1 @@ -462,6 +488,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.8.2 @@ -478,6 +505,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.9.2 @@ -494,6 +522,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.10.4 @@ -510,6 +539,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.11.4 @@ -526,6 +556,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.12.2 @@ -542,6 +573,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.13.4 @@ -558,6 +590,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.14.2 @@ -574,6 +607,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.15.0 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index d8c5d55fc7e4f..867ebe41ed6af 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -30,6 +30,7 @@ steps: localSsds: 1 localSsdInterface: nvme machineType: custom-32-98304 + diskSizeGb: 250 env: {} - group: platform-support-windows steps: diff --git a/.buildkite/pipelines/periodic.bwc.template.yml b/.buildkite/pipelines/periodic.bwc.template.yml index 43a0a7438d656..b22270dbf221c 100644 --- a/.buildkite/pipelines/periodic.bwc.template.yml +++ b/.buildkite/pipelines/periodic.bwc.template.yml @@ -7,6 +7,7 @@ machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION retry: diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml index 207a332ed6717..87e30a0ea73ba 100644 --- a/.buildkite/pipelines/periodic.template.yml +++ b/.buildkite/pipelines/periodic.template.yml @@ -25,6 +25,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: example-plugins command: |- cd $$WORKSPACE/plugins/examples @@ -36,6 +37,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: java-fips-matrix steps: - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" @@ -57,6 +59,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -73,6 +76,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -101,6 +105,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -121,6 +126,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -156,6 +162,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / azure command: | export azure_storage_container=elasticsearch-ci-thirdparty @@ -170,6 +177,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / gcs command: | export google_storage_bucket=elasticsearch-ci-thirdparty @@ -184,6 +192,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / geoip command: | .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" @@ -193,6 +202,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 @@ -207,6 +217,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: @@ -217,6 +228,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 if: build.branch == "main" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency @@ -225,6 +237,7 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-2 + diskSizeGb: 250 - label: check-branch-protection-rules command: .buildkite/scripts/branch-protection.sh timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 32da1db652239..06e7ffbc8fb1c 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -11,6 +11,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.0.1 retry: @@ -30,6 +31,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.1.1 retry: @@ -49,6 +51,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.2.1 retry: @@ -68,6 +71,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.3.2 retry: @@ -87,6 +91,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.4.2 retry: @@ -106,6 +111,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.5.2 retry: @@ -125,6 +131,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.6.2 retry: @@ -144,6 +151,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.7.1 retry: @@ -163,6 +171,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.8.1 retry: @@ -182,6 +191,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.9.3 retry: @@ -201,6 +211,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.10.2 retry: @@ -220,6 +231,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.11.2 retry: @@ -239,6 +251,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.12.1 retry: @@ -258,6 +271,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.13.4 retry: @@ -277,6 +291,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.14.2 retry: @@ -296,6 +311,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.15.2 retry: @@ -315,6 +331,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.16.3 retry: @@ -334,6 +351,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.17.23 retry: @@ -353,6 +371,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.0.1 retry: @@ -372,6 +391,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.1.3 retry: @@ -391,6 +411,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.2.3 retry: @@ -410,6 +431,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.3.3 retry: @@ -429,6 +451,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.4.3 retry: @@ -448,6 +471,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.5.3 retry: @@ -467,6 +491,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.6.2 retry: @@ -486,6 +511,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.7.1 retry: @@ -505,6 +531,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.8.2 retry: @@ -524,6 +551,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.9.2 retry: @@ -543,6 +571,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.10.4 retry: @@ -562,6 +591,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.11.4 retry: @@ -581,6 +611,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.12.2 retry: @@ -600,6 +631,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.13.4 retry: @@ -619,6 +651,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.14.2 retry: @@ -638,6 +671,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.15.0 retry: @@ -672,6 +706,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: example-plugins command: |- cd $$WORKSPACE/plugins/examples @@ -683,6 +718,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: java-fips-matrix steps: - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" @@ -704,6 +740,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -720,6 +757,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -748,6 +786,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -768,6 +807,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -803,6 +843,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / azure command: | export azure_storage_container=elasticsearch-ci-thirdparty @@ -817,6 +858,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / gcs command: | export google_storage_bucket=elasticsearch-ci-thirdparty @@ -831,6 +873,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / geoip command: | .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" @@ -840,6 +883,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 @@ -854,6 +898,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: @@ -864,6 +909,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 if: build.branch == "main" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency @@ -872,6 +918,7 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-2 + diskSizeGb: 250 - label: check-branch-protection-rules command: .buildkite/scripts/branch-protection.sh timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/pull-request/build-benchmark.yml b/.buildkite/pipelines/pull-request/build-benchmark.yml index 8d3215b8393ce..96330bee03638 100644 --- a/.buildkite/pipelines/pull-request/build-benchmark.yml +++ b/.buildkite/pipelines/pull-request/build-benchmark.yml @@ -22,3 +22,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/bwc-snapshots.yml b/.buildkite/pipelines/pull-request/bwc-snapshots.yml index 5a9fc2d938ac0..8f59e593b286f 100644 --- a/.buildkite/pipelines/pull-request/bwc-snapshots.yml +++ b/.buildkite/pipelines/pull-request/bwc-snapshots.yml @@ -18,3 +18,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/cloud-deploy.yml b/.buildkite/pipelines/pull-request/cloud-deploy.yml index ce8e8206d51ff..2932f874c5cf8 100644 --- a/.buildkite/pipelines/pull-request/cloud-deploy.yml +++ b/.buildkite/pipelines/pull-request/cloud-deploy.yml @@ -11,3 +11,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/docs-check.yml b/.buildkite/pipelines/pull-request/docs-check.yml index 2201eb2d1e4ea..3bf1e43697a7c 100644 --- a/.buildkite/pipelines/pull-request/docs-check.yml +++ b/.buildkite/pipelines/pull-request/docs-check.yml @@ -12,3 +12,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/eql-correctness.yml b/.buildkite/pipelines/pull-request/eql-correctness.yml index 8f7ca6942c0e9..d85827d10e886 100644 --- a/.buildkite/pipelines/pull-request/eql-correctness.yml +++ b/.buildkite/pipelines/pull-request/eql-correctness.yml @@ -7,3 +7,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/example-plugins.yml b/.buildkite/pipelines/pull-request/example-plugins.yml index 18d0de6594980..fb4a17fb214cb 100644 --- a/.buildkite/pipelines/pull-request/example-plugins.yml +++ b/.buildkite/pipelines/pull-request/example-plugins.yml @@ -16,3 +16,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/full-bwc.yml b/.buildkite/pipelines/pull-request/full-bwc.yml index d3fa8eccaf7d9..c404069bd0e60 100644 --- a/.buildkite/pipelines/pull-request/full-bwc.yml +++ b/.buildkite/pipelines/pull-request/full-bwc.yml @@ -13,3 +13,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml b/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml index c62cf23310422..970dafbb28647 100644 --- a/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml +++ b/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml @@ -18,5 +18,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION diff --git a/.buildkite/pipelines/pull-request/part-1-fips.yml b/.buildkite/pipelines/pull-request/part-1-fips.yml index 42f930c1bde9a..99544e7f5a80b 100644 --- a/.buildkite/pipelines/pull-request/part-1-fips.yml +++ b/.buildkite/pipelines/pull-request/part-1-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-1.yml b/.buildkite/pipelines/pull-request/part-1.yml index 3d467c6c41e43..b4b9d5469ec41 100644 --- a/.buildkite/pipelines/pull-request/part-1.yml +++ b/.buildkite/pipelines/pull-request/part-1.yml @@ -7,3 +7,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-2-fips.yml b/.buildkite/pipelines/pull-request/part-2-fips.yml index 6a3647ceb50ae..36a9801547d78 100644 --- a/.buildkite/pipelines/pull-request/part-2-fips.yml +++ b/.buildkite/pipelines/pull-request/part-2-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-2.yml b/.buildkite/pipelines/pull-request/part-2.yml index 43de69bbcd945..12bd78cf895fd 100644 --- a/.buildkite/pipelines/pull-request/part-2.yml +++ b/.buildkite/pipelines/pull-request/part-2.yml @@ -7,3 +7,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-3-fips.yml b/.buildkite/pipelines/pull-request/part-3-fips.yml index cee3ea153acb9..4a2df3026e782 100644 --- a/.buildkite/pipelines/pull-request/part-3-fips.yml +++ b/.buildkite/pipelines/pull-request/part-3-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-3.yml b/.buildkite/pipelines/pull-request/part-3.yml index 12abae7634822..6991c05da85c6 100644 --- a/.buildkite/pipelines/pull-request/part-3.yml +++ b/.buildkite/pipelines/pull-request/part-3.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-4-fips.yml b/.buildkite/pipelines/pull-request/part-4-fips.yml index 11a50456ca4c0..734f8af816895 100644 --- a/.buildkite/pipelines/pull-request/part-4-fips.yml +++ b/.buildkite/pipelines/pull-request/part-4-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-4.yml b/.buildkite/pipelines/pull-request/part-4.yml index af11f08953d07..59f2f2898a590 100644 --- a/.buildkite/pipelines/pull-request/part-4.yml +++ b/.buildkite/pipelines/pull-request/part-4.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-5-fips.yml b/.buildkite/pipelines/pull-request/part-5-fips.yml index 4e193ac751086..801b812bb99c0 100644 --- a/.buildkite/pipelines/pull-request/part-5-fips.yml +++ b/.buildkite/pipelines/pull-request/part-5-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-5.yml b/.buildkite/pipelines/pull-request/part-5.yml index 306ce7533d0ed..c7e50631d1cdd 100644 --- a/.buildkite/pipelines/pull-request/part-5.yml +++ b/.buildkite/pipelines/pull-request/part-5.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/precommit.yml b/.buildkite/pipelines/pull-request/precommit.yml index f6548dfeed9b2..8d1458b1b60c8 100644 --- a/.buildkite/pipelines/pull-request/precommit.yml +++ b/.buildkite/pipelines/pull-request/precommit.yml @@ -10,3 +10,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/rest-compatibility.yml b/.buildkite/pipelines/pull-request/rest-compatibility.yml index a69810e23d960..16144a2a0780f 100644 --- a/.buildkite/pipelines/pull-request/rest-compatibility.yml +++ b/.buildkite/pipelines/pull-request/rest-compatibility.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/validate-changelogs.yml b/.buildkite/pipelines/pull-request/validate-changelogs.yml index 9451d321a9b39..296ef11637118 100644 --- a/.buildkite/pipelines/pull-request/validate-changelogs.yml +++ b/.buildkite/pipelines/pull-request/validate-changelogs.yml @@ -7,3 +7,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 From e421a29a801d48c63400b72f077f787431316283 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 20 Jun 2024 07:44:31 -0700 Subject: [PATCH 11/16] Unmute test --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 9b58a9446b3ca..8e9ed04038074 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -67,9 +67,6 @@ tests: - class: "org.elasticsearch.xpack.shutdown.NodeShutdownReadinessIT" issue: "https://github.com/elastic/elasticsearch/issues/109838" method: "testShutdownReadinessService" -- class: "org.elasticsearch.packaging.test.RpmPreservationTests" - issue: "https://github.com/elastic/elasticsearch/issues/109898" - method: "test30PreserveConfig" - class: "org.elasticsearch.xpack.security.ScrollHelperIntegTests" issue: "https://github.com/elastic/elasticsearch/issues/109905" method: "testFetchAllEntities" From 1f504076f3e71f8dddcc32fce1c79656f9205ae4 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 20 Jun 2024 17:27:13 +0200 Subject: [PATCH 12/16] [Inference API] Extend AbstractBWCWireSerializingTestCase for AzureAiStudioChatCompletionServiceSettingsTests (#109919) --- ...dioChatCompletionServiceSettingsTests.java | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java index d46a5f190017a..95be365706ccb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java @@ -7,15 +7,18 @@ package org.elasticsearch.xpack.inference.services.azureaistudio.completion; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.Strings; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioEndpointType; import org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioProvider; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; import org.hamcrest.CoreMatchers; import java.io.IOException; @@ -27,7 +30,8 @@ import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.TARGET_FIELD; import static org.hamcrest.Matchers.is; -public class AzureAiStudioChatCompletionServiceSettingsTests extends ESTestCase { +public class AzureAiStudioChatCompletionServiceSettingsTests extends AbstractBWCWireSerializationTestCase< + AzureAiStudioChatCompletionServiceSettings> { public void testFromMap_Request_CreatesSettingsCorrectly() { var target = "http://sometarget.local"; var provider = "openai"; @@ -119,4 +123,38 @@ public void testToFilteredXContent_WritesAllValues() throws IOException { public static HashMap createRequestSettingsMap(String target, String provider, String endpointType) { return new HashMap<>(Map.of(TARGET_FIELD, target, PROVIDER_FIELD, provider, ENDPOINT_TYPE_FIELD, endpointType)); } + + @Override + protected Writeable.Reader instanceReader() { + return AzureAiStudioChatCompletionServiceSettings::new; + } + + @Override + protected AzureAiStudioChatCompletionServiceSettings createTestInstance() { + return createRandom(); + } + + @Override + protected AzureAiStudioChatCompletionServiceSettings mutateInstance(AzureAiStudioChatCompletionServiceSettings instance) + throws IOException { + return randomValueOtherThan(instance, AzureAiStudioChatCompletionServiceSettingsTests::createRandom); + } + + @Override + protected AzureAiStudioChatCompletionServiceSettings mutateInstanceForVersion( + AzureAiStudioChatCompletionServiceSettings instance, + TransportVersion version + ) { + return instance; + } + + private static AzureAiStudioChatCompletionServiceSettings createRandom() { + return new AzureAiStudioChatCompletionServiceSettings( + randomAlphaOfLength(10), + randomFrom(AzureAiStudioProvider.values()), + randomFrom(AzureAiStudioEndpointType.values()), + RateLimitSettingsTests.createRandom() + ); + } + } From 2c205e38f6ce036916efc9e06ba49b5e772f4879 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Thu, 20 Jun 2024 16:50:00 +0100 Subject: [PATCH 13/16] ES|QL Add primitive float variants of all aggregators to the compute engine (#109781) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds primitive float variants of all ES|QL aggregators to the compute engine. Much of the changes are mechanical and/or are generated, and retain similar type input/output params as other aggregators. Co-authored-by: Iván Cea Fontenla --- docs/changelog/109781.yaml | 5 + x-pack/plugin/esql/compute/build.gradle | 30 ++ .../compute/gen/AggregatorImplementer.java | 13 +- .../gen/GroupingAggregatorImplementer.java | 2 +- .../elasticsearch/compute/gen/Methods.java | 1 + .../org/elasticsearch/compute/gen/Types.java | 26 ++ .../compute/aggregation/FloatArrayState.java | 120 ++++++ .../compute/aggregation/FloatState.java | 55 +++ .../aggregation/RateFloatAggregator.java | 341 +++++++++++++++++ .../aggregation/TopListFloatAggregator.java | 137 +++++++ .../aggregation/ValuesFloatAggregator.java | 187 ++++++++++ .../compute/data/sort/FloatBucketedSort.java | 346 ++++++++++++++++++ .../CountDistinctFloatAggregatorFunction.java | 127 +++++++ ...stinctFloatAggregatorFunctionSupplier.java | 42 +++ ...stinctFloatGroupingAggregatorFunction.java | 202 ++++++++++ .../MaxFloatAggregatorFunction.java | 138 +++++++ .../MaxFloatAggregatorFunctionSupplier.java | 38 ++ .../MaxFloatGroupingAggregatorFunction.java | 208 +++++++++++ ...oluteDeviationFloatAggregatorFunction.java | 124 +++++++ ...iationFloatAggregatorFunctionSupplier.java | 39 ++ ...iationFloatGroupingAggregatorFunction.java | 199 ++++++++++ .../MinFloatAggregatorFunction.java | 138 +++++++ .../MinFloatAggregatorFunctionSupplier.java | 38 ++ .../MinFloatGroupingAggregatorFunction.java | 208 +++++++++++ .../PercentileFloatAggregatorFunction.java | 127 +++++++ ...entileFloatAggregatorFunctionSupplier.java | 41 +++ ...entileFloatGroupingAggregatorFunction.java | 202 ++++++++++ .../RateFloatAggregatorFunctionSupplier.java | 41 +++ .../RateFloatGroupingAggregatorFunction.java | 227 ++++++++++++ .../SumFloatAggregatorFunction.java | 144 ++++++++ .../SumFloatAggregatorFunctionSupplier.java | 38 ++ .../SumFloatGroupingAggregatorFunction.java | 212 +++++++++++ .../TopListFloatAggregatorFunction.java | 126 +++++++ ...opListFloatAggregatorFunctionSupplier.java | 45 +++ ...opListFloatGroupingAggregatorFunction.java | 202 ++++++++++ .../ValuesFloatAggregatorFunction.java | 120 ++++++ ...ValuesFloatAggregatorFunctionSupplier.java | 38 ++ ...ValuesFloatGroupingAggregatorFunction.java | 195 ++++++++++ .../CountDistinctFloatAggregator.java | 72 ++++ .../aggregation/MaxFloatAggregator.java | 25 ++ ...edianAbsoluteDeviationFloatAggregator.java | 63 ++++ .../aggregation/MinFloatAggregator.java | 25 ++ .../PercentileFloatAggregator.java | 63 ++++ .../aggregation/SumFloatAggregator.java | 31 ++ .../compute/aggregation/X-ArrayState.java.st | 2 +- .../aggregation/X-RateAggregator.java.st | 4 +- .../aggregation/X-ValuesAggregator.java.st | 30 +- .../AggregatorFunctionTestCase.java | 6 + ...tDistinctFloatAggregatorFunctionTests.java | 56 +++ ...tFloatGroupingAggregatorFunctionTests.java | 60 +++ .../GroupingAggregatorFunctionTestCase.java | 7 + .../MaxFloatAggregatorFunctionTests.java | 43 +++ ...xFloatGroupingAggregatorFunctionTests.java | 54 +++ ...DeviationFloatAggregatorFunctionTests.java | 45 +++ ...nFloatGroupingAggregatorFunctionTests.java | 79 ++++ .../MinFloatAggregatorFunctionTests.java | 43 +++ ...nFloatGroupingAggregatorFunctionTests.java | 53 +++ ...ercentileFloatAggregatorFunctionTests.java | 56 +++ ...eFloatGroupingAggregatorFunctionTests.java | 64 ++++ .../SumFloatAggregatorFunctionTests.java | 156 ++++++++ ...mFloatGroupingAggregatorFunctionTests.java | 50 +++ .../TopListFloatAggregatorFunctionTests.java | 44 +++ .../ValuesFloatAggregatorFunctionTests.java | 43 +++ ...sFloatGroupingAggregatorFunctionTests.java | 55 +++ .../data/sort/FloatBucketedSortTests.java | 58 +++ .../operator/ForkingOperatorTestCase.java | 23 ++ .../LongFloatTupleBlockSourceOperator.java | 70 ++++ .../operator/NullInsertingSourceOperator.java | 4 + .../PositionMergingSourceOperator.java | 2 + .../SequenceFloatBlockSourceOperator.java | 62 ++++ 70 files changed, 5962 insertions(+), 8 deletions(-) create mode 100644 docs/changelog/109781.yaml create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatArrayState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/SumFloatAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunctionTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LongFloatTupleBlockSourceOperator.java create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceFloatBlockSourceOperator.java diff --git a/docs/changelog/109781.yaml b/docs/changelog/109781.yaml new file mode 100644 index 0000000000000..df74645b53d84 --- /dev/null +++ b/docs/changelog/109781.yaml @@ -0,0 +1,5 @@ +pr: 109781 +summary: ES|QL Add primitive float variants of all aggregators to the compute engine +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle index bc206ee1d78d6..c7fa29c6a91f0 100644 --- a/x-pack/plugin/esql/compute/build.gradle +++ b/x-pack/plugin/esql/compute/build.gradle @@ -410,6 +410,11 @@ tasks.named('stringTemplates').configure { it.inputFile = stateInputFile it.outputFile = "org/elasticsearch/compute/aggregation/LongState.java" } + template { + it.properties = floatProperties + it.inputFile = stateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/FloatState.java" + } template { it.properties = doubleProperties it.inputFile = stateInputFile @@ -463,6 +468,11 @@ tasks.named('stringTemplates').configure { it.inputFile = arrayStateInputFile it.outputFile = "org/elasticsearch/compute/aggregation/DoubleArrayState.java" } + template { + it.properties = floatProperties + it.inputFile = arrayStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/FloatArrayState.java" + } File valuesAggregatorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st") template { it.properties = intProperties @@ -474,6 +484,11 @@ tasks.named('stringTemplates').configure { it.inputFile = valuesAggregatorInputFile it.outputFile = "org/elasticsearch/compute/aggregation/ValuesLongAggregator.java" } + template { + it.properties = floatProperties + it.inputFile = valuesAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java" + } template { it.properties = doubleProperties it.inputFile = valuesAggregatorInputFile @@ -496,6 +511,11 @@ tasks.named('stringTemplates').configure { it.inputFile = rateAggregatorInputFile it.outputFile = "org/elasticsearch/compute/aggregation/RateLongAggregator.java" } + template { + it.properties = floatProperties + it.inputFile = rateAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/RateFloatAggregator.java" + } template { it.properties = doubleProperties it.inputFile = rateAggregatorInputFile @@ -514,6 +534,11 @@ tasks.named('stringTemplates').configure { it.inputFile = topListAggregatorInputFile it.outputFile = "org/elasticsearch/compute/aggregation/TopListLongAggregator.java" } + template { + it.properties = floatProperties + it.inputFile = topListAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/TopListFloatAggregator.java" + } template { it.properties = doubleProperties it.inputFile = topListAggregatorInputFile @@ -667,6 +692,11 @@ tasks.named('stringTemplates').configure { it.inputFile = bucketedSortInputFile it.outputFile = "org/elasticsearch/compute/data/sort/LongBucketedSort.java" } + template { + it.properties = floatProperties + it.inputFile = bucketedSortInputFile + it.outputFile = "org/elasticsearch/compute/data/sort/FloatBucketedSort.java" + } template { it.properties = doubleProperties it.inputFile = bucketedSortInputFile diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index d3fe51b4cc225..1127d4b4ccb72 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -44,6 +44,8 @@ import static org.elasticsearch.compute.gen.Types.DOUBLE_VECTOR; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.ELEMENT_TYPE; +import static org.elasticsearch.compute.gen.Types.FLOAT_BLOCK; +import static org.elasticsearch.compute.gen.Types.FLOAT_VECTOR; import static org.elasticsearch.compute.gen.Types.INTERMEDIATE_STATE_DESC; import static org.elasticsearch.compute.gen.Types.INT_BLOCK; import static org.elasticsearch.compute.gen.Types.INT_VECTOR; @@ -136,6 +138,8 @@ static String valueType(ExecutableElement init, ExecutableElement combine) { switch (initReturn) { case "double": return "double"; + case "float": + return "float"; case "long": return "long"; case "int": @@ -151,6 +155,7 @@ static ClassName valueBlockType(ExecutableElement init, ExecutableElement combin return switch (valueType(init, combine)) { case "boolean" -> BOOLEAN_BLOCK; case "double" -> DOUBLE_BLOCK; + case "float" -> FLOAT_BLOCK; case "long" -> LONG_BLOCK; case "int" -> INT_BLOCK; case "org.apache.lucene.util.BytesRef" -> BYTES_REF_BLOCK; @@ -162,6 +167,7 @@ static ClassName valueVectorType(ExecutableElement init, ExecutableElement combi return switch (valueType(init, combine)) { case "boolean" -> BOOLEAN_VECTOR; case "double" -> DOUBLE_VECTOR; + case "float" -> FLOAT_VECTOR; case "long" -> LONG_VECTOR; case "int" -> INT_VECTOR; case "org.apache.lucene.util.BytesRef" -> BYTES_REF_VECTOR; @@ -445,6 +451,8 @@ private String primitiveStateMethod() { return "longValue"; case "org.elasticsearch.compute.aggregation.DoubleState": return "doubleValue"; + case "org.elasticsearch.compute.aggregation.FloatState": + return "floatValue"; default: throw new IllegalArgumentException( "don't know how to fetch primitive values from " + stateType + ". define combineIntermediate." @@ -495,6 +503,9 @@ private void primitiveStateToResult(MethodSpec.Builder builder) { case "org.elasticsearch.compute.aggregation.DoubleState": builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantDoubleBlockWith(state.doubleValue(), 1)"); return; + case "org.elasticsearch.compute.aggregation.FloatState": + builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantFloatBlockWith(state.floatValue(), 1)"); + return; default: throw new IllegalArgumentException("don't know how to convert state to result: " + stateType); } @@ -521,7 +532,7 @@ private MethodSpec close() { private boolean hasPrimitiveState() { return switch (stateType.toString()) { case "org.elasticsearch.compute.aggregation.IntState", "org.elasticsearch.compute.aggregation.LongState", - "org.elasticsearch.compute.aggregation.DoubleState" -> true; + "org.elasticsearch.compute.aggregation.DoubleState", "org.elasticsearch.compute.aggregation.FloatState" -> true; default -> false; }; } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index cb65d2337d588..c9cdcfe42fddd 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -585,7 +585,7 @@ private MethodSpec close() { private boolean hasPrimitiveState() { return switch (stateType.toString()) { case "org.elasticsearch.compute.aggregation.IntArrayState", "org.elasticsearch.compute.aggregation.LongArrayState", - "org.elasticsearch.compute.aggregation.DoubleArrayState" -> true; + "org.elasticsearch.compute.aggregation.DoubleArrayState", "org.elasticsearch.compute.aggregation.FloatArrayState" -> true; default -> false; }; } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java index 741a1294e6fb5..6f98f1f797ab0 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java @@ -211,6 +211,7 @@ static String vectorAccessorName(String elementTypeName) { case "INT" -> "getInt"; case "LONG" -> "getLong"; case "DOUBLE" -> "getDouble"; + case "FLOAT" -> "getFloat"; case "BYTES_REF" -> "getBytesRef"; default -> throw new IllegalArgumentException( "don't know how to fetch primitive values from " + elementTypeName + ". define combineIntermediate." diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index 6618d9e4f41b5..3150741ddcb05 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -41,12 +41,14 @@ public class Types { static final ClassName INT_BLOCK = ClassName.get(DATA_PACKAGE, "IntBlock"); static final ClassName LONG_BLOCK = ClassName.get(DATA_PACKAGE, "LongBlock"); static final ClassName DOUBLE_BLOCK = ClassName.get(DATA_PACKAGE, "DoubleBlock"); + static final ClassName FLOAT_BLOCK = ClassName.get(DATA_PACKAGE, "FloatBlock"); static final ClassName BOOLEAN_BLOCK_BUILDER = BOOLEAN_BLOCK.nestedClass("Builder"); static final ClassName BYTES_REF_BLOCK_BUILDER = BYTES_REF_BLOCK.nestedClass("Builder"); static final ClassName INT_BLOCK_BUILDER = INT_BLOCK.nestedClass("Builder"); static final ClassName LONG_BLOCK_BUILDER = LONG_BLOCK.nestedClass("Builder"); static final ClassName DOUBLE_BLOCK_BUILDER = DOUBLE_BLOCK.nestedClass("Builder"); + static final ClassName FLOAT_BLOCK_BUILDER = FLOAT_BLOCK.nestedClass("Builder"); static final ClassName ELEMENT_TYPE = ClassName.get(DATA_PACKAGE, "ElementType"); @@ -55,35 +57,41 @@ public class Types { static final ClassName INT_VECTOR = ClassName.get(DATA_PACKAGE, "IntVector"); static final ClassName LONG_VECTOR = ClassName.get(DATA_PACKAGE, "LongVector"); static final ClassName DOUBLE_VECTOR = ClassName.get(DATA_PACKAGE, "DoubleVector"); + static final ClassName FLOAT_VECTOR = ClassName.get(DATA_PACKAGE, "FloatVector"); static final ClassName BOOLEAN_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "BooleanVector", "Builder"); static final ClassName BYTES_REF_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "BytesRefVector", "Builder"); static final ClassName INT_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "IntVector", "Builder"); static final ClassName LONG_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "LongVector", "Builder"); static final ClassName DOUBLE_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "DoubleVector", "Builder"); + static final ClassName FLOAT_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "FloatVector", "Builder"); static final ClassName BOOLEAN_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "BooleanVector", "FixedBuilder"); static final ClassName INT_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "IntVector", "FixedBuilder"); static final ClassName LONG_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "LongVector", "FixedBuilder"); static final ClassName DOUBLE_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "DoubleVector", "FixedBuilder"); + static final ClassName FLOAT_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "FloatVector", "FixedBuilder"); static final ClassName BOOLEAN_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "BooleanArrayVector"); static final ClassName BYTES_REF_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "BytesRefArrayVector"); static final ClassName INT_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "IntArrayVector"); static final ClassName LONG_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "LongArrayVector"); static final ClassName DOUBLE_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "DoubleArrayVector"); + static final ClassName FLOAT_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "FloatArrayVector"); static final ClassName BOOLEAN_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "BooleanArrayBlock"); static final ClassName BYTES_REF_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "BytesRefArrayBlock"); static final ClassName INT_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "IntArrayBlock"); static final ClassName LONG_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "LongArrayBlock"); static final ClassName DOUBLE_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "DoubleArrayBlock"); + static final ClassName FLOAT_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "FloatArrayBlock"); static final ClassName BOOLEAN_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantBooleanVector"); static final ClassName BYTES_REF_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantBytesRefVector"); static final ClassName INT_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantIntVector"); static final ClassName LONG_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantLongVector"); static final ClassName DOUBLE_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantDoubleVector"); + static final ClassName FLOAT_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantFloatVector"); static final ClassName AGGREGATOR_FUNCTION = ClassName.get(AGGREGATION_PACKAGE, "AggregatorFunction"); static final ClassName AGGREGATOR_FUNCTION_SUPPLIER = ClassName.get(AGGREGATION_PACKAGE, "AggregatorFunctionSupplier"); @@ -162,6 +170,9 @@ static ClassName blockType(String elementType) { if (elementType.equalsIgnoreCase(TypeName.DOUBLE.toString())) { return DOUBLE_BLOCK; } + if (elementType.equalsIgnoreCase(TypeName.FLOAT.toString())) { + return FLOAT_BLOCK; + } throw new IllegalArgumentException("unknown vector type for [" + elementType + "]"); } @@ -181,6 +192,9 @@ static ClassName vectorType(TypeName elementType) { if (elementType.equals(TypeName.DOUBLE)) { return DOUBLE_VECTOR; } + if (elementType.equals(TypeName.FLOAT)) { + return FLOAT_VECTOR; + } throw new IllegalArgumentException("unknown vector type for [" + elementType + "]"); } @@ -200,6 +214,9 @@ static ClassName vectorType(String elementType) { if (elementType.equalsIgnoreCase(TypeName.DOUBLE.toString())) { return DOUBLE_VECTOR; } + if (elementType.equalsIgnoreCase(TypeName.FLOAT.toString())) { + return FLOAT_VECTOR; + } throw new IllegalArgumentException("unknown vector type for [" + elementType + "]"); } @@ -234,6 +251,12 @@ static ClassName builderType(TypeName resultType) { if (resultType.equals(DOUBLE_VECTOR)) { return DOUBLE_VECTOR_BUILDER; } + if (resultType.equals(FLOAT_BLOCK)) { + return FLOAT_BLOCK_BUILDER; + } + if (resultType.equals(FLOAT_VECTOR)) { + return FLOAT_VECTOR_BUILDER; + } throw new IllegalArgumentException("unknown builder type for [" + resultType + "]"); } @@ -250,6 +273,9 @@ static ClassName vectorFixedBuilderType(TypeName elementType) { if (elementType.equals(TypeName.DOUBLE)) { return DOUBLE_VECTOR_FIXED_BUILDER; } + if (elementType.equals(TypeName.FLOAT)) { + return FLOAT_VECTOR_FIXED_BUILDER; + } throw new IllegalArgumentException("unknown vector fixed builder type for [" + elementType + "]"); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatArrayState.java new file mode 100644 index 0000000000000..b3767828f00db --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatArrayState.java @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.FloatArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of floats. It is created in a mode where it + * won't track the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

+ * This class is generated. Do not edit it. + *

+ */ +final class FloatArrayState extends AbstractArrayState implements GroupingAggregatorState { + private final float init; + + private FloatArray values; + + FloatArrayState(BigArrays bigArrays, float init) { + super(bigArrays); + this.values = bigArrays.newFloatArray(1, false); + this.values.set(0, init); + this.init = init; + } + + float get(int groupId) { + return values.get(groupId); + } + + float getOrDefault(int groupId) { + return groupId < values.size() ? values.get(groupId) : init; + } + + void set(int groupId, float value) { + ensureCapacity(groupId); + values.set(groupId, value); + trackGroupId(groupId); + } + + Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) { + if (false == trackingGroupIds()) { + try (var builder = driverContext.blockFactory().newFloatVectorFixedBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + builder.appendFloat(i, values.get(selected.getInt(i))); + } + return builder.build().asBlock(); + } + } + try (FloatBlock.Builder builder = driverContext.blockFactory().newFloatBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group)) { + builder.appendFloat(values.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + if (groupId >= values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, groupId + 1); + values.fill(prevSize, values.size(), init); + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate( + Block[] blocks, + int offset, + IntVector selected, + org.elasticsearch.compute.operator.DriverContext driverContext + ) { + assert blocks.length >= offset + 2; + try ( + var valuesBuilder = driverContext.blockFactory().newFloatBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (group < values.size()) { + valuesBuilder.appendFloat(values.get(group)); + } else { + valuesBuilder.appendFloat(0); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + } + blocks[offset + 0] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java new file mode 100644 index 0000000000000..81bdd39e51b6e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * Aggregator state for a single float. + * This class is generated. Do not edit it. + */ +final class FloatState implements AggregatorState { + private float value; + private boolean seen; + + FloatState() { + this(0); + } + + FloatState(float init) { + this.value = init; + } + + float floatValue() { + return value; + } + + void floatValue(float value) { + this.value = value; + } + + boolean seen() { + return seen; + } + + void seen(boolean seen) { + this.seen = seen; + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + assert blocks.length >= offset + 2; + blocks[offset + 0] = driverContext.blockFactory().newConstantFloatBlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + } + + @Override + public void close() {} +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java new file mode 100644 index 0000000000000..b50b125d98331 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java @@ -0,0 +1,341 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ObjectArray; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; + +import java.util.Arrays; + +/** + * A rate grouping aggregation definition for float. + * This class is generated. Edit `X-RateAggregator.java.st` instead. + */ +@GroupingAggregator( + includeTimestamps = true, + value = { + @IntermediateState(name = "timestamps", type = "LONG_BLOCK"), + @IntermediateState(name = "values", type = "FLOAT_BLOCK"), + @IntermediateState(name = "resets", type = "DOUBLE") } +) +public class RateFloatAggregator { + + public static FloatRateGroupingState initGrouping(DriverContext driverContext, long unitInMillis) { + return new FloatRateGroupingState(driverContext.bigArrays(), driverContext.breaker(), unitInMillis); + } + + public static void combine(FloatRateGroupingState current, int groupId, long timestamp, float value) { + current.append(groupId, timestamp, value); + } + + public static void combineIntermediate( + FloatRateGroupingState current, + int groupId, + LongBlock timestamps, + FloatBlock values, + double reset, + int otherPosition + ) { + current.combine(groupId, timestamps, values, reset, otherPosition); + } + + public static void combineStates( + FloatRateGroupingState current, + int currentGroupId, // make the stylecheck happy + FloatRateGroupingState otherState, + int otherGroupId + ) { + current.combineState(currentGroupId, otherState, otherGroupId); + } + + public static Block evaluateFinal(FloatRateGroupingState state, IntVector selected, DriverContext driverContext) { + return state.evaluateFinal(selected, driverContext.blockFactory()); + } + + private static class FloatRateState { + static final long BASE_RAM_USAGE = RamUsageEstimator.sizeOfObject(FloatRateState.class); + final long[] timestamps; // descending order + final float[] values; + double reset = 0; + + FloatRateState(int initialSize) { + this.timestamps = new long[initialSize]; + this.values = new float[initialSize]; + } + + FloatRateState(long[] ts, float[] vs) { + this.timestamps = ts; + this.values = vs; + } + + private float dv(float v0, float v1) { + // counter reset detection + return v0 > v1 ? v1 : v1 - v0; + } + + void append(long t, float v) { + assert timestamps.length == 2 : "expected two timestamps; got " + timestamps.length; + assert t < timestamps[1] : "@timestamp goes backward: " + t + " >= " + timestamps[1]; + reset += dv(v, values[1]) + dv(values[1], values[0]) - dv(v, values[0]); + timestamps[1] = t; + values[1] = v; + } + + int entries() { + return timestamps.length; + } + + static long bytesUsed(int entries) { + var ts = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Long.BYTES * entries); + var vs = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Float.BYTES * entries); + return BASE_RAM_USAGE + ts + vs; + } + } + + public static final class FloatRateGroupingState implements Releasable, Accountable, GroupingAggregatorState { + private ObjectArray states; + private final long unitInMillis; + private final BigArrays bigArrays; + private final CircuitBreaker breaker; + private long stateBytes; // for individual states + + FloatRateGroupingState(BigArrays bigArrays, CircuitBreaker breaker, long unitInMillis) { + this.bigArrays = bigArrays; + this.breaker = breaker; + this.states = bigArrays.newObjectArray(1); + this.unitInMillis = unitInMillis; + } + + void ensureCapacity(int groupId) { + states = bigArrays.grow(states, groupId + 1); + } + + void adjustBreaker(long bytes) { + breaker.addEstimateBytesAndMaybeBreak(bytes, "<>"); + stateBytes += bytes; + assert stateBytes >= 0 : stateBytes; + } + + void append(int groupId, long timestamp, float value) { + ensureCapacity(groupId); + var state = states.get(groupId); + if (state == null) { + adjustBreaker(FloatRateState.bytesUsed(1)); + state = new FloatRateState(new long[] { timestamp }, new float[] { value }); + states.set(groupId, state); + } else { + if (state.entries() == 1) { + adjustBreaker(FloatRateState.bytesUsed(2)); + state = new FloatRateState(new long[] { state.timestamps[0], timestamp }, new float[] { state.values[0], value }); + states.set(groupId, state); + adjustBreaker(-FloatRateState.bytesUsed(1)); // old state + } else { + state.append(timestamp, value); + } + } + } + + void combine(int groupId, LongBlock timestamps, FloatBlock values, double reset, int otherPosition) { + final int valueCount = timestamps.getValueCount(otherPosition); + if (valueCount == 0) { + return; + } + final int firstIndex = timestamps.getFirstValueIndex(otherPosition); + ensureCapacity(groupId); + var state = states.get(groupId); + if (state == null) { + adjustBreaker(FloatRateState.bytesUsed(valueCount)); + state = new FloatRateState(valueCount); + state.reset = reset; + states.set(groupId, state); + // TODO: add bulk_copy to Block + for (int i = 0; i < valueCount; i++) { + state.timestamps[i] = timestamps.getLong(firstIndex + i); + state.values[i] = values.getFloat(firstIndex + i); + } + } else { + adjustBreaker(FloatRateState.bytesUsed(state.entries() + valueCount)); + var newState = new FloatRateState(state.entries() + valueCount); + newState.reset = state.reset + reset; + states.set(groupId, newState); + merge(state, newState, firstIndex, valueCount, timestamps, values); + adjustBreaker(-FloatRateState.bytesUsed(state.entries())); // old state + } + } + + void merge(FloatRateState curr, FloatRateState dst, int firstIndex, int rightCount, LongBlock timestamps, FloatBlock values) { + int i = 0, j = 0, k = 0; + final int leftCount = curr.entries(); + while (i < leftCount && j < rightCount) { + final var t1 = curr.timestamps[i]; + final var t2 = timestamps.getLong(firstIndex + j); + if (t1 > t2) { + dst.timestamps[k] = t1; + dst.values[k] = curr.values[i]; + ++i; + } else { + dst.timestamps[k] = t2; + dst.values[k] = values.getFloat(firstIndex + j); + ++j; + } + ++k; + } + if (i < leftCount) { + System.arraycopy(curr.timestamps, i, dst.timestamps, k, leftCount - i); + System.arraycopy(curr.values, i, dst.values, k, leftCount - i); + } + while (j < rightCount) { + dst.timestamps[k] = timestamps.getLong(firstIndex + j); + dst.values[k] = values.getFloat(firstIndex + j); + ++k; + ++j; + } + } + + void combineState(int groupId, FloatRateGroupingState otherState, int otherGroupId) { + var other = otherGroupId < otherState.states.size() ? otherState.states.get(otherGroupId) : null; + if (other == null) { + return; + } + ensureCapacity(groupId); + var curr = states.get(groupId); + if (curr == null) { + var len = other.entries(); + adjustBreaker(FloatRateState.bytesUsed(len)); + curr = new FloatRateState(Arrays.copyOf(other.timestamps, len), Arrays.copyOf(other.values, len)); + curr.reset = other.reset; + states.set(groupId, curr); + } else { + states.set(groupId, mergeState(curr, other)); + } + } + + FloatRateState mergeState(FloatRateState s1, FloatRateState s2) { + var newLen = s1.entries() + s2.entries(); + adjustBreaker(FloatRateState.bytesUsed(newLen)); + var dst = new FloatRateState(newLen); + dst.reset = s1.reset + s2.reset; + int i = 0, j = 0, k = 0; + while (i < s1.entries() && j < s2.entries()) { + if (s1.timestamps[i] > s2.timestamps[j]) { + dst.timestamps[k] = s1.timestamps[i]; + dst.values[k] = s1.values[i]; + ++i; + } else { + dst.timestamps[k] = s2.timestamps[j]; + dst.values[k] = s2.values[j]; + ++j; + } + ++k; + } + System.arraycopy(s1.timestamps, i, dst.timestamps, k, s1.entries() - i); + System.arraycopy(s1.values, i, dst.values, k, s1.entries() - i); + System.arraycopy(s2.timestamps, j, dst.timestamps, k, s2.entries() - j); + System.arraycopy(s2.values, j, dst.values, k, s2.entries() - j); + return dst; + } + + @Override + public long ramBytesUsed() { + return states.ramBytesUsed() + stateBytes; + } + + @Override + public void close() { + Releasables.close(states, () -> adjustBreaker(-stateBytes)); + } + + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + assert blocks.length >= offset + 3 : "blocks=" + blocks.length + ",offset=" + offset; + final BlockFactory blockFactory = driverContext.blockFactory(); + final int positionCount = selected.getPositionCount(); + try ( + LongBlock.Builder timestamps = blockFactory.newLongBlockBuilder(positionCount * 2); + FloatBlock.Builder values = blockFactory.newFloatBlockBuilder(positionCount * 2); + DoubleVector.FixedBuilder resets = blockFactory.newDoubleVectorFixedBuilder(positionCount) + ) { + for (int i = 0; i < positionCount; i++) { + final var groupId = selected.getInt(i); + final var state = groupId < states.size() ? states.get(groupId) : null; + if (state != null) { + timestamps.beginPositionEntry(); + for (long t : state.timestamps) { + timestamps.appendLong(t); + } + timestamps.endPositionEntry(); + + values.beginPositionEntry(); + for (float v : state.values) { + values.appendFloat(v); + } + values.endPositionEntry(); + + resets.appendDouble(i, state.reset); + } else { + timestamps.appendNull(); + values.appendNull(); + resets.appendDouble(i, 0); + } + } + blocks[offset] = timestamps.build(); + blocks[offset + 1] = values.build(); + blocks[offset + 2] = resets.build().asBlock(); + } + } + + Block evaluateFinal(IntVector selected, BlockFactory blockFactory) { + int positionCount = selected.getPositionCount(); + try (DoubleBlock.Builder rates = blockFactory.newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + final var groupId = selected.getInt(p); + final var state = groupId < states.size() ? states.get(groupId) : null; + if (state == null) { + rates.appendNull(); + continue; + } + int len = state.entries(); + long dt = state.timestamps[0] - state.timestamps[len - 1]; + if (dt == 0) { + // TODO: maybe issue warning when we don't have enough sample? + rates.appendNull(); + } else { + double reset = state.reset; + for (int i = 1; i < len; i++) { + if (state.values[i - 1] < state.values[i]) { + reset += state.values[i]; + } + } + double dv = state.values[0] - state.values[len - 1] + reset; + rates.appendDouble(dv * unitInMillis / dt); + } + } + return rates.build(); + } + } + + void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + // noop - we handle the null states inside `toIntermediate` and `evaluateFinal` + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListFloatAggregator.java new file mode 100644 index 0000000000000..c5fc51d5ba13f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListFloatAggregator.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.sort.FloatBucketedSort; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +/** + * Aggregates the top N field values for float. + */ +@Aggregator({ @IntermediateState(name = "topList", type = "FLOAT_BLOCK") }) +@GroupingAggregator +class TopListFloatAggregator { + public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { + return new SingleState(bigArrays, limit, ascending); + } + + public static void combine(SingleState state, float v) { + state.add(v); + } + + public static void combineIntermediate(SingleState state, FloatBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.getFloat(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) { + return new GroupingState(bigArrays, limit, ascending); + } + + public static void combine(GroupingState state, int groupId, float v) { + state.add(groupId, v); + } + + public static void combineIntermediate(GroupingState state, int groupId, FloatBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getFloat(i)); + } + } + + public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { + current.merge(groupId, state, statePosition); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class GroupingState implements Releasable { + private final FloatBucketedSort sort; + + private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { + this.sort = new FloatBucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + } + + public void add(int groupId, float value) { + sort.collect(value, groupId); + } + + public void merge(int groupId, GroupingState other, int otherGroupId) { + sort.merge(groupId, other.sort, otherGroupId); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + Releasables.closeExpectNoException(sort); + } + } + + public static class SingleState implements Releasable { + private final GroupingState internalState; + + private SingleState(BigArrays bigArrays, int limit, boolean ascending) { + this.internalState = new GroupingState(bigArrays, limit, ascending); + } + + public void add(float value) { + internalState.add(0, value); + } + + public void merge(GroupingState other) { + internalState.merge(0, other, 0); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + try (var intValues = blockFactory.newConstantIntVector(0, 1)) { + return internalState.toBlock(blockFactory, intValues); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java new file mode 100644 index 0000000000000..f9e5e1b7b283a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.LongHash; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; + +/** + * Aggregates field values for float. + * This class is generated. Edit @{code X-ValuesAggregator.java.st} instead + * of this file. + */ +@Aggregator({ @IntermediateState(name = "values", type = "FLOAT_BLOCK") }) +@GroupingAggregator +class ValuesFloatAggregator { + public static SingleState initSingle(BigArrays bigArrays) { + return new SingleState(bigArrays); + } + + public static void combine(SingleState state, float v) { + state.values.add(Float.floatToIntBits(v)); + } + + public static void combineIntermediate(SingleState state, FloatBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.getFloat(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays) { + return new GroupingState(bigArrays); + } + + public static void combine(GroupingState state, int groupId, float v) { + /* + * Encode the groupId and value into a single long - + * the top 32 bits for the group, the bottom 32 for the value. + */ + state.values.add((((long) groupId) << Float.SIZE) | (Float.floatToIntBits(v) & 0xFFFFFFFFL)); + } + + public static void combineIntermediate(GroupingState state, int groupId, FloatBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getFloat(i)); + } + } + + public static void combineStates(GroupingState current, int currentGroupId, GroupingState state, int statePosition) { + for (int id = 0; id < state.values.size(); id++) { + long both = state.values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group == statePosition) { + float value = Float.intBitsToFloat((int) both); + combine(current, currentGroupId, value); + } + } + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class SingleState implements Releasable { + private final LongHash values; + + private SingleState(BigArrays bigArrays) { + values = new LongHash(1, bigArrays); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + if (values.size() == 0) { + return blockFactory.newConstantNullBlock(1); + } + if (values.size() == 1) { + return blockFactory.newConstantFloatBlockWith(Float.intBitsToFloat((int) values.get(0)), 1); + } + try (FloatBlock.Builder builder = blockFactory.newFloatBlockBuilder((int) values.size())) { + builder.beginPositionEntry(); + for (int id = 0; id < values.size(); id++) { + builder.appendFloat(Float.intBitsToFloat((int) values.get(id))); + } + builder.endPositionEntry(); + return builder.build(); + } + } + + @Override + public void close() { + values.close(); + } + } + + /** + * State for a grouped {@code VALUES} aggregation. This implementation + * emphasizes collect-time performance over the performance of rendering + * results. That's good, but it's a pretty intensive emphasis, requiring + * an {@code O(n^2)} operation for collection to support a {@code O(1)} + * collector operation. But at least it's fairly simple. + */ + public static class GroupingState implements Releasable { + private final LongHash values; + + private GroupingState(BigArrays bigArrays) { + values = new LongHash(1, bigArrays); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + if (values.size() == 0) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + try (FloatBlock.Builder builder = blockFactory.newFloatBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int selectedGroup = selected.getInt(s); + /* + * Count can effectively be in three states - 0, 1, many. We use those + * states to buffer the first value, so we can avoid calling + * beginPositionEntry on single valued fields. + */ + int count = 0; + float first = 0; + for (int id = 0; id < values.size(); id++) { + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group == selectedGroup) { + float value = Float.intBitsToFloat((int) both); + switch (count) { + case 0 -> first = value; + case 1 -> { + builder.beginPositionEntry(); + builder.appendFloat(first); + builder.appendFloat(value); + } + default -> builder.appendFloat(value); + } + count++; + } + } + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> builder.appendFloat(first); + default -> builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + values.close(); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java new file mode 100644 index 0000000000000..b490fe193c33f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java @@ -0,0 +1,346 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.FloatArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Aggregates the top N float values per bucket. + * See {@link BucketedSort} for more information. + * This class is generated. Edit @{code X-BucketedSort.java.st} instead of this file. + */ +public class FloatBucketedSort implements Releasable { + + private final BigArrays bigArrays; + private final SortOrder order; + private final int bucketSize; + /** + * {@code true} if the bucket is in heap mode, {@code false} if + * it is still gathering. + */ + private final BitArray heapMode; + /** + * An array containing all the values on all buckets. The structure is as follows: + *

+ * For each bucket, there are bucketSize elements, based on the bucket id (0, 1, 2...). + * Then, for each bucket, it can be in 2 states: + *

+ *
    + *
  • + * Gather mode: All buckets start in gather mode, and remain here while they have less than bucketSize elements. + * In gather mode, the elements are stored in the array from the highest index to the lowest index. + * The lowest index contains the offset to the next slot to be filled. + *

    + * This allows us to insert elements in O(1) time. + *

    + *

    + * When the bucketSize-th element is collected, the bucket transitions to heap mode, by heapifying its contents. + *

    + *
  • + *
  • + * Heap mode: The bucket slots are organized as a min heap structure. + *

    + * The root of the heap is the minimum value in the bucket, + * which allows us to quickly discard new values that are not in the top N. + *

    + *
  • + *
+ */ + private FloatArray values; + + public FloatBucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) { + this.bigArrays = bigArrays; + this.order = order; + this.bucketSize = bucketSize; + heapMode = new BitArray(0, bigArrays); + + boolean success = false; + try { + values = bigArrays.newFloatArray(0, false); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + /** + * Collects a {@code value} into a {@code bucket}. + *

+ * It may or may not be inserted in the heap, depending on if it is better than the current root. + *

+ */ + public void collect(float value, int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (inHeapMode(bucket)) { + if (betterThan(value, values.get(rootIndex))) { + values.set(rootIndex, value); + downHeap(rootIndex, 0); + } + return; + } + // Gathering mode + long requiredSize = rootIndex + bucketSize; + if (values.size() < requiredSize) { + grow(requiredSize); + } + int next = getNextGatherOffset(rootIndex); + assert 0 <= next && next < bucketSize + : "Expected next to be in the range of valid buckets [0 <= " + next + " < " + bucketSize + "]"; + long index = next + rootIndex; + values.set(index, value); + if (next == 0) { + heapMode.set(bucket); + heapify(rootIndex); + } else { + setNextGatherOffset(rootIndex, next - 1); + } + } + + /** + * The order of the sort. + */ + public SortOrder getOrder() { + return order; + } + + /** + * The number of values to store per bucket. + */ + public int getBucketSize() { + return bucketSize; + } + + /** + * Get the first and last indexes (inclusive, exclusive) of the values for a bucket. + * Returns [0, 0] if the bucket has never been collected. + */ + private Tuple getBucketValuesIndexes(int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (rootIndex >= values.size()) { + // We've never seen this bucket. + return Tuple.tuple(0L, 0L); + } + long start = inHeapMode(bucket) ? rootIndex : (rootIndex + getNextGatherOffset(rootIndex) + 1); + long end = rootIndex + bucketSize; + return Tuple.tuple(start, end); + } + + /** + * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}. + */ + public void merge(int groupId, FloatBucketedSort other, int otherGroupId) { + var otherBounds = other.getBucketValuesIndexes(otherGroupId); + + // TODO: This can be improved for heapified buckets by making use of the heap structures + for (long i = otherBounds.v1(); i < otherBounds.v2(); i++) { + collect(other.values.get(i), groupId); + } + } + + /** + * Creates a block with the values from the {@code selected} groups. + */ + public Block toBlock(BlockFactory blockFactory, IntVector selected) { + // Check if the selected groups are all empty, to avoid allocating extra memory + if (IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> { + var bounds = this.getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + return size > 0; + })) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + + // Used to sort the values in the bucket. + var bucketValues = new float[bucketSize]; + + try (var builder = blockFactory.newFloatBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int bucket = selected.getInt(s); + + var bounds = getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + if (size == 0) { + builder.appendNull(); + continue; + } + + if (size == 1) { + builder.appendFloat(values.get(bounds.v1())); + continue; + } + + for (int i = 0; i < size; i++) { + bucketValues[i] = values.get(bounds.v1() + i); + } + + // TODO: Make use of heap structures to faster iterate in order instead of copying and sorting + Arrays.sort(bucketValues, 0, (int) size); + + builder.beginPositionEntry(); + if (order == SortOrder.ASC) { + for (int i = 0; i < size; i++) { + builder.appendFloat(bucketValues[i]); + } + } else { + for (int i = (int) size - 1; i >= 0; i--) { + builder.appendFloat(bucketValues[i]); + } + } + builder.endPositionEntry(); + } + return builder.build(); + } + } + + /** + * Is this bucket a min heap {@code true} or in gathering mode {@code false}? + */ + private boolean inHeapMode(int bucket) { + return heapMode.get(bucket); + } + + /** + * Get the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private int getNextGatherOffset(long rootIndex) { + return (int) values.get(rootIndex); + } + + /** + * Set the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private void setNextGatherOffset(long rootIndex, int offset) { + values.set(rootIndex, offset); + } + + /** + * {@code true} if the entry at index {@code lhs} is "better" than + * the entry at {@code rhs}. "Better" in this means "lower" for + * {@link SortOrder#ASC} and "higher" for {@link SortOrder#DESC}. + */ + private boolean betterThan(float lhs, float rhs) { + return getOrder().reverseMul() * Float.compare(lhs, rhs) < 0; + } + + /** + * Swap the data at two indices. + */ + private void swap(long lhs, long rhs) { + var tmp = values.get(lhs); + values.set(lhs, values.get(rhs)); + values.set(rhs, tmp); + } + + /** + * Allocate storage for more buckets and store the "next gather offset" + * for those new buckets. + */ + private void grow(long minSize) { + long oldMax = values.size(); + values = bigArrays.grow(values, minSize); + // Set the next gather offsets for all newly allocated buckets. + setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + } + + /** + * Maintain the "next gather offsets" for newly allocated buckets. + */ + private void setNextGatherOffsets(long startingAt) { + int nextOffset = getBucketSize() - 1; + for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { + setNextGatherOffset(bucketRoot, nextOffset); + } + } + + /** + * Heapify a bucket whose entries are in random order. + *

+ * This works by validating the heap property on each node, iterating + * "upwards", pushing any out of order parents "down". Check out the + * wikipedia + * entry on binary heaps for more about this. + *

+ *

+ * While this *looks* like it could easily be {@code O(n * log n)}, it is + * a fairly well studied algorithm attributed to Floyd. There's + * been a bunch of work that puts this at {@code O(n)}, close to 1.88n worst + * case. + *

+ * + * @param rootIndex the index the start of the bucket + */ + private void heapify(long rootIndex) { + int maxParent = bucketSize / 2 - 1; + for (int parent = maxParent; parent >= 0; parent--) { + downHeap(rootIndex, parent); + } + } + + /** + * Correct the heap invariant of a parent and its children. This + * runs in {@code O(log n)} time. + * @param rootIndex index of the start of the bucket + * @param parent Index within the bucket of the parent to check. + * For example, 0 is the "root". + */ + private void downHeap(long rootIndex, int parent) { + while (true) { + long parentIndex = rootIndex + parent; + int worst = parent; + long worstIndex = parentIndex; + int leftChild = parent * 2 + 1; + long leftIndex = rootIndex + leftChild; + if (leftChild < bucketSize) { + if (betterThan(values.get(worstIndex), values.get(leftIndex))) { + worst = leftChild; + worstIndex = leftIndex; + } + int rightChild = leftChild + 1; + long rightIndex = rootIndex + rightChild; + if (rightChild < bucketSize && betterThan(values.get(worstIndex), values.get(rightIndex))) { + worst = rightChild; + worstIndex = rightIndex; + } + } + if (worst == parent) { + break; + } + swap(worstIndex, parentIndex); + parent = worst; + } + } + + @Override + public final void close() { + Releasables.close(values, heapMode); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java new file mode 100644 index 0000000000000..aad616eac95a1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link CountDistinctFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class CountDistinctFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("hll", ElementType.BYTES_REF) ); + + private final DriverContext driverContext; + + private final HllStates.SingleState state; + + private final List channels; + + private final int precision; + + public CountDistinctFloatAggregatorFunction(DriverContext driverContext, List channels, + HllStates.SingleState state, int precision) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.precision = precision; + } + + public static CountDistinctFloatAggregatorFunction create(DriverContext driverContext, + List channels, int precision) { + return new CountDistinctFloatAggregatorFunction(driverContext, channels, CountDistinctFloatAggregator.initSingle(driverContext.bigArrays(), precision), precision); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + CountDistinctFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + CountDistinctFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block hllUncast = page.getBlock(channels.get(0)); + if (hllUncast.areAllValuesNull()) { + return; + } + BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); + assert hll.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + CountDistinctFloatAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = CountDistinctFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..4c2aad00a7a72 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionSupplier.java @@ -0,0 +1,42 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link CountDistinctFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class CountDistinctFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int precision; + + public CountDistinctFloatAggregatorFunctionSupplier(List channels, int precision) { + this.channels = channels; + this.precision = precision; + } + + @Override + public CountDistinctFloatAggregatorFunction aggregator(DriverContext driverContext) { + return CountDistinctFloatAggregatorFunction.create(driverContext, channels, precision); + } + + @Override + public CountDistinctFloatGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return CountDistinctFloatGroupingAggregatorFunction.create(channels, driverContext, precision); + } + + @Override + public String describe() { + return "count_distinct of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..60c1755b88c6a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link CountDistinctFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class CountDistinctFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("hll", ElementType.BYTES_REF) ); + + private final HllStates.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int precision; + + public CountDistinctFloatGroupingAggregatorFunction(List channels, + HllStates.GroupingState state, DriverContext driverContext, int precision) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.precision = precision; + } + + public static CountDistinctFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int precision) { + return new CountDistinctFloatGroupingAggregatorFunction(channels, CountDistinctFloatAggregator.initGrouping(driverContext.bigArrays(), precision), driverContext, precision); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block hllUncast = page.getBlock(channels.get(0)); + if (hllUncast.areAllValuesNull()) { + return; + } + BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + HllStates.GroupingState inState = ((CountDistinctFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + CountDistinctFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = CountDistinctFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java new file mode 100644 index 0000000000000..0dcef4341727d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java @@ -0,0 +1,138 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MaxFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.FLOAT), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final FloatState state; + + private final List channels; + + public MaxFloatAggregatorFunction(DriverContext driverContext, List channels, + FloatState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MaxFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MaxFloatAggregatorFunction(driverContext, channels, new FloatState(MaxFloatAggregator.init())); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + state.seen(true); + for (int i = 0; i < vector.getPositionCount(); i++) { + state.floatValue(MaxFloatAggregator.combine(state.floatValue(), vector.getFloat(i))); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + state.seen(true); + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + state.floatValue(MaxFloatAggregator.combine(state.floatValue(), block.getFloat(i))); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + FloatVector max = ((FloatBlock) maxUncast).asVector(); + assert max.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + if (seen.getBoolean(0)) { + state.floatValue(MaxFloatAggregator.combine(state.floatValue(), max.getFloat(0))); + state.seen(true); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + if (state.seen() == false) { + blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1); + return; + } + blocks[offset] = driverContext.blockFactory().newConstantFloatBlockWith(state.floatValue(), 1); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..a3aa44f432430 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MaxFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MaxFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MaxFloatAggregatorFunction aggregator(DriverContext driverContext) { + return MaxFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public MaxFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return MaxFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "max of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..85708792732a7 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java @@ -0,0 +1,208 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MaxFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.FLOAT), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final FloatArrayState state; + + private final List channels; + + private final DriverContext driverContext; + + public MaxFloatGroupingAggregatorFunction(List channels, FloatArrayState state, + DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MaxFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new MaxFloatGroupingAggregatorFunction(channels, new FloatArrayState(driverContext.bigArrays(), MaxFloatAggregator.init()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(v))); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset))); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(v))); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + FloatVector max = ((FloatBlock) maxUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert max.getPositionCount() == seen.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (seen.getBoolean(groupPosition + positionOffset)) { + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), max.getFloat(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + FloatArrayState inState = ((MaxFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + if (inState.hasValue(position)) { + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), inState.get(position))); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = state.toValuesBlock(selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java new file mode 100644 index 0000000000000..38a16859140f8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java @@ -0,0 +1,124 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MedianAbsoluteDeviationFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MedianAbsoluteDeviationFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("quart", ElementType.BYTES_REF) ); + + private final DriverContext driverContext; + + private final QuantileStates.SingleState state; + + private final List channels; + + public MedianAbsoluteDeviationFloatAggregatorFunction(DriverContext driverContext, + List channels, QuantileStates.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MedianAbsoluteDeviationFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MedianAbsoluteDeviationFloatAggregatorFunction(driverContext, channels, MedianAbsoluteDeviationFloatAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + MedianAbsoluteDeviationFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + MedianAbsoluteDeviationFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block quartUncast = page.getBlock(channels.get(0)); + if (quartUncast.areAllValuesNull()) { + return; + } + BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); + assert quart.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = MedianAbsoluteDeviationFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..1fad0faafad4e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionSupplier.java @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MedianAbsoluteDeviationFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MedianAbsoluteDeviationFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MedianAbsoluteDeviationFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MedianAbsoluteDeviationFloatAggregatorFunction aggregator(DriverContext driverContext) { + return MedianAbsoluteDeviationFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public MedianAbsoluteDeviationFloatGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return MedianAbsoluteDeviationFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "median_absolute_deviation of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..84646476fcee0 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java @@ -0,0 +1,199 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MedianAbsoluteDeviationFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MedianAbsoluteDeviationFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("quart", ElementType.BYTES_REF) ); + + private final QuantileStates.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public MedianAbsoluteDeviationFloatGroupingAggregatorFunction(List channels, + QuantileStates.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MedianAbsoluteDeviationFloatGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new MedianAbsoluteDeviationFloatGroupingAggregatorFunction(channels, MedianAbsoluteDeviationFloatAggregator.initGrouping(driverContext.bigArrays()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block quartUncast = page.getBlock(channels.get(0)); + if (quartUncast.areAllValuesNull()) { + return; + } + BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + QuantileStates.GroupingState inState = ((MedianAbsoluteDeviationFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + MedianAbsoluteDeviationFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = MedianAbsoluteDeviationFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java new file mode 100644 index 0000000000000..ecabcbdcf57bb --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java @@ -0,0 +1,138 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MinFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("min", ElementType.FLOAT), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final FloatState state; + + private final List channels; + + public MinFloatAggregatorFunction(DriverContext driverContext, List channels, + FloatState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MinFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MinFloatAggregatorFunction(driverContext, channels, new FloatState(MinFloatAggregator.init())); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + state.seen(true); + for (int i = 0; i < vector.getPositionCount(); i++) { + state.floatValue(MinFloatAggregator.combine(state.floatValue(), vector.getFloat(i))); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + state.seen(true); + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + state.floatValue(MinFloatAggregator.combine(state.floatValue(), block.getFloat(i))); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block minUncast = page.getBlock(channels.get(0)); + if (minUncast.areAllValuesNull()) { + return; + } + FloatVector min = ((FloatBlock) minUncast).asVector(); + assert min.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + if (seen.getBoolean(0)) { + state.floatValue(MinFloatAggregator.combine(state.floatValue(), min.getFloat(0))); + state.seen(true); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + if (state.seen() == false) { + blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1); + return; + } + blocks[offset] = driverContext.blockFactory().newConstantFloatBlockWith(state.floatValue(), 1); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..a8ccc70f9996a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MinFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MinFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MinFloatAggregatorFunction aggregator(DriverContext driverContext) { + return MinFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public MinFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return MinFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "min of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..2f00bbf1335ed --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java @@ -0,0 +1,208 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MinFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("min", ElementType.FLOAT), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final FloatArrayState state; + + private final List channels; + + private final DriverContext driverContext; + + public MinFloatGroupingAggregatorFunction(List channels, FloatArrayState state, + DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MinFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new MinFloatGroupingAggregatorFunction(channels, new FloatArrayState(driverContext.bigArrays(), MinFloatAggregator.init()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(v))); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset))); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(v))); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block minUncast = page.getBlock(channels.get(0)); + if (minUncast.areAllValuesNull()) { + return; + } + FloatVector min = ((FloatBlock) minUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert min.getPositionCount() == seen.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (seen.getBoolean(groupPosition + positionOffset)) { + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), min.getFloat(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + FloatArrayState inState = ((MinFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + if (inState.hasValue(position)) { + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), inState.get(position))); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = state.toValuesBlock(selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java new file mode 100644 index 0000000000000..8f0ffd81e64b6 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link PercentileFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class PercentileFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("quart", ElementType.BYTES_REF) ); + + private final DriverContext driverContext; + + private final QuantileStates.SingleState state; + + private final List channels; + + private final double percentile; + + public PercentileFloatAggregatorFunction(DriverContext driverContext, List channels, + QuantileStates.SingleState state, double percentile) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.percentile = percentile; + } + + public static PercentileFloatAggregatorFunction create(DriverContext driverContext, + List channels, double percentile) { + return new PercentileFloatAggregatorFunction(driverContext, channels, PercentileFloatAggregator.initSingle(percentile), percentile); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + PercentileFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + PercentileFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block quartUncast = page.getBlock(channels.get(0)); + if (quartUncast.areAllValuesNull()) { + return; + } + BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); + assert quart.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + PercentileFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = PercentileFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..1d1678f15448c --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionSupplier.java @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link PercentileFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class PercentileFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final double percentile; + + public PercentileFloatAggregatorFunctionSupplier(List channels, double percentile) { + this.channels = channels; + this.percentile = percentile; + } + + @Override + public PercentileFloatAggregatorFunction aggregator(DriverContext driverContext) { + return PercentileFloatAggregatorFunction.create(driverContext, channels, percentile); + } + + @Override + public PercentileFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return PercentileFloatGroupingAggregatorFunction.create(channels, driverContext, percentile); + } + + @Override + public String describe() { + return "percentile of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..564e0e90018c2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link PercentileFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class PercentileFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("quart", ElementType.BYTES_REF) ); + + private final QuantileStates.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final double percentile; + + public PercentileFloatGroupingAggregatorFunction(List channels, + QuantileStates.GroupingState state, DriverContext driverContext, double percentile) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.percentile = percentile; + } + + public static PercentileFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext, double percentile) { + return new PercentileFloatGroupingAggregatorFunction(channels, PercentileFloatAggregator.initGrouping(driverContext.bigArrays(), percentile), driverContext, percentile); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + PercentileFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + PercentileFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + PercentileFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + PercentileFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block quartUncast = page.getBlock(channels.get(0)); + if (quartUncast.areAllValuesNull()) { + return; + } + BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + QuantileStates.GroupingState inState = ((PercentileFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + PercentileFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = PercentileFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..4b1546314a9cb --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatAggregatorFunctionSupplier.java @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link RateFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class RateFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final long unitInMillis; + + public RateFloatAggregatorFunctionSupplier(List channels, long unitInMillis) { + this.channels = channels; + this.unitInMillis = unitInMillis; + } + + @Override + public AggregatorFunction aggregator(DriverContext driverContext) { + throw new UnsupportedOperationException("non-grouping aggregator is not supported"); + } + + @Override + public RateFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return RateFloatGroupingAggregatorFunction.create(channels, driverContext, unitInMillis); + } + + @Override + public String describe() { + return "rate of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..40f53741bf3da --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java @@ -0,0 +1,227 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link RateFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class RateFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("timestamps", ElementType.LONG), + new IntermediateStateDesc("values", ElementType.FLOAT), + new IntermediateStateDesc("resets", ElementType.DOUBLE) ); + + private final RateFloatAggregator.FloatRateGroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final long unitInMillis; + + public RateFloatGroupingAggregatorFunction(List channels, + RateFloatAggregator.FloatRateGroupingState state, DriverContext driverContext, + long unitInMillis) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.unitInMillis = unitInMillis; + } + + public static RateFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext, long unitInMillis) { + return new RateFloatGroupingAggregatorFunction(channels, RateFloatAggregator.initGrouping(driverContext, unitInMillis), driverContext, unitInMillis); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + LongBlock timestampsBlock = page.getBlock(channels.get(1)); + LongVector timestampsVector = timestampsBlock.asVector(); + if (timestampsVector == null) { + throw new IllegalStateException("expected @timestamp vector; but got a block"); + } + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock, timestampsVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock, timestampsVector); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector, timestampsVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector, timestampsVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + RateFloatAggregator.combine(state, groupId, timestamps.getLong(v), values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + var valuePosition = groupPosition + positionOffset; + RateFloatAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getFloat(valuePosition)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + RateFloatAggregator.combine(state, groupId, timestamps.getLong(v), values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + var valuePosition = groupPosition + positionOffset; + RateFloatAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getFloat(valuePosition)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block timestampsUncast = page.getBlock(channels.get(0)); + if (timestampsUncast.areAllValuesNull()) { + return; + } + LongBlock timestamps = (LongBlock) timestampsUncast; + Block valuesUncast = page.getBlock(channels.get(1)); + if (valuesUncast.areAllValuesNull()) { + return; + } + FloatBlock values = (FloatBlock) valuesUncast; + Block resetsUncast = page.getBlock(channels.get(2)); + if (resetsUncast.areAllValuesNull()) { + return; + } + DoubleVector resets = ((DoubleBlock) resetsUncast).asVector(); + assert timestamps.getPositionCount() == values.getPositionCount() && timestamps.getPositionCount() == resets.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + RateFloatAggregator.combineIntermediate(state, groupId, timestamps, values, resets.getDouble(groupPosition + positionOffset), groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + RateFloatAggregator.FloatRateGroupingState inState = ((RateFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + RateFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = RateFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java new file mode 100644 index 0000000000000..3dedc327294d5 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java @@ -0,0 +1,144 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link SumFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class SumFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("value", ElementType.DOUBLE), + new IntermediateStateDesc("delta", ElementType.DOUBLE), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final SumDoubleAggregator.SumState state; + + private final List channels; + + public SumFloatAggregatorFunction(DriverContext driverContext, List channels, + SumDoubleAggregator.SumState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static SumFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new SumFloatAggregatorFunction(driverContext, channels, SumFloatAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + state.seen(true); + for (int i = 0; i < vector.getPositionCount(); i++) { + SumFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + state.seen(true); + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + SumFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block valueUncast = page.getBlock(channels.get(0)); + if (valueUncast.areAllValuesNull()) { + return; + } + DoubleVector value = ((DoubleBlock) valueUncast).asVector(); + assert value.getPositionCount() == 1; + Block deltaUncast = page.getBlock(channels.get(1)); + if (deltaUncast.areAllValuesNull()) { + return; + } + DoubleVector delta = ((DoubleBlock) deltaUncast).asVector(); + assert delta.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(2)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + SumFloatAggregator.combineIntermediate(state, value.getDouble(0), delta.getDouble(0), seen.getBoolean(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + if (state.seen() == false) { + blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1); + return; + } + blocks[offset] = SumFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..515122ec08ac0 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link SumFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class SumFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public SumFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public SumFloatAggregatorFunction aggregator(DriverContext driverContext) { + return SumFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public SumFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return SumFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "sum of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..c69ce16f0bccb --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java @@ -0,0 +1,212 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link SumFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class SumFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("value", ElementType.DOUBLE), + new IntermediateStateDesc("delta", ElementType.DOUBLE), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final SumDoubleAggregator.GroupingSumState state; + + private final List channels; + + private final DriverContext driverContext; + + public SumFloatGroupingAggregatorFunction(List channels, + SumDoubleAggregator.GroupingSumState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static SumFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new SumFloatGroupingAggregatorFunction(channels, SumFloatAggregator.initGrouping(driverContext.bigArrays()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + SumFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + SumFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + SumFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + SumFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block valueUncast = page.getBlock(channels.get(0)); + if (valueUncast.areAllValuesNull()) { + return; + } + DoubleVector value = ((DoubleBlock) valueUncast).asVector(); + Block deltaUncast = page.getBlock(channels.get(1)); + if (deltaUncast.areAllValuesNull()) { + return; + } + DoubleVector delta = ((DoubleBlock) deltaUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(2)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert value.getPositionCount() == delta.getPositionCount() && value.getPositionCount() == seen.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + SumFloatAggregator.combineIntermediate(state, groupId, value.getDouble(groupPosition + positionOffset), delta.getDouble(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + SumDoubleAggregator.GroupingSumState inState = ((SumFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + SumFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = SumFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunction.java new file mode 100644 index 0000000000000..6232d6ff21fc9 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunction.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link TopListFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.FLOAT) ); + + private final DriverContext driverContext; + + private final TopListFloatAggregator.SingleState state; + + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListFloatAggregatorFunction(DriverContext driverContext, List channels, + TopListFloatAggregator.SingleState state, int limit, boolean ascending) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListFloatAggregatorFunction create(DriverContext driverContext, + List channels, int limit, boolean ascending) { + return new TopListFloatAggregatorFunction(driverContext, channels, TopListFloatAggregator.initSingle(driverContext.bigArrays(), limit, ascending), limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + TopListFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + TopListFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + FloatBlock topList = (FloatBlock) topListUncast; + assert topList.getPositionCount() == 1; + TopListFloatAggregator.combineIntermediate(state, topList); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = TopListFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..ff1c3e8df4b46 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionSupplier.java @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link TopListFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListFloatAggregatorFunctionSupplier(List channels, int limit, + boolean ascending) { + this.channels = channels; + this.limit = limit; + this.ascending = ascending; + } + + @Override + public TopListFloatAggregatorFunction aggregator(DriverContext driverContext) { + return TopListFloatAggregatorFunction.create(driverContext, channels, limit, ascending); + } + + @Override + public TopListFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return TopListFloatGroupingAggregatorFunction.create(channels, driverContext, limit, ascending); + } + + @Override + public String describe() { + return "top_list of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..66f8fa7eeb35d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link TopListFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.FLOAT) ); + + private final TopListFloatAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int limit; + + private final boolean ascending; + + public TopListFloatGroupingAggregatorFunction(List channels, + TopListFloatAggregator.GroupingState state, DriverContext driverContext, int limit, + boolean ascending) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int limit, boolean ascending) { + return new TopListFloatGroupingAggregatorFunction(channels, TopListFloatAggregator.initGrouping(driverContext.bigArrays(), limit, ascending), driverContext, limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + TopListFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + FloatBlock topList = (FloatBlock) topListUncast; + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListFloatAggregator.combineIntermediate(state, groupId, topList, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + TopListFloatAggregator.GroupingState inState = ((TopListFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + TopListFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = TopListFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java new file mode 100644 index 0000000000000..c7385e87bfbf2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java @@ -0,0 +1,120 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link ValuesFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class ValuesFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("values", ElementType.FLOAT) ); + + private final DriverContext driverContext; + + private final ValuesFloatAggregator.SingleState state; + + private final List channels; + + public ValuesFloatAggregatorFunction(DriverContext driverContext, List channels, + ValuesFloatAggregator.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static ValuesFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new ValuesFloatAggregatorFunction(driverContext, channels, ValuesFloatAggregator.initSingle(driverContext.bigArrays())); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + ValuesFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + ValuesFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block valuesUncast = page.getBlock(channels.get(0)); + if (valuesUncast.areAllValuesNull()) { + return; + } + FloatBlock values = (FloatBlock) valuesUncast; + assert values.getPositionCount() == 1; + ValuesFloatAggregator.combineIntermediate(state, values); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = ValuesFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..b4b0c2f1a0444 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link ValuesFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class ValuesFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public ValuesFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public ValuesFloatAggregatorFunction aggregator(DriverContext driverContext) { + return ValuesFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public ValuesFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return ValuesFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "values of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..54cc06072cd24 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java @@ -0,0 +1,195 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link ValuesFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class ValuesFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("values", ElementType.FLOAT) ); + + private final ValuesFloatAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public ValuesFloatGroupingAggregatorFunction(List channels, + ValuesFloatAggregator.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static ValuesFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new ValuesFloatGroupingAggregatorFunction(channels, ValuesFloatAggregator.initGrouping(driverContext.bigArrays()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + ValuesFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + ValuesFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + ValuesFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + ValuesFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block valuesUncast = page.getBlock(channels.get(0)); + if (valuesUncast.areAllValuesNull()) { + return; + } + FloatBlock values = (FloatBlock) valuesUncast; + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + ValuesFloatAggregator.combineIntermediate(state, groupId, values, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + ValuesFloatAggregator.GroupingState inState = ((ValuesFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + ValuesFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = ValuesFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregator.java new file mode 100644 index 0000000000000..2159f0864e1cf --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregator.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.DriverContext; + +@Aggregator({ @IntermediateState(name = "hll", type = "BYTES_REF") }) +@GroupingAggregator +public class CountDistinctFloatAggregator { + + public static HllStates.SingleState initSingle(BigArrays bigArrays, int precision) { + return new HllStates.SingleState(bigArrays, precision); + } + + public static void combine(HllStates.SingleState current, float v) { + current.collect(v); + } + + public static void combineIntermediate(HllStates.SingleState current, BytesRef inValue) { + current.merge(0, inValue, 0); + } + + public static Block evaluateFinal(HllStates.SingleState state, DriverContext driverContext) { + long result = state.cardinality(); + return driverContext.blockFactory().newConstantLongBlockWith(result, 1); + } + + public static HllStates.GroupingState initGrouping(BigArrays bigArrays, int precision) { + return new HllStates.GroupingState(bigArrays, precision); + } + + public static void combine(HllStates.GroupingState current, int groupId, float v) { + current.collect(groupId, v); + } + + public static void combineIntermediate(HllStates.GroupingState current, int groupId, BytesRef inValue) { + current.merge(groupId, inValue, 0); + } + + public static void combineStates( + HllStates.GroupingState current, + int currentGroupId, + HllStates.GroupingState state, + int statePosition + ) { + current.merge(currentGroupId, state.hll, statePosition); + } + + public static Block evaluateFinal(HllStates.GroupingState state, IntVector selected, DriverContext driverContext) { + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + long count = state.cardinality(group); + builder.appendLong(count); + } + return builder.build(); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxFloatAggregator.java new file mode 100644 index 0000000000000..eea436541069e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxFloatAggregator.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +@Aggregator({ @IntermediateState(name = "max", type = "FLOAT"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class MaxFloatAggregator { + + public static float init() { + return Float.MIN_VALUE; + } + + public static float combine(float current, float v) { + return Math.max(current, v); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregator.java new file mode 100644 index 0000000000000..b81cc945f0695 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregator.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; + +@Aggregator({ @IntermediateState(name = "quart", type = "BYTES_REF") }) +@GroupingAggregator +class MedianAbsoluteDeviationFloatAggregator { + + public static QuantileStates.SingleState initSingle() { + return new QuantileStates.SingleState(QuantileStates.MEDIAN); + } + + public static void combine(QuantileStates.SingleState current, float v) { + current.add(v); + } + + public static void combineIntermediate(QuantileStates.SingleState state, BytesRef inValue) { + state.add(inValue); + } + + public static Block evaluateFinal(QuantileStates.SingleState state, DriverContext driverContext) { + return state.evaluateMedianAbsoluteDeviation(driverContext); + } + + public static QuantileStates.GroupingState initGrouping(BigArrays bigArrays) { + return new QuantileStates.GroupingState(bigArrays, QuantileStates.MEDIAN); + } + + public static void combine(QuantileStates.GroupingState state, int groupId, float v) { + state.add(groupId, v); + } + + public static void combineIntermediate(QuantileStates.GroupingState state, int groupId, BytesRef inValue) { + state.add(groupId, inValue); + } + + public static void combineStates( + QuantileStates.GroupingState current, + int currentGroupId, + QuantileStates.GroupingState state, + int statePosition + ) { + current.add(currentGroupId, state.getOrNull(statePosition)); + } + + public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected, DriverContext driverContext) { + return state.evaluateMedianAbsoluteDeviation(selected, driverContext); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinFloatAggregator.java new file mode 100644 index 0000000000000..9ea52eab846c1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinFloatAggregator.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +@Aggregator({ @IntermediateState(name = "min", type = "FLOAT"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class MinFloatAggregator { + + public static float init() { + return Float.POSITIVE_INFINITY; + } + + public static float combine(float current, float v) { + return Math.min(current, v); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregator.java new file mode 100644 index 0000000000000..37b68b3c31335 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregator.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; + +@Aggregator({ @IntermediateState(name = "quart", type = "BYTES_REF") }) +@GroupingAggregator +class PercentileFloatAggregator { + + public static QuantileStates.SingleState initSingle(double percentile) { + return new QuantileStates.SingleState(percentile); + } + + public static void combine(QuantileStates.SingleState current, float v) { + current.add(v); + } + + public static void combineIntermediate(QuantileStates.SingleState state, BytesRef inValue) { + state.add(inValue); + } + + public static Block evaluateFinal(QuantileStates.SingleState state, DriverContext driverContext) { + return state.evaluatePercentile(driverContext); + } + + public static QuantileStates.GroupingState initGrouping(BigArrays bigArrays, double percentile) { + return new QuantileStates.GroupingState(bigArrays, percentile); + } + + public static void combine(QuantileStates.GroupingState state, int groupId, float v) { + state.add(groupId, v); + } + + public static void combineIntermediate(QuantileStates.GroupingState state, int groupId, BytesRef inValue) { + state.add(groupId, inValue); + } + + public static void combineStates( + QuantileStates.GroupingState current, + int currentGroupId, + QuantileStates.GroupingState state, + int statePosition + ) { + current.add(currentGroupId, state.getOrNull(statePosition)); + } + + public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected, DriverContext driverContext) { + return state.evaluatePercentile(selected, driverContext); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/SumFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/SumFloatAggregator.java new file mode 100644 index 0000000000000..ea6b55b949f15 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/SumFloatAggregator.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +@Aggregator( + { + @IntermediateState(name = "value", type = "DOUBLE"), + @IntermediateState(name = "delta", type = "DOUBLE"), + @IntermediateState(name = "seen", type = "BOOLEAN") } +) +@GroupingAggregator +class SumFloatAggregator extends SumDoubleAggregator { + + public static void combine(SumState current, float v) { + current.add(v); + } + + public static void combine(GroupingSumState current, int groupId, float v) { + current.add(v, groupId); + } + +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st index 246aebe2c08ec..18686928f14a8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st @@ -17,7 +17,7 @@ import org.elasticsearch.compute.data.$Type$Block; $if(int)$ import org.elasticsearch.compute.data.$Type$Vector; $endif$ -$if(double)$ +$if(double||float)$ import org.elasticsearch.compute.data.IntVector; $endif$ import org.elasticsearch.compute.operator.DriverContext; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st index 212a017cb300d..2581d3ebbf80b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st @@ -18,7 +18,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.DoubleVector; -$if(int)$ +$if(float)$ +import org.elasticsearch.compute.data.FloatBlock; +$elseif(int)$ import org.elasticsearch.compute.data.IntBlock; $endif$ import org.elasticsearch.compute.data.IntVector; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st index f9b15ccd34092..ea62dcf295825 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st @@ -27,7 +27,7 @@ import org.elasticsearch.compute.ann.GroupingAggregator; import org.elasticsearch.compute.ann.IntermediateState; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; -$if(int||double||BytesRef)$ +$if(int||double||float||BytesRef)$ import org.elasticsearch.compute.data.$Type$Block; $endif$ import org.elasticsearch.compute.data.IntVector; @@ -55,7 +55,9 @@ class Values$Type$Aggregator { } public static void combine(SingleState state, $type$ v) { -$if(double)$ +$if(float)$ + state.values.add(Float.floatToIntBits(v)); +$elseif(double)$ state.values.add(Double.doubleToLongBits(v)); $else$ state.values.add(v); @@ -98,6 +100,12 @@ $elseif(int)$ * the top 32 bits for the group, the bottom 32 for the value. */ state.values.add((((long) groupId) << Integer.SIZE) | (v & 0xFFFFFFFFL)); +$elseif(float)$ + /* + * Encode the groupId and value into a single long - + * the top 32 bits for the group, the bottom 32 for the value. + */ + state.values.add((((long) groupId) << Float.SIZE) | (Float.floatToIntBits(v) & 0xFFFFFFFFL)); $endif$ } @@ -132,6 +140,11 @@ $elseif(int)$ int group = (int) (both >>> Integer.SIZE); if (group == statePosition) { int value = (int) both; +$elseif(float)$ + long both = state.values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group == statePosition) { + float value = Float.intBitsToFloat((int) both); $endif$ combine(current, currentGroupId, $if(BytesRef)$state.bytes.get(value, scratch)$else$value$endif$); } @@ -172,6 +185,8 @@ $endif$ if (values.size() == 1) { $if(long)$ return blockFactory.newConstantLongBlockWith(values.get(0), 1); +$elseif(float)$ + return blockFactory.newConstantFloatBlockWith(Float.intBitsToFloat((int) values.get(0)), 1); $elseif(double)$ return blockFactory.newConstantDoubleBlockWith(Double.longBitsToDouble(values.get(0)), 1); $elseif(int)$ @@ -185,6 +200,8 @@ $endif$ for (int id = 0; id < values.size(); id++) { $if(long)$ builder.appendLong(values.get(id)); +$elseif(float)$ + builder.appendFloat(Float.intBitsToFloat((int) values.get(id))); $elseif(double)$ builder.appendDouble(Double.longBitsToDouble(values.get(id))); $elseif(int)$ @@ -219,7 +236,7 @@ $elseif(BytesRef)$ private final LongLongHash values; private final BytesRefHash bytes; -$elseif(int)$ +$elseif(int||float)$ private final LongHash values; $endif$ @@ -229,7 +246,7 @@ $if(long||double)$ $elseif(BytesRef)$ values = new LongLongHash(1, bigArrays); bytes = new BytesRefHash(1, bigArrays); -$elseif(int)$ +$elseif(int||float)$ values = new LongHash(1, bigArrays); $endif$ } @@ -262,6 +279,11 @@ $if(long||BytesRef)$ $elseif(double)$ if (values.getKey1(id) == selectedGroup) { double value = Double.longBitsToDouble(values.getKey2(id)); +$elseif(float)$ + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group == selectedGroup) { + float value = Float.intBitsToFloat((int) both); $elseif(int)$ long both = values.get(id); int group = (int) (both >>> Integer.SIZE); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java index 08e8bca64bbe7..6e56b96bda06e 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -223,6 +224,11 @@ protected static Stream allBooleans(Block input) { return allValueOffsets(b).mapToObj(i -> b.getBoolean(i)); } + protected static Stream allFloats(Block input) { + FloatBlock b = (FloatBlock) input; + return allValueOffsets(b).mapToObj(b::getFloat); + } + protected static DoubleStream allDoubles(Block input) { DoubleBlock b = (DoubleBlock) input; return allValueOffsets(b).mapToDouble(i -> b.getDouble(i)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..7f520de393d73 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.BasicBlockTests; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class CountDistinctFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new CountDistinctFloatAggregatorFunctionSupplier(inputChannels, 40000); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "count_distinct of floats"; + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + long expected = input.stream().flatMap(AggregatorFunctionTestCase::allFloats).distinct().count(); + + long count = ((LongBlock) result).getLong(0); + // HLL is an approximation algorithm and precision depends on the number of values computed and the precision_threshold param + // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-cardinality-aggregation.html + // For a number of values close to 10k and precision_threshold=1000, precision should be less than 10% + assertThat((double) count, closeTo(expected, expected * .1)); + } + + @Override + protected void assertOutputFromEmpty(Block b) { + assertThat(b.getPositionCount(), equalTo(1)); + assertThat(BasicBlockTests.valuesAtPositions(b, 0, 1), equalTo(List.of(List.of(0L)))); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..03a11bb976b21 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class CountDistinctFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new CountDistinctFloatAggregatorFunctionSupplier(inputChannels, 40000); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "count_distinct of floats"; + } + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, size).mapToObj(l -> Tuple.tuple(randomGroupId(size), randomFloatBetween(0, 100, true))) + ); + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + long distinct = input.stream().flatMap(p -> allFloats(p, group)).distinct().count(); + long count = ((LongBlock) result).getLong(position); + // HLL is an approximation algorithm and precision depends on the number of values computed and the precision_threshold param + // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-cardinality-aggregation.html + // For a number of values close to 10k and precision_threshold=1000, precision should be less than 10% + assertThat((double) count, closeTo(distinct, distinct * 0.1)); + } + + @Override + protected void assertOutputFromNullOnly(Block b, int position) { + assertThat(b.isNull(position), equalTo(false)); + assertThat(b.getValueCount(position), equalTo(1)); + assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java index 65cd2585b8182..3436d6b537611 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; @@ -202,6 +203,7 @@ protected void appendNull(ElementType elementType, Block.Builder builder, int bl append(builder, switch (elementType) { case BOOLEAN -> randomBoolean(); case BYTES_REF -> new BytesRef(randomAlphaOfLength(3)); + case FLOAT -> randomFloat(); case DOUBLE -> randomDouble(); case INT -> 1; case LONG -> 1L; @@ -479,6 +481,11 @@ protected static Stream allBooleans(Page page, Long group) { return allValueOffsets(page, group).mapToObj(i -> b.getBoolean(i)); } + protected static Stream allFloats(Page page, Long group) { + FloatBlock b = page.getBlock(1); + return allValueOffsets(page, group).mapToObj(b::getFloat); + } + protected static DoubleStream allDoubles(Page page, Long group) { DoubleBlock b = page.getBlock(1); return allValueOffsets(page, group).mapToDouble(i -> b.getDouble(i)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..5e14a99fd0fa2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MaxFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MaxFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "max of floats"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Float max = input.stream().flatMap(AggregatorFunctionTestCase::allFloats).max(floatComparator()).get(); + assertThat(((FloatBlock) result).getFloat(0), equalTo(max)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..0abcb05a91af6 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.Optional; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MaxFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, end).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MaxFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "max of floats"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Optional max = input.stream().flatMap(p -> allFloats(p, group)).max(floatComparator()); + if (max.isEmpty()) { + assertThat(result.isNull(position), equalTo(true)); + return; + } + assertThat(result.isNull(position), equalTo(false)); + assertThat(((FloatBlock) result).getFloat(position), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..786603e12f9c8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.Randomness; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.Matchers.closeTo; + +public class MedianAbsoluteDeviationFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + List values = Arrays.asList(1.2f, 1.25f, 2.0f, 2.0f, 4.3f, 6.0f, 9.0f); + Randomness.shuffle(values); + return new SequenceFloatBlockSourceOperator(blockFactory, values.subList(0, Math.min(values.size(), end))); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MedianAbsoluteDeviationFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "median_absolute_deviation of floats"; + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + assertThat(((DoubleBlock) result).getDouble(0), closeTo(0.8, 0.001d)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..14416b3aec1ee --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.Randomness; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.closeTo; + +public class MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + Float[][] samples = new Float[][] { + { 1.2f, 1.25f, 2.0f, 2.0f, 4.3f, 6.0f, 9.0f }, + { 0.1f, 1.5f, 2.0f, 3.0f, 4.0f, 7.5f, 100.0f }, + { 0.2f, 1.75f, 2.0f, 2.5f }, + { 0.5f, 3.0f, 3.0f, 3.0f, 4.3f }, + { 0.25f, 1.5f, 3.0f } }; + List> values = new ArrayList<>(); + for (int i = 0; i < samples.length; i++) { + List list = Arrays.stream(samples[i]).collect(Collectors.toList()); + Randomness.shuffle(list); + for (float v : list) { + values.add(Tuple.tuple((long) i, v)); + } + } + return new LongFloatTupleBlockSourceOperator(blockFactory, values.subList(0, Math.min(values.size(), end))); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MedianAbsoluteDeviationFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "median_absolute_deviation of floats"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + double medianAbsoluteDeviation = medianAbsoluteDeviation(input.stream().flatMap(p -> allFloats(p, group))); + assertThat(((DoubleBlock) result).getDouble(position), closeTo(medianAbsoluteDeviation, medianAbsoluteDeviation * .000001)); + } + + static double medianAbsoluteDeviation(Stream s) { + Float[] data = s.toArray(Float[]::new); + float median = median(Arrays.stream(data)); + return median(Arrays.stream(data).map(d -> Math.abs(median - d))); + } + + static float median(Stream s) { + // The input data is small enough that tdigest will find the actual median. + Float[] data = s.sorted().toArray(Float[]::new); + if (data.length == 0) { + return 0; + } + int c = data.length / 2; + return data.length % 2 == 0 ? (data[c - 1] + data[c]) / 2 : data[c]; + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..59a09569c65a2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MinFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MinFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "min of floats"; + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + Float min = input.stream().flatMap(b -> allFloats(b)).min(floatComparator()).get(); + assertThat(((FloatBlock) result).getFloat(0), equalTo(min)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..be41e058f60da --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.Optional; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MinFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, end).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MinFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "min of floats"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Optional min = input.stream().flatMap(p -> allFloats(p, group)).min(floatComparator()); + if (min.isEmpty()) { + assertThat(result.isNull(position), equalTo(true)); + return; + } + assertThat(result.isNull(position), equalTo(false)); + assertThat(((FloatBlock) result).getFloat(position), equalTo(min.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..6e4a1e09640dc --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; + +public class PercentileFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + + private double percentile; + + @Before + public void initParameters() { + percentile = randomFrom(0, 1, 5, 10, 25, 50, 75, 90, 95, 99, 100); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new PercentileFloatAggregatorFunctionSupplier(inputChannels, percentile); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "percentile of floats"; + } + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + TDigestState td = TDigestState.create(QuantileStates.DEFAULT_COMPRESSION); + input.stream().flatMap(AggregatorFunctionTestCase::allFloats).forEach(td::add); + double expected = td.quantile(percentile / 100); + double value = ((DoubleBlock) result).getDouble(0); + assertThat(value, closeTo(expected, expected * 0.1)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..84a97a7cd30ac --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.junit.Before; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; + +public class PercentileFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + private double percentile; + + @Before + public void initParameters() { + percentile = randomFrom(0, 1, 5, 10, 25, 50, 75, 90, 95, 99, 100); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new PercentileFloatAggregatorFunctionSupplier(inputChannels, percentile); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "percentile of floats"; + } + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, end).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + TDigestState td = TDigestState.create(QuantileStates.DEFAULT_COMPRESSION); + input.stream().flatMap(p -> allFloats(p, group)).forEach(td::add); + if (td.size() > 0) { + double expected = td.quantile(percentile / 100); + double value = ((DoubleBlock) result).getDouble(position); + assertThat(value, closeTo(expected, expected * 0.1)); + } else { + assertTrue(result.isNull(position)); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..d365f02d289c8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionTests.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.Driver; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.compute.operator.TestResultPageSinkOperator; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.LongStream; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class SumFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new SumFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "sum of floats"; + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + double sum = input.stream().flatMap(AggregatorFunctionTestCase::allFloats).mapToDouble(f -> (double) f).sum(); + assertThat(((DoubleBlock) result).getDouble(0), closeTo(sum, .0001)); + } + + public void testOverflowSucceeds() { + DriverContext driverContext = driverContext(); + List results = new ArrayList<>(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator(driverContext.blockFactory(), Stream.of(Float.MAX_VALUE - 1, 2f)), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertThat(results.get(0).getBlock(0).getDouble(0), equalTo((double) Float.MAX_VALUE + 1)); + assertDriverContext(driverContext); + } + + public void testSummationAccuracy() { + DriverContext driverContext = driverContext(); + List results = new ArrayList<>(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator( + driverContext.blockFactory(), + Stream.of(0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.0f, 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, 1.6f, 1.7f) + ), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertEquals(15.3, results.get(0).getBlock(0).getDouble(0), 0.001); + assertDriverContext(driverContext); + + // Summing up an array which contains NaN and infinities and expect a result same as naive summation + results.clear(); + int n = randomIntBetween(5, 10); + Float[] values = new Float[n]; + float sum = 0; + for (int i = 0; i < n; i++) { + values[i] = frequently() + ? randomFrom(Float.NaN, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY) + : randomFloatBetween(Float.MIN_VALUE, Float.MAX_VALUE, true); + sum += values[i]; + } + driverContext = driverContext(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator(driverContext.blockFactory(), Stream.of(values)), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertEquals(sum, results.get(0).getBlock(0).getDouble(0), 1e-10); + assertDriverContext(driverContext); + + // Summing up some big float values and expect a big double result + results.clear(); + n = randomIntBetween(5, 10); + Float[] largeValues = new Float[n]; + for (int i = 0; i < n; i++) { + largeValues[i] = Float.MAX_VALUE; + } + driverContext = driverContext(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator(driverContext.blockFactory(), Stream.of(largeValues)), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertEquals((double) Float.MAX_VALUE * n, results.get(0).getBlock(0).getDouble(0), 0d); + assertDriverContext(driverContext); + + results.clear(); + for (int i = 0; i < n; i++) { + largeValues[i] = -Float.MAX_VALUE; + } + driverContext = driverContext(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator(driverContext.blockFactory(), Stream.of(largeValues)), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertEquals((double) -Float.MAX_VALUE * n, results.get(0).getBlock(0).getDouble(0), 0d); + assertDriverContext(driverContext); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..54bd92cbfff21 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.aggregations.metrics.CompensatedSum; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; + +public class SumFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, end).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new SumFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "sum of floats"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + CompensatedSum sum = new CompensatedSum(); + input.stream().flatMap(p -> allFloats(p, group)).mapToDouble(f -> (double) f).forEach(sum::add); + // Won't precisely match in distributed case but will be close + assertThat(((DoubleBlock) result).getDouble(position), closeTo(sum.value(), 0.01)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..98a016783955e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.contains; + +public class TopListFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + private static final int LIMIT = 100; + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, IntStream.range(0, size).mapToObj(l -> randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new TopListFloatAggregatorFunctionSupplier(inputChannels, LIMIT, true); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "top_list of floats"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMap(b -> allFloats(b)).sorted().limit(LIMIT).toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), contains(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..899a89dd993a4 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class ValuesFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, IntStream.range(0, size).mapToObj(i -> randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new ValuesFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "values of floats"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMap(AggregatorFunctionTestCase::allFloats).collect(Collectors.toSet()).toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), containsInAnyOrder(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..787b6fd4c75be --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class ValuesFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new ValuesFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "values of floats"; + } + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, size).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + public void assertSimpleGroup(List input, Block result, int position, Long group) { + Object[] values = input.stream().flatMap(p -> allFloats(p, group)).collect(Collectors.toSet()).toArray(Object[]::new); + Object resultValue = BlockUtils.toJavaObject(result, position); + switch (values.length) { + case 0 -> assertThat(resultValue, nullValue()); + case 1 -> assertThat(resultValue, equalTo(values[0])); + default -> assertThat((List) resultValue, containsInAnyOrder(values)); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java new file mode 100644 index 0000000000000..8b3d288339037 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.search.sort.SortOrder; + +import static org.hamcrest.Matchers.equalTo; + +public class FloatBucketedSortTests extends BucketedSortTestCase { + @Override + protected FloatBucketedSort build(SortOrder sortOrder, int bucketSize) { + return new FloatBucketedSort(bigArrays(), sortOrder, bucketSize); + } + + @Override + protected Object expectedValue(double v) { + return v; + } + + @Override + protected double randomValue() { + return randomFloatBetween(Float.MIN_VALUE, Float.MAX_VALUE, true); + } + + @Override + protected void collect(FloatBucketedSort sort, double value, int bucket) { + sort.collect((float) value, bucket); + } + + @Override + protected void merge(FloatBucketedSort sort, int groupId, FloatBucketedSort other, int otherGroupId) { + sort.merge(groupId, other, otherGroupId); + } + + @Override + protected Block toBlock(FloatBucketedSort sort, BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + @Override + protected void assertBlockTypeAndValues(Block block, Object... values) { + assertThat(block.elementType(), equalTo(ElementType.FLOAT)); + var typedBlock = (FloatBlock) block; + for (int i = 0; i < values.length; i++) { + assertThat((double) typedBlock.getFloat(i), equalTo(values[i])); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java index 848f3750a4b20..fa72721545ab9 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -353,4 +354,26 @@ public void setThreadPool() { public void shutdownThreadPool() { terminate(threadPool); } + + protected Comparator floatComparator() { + return FloatComparator.INSTANCE; + } + + static final class FloatComparator implements Comparator { + + static final FloatComparator INSTANCE = new FloatComparator(); + + @Override + public int compare(Float o1, Float o2) { + float first = o1; + float second = o2; + if (first < second) { + return -1; + } else if (first == second) { + return 0; + } else { + return 1; + } + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LongFloatTupleBlockSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LongFloatTupleBlockSourceOperator.java new file mode 100644 index 0000000000000..9276174c9dbb1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LongFloatTupleBlockSourceOperator.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.stream.Stream; + +/** + * A source operator whose output is the given tuple values. This operator produces pages + * with two Blocks. The returned pages preserve the order of values as given in the in initial list. + */ +public class LongFloatTupleBlockSourceOperator extends AbstractBlockSourceOperator { + + private static final int DEFAULT_MAX_PAGE_POSITIONS = 8 * 1024; + + private final List> values; + + public LongFloatTupleBlockSourceOperator(BlockFactory blockFactory, Stream> values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public LongFloatTupleBlockSourceOperator(BlockFactory blockFactory, Stream> values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + this.values = values.toList(); + } + + public LongFloatTupleBlockSourceOperator(BlockFactory blockFactory, List> values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public LongFloatTupleBlockSourceOperator(BlockFactory blockFactory, List> values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + this.values = values; + } + + @Override + protected Page createPage(int positionOffset, int length) { + var blockBuilder1 = blockFactory.newLongBlockBuilder(length); + var blockBuilder2 = blockFactory.newFloatBlockBuilder(length); + for (int i = 0; i < length; i++) { + Tuple item = values.get(positionOffset + i); + if (item.v1() == null) { + blockBuilder1.appendNull(); + } else { + blockBuilder1.appendLong(item.v1()); + } + if (item.v2() == null) { + blockBuilder2.appendNull(); + } else { + blockBuilder2.appendFloat(item.v2()); + } + } + currentPosition += length; + return new Page(blockBuilder1.build(), blockBuilder2.build()); + } + + @Override + protected int remaining() { + return values.size() - currentPosition; + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/NullInsertingSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/NullInsertingSourceOperator.java index 260918396dcd3..c8444551f415b 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/NullInsertingSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/NullInsertingSourceOperator.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -99,6 +100,9 @@ private void copyValue(Block from, int valueIndex, Block.Builder into) { case DOUBLE: ((DoubleBlock.Builder) into).appendDouble(((DoubleBlock) from).getDouble(valueIndex)); break; + case FLOAT: + ((FloatBlock.Builder) into).appendFloat(((FloatBlock) from).getFloat(valueIndex)); + break; default: throw new IllegalArgumentException("unknown block type " + elementType); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/PositionMergingSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/PositionMergingSourceOperator.java index 4bbd6d0af0c2a..651ec6dc191a9 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/PositionMergingSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/PositionMergingSourceOperator.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -74,6 +75,7 @@ private void copyTo(Block.Builder builder, Block in, int position, int valueCoun switch (in.elementType()) { case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(((BooleanBlock) in).getBoolean(i)); case BYTES_REF -> ((BytesRefBlock.Builder) builder).appendBytesRef(((BytesRefBlock) in).getBytesRef(i, scratch)); + case FLOAT -> ((FloatBlock.Builder) builder).appendFloat(((FloatBlock) in).getFloat(i)); case DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((DoubleBlock) in).getDouble(i)); case INT -> ((IntBlock.Builder) builder).appendInt(((IntBlock) in).getInt(i)); case LONG -> ((LongBlock.Builder) builder).appendLong(((LongBlock) in).getLong(i)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceFloatBlockSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceFloatBlockSourceOperator.java new file mode 100644 index 0000000000000..db524366b381e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceFloatBlockSourceOperator.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; + +import java.util.List; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +/** + * A source operator whose output is the given float values. This operator produces pages + * containing a single Block. The Block contains the float values from the given list, in order. + */ +public class SequenceFloatBlockSourceOperator extends AbstractBlockSourceOperator { + + static final int DEFAULT_MAX_PAGE_POSITIONS = 8 * 1024; + + private final float[] values; + + public SequenceFloatBlockSourceOperator(BlockFactory blockFactory, Stream values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public SequenceFloatBlockSourceOperator(BlockFactory blockFactory, Stream values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + var l = values.toList(); + this.values = new float[l.size()]; + IntStream.range(0, l.size()).forEach(i -> this.values[i] = l.get(i)); + } + + public SequenceFloatBlockSourceOperator(BlockFactory blockFactory, List values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public SequenceFloatBlockSourceOperator(BlockFactory blockFactory, List values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + this.values = new float[values.size()]; + IntStream.range(0, this.values.length).forEach(i -> this.values[i] = values.get(i)); + } + + @Override + protected Page createPage(int positionOffset, int length) { + FloatVector.FixedBuilder builder = blockFactory.newFloatVectorFixedBuilder(length); + for (int i = 0; i < length; i++) { + builder.appendFloat(values[positionOffset + i]); + } + currentPosition += length; + return new Page(builder.build().asBlock()); + } + + protected int remaining() { + return values.length - currentPosition; + } +} From 7dd726742df70b9db3631657f8f15728823edbd4 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 20 Jun 2024 17:54:12 +0200 Subject: [PATCH 14/16] ES|QL: register TABLES_TYPES capability and fix CSV tests (#109926) Registering `TABLES_TYPES` capability, that was created (but not registered) in https://github.com/elastic/elasticsearch/pull/109489 Also using it as a condition for `lookup.csv-spec` tests. These tests failed badly in Serverless multi-cluster due to different output format in two different nodes. Probably (if I got it right) the YAML tests that were using this capability were not running at all. This PR should re-enable them. --- .../src/main/resources/lookup.csv-spec | 50 +++++++++---------- .../xpack/esql/action/EsqlCapabilities.java | 7 +-- .../rest-api-spec/test/esql/150_lookup.yml | 12 ++--- 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec index 377d6d6678032..35e1101becbf9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec @@ -1,5 +1,5 @@ keywordByInt -required_capability: lookup_command +required_capability: tables_types FROM employees | SORT emp_no | LIMIT 4 @@ -17,7 +17,7 @@ emp_no:integer | languages:integer | lang_name:keyword ; keywordByMvInt -required_capability: lookup_command +required_capability: tables_types ROW int=[1, 2, 3] | LOOKUP int_number_names ON int ; @@ -27,7 +27,7 @@ int:integer | name:keyword ; keywordByDupeInt -required_capability: lookup_command +required_capability: tables_types ROW int=[1, 1, 1] | LOOKUP int_number_names ON int ; @@ -37,7 +37,7 @@ int:integer | name:keyword ; intByKeyword -required_capability: lookup_command +required_capability: tables_types ROW name="two" | LOOKUP int_number_names ON name ; @@ -48,7 +48,7 @@ name:keyword | int:integer keywordByLong -required_capability: lookup_command +required_capability: tables_types FROM employees | SORT emp_no | LIMIT 4 @@ -66,7 +66,7 @@ emp_no:integer | languages:long | lang_name:keyword ; longByKeyword -required_capability: lookup_command +required_capability: tables_types ROW name="two" | LOOKUP long_number_names ON name ; @@ -76,7 +76,7 @@ name:keyword | long:long ; keywordByFloat -required_capability: lookup_command +required_capability: tables_types FROM employees | SORT emp_no | LIMIT 4 @@ -94,7 +94,7 @@ emp_no:integer | height:double | height_name:keyword ; floatByKeyword -required_capability: lookup_command +required_capability: tables_types ROW name="two point zero eight" | LOOKUP double_number_names ON name ; @@ -104,7 +104,7 @@ two point zero eight | 2.08 ; floatByNullMissing -required_capability: lookup_command +required_capability: tables_types ROW name=null | LOOKUP double_number_names ON name ; @@ -114,7 +114,7 @@ name:null | double:double ; floatByNullMatching -required_capability: lookup_command +required_capability: tables_types ROW name=null | LOOKUP double_number_names_with_null ON name ; @@ -124,7 +124,7 @@ name:null | double:double ; intIntByKeywordKeyword -required_capability: lookup_command +required_capability: tables_types ROW aa="foo", ab="zoo" | LOOKUP big ON aa, ab ; @@ -134,7 +134,7 @@ foo | zoo | 1 | -1 ; intIntByKeywordKeywordMissing -required_capability: lookup_command +required_capability: tables_types ROW aa="foo", ab="zoi" | LOOKUP big ON aa, ab ; @@ -144,7 +144,7 @@ foo | zoi | null | null ; intIntByKeywordKeywordSameValues -required_capability: lookup_command +required_capability: tables_types ROW aa="foo", ab="foo" | LOOKUP big ON aa, ab ; @@ -154,7 +154,7 @@ foo | foo | 2 | -2 ; intIntByKeywordKeywordSameValuesMissing -required_capability: lookup_command +required_capability: tables_types ROW aa="bar", ab="bar" | LOOKUP big ON aa, ab ; @@ -164,7 +164,7 @@ bar | bar | null | null ; lookupBeforeStats -required_capability: lookup_command +required_capability: tables_types FROM employees | RENAME languages AS int | LOOKUP int_number_names ON int @@ -182,7 +182,7 @@ height:double | languages:keyword ; lookupAfterStats -required_capability: lookup_command +required_capability: tables_types FROM employees | STATS int=TO_INT(AVG(height)) | LOOKUP int_number_names ON int @@ -194,7 +194,7 @@ two // Makes sure the LOOKUP squashes previous names doesNotDuplicateNames -required_capability: lookup_command +required_capability: tables_types FROM employees | SORT emp_no | LIMIT 4 @@ -213,7 +213,7 @@ emp_no:integer | languages:long | name:keyword ; lookupBeforeSort -required_capability: lookup_command +required_capability: tables_types FROM employees | WHERE emp_no < 10005 | RENAME languages AS int @@ -231,7 +231,7 @@ languages:keyword | emp_no:integer ; lookupAfterSort -required_capability: lookup_command +required_capability: tables_types FROM employees | WHERE emp_no < 10005 | SORT languages ASC, emp_no ASC @@ -253,7 +253,7 @@ languages:keyword | emp_no:integer // named "lookup" // rowNamedLookup -required_capability: lookup_command +required_capability: tables_types ROW lookup = "a" ; @@ -262,7 +262,7 @@ lookup:keyword ; rowNamedLOOKUP -required_capability: lookup_command +required_capability: tables_types ROW LOOKUP = "a" ; @@ -271,7 +271,7 @@ LOOKUP:keyword ; evalNamedLookup -required_capability: lookup_command +required_capability: tables_types ROW a = "a" | EVAL lookup = CONCAT(a, "1") ; @@ -280,7 +280,7 @@ a:keyword | lookup:keyword ; dissectNamedLookup -required_capability: lookup_command +required_capability: tables_types row a = "foo bar" | dissect a "foo %{lookup}"; a:keyword | lookup:keyword @@ -288,7 +288,7 @@ a:keyword | lookup:keyword ; renameIntoLookup -required_capability: lookup_command +required_capability: tables_types row a = "foo bar" | RENAME a AS lookup; lookup:keyword @@ -296,7 +296,7 @@ lookup:keyword ; sortOnLookup -required_capability: lookup_command +required_capability: tables_types ROW lookup = "a" | SORT lookup ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 654c1ffd8a5e9..ebfcdacd7587a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -57,11 +57,6 @@ public class EsqlCapabilities { */ private static final String METADATA_IGNORED_FIELD = "metadata_field_ignored"; - /** - * Support for the "LOOKUP" command. - */ - private static final String LOOKUP_COMMAND = "lookup_command"; - /** * Support for the syntax {@code "tables": {"type": []}}. */ @@ -103,7 +98,7 @@ private static Set capabilities() { caps.add(NAMED_POSITIONAL_PARAMETER); if (Build.current().isSnapshot()) { - caps.add(LOOKUP_COMMAND); + caps.add(TABLES_TYPES); } /* diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml index 5f76954e57c89..96bfabf862f50 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml @@ -38,7 +38,7 @@ basic: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -66,7 +66,7 @@ read multivalue keyword: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -98,7 +98,7 @@ keyword matches text: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -144,7 +144,7 @@ duplicate keys: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -167,7 +167,7 @@ multivalued keys: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -209,7 +209,7 @@ on function: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: From 8bc5ecdc312ad3ce6b7ec15d4b193623d48d27f9 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 20 Jun 2024 09:09:45 -0700 Subject: [PATCH 15/16] Support synthetic source together with ignore_malformed in histogram fields (#109882) --- docs/changelog/109882.yaml | 5 + .../mapping/types/histogram.asciidoc | 3 +- .../mapper/CompositeSyntheticFieldLoader.java | 174 ++++++++++++++++++ .../mapper/IgnoreMalformedStoredValues.java | 2 +- .../mapper/HistogramFieldMapper.java | 148 +++++++++------ .../mapper/HistogramFieldMapperTests.java | 91 +++++++-- .../test/analytics/histogram.yml | 51 +++++ 7 files changed, 398 insertions(+), 76 deletions(-) create mode 100644 docs/changelog/109882.yaml create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java diff --git a/docs/changelog/109882.yaml b/docs/changelog/109882.yaml new file mode 100644 index 0000000000000..0f0fed01c5a7a --- /dev/null +++ b/docs/changelog/109882.yaml @@ -0,0 +1,5 @@ +pr: 109882 +summary: Support synthetic source together with `ignore_malformed` in histogram fields +area: Mapping +type: enhancement +issues: [] diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc index 3e221b11182ad..8cd30110250bf 100644 --- a/docs/reference/mapping/types/histogram.asciidoc +++ b/docs/reference/mapping/types/histogram.asciidoc @@ -79,8 +79,7 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. `histogram` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -<> or <>. +default configuration. Synthetic `_source` cannot be used together with <>. NOTE: To save space, zero-count buckets are not stored in the histogram doc values. As a result, when indexing a histogram field in an index with synthetic source enabled, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java new file mode 100644 index 0000000000000..efc3c7b507300 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Collections.emptyList; + +/** + * A {@link SourceLoader.SyntheticFieldLoader} that uses a set of sub-loaders + * to produce synthetic source for the field. + * Typical use case is to gather field values from doc_values and append malformed values + * stored in a different field in case of ignore_malformed being enabled. + */ +public class CompositeSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader { + private final String fieldName; + private final String fullFieldName; + private final SyntheticFieldLoaderLayer[] parts; + private boolean hasValue; + + public CompositeSyntheticFieldLoader(String fieldName, String fullFieldName, SyntheticFieldLoaderLayer... parts) { + this.fieldName = fieldName; + this.fullFieldName = fullFieldName; + this.parts = parts; + this.hasValue = false; + } + + @Override + public Stream> storedFieldLoaders() { + return Arrays.stream(parts).flatMap(SyntheticFieldLoaderLayer::storedFieldLoaders).map(e -> Map.entry(e.getKey(), values -> { + hasValue = true; + e.getValue().load(values); + })); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + var loaders = new ArrayList(parts.length); + for (var part : parts) { + var partLoader = part.docValuesLoader(leafReader, docIdsInLeaf); + if (partLoader != null) { + loaders.add(partLoader); + } + } + + if (loaders.isEmpty()) { + return null; + } + + return docId -> { + boolean hasDocs = false; + for (var loader : loaders) { + hasDocs |= loader.advanceToDoc(docId); + } + + this.hasValue |= hasDocs; + return hasDocs; + }; + } + + @Override + public boolean hasValue() { + return hasValue; + } + + @Override + public void write(XContentBuilder b) throws IOException { + var totalCount = Arrays.stream(parts).mapToLong(SyntheticFieldLoaderLayer::valueCount).sum(); + + if (totalCount == 0) { + return; + } + + if (totalCount == 1) { + b.field(fieldName); + for (var part : parts) { + part.write(b); + } + return; + } + + b.startArray(fieldName); + for (var part : parts) { + part.write(b); + } + b.endArray(); + } + + @Override + public String fieldName() { + return this.fullFieldName; + } + + /** + * Represents one layer of loading synthetic source values for a field + * as a part of {@link CompositeSyntheticFieldLoader}. + *
+ * Note that the contract of {@link SourceLoader.SyntheticFieldLoader#write(XContentBuilder)} + * is slightly different here since it only needs to write field values without encompassing object or array. + */ + public interface SyntheticFieldLoaderLayer extends SourceLoader.SyntheticFieldLoader { + /** + * Number of values that this loader will write. + * @return + */ + long valueCount(); + } + + /** + * Layer that loads malformed values stored in a dedicated field with a conventional name. + * @see IgnoreMalformedStoredValues + */ + public static class MalformedValuesLayer implements SyntheticFieldLoaderLayer { + private final String fieldName; + private List values; + + public MalformedValuesLayer(String fieldName) { + this.fieldName = IgnoreMalformedStoredValues.name(fieldName); + this.values = emptyList(); + } + + @Override + public long valueCount() { + return values.size(); + } + + @Override + public Stream> storedFieldLoaders() { + return Stream.of(Map.entry(fieldName, values -> this.values = values)); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + return null; + } + + @Override + public boolean hasValue() { + return values.isEmpty() == false; + } + + @Override + public void write(XContentBuilder b) throws IOException { + for (Object v : values) { + if (v instanceof BytesRef r) { + XContentDataHelper.decodeAndWrite(b, r); + } else { + b.value(v); + } + } + values = emptyList(); + } + + @Override + public String fieldName() { + return fieldName; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java index 6ad0823738ba0..3d2c51fb5b8af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java @@ -128,7 +128,7 @@ public void write(XContentBuilder b) throws IOException { } } - private static String name(String fieldName) { + public static String name(String fieldName) { return fieldName + "._ignore_malformed"; } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index cdcc0e495582a..9fe7fc647455e 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -29,9 +29,11 @@ import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.LeafHistogramFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.SourceLoader; @@ -44,6 +46,7 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xcontent.CopyingXContentParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -287,8 +290,12 @@ protected boolean supportsParsingObject() { @Override public void parse(DocumentParserContext context) throws IOException { context.path().add(simpleName()); + + boolean shouldStoreMalformedDataForSyntheticSource = context.mappingLookup().isSourceSynthetic() && ignoreMalformed(); XContentParser.Token token; XContentSubParser subParser = null; + XContentBuilder malformedDataForSyntheticSource = null; + try { token = context.parser().currentToken(); if (token == XContentParser.Token.VALUE_NULL) { @@ -299,10 +306,16 @@ public void parse(DocumentParserContext context) throws IOException { ArrayList counts = null; // should be an object ensureExpectedToken(XContentParser.Token.START_OBJECT, token, context.parser()); - subParser = new XContentSubParser(context.parser()); + if (shouldStoreMalformedDataForSyntheticSource) { + var copyingParser = new CopyingXContentParser(context.parser()); + malformedDataForSyntheticSource = copyingParser.getBuilder(); + subParser = new XContentSubParser(copyingParser); + } else { + subParser = new XContentSubParser(context.parser()); + } token = subParser.nextToken(); while (token != XContentParser.Token.END_OBJECT) { - // should be an field + // should be a field ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, subParser); String fieldName = subParser.currentName(); if (fieldName.equals(VALUES_FIELD.getPreferredName())) { @@ -427,7 +440,17 @@ public void parse(DocumentParserContext context) throws IOException { if (subParser != null) { // close the subParser so we advance to the end of the object subParser.close(); + } else if (shouldStoreMalformedDataForSyntheticSource) { + // We have a malformed value, but it's not an object given that `subParser` is null. + // So we just remember whatever it is. + malformedDataForSyntheticSource = XContentBuilder.builder(context.parser().contentType().xContent()) + .copyCurrentStructure(context.parser()); + } + + if (malformedDataForSyntheticSource != null) { + context.doc().add(IgnoreMalformedStoredValues.storedField(name(), malformedDataForSyntheticSource)); } + context.addIgnoredField(fieldType().name()); } context.path().remove(); @@ -491,76 +514,85 @@ protected SyntheticSourceMode syntheticSourceMode() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (ignoreMalformed.value()) { - throw new IllegalArgumentException( - "field [" + name() + "] of type [histogram] doesn't support synthetic source because it ignores malformed histograms" - ); - } if (copyTo.copyToFields().isEmpty() != true) { throw new IllegalArgumentException( "field [" + name() + "] of type [histogram] doesn't support synthetic source because it declares copy_to" ); } - return new SourceLoader.SyntheticFieldLoader() { - private final InternalHistogramValue value = new InternalHistogramValue(); - private BytesRef binaryValue; - @Override - public Stream> storedFieldLoaders() { - return Stream.of(); - } + return new CompositeSyntheticFieldLoader( + simpleName(), + name(), + new HistogramSyntheticFieldLoader(), + new CompositeSyntheticFieldLoader.MalformedValuesLayer(name()) + ); + } - @Override - public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { - BinaryDocValues docValues = leafReader.getBinaryDocValues(fieldType().name()); - if (docValues == null) { - // No values in this leaf - binaryValue = null; - return null; - } - return docId -> { - if (docValues.advanceExact(docId)) { - binaryValue = docValues.binaryValue(); - return true; - } - binaryValue = null; - return false; - }; - } + private class HistogramSyntheticFieldLoader implements CompositeSyntheticFieldLoader.SyntheticFieldLoaderLayer { + private final InternalHistogramValue value = new InternalHistogramValue(); + private BytesRef binaryValue; - @Override - public boolean hasValue() { - return binaryValue != null; - } + @Override + public Stream> storedFieldLoaders() { + return Stream.of(); + } - @Override - public void write(XContentBuilder b) throws IOException { - if (binaryValue == null) { - return; + @Override + public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) + throws IOException { + BinaryDocValues docValues = leafReader.getBinaryDocValues(fieldType().name()); + if (docValues == null) { + // No values in this leaf + binaryValue = null; + return null; + } + return docId -> { + if (docValues.advanceExact(docId)) { + binaryValue = docValues.binaryValue(); + return true; } - b.startObject(simpleName()); + binaryValue = null; + return false; + }; + } - value.reset(binaryValue); - b.startArray("values"); - while (value.next()) { - b.value(value.value()); - } - b.endArray(); + @Override + public boolean hasValue() { + return binaryValue != null; + } - value.reset(binaryValue); - b.startArray("counts"); - while (value.next()) { - b.value(value.count()); - } - b.endArray(); + @Override + public void write(XContentBuilder b) throws IOException { + if (binaryValue == null) { + return; + } + b.startObject(); - b.endObject(); + value.reset(binaryValue); + b.startArray("values"); + while (value.next()) { + b.value(value.value()); } + b.endArray(); - @Override - public String fieldName() { - return name(); + value.reset(binaryValue); + b.startArray("counts"); + while (value.next()) { + b.value(value.count()); } - }; - } + b.endArray(); + + b.endObject(); + } + + @Override + public String fieldName() { + return name(); + } + + @Override + public long valueCount() { + return binaryValue != null ? 1 : 0; + } + }; } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 5e2bdaf2d465e..6fcbf20b8657f 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.analytics.mapper; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; @@ -15,6 +17,7 @@ import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.junit.AssumptionViolatedException; @@ -26,7 +29,6 @@ import java.util.Map; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.matchesPattern; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -121,9 +123,44 @@ protected boolean supportsIgnoreMalformed() { @Override protected List exampleMalformedValues() { + var randomString = randomAlphaOfLengthBetween(1, 10); + var randomLong = randomLong(); + var randomDouble = randomDouble(); + var randomBoolean = randomBoolean(); + return List.of( + exampleMalformedValue(b -> b.value(randomString)).errorMatches( + "Failed to parse object: expecting token of type [START_OBJECT]" + ), + exampleMalformedValue(b -> b.value(randomLong)).errorMatches("Failed to parse object: expecting token of type [START_OBJECT]"), + exampleMalformedValue(b -> b.value(randomDouble)).errorMatches( + "Failed to parse object: expecting token of type [START_OBJECT]" + ), + exampleMalformedValue(b -> b.value(randomBoolean)).errorMatches( + "Failed to parse object: expecting token of type [START_OBJECT]" + ), + exampleMalformedValue(b -> b.startObject().endObject()).errorMatches("expected field called [values]"), exampleMalformedValue(b -> b.startObject().startArray("values").value(2).value(2).endArray().endObject()).errorMatches( "expected field called [counts]" + ), + exampleMalformedValue(b -> b.startObject().startArray("counts").value(2).value(2).endArray().endObject()).errorMatches( + "expected field called [values]" + ), + // Make sure that entire sub-object is preserved in synthetic source + exampleMalformedValue( + b -> b.startObject() + .startArray("values") + .value(2) + .endArray() + .field("somefield", randomString) + .array("somearray", randomLong, randomLong) + .startObject("someobject") + .field("nestedfield", randomDouble) + .endObject() + .endObject() + ).errorMatches("unknown parameter [somefield]"), + exampleMalformedValue(b -> b.startArray().value(randomLong).value(randomLong).endArray()).errorMatches( + "expecting token of type [START_OBJECT] but found [VALUE_NUMBER]" ) ); } @@ -336,13 +373,44 @@ protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + public void testArrayValueSyntheticSource() throws Exception { + DocumentMapper mapper = createDocumentMapper( + syntheticSourceFieldMapping(b -> b.field("type", "histogram").field("ignore_malformed", "true")) + ); + + var randomString = randomAlphaOfLength(10); + CheckedConsumer arrayValue = b -> { + b.startArray("field"); + { + b.startObject().field("counts", new int[] { 1, 2, 3 }).field("values", new double[] { 1, 2, 3 }).endObject(); + b.startObject().field("counts", new int[] { 4, 5, 6 }).field("values", new double[] { 4, 5, 6 }).endObject(); + b.value(randomString); + } + b.endArray(); + }; + + var expected = JsonXContent.contentBuilder().startObject(); + // First value comes from synthetic field loader and so is formatted in a specific format (e.g. values always come first). + // Other values are stored as is as part of ignore_malformed logic for synthetic source. + { + expected.startArray("field"); + expected.startObject().field("values", new double[] { 1, 2, 3 }).field("counts", new int[] { 1, 2, 3 }).endObject(); + expected.startObject().field("counts", new int[] { 4, 5, 6 }).field("values", new double[] { 4, 5, 6 }).endObject(); + expected.value(randomString); + expected.endArray(); + } + expected.endObject(); + + var syntheticSource = syntheticSource(mapper, arrayValue); + assertEquals(Strings.toString(expected), syntheticSource); + } + @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { - assumeFalse("synthetic _source support for histogram doesn't support ignore_malformed", ignoreMalformed); - return new HistogramFieldSyntheticSourceSupport(); + return new HistogramFieldSyntheticSourceSupport(ignoreMalformed); } - private static class HistogramFieldSyntheticSourceSupport implements SyntheticSourceSupport { + private record HistogramFieldSyntheticSourceSupport(boolean ignoreMalformed) implements SyntheticSourceSupport { @Override public SyntheticSourceExample example(int maxVals) { if (randomBoolean()) { @@ -371,21 +439,14 @@ private int randomCount() { private void mapping(XContentBuilder b) throws IOException { b.field("type", "histogram"); + if (ignoreMalformed) { + b.field("ignore_malformed", true); + } } @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - matchesPattern( - "field \\[field] of type \\[histogram] doesn't support synthetic source because it ignores malformed histograms" - ), - b -> { - b.field("type", "histogram"); - b.field("ignore_malformed", true); - } - ) - ); + return List.of(); } } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml index b719502ae8f28..726b9d153025e 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml @@ -306,3 +306,54 @@ histogram with large count values: - match: { aggregations.percent.values.1\.0: 0.2 } - match: { aggregations.percent.values.5\.0: 0.2 } - match: { aggregations.percent.values.25\.0: 0.2 } + +--- +histogram with synthetic source and ignore_malformed: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: introduced in 8.15.0 + + - do: + indices.create: + index: histo_synthetic + body: + mappings: + _source: + mode: synthetic + properties: + latency: + type: histogram + ignore_malformed: true + + - do: + index: + index: histo_synthetic + id: "1" + body: + latency: "quick brown fox" + + - do: + index: + index: histo_synthetic + id: "2" + body: + latency: [{"values": [1.0], "counts": [1], "hello": "world"}, [123, 456], {"values": [2.0], "counts": [2]}, "fox"] + + - do: + indices.refresh: {} + + - do: + get: + index: histo_synthetic + id: 1 + - match: + _source: + latency: "quick brown fox" + + - do: + get: + index: histo_synthetic + id: 2 + - match: + _source: + latency: [{"values": [2.0], "counts": [2]}, {"values": [1.0], "counts": [1], "hello": "world"}, 123, 456, "fox"] From 1f76deef7fe6910bdd6619c216eebe622969fe86 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 20 Jun 2024 18:17:58 +0200 Subject: [PATCH 16/16] Do not create Weights concurrently in ES|QL lucene operator (#109907) --- .../elasticsearch/compute/lucene/LuceneOperator.java | 2 +- .../elasticsearch/compute/lucene/LuceneSlice.java | 3 +-- .../compute/lucene/LuceneSliceQueue.java | 12 +----------- .../TimeSeriesSortedSourceOperatorFactory.java | 2 +- .../esql/planner/LocalExecutionPlannerTests.java | 3 ++- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java index 184f28e750aec..11ccfb55a77aa 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneOperator.java @@ -140,7 +140,7 @@ LuceneScorer getCurrentOrLoadNextScorer() { logger.trace("Starting {}", partialLeaf); final LeafReaderContext leaf = partialLeaf.leafReaderContext(); if (currentScorer == null || currentScorer.leafReaderContext() != leaf) { - final Weight weight = currentSlice.weight().get(); + final Weight weight = currentSlice.weight(); processedQueries.add(weight.getQuery()); currentScorer = new LuceneScorer(currentSlice.shardContext(), weight, leaf); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSlice.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSlice.java index 716df6844e79f..d7b6a86e07905 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSlice.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSlice.java @@ -10,12 +10,11 @@ import org.apache.lucene.search.Weight; import java.util.List; -import java.util.function.Supplier; /** * Holds a list of multiple partial Lucene segments */ -public record LuceneSlice(ShardContext shardContext, List leaves, Supplier weight) { +public record LuceneSlice(ShardContext shardContext, List leaves, Weight weight) { int numLeaves() { return leaves.size(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSliceQueue.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSliceQueue.java index f3bcdc7593dab..1c9c97a364fc7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSliceQueue.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneSliceQueue.java @@ -20,7 +20,6 @@ import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Function; -import java.util.function.Supplier; /** * Shared Lucene slices between Lucene operators. @@ -64,16 +63,7 @@ public static LuceneSliceQueue create( case SEGMENT -> segmentSlices(leafContexts); case DOC -> docSlices(ctx.searcher().getIndexReader(), taskConcurrency); }; - final Weight[] cachedWeight = new Weight[1]; - final Supplier weight = () -> { - if (cachedWeight[0] == null) { - cachedWeight[0] = weightFunction.apply(ctx); - } - return cachedWeight[0]; - }; - if (groups.size() > 1) { - weight.get(); // eagerly build Weight once - } + final Weight weight = weightFunction.apply(ctx); for (List group : groups) { if (group.isEmpty() == false) { slices.add(new LuceneSlice(ctx, group, weight)); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/TimeSeriesSortedSourceOperatorFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/TimeSeriesSortedSourceOperatorFactory.java index 3dde3ba75be78..cf8ac05e14a1d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/TimeSeriesSortedSourceOperatorFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/TimeSeriesSortedSourceOperatorFactory.java @@ -246,7 +246,7 @@ class TimeSeriesIterator { TimeSeriesIterator(LuceneSlice slice) throws IOException { this.slice = slice; - Weight weight = slice.weight().get(); + Weight weight = slice.weight(); if (slice.numLeaves() == 1) { queue = null; leaf = new Leaf(weight, slice.getLeaf(0).leafReaderContext()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java index 853096626179e..5ecdb4864a9cc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy; import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; @@ -164,7 +165,7 @@ private EsPhysicalOperationProviders esPhysicalOperationProviders() throws IOExc new EsPhysicalOperationProviders.DefaultShardContext( i, createSearchExecutionContext(createMapperService(mapping(b -> {})), searcher), - null + AliasFilter.EMPTY ) ); }