From 82e5c6e05720edebdc2ecabe5a66a25c2e5e2767 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Fri, 19 Jan 2024 14:05:17 -0500 Subject: [PATCH] Add opensearch-java client support Signed-off-by: Andriy Redko --- settings.gradle.kts | 1 + spring-data-opensearch/build.gradle.kts | 1 + .../data/client/osc/Aggregation.java | 44 + .../osc/AutoCloseableOpenSearchClient.java | 47 + .../data/client/osc/ChildTemplate.java | 74 + .../data/client/osc/ClusterTemplate.java | 47 + .../client/osc/CriteriaFilterProcessor.java | 336 +++ .../client/osc/CriteriaQueryException.java | 28 + .../client/osc/CriteriaQueryProcessor.java | 392 ++++ .../data/client/osc/DocumentAdapters.java | 239 ++ .../data/client/osc/EntityAsMap.java | 27 + .../client/osc/HighlightQueryBuilder.java | 228 ++ .../data/client/osc/IndicesTemplate.java | 447 ++++ .../opensearch/data/client/osc/JsonUtils.java | 68 + .../data/client/osc/JsonpUtils.java | 70 + .../data/client/osc/NativeQuery.java | 133 ++ .../data/client/osc/NativeQueryBuilder.java | 214 ++ .../client/osc/OpenSearchAggregation.java | 37 + .../client/osc/OpenSearchAggregations.java | 76 + .../OpenSearchClientBeanDefinitionParser.java | 47 + .../osc/OpenSearchClientFactoryBean.java | 96 + .../data/client/osc/OpenSearchClients.java | 402 ++++ .../client/osc/OpenSearchConfiguration.java | 129 ++ .../osc/OpenSearchExceptionTranslator.java | 136 ++ .../data/client/osc/OpenSearchTemplate.java | 723 ++++++ .../opensearch/data/client/osc/Queries.java | 194 ++ .../data/client/osc/QueryBuilders.java | 172 ++ .../data/client/osc/RequestConverter.java | 1951 +++++++++++++++++ .../data/client/osc/ResponseConverter.java | 567 +++++ .../osc/SearchDocumentResponseBuilder.java | 264 +++ .../opensearch/data/client/osc/TypeUtils.java | 480 ++++ .../data/client/osc/package-info.java | 23 + 32 files changed, 7693 insertions(+) create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Aggregation.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClient.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ChildTemplate.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ClusterTemplate.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaFilterProcessor.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryException.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryProcessor.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/DocumentAdapters.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/EntityAsMap.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/HighlightQueryBuilder.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/IndicesTemplate.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonUtils.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonpUtils.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQuery.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQueryBuilder.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregation.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregations.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientBeanDefinitionParser.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientFactoryBean.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClients.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchConfiguration.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchExceptionTranslator.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchTemplate.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Queries.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/QueryBuilders.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/RequestConverter.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ResponseConverter.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilder.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/TypeUtils.java create mode 100644 spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/package-info.java diff --git a/settings.gradle.kts b/settings.gradle.kts index 018909b..2bde81d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -31,6 +31,7 @@ dependencyResolutionManagement { create("opensearchLibs") { version("opensearch", "2.13.0") + library("java-client", "org.opensearch.client:opensearch-java:2.10.1") library("client", "org.opensearch.client", "opensearch-rest-client").versionRef("opensearch") library("high-level-client", "org.opensearch.client", "opensearch-rest-high-level-client").versionRef("opensearch") library("sniffer", "org.opensearch.client", "opensearch-rest-client-sniffer").versionRef("opensearch") diff --git a/spring-data-opensearch/build.gradle.kts b/spring-data-opensearch/build.gradle.kts index d42e524..da25b29 100644 --- a/spring-data-opensearch/build.gradle.kts +++ b/spring-data-opensearch/build.gradle.kts @@ -32,6 +32,7 @@ dependencies { implementation(springLibs.context) implementation(springLibs.tx) compileOnly(springLibs.web) + compileOnly(opensearchLibs.java.client) testImplementation("jakarta.enterprise:jakarta.enterprise.cdi-api:3.0.0") testImplementation("org.slf4j:log4j-over-slf4j:2.0.13") diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Aggregation.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Aggregation.java new file mode 100644 index 0000000..bc1382a --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Aggregation.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.opensearch._types.aggregations.Aggregate; + +/** + * Class to combine an OpenSearch {@link org.opensearch.client.opensearch._types.aggregations.Aggregate} with its + * name. Necessary as the OpenSearch Aggregate does not know its name. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class Aggregation { + + private final String name; + private final Aggregate aggregate; + + public Aggregation(String name, Aggregate aggregate) { + this.name = name; + this.aggregate = aggregate; + } + + public String getName() { + return name; + } + + public Aggregate getAggregate() { + return aggregate; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClient.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClient.java new file mode 100644 index 0000000..5c29335 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClient.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.RestClient; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.cluster.OpenSearchClusterClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.springframework.util.Assert; + +/** + * Extension of the {@link OpenSearchClient} class that implements {@link AutoCloseable}. As the underlying + * {@link RestClient} must be closed properly this is handled in the {@link #close()} method. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class AutoCloseableOpenSearchClient extends OpenSearchClient implements AutoCloseable { + + public AutoCloseableOpenSearchClient(OpenSearchTransport transport) { + super(transport); + Assert.notNull(transport, "transport must not be null"); + } + + @Override + public void close() throws Exception { + transport.close(); + } + + @Override + public OpenSearchClusterClient cluster() { + return super.cluster(); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ChildTemplate.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ChildTemplate.java new file mode 100644 index 0000000..58461b4 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ChildTemplate.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.io.IOException; +import org.opensearch.client.ApiClient; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch.cluster.OpenSearchClusterClient; +import org.opensearch.client.transport.Transport; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.util.Assert; + +/** + * base class for a template that uses one of the {@link org.opensearch.client.opensearch.OpenSearchClient}'s child + * clients like {@link OpenSearchClusterClient} or + * {@link org.opensearch.client.opensearch.indices.OpenSearchIndicesClient}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public abstract class ChildTemplate> { + + protected final CLIENT client; + protected final RequestConverter requestConverter; + protected final ResponseConverter responseConverter; + protected final OpenSearchExceptionTranslator exceptionTranslator; + + public ChildTemplate(CLIENT client, ElasticsearchConverter elasticsearchConverter) { + this.client = client; + JsonpMapper jsonpMapper = client._transport().jsonpMapper(); + requestConverter = new RequestConverter(elasticsearchConverter, jsonpMapper); + responseConverter = new ResponseConverter(jsonpMapper); + exceptionTranslator = new OpenSearchExceptionTranslator(jsonpMapper); + } + + /** + * Callback interface to be used with {@link #execute(ClientCallback)} for operating directly on the client. + */ + @FunctionalInterface + public interface ClientCallback { + RESULT doWithClient(CLIENT client) throws IOException; + } + + /** + * Execute a callback with the client and provide exception translation. + * + * @param callback the callback to execute, must not be {@literal null} + * @param the type returned from the callback + * @return the callback result + */ + public RESULT execute(ClientCallback callback) { + + Assert.notNull(callback, "callback must not be null"); + + try { + return callback.doWithClient(client); + } catch (IOException | RuntimeException e) { + throw exceptionTranslator.translateException(e); + } + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ClusterTemplate.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ClusterTemplate.java new file mode 100644 index 0000000..cc94b97 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ClusterTemplate.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.opensearch.cluster.HealthRequest; +import org.opensearch.client.opensearch.cluster.HealthResponse; +import org.opensearch.client.opensearch.cluster.OpenSearchClusterClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.springframework.data.elasticsearch.core.cluster.ClusterHealth; +import org.springframework.data.elasticsearch.core.cluster.ClusterOperations; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; + +/** + * Implementation of the {@link ClusterOperations} interface using en {@link OpenSearchClusterClient}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class ClusterTemplate extends ChildTemplate + implements ClusterOperations { + + public ClusterTemplate(OpenSearchClusterClient client, ElasticsearchConverter elasticsearchConverter) { + super(client, elasticsearchConverter); + } + + @Override + public ClusterHealth health() { + + HealthRequest healthRequest = requestConverter.clusterHealthRequest(); + HealthResponse healthResponse = execute(client -> client.health(healthRequest)); + return responseConverter.clusterHealth(healthResponse); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaFilterProcessor.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaFilterProcessor.java new file mode 100644 index 0000000..a14910c --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaFilterProcessor.java @@ -0,0 +1,336 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.GeoDistanceType; +import org.opensearch.client.opensearch._types.GeoShapeRelation; +import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.GeoBoundingBoxQuery; +import org.opensearch.client.opensearch._types.query_dsl.GeoDistanceQuery; +import org.opensearch.client.opensearch._types.query_dsl.GeoShapeQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders; +import org.opensearch.client.opensearch._types.query_dsl.QueryVariant; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.data.elasticsearch.core.convert.GeoConverters; +import org.springframework.data.elasticsearch.core.geo.GeoBox; +import org.springframework.data.elasticsearch.core.geo.GeoJson; +import org.springframework.data.elasticsearch.core.geo.GeoPoint; +import org.springframework.data.elasticsearch.core.query.Criteria; +import org.springframework.data.geo.Box; +import org.springframework.data.geo.Distance; +import org.springframework.data.geo.Metrics; +import org.springframework.data.geo.Point; +import org.springframework.util.Assert; + +/** + * Class to convert a {@link org.springframework.data.elasticsearch.core.query.CriteriaQuery} into an OpenSearch + * filter. + * + * @author Peter-Josef Meisch + * @author Junghoon Ban + * @since 4.4 + */ +class CriteriaFilterProcessor { + /** + * Creates a filter query from the given criteria. + * + * @param criteria the criteria to process + * @return the optional query, empty if the criteria did not contain filter relevant elements + */ + public static Optional createQuery(Criteria criteria) { + + Assert.notNull(criteria, "criteria must not be null"); + + List filterQueries = new ArrayList<>(); + + for (Criteria chainedCriteria : criteria.getCriteriaChain()) { + + if (chainedCriteria.isOr()) { + BoolQuery.Builder boolQueryBuilder = QueryBuilders.bool(); + queriesForEntries(chainedCriteria).forEach(boolQueryBuilder::should); + filterQueries.add(new Query(boolQueryBuilder.build())); + } else if (chainedCriteria.isNegating()) { + Collection negatingFilters = buildNegatingFilter(criteria.getField().getName(), + criteria.getFilterCriteriaEntries()); + filterQueries.addAll(negatingFilters); + } else { + filterQueries.addAll(queriesForEntries(chainedCriteria)); + } + } + + if (filterQueries.isEmpty()) { + return Optional.empty(); + } else { + + if (filterQueries.size() == 1) { + return Optional.of(filterQueries.get(0)); + } else { + BoolQuery.Builder boolQueryBuilder = QueryBuilders.bool(); + filterQueries.forEach(boolQueryBuilder::must); + BoolQuery boolQuery = boolQueryBuilder.build(); + return Optional.of(new Query(boolQuery)); + } + } + } + + private static Collection buildNegatingFilter(String fieldName, + Set filterCriteriaEntries) { + + List negationFilters = new ArrayList<>(); + + filterCriteriaEntries.forEach(criteriaEntry -> { + Optional query = queryFor(criteriaEntry.getKey(), criteriaEntry.getValue(), fieldName); + + if (query.isPresent()) { + BoolQuery negatingFilter = QueryBuilders.bool().mustNot(query.get()).build(); + negationFilters.add(new Query(negatingFilter)); + } + }); + + return negationFilters; + } + + private static Collection queriesForEntries(Criteria criteria) { + + Assert.notNull(criteria.getField(), "criteria must have a field"); + String fieldName = criteria.getField().getName(); + Assert.notNull(fieldName, "Unknown field"); + + return criteria.getFilterCriteriaEntries().stream() + .map(entry -> queryFor(entry.getKey(), entry.getValue(), fieldName)) // + .filter(Optional::isPresent) // + .map(Optional::get) // + .collect(Collectors.toList()); + } + + private static Optional queryFor(Criteria.OperationKey key, Object value, String fieldName) { + + ObjectBuilder queryBuilder = null; + + switch (key) { + case WITHIN -> { + Assert.isTrue(value instanceof Object[], "Value of a geo distance filter should be an array of two values."); + queryBuilder = withinQuery(fieldName, (Object[]) value); + } + case BBOX -> { + Assert.isTrue(value instanceof Object[], + "Value of a boundedBy filter should be an array of one or two values."); + queryBuilder = boundingBoxQuery(fieldName, (Object[]) value); + } + case GEO_INTERSECTS -> { + Assert.isTrue(value instanceof GeoJson, "value of a GEO_INTERSECTS filter must be a GeoJson object"); + queryBuilder = geoJsonQuery(fieldName, (GeoJson) value, "intersects"); + } + case GEO_IS_DISJOINT -> { + Assert.isTrue(value instanceof GeoJson, "value of a GEO_IS_DISJOINT filter must be a GeoJson object"); + queryBuilder = geoJsonQuery(fieldName, (GeoJson) value, "disjoint"); + } + case GEO_WITHIN -> { + Assert.isTrue(value instanceof GeoJson, "value of a GEO_WITHIN filter must be a GeoJson object"); + queryBuilder = geoJsonQuery(fieldName, (GeoJson) value, "within"); + } + case GEO_CONTAINS -> { + Assert.isTrue(value instanceof GeoJson, "value of a GEO_CONTAINS filter must be a GeoJson object"); + queryBuilder = geoJsonQuery(fieldName, (GeoJson) value, "contains"); + } + } + + return Optional.ofNullable(queryBuilder != null ? queryBuilder.build()._toQuery() : null); + } + + private static ObjectBuilder withinQuery(String fieldName, Object... values) { + + Assert.noNullElements(values, "Geo distance filter takes 2 not null elements array as parameter."); + Assert.isTrue(values.length == 2, "Geo distance filter takes a 2-elements array as parameter."); + Assert.isTrue(values[0] instanceof GeoPoint || values[0] instanceof String || values[0] instanceof Point, + "First element of a geo distance filter must be a GeoPoint, a Point or a text"); + Assert.isTrue(values[1] instanceof String || values[1] instanceof Distance, + "Second element of a geo distance filter must be a text or a Distance"); + + String dist = (values[1] instanceof Distance distance) ? extractDistanceString(distance) : (String) values[1]; + + return QueryBuilders.geoDistance() // + .field(fieldName) // + .distance(dist) // + .distanceType(GeoDistanceType.Plane) // + .location(location -> { + if (values[0]instanceof GeoPoint loc) { + location.latlon(latlon -> latlon.lat(loc.getLat()).lon(loc.getLon())); + } else if (values[0] instanceof Point point) { + GeoPoint loc = GeoPoint.fromPoint(point); + location.latlon(latlon -> latlon.lat(loc.getLat()).lon(loc.getLon())); + } else { + String loc = (String) values[0]; + if (loc.contains(",")) { + String[] c = loc.split(","); + location.latlon(latlon -> latlon.lat(Double.parseDouble(c[0])).lon(Double.parseDouble(c[1]))); + } else { + location.geohash(geohash -> geohash.geohash(loc)); + } + } + return location; + }); + } + + private static ObjectBuilder boundingBoxQuery(String fieldName, Object... values) { + + Assert.noNullElements(values, "Geo boundedBy filter takes a not null element array as parameter."); + + GeoBoundingBoxQuery.Builder queryBuilder = QueryBuilders.geoBoundingBox() // + .field(fieldName); + + if (values.length == 1) { + // GeoEnvelop + oneParameterBBox(queryBuilder, values[0]); + } else if (values.length == 2) { + // 2x GeoPoint + // 2x text + twoParameterBBox(queryBuilder, values); + } else { + throw new IllegalArgumentException( + "Geo distance filter takes a 1-elements array(GeoBox) or 2-elements array(GeoPoints or Strings(format lat,lon or geohash))."); + } + return queryBuilder; + } + + private static void oneParameterBBox(GeoBoundingBoxQuery.Builder queryBuilder, Object value) { + Assert.isTrue(value instanceof GeoBox || value instanceof Box, + "single-element of boundedBy filter must be type of GeoBox or Box"); + + GeoBox geoBBox; + if (value instanceof Box box) { + geoBBox = GeoBox.fromBox(box); + } else { + geoBBox = (GeoBox) value; + } + + queryBuilder.boundingBox(bb -> bb // + .tlbr(tlbr -> tlbr // + .topLeft(glb -> glb // + .latlon(latlon -> latlon // + .lat(geoBBox.getTopLeft().getLat()) // + .lon(geoBBox.getTopLeft().getLon()))) // + .bottomRight(glb -> glb // + .latlon(latlon -> latlon // + .lat(geoBBox.getBottomRight().getLat())// + .lon(geoBBox.getBottomRight().getLon()// ) + ))))); + } + + private static void twoParameterBBox(GeoBoundingBoxQuery.Builder queryBuilder, Object... values) { + + Assert.isTrue(allElementsAreOfType(values, GeoPoint.class) || allElementsAreOfType(values, String.class), + " both elements of boundedBy filter must be type of GeoPoint or text(format lat,lon or geohash)"); + + if (values[0]instanceof GeoPoint topLeft) { + GeoPoint bottomRight = (GeoPoint) values[1]; + queryBuilder.boundingBox(bb -> bb // + .tlbr(tlbr -> tlbr // + .topLeft(glb -> glb // + .latlon(latlon -> latlon // + .lat(topLeft.getLat()) // + .lon(topLeft.getLon()))) // + .bottomRight(glb -> glb // + .latlon(latlon -> latlon // + .lat(bottomRight.getLat()) // + .lon(bottomRight.getLon()))) // + ) // + ); + } else { + String topLeft = (String) values[0]; + String bottomRight = (String) values[1]; + boolean isGeoHash = !topLeft.contains(","); + queryBuilder.boundingBox(bb -> bb // + .tlbr(tlbr -> tlbr // + .topLeft(glb -> { + if (isGeoHash) { + glb.geohash(gh -> gh.geohash(topLeft)); + } else { + glb.text(topLeft); + } + return glb; + }) // + .bottomRight(glb -> { + if (isGeoHash) { + glb.geohash(gh -> gh.geohash(bottomRight)); + } else { + glb.text(bottomRight); + } + return glb; + }) // + )); + } + } + + private static boolean allElementsAreOfType(Object[] array, Class clazz) { + for (Object o : array) { + if (!clazz.isInstance(o)) { + return false; + } + } + return true; + } + + private static ObjectBuilder geoJsonQuery(String fieldName, GeoJson geoJson, + String relation) { + return buildGeoShapeQuery(fieldName, geoJson, relation); + } + + private static ObjectBuilder buildGeoShapeQuery(String fieldName, GeoJson geoJson, + String relation) { + return QueryBuilders.geoShape().field(fieldName) // + .shape(gsf -> gsf // + .shape(JsonData.of(GeoConverters.GeoJsonToMapConverter.INSTANCE.convert(geoJson))) // + .relation(toRelation(relation))); // + } + + private static GeoShapeRelation toRelation(String relation) { + + for (GeoShapeRelation geoShapeRelation : GeoShapeRelation.values()) { + + if (geoShapeRelation.name().equalsIgnoreCase(relation)) { + return geoShapeRelation; + } + } + throw new IllegalArgumentException("Unknown geo_shape relation: " + relation); + } + + /** + * extract the distance string from a {@link org.springframework.data.geo.Distance} object. + * + * @param distance distance object to extract string from + */ + private static String extractDistanceString(Distance distance) { + + StringBuilder sb = new StringBuilder(); + sb.append((int) distance.getValue()); + switch ((Metrics) distance.getMetric()) { + case KILOMETERS -> sb.append("km"); + case MILES -> sb.append("mi"); + } + + return sb.toString(); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryException.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryException.java new file mode 100644 index 0000000..982eecf --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.springframework.dao.UncategorizedDataAccessException; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class CriteriaQueryException extends UncategorizedDataAccessException { + public CriteriaQueryException(String msg) { + super(msg, null); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryProcessor.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryProcessor.java new file mode 100644 index 0000000..f5d7a60 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryProcessor.java @@ -0,0 +1,392 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.Queries.*; +import static org.springframework.util.StringUtils.*; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.ChildScoreMode; +import org.opensearch.client.opensearch._types.query_dsl.Operator; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.springframework.data.elasticsearch.annotations.FieldType; +import org.springframework.data.elasticsearch.core.query.Criteria; +import org.springframework.data.elasticsearch.core.query.Field; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Class to convert a {@link org.springframework.data.elasticsearch.core.query.CriteriaQuery} into an OpenSearch + * query. + * + * @author Peter-Josef Meisch + * @author Ezequiel Antúnez Camacho + * @since 4.4 + */ +class CriteriaQueryProcessor { + + /** + * creates a query from the criteria + * + * @param criteria the {@link Criteria} + * @return the optional query, null if the criteria did not contain filter relevant elements + */ + @Nullable + public static Query createQuery(Criteria criteria) { + + Assert.notNull(criteria, "criteria must not be null"); + + List shouldQueries = new ArrayList<>(); + List mustNotQueries = new ArrayList<>(); + List mustQueries = new ArrayList<>(); + + Query firstQuery = null; + boolean negateFirstQuery = false; + + for (Criteria chainedCriteria : criteria.getCriteriaChain()) { + Query queryFragment = queryForEntries(chainedCriteria); + + if (queryFragment != null) { + + if (firstQuery == null) { + firstQuery = queryFragment; + negateFirstQuery = chainedCriteria.isNegating(); + continue; + } + + if (chainedCriteria.isOr()) { + shouldQueries.add(queryFragment); + } else if (chainedCriteria.isNegating()) { + mustNotQueries.add(queryFragment); + } else { + mustQueries.add(queryFragment); + } + } + } + + for (Criteria subCriteria : criteria.getSubCriteria()) { + Query subQuery = createQuery(subCriteria); + if (subQuery != null) { + if (criteria.isOr()) { + shouldQueries.add(subQuery); + } else if (criteria.isNegating()) { + mustNotQueries.add(subQuery); + } else { + mustQueries.add(subQuery); + } + } + } + + if (firstQuery != null) { + + if (!shouldQueries.isEmpty() && mustNotQueries.isEmpty() && mustQueries.isEmpty()) { + shouldQueries.add(0, firstQuery); + } else { + + if (negateFirstQuery) { + mustNotQueries.add(0, firstQuery); + } else { + mustQueries.add(0, firstQuery); + } + } + } + + if (shouldQueries.isEmpty() && mustNotQueries.isEmpty() && mustQueries.isEmpty()) { + return null; + } + + Query query = new Query.Builder().bool(boolQueryBuilder -> { + + if (!shouldQueries.isEmpty()) { + boolQueryBuilder.should(shouldQueries); + } + + if (!mustNotQueries.isEmpty()) { + boolQueryBuilder.mustNot(mustNotQueries); + } + + if (!mustQueries.isEmpty()) { + boolQueryBuilder.must(mustQueries); + } + + return boolQueryBuilder; + }).build(); + + return query; + } + + @Nullable + private static Query queryForEntries(Criteria criteria) { + + Field field = criteria.getField(); + + if (field == null || criteria.getQueryCriteriaEntries().isEmpty()) + return null; + + String fieldName = field.getName(); + Assert.notNull(fieldName, "Unknown field " + fieldName); + + Iterator it = criteria.getQueryCriteriaEntries().iterator(); + + Float boost = Float.isNaN(criteria.getBoost()) ? null : criteria.getBoost(); + Query.Builder queryBuilder; + + if (criteria.getQueryCriteriaEntries().size() == 1) { + queryBuilder = queryFor(it.next(), field, boost); + } else { + queryBuilder = new Query.Builder(); + queryBuilder.bool(boolQueryBuilder -> { + while (it.hasNext()) { + Criteria.CriteriaEntry entry = it.next(); + boolQueryBuilder.must(queryFor(entry, field, null).build()); + } + boolQueryBuilder.boost(boost); + return boolQueryBuilder; + }); + + } + + if (hasText(field.getPath())) { + final Query query = queryBuilder.build(); + queryBuilder = new Query.Builder(); + queryBuilder.nested(nqb -> nqb // + .path(field.getPath()) // + .query(query) // + .scoreMode(ChildScoreMode.Avg)); + } + + return queryBuilder.build(); + } + + private static Query.Builder queryFor(Criteria.CriteriaEntry entry, Field field, @Nullable Float boost) { + + String fieldName = field.getName(); + boolean isKeywordField = FieldType.Keyword == field.getFieldType(); + + Criteria.OperationKey key = entry.getKey(); + Object value = key.hasValue() ? entry.getValue() : null; + String searchText = value != null ? escape(value.toString()) : "UNKNOWN_VALUE"; + + Query.Builder queryBuilder = new Query.Builder(); + switch (key) { + case EXISTS: + queryBuilder // + .exists(eb -> eb // + .field(fieldName) // + .boost(boost)); + break; + case EMPTY: + queryBuilder // + .bool(bb -> bb // + .must(mb -> mb // + .exists(eb -> eb // + .field(fieldName) // + )) // + .mustNot(mnb -> mnb // + .wildcard(wb -> wb // + .field(fieldName) // + .wildcard("*"))) // + .boost(boost)); + break; + case NOT_EMPTY: + queryBuilder // + .wildcard(wb -> wb // + .field(fieldName) // + .wildcard("*") // + .boost(boost)); + break; + case EQUALS: + queryBuilder.queryString(queryStringQuery(fieldName, searchText, Operator.And, boost)); + break; + case CONTAINS: + queryBuilder.queryString(queryStringQuery(fieldName, '*' + searchText + '*', true, boost)); + break; + case STARTS_WITH: + queryBuilder.queryString(queryStringQuery(fieldName, searchText + '*', true, boost)); + break; + case ENDS_WITH: + queryBuilder.queryString(queryStringQuery(fieldName, '*' + searchText, true, boost)); + break; + case EXPRESSION: + queryBuilder.queryString(queryStringQuery(fieldName, value.toString(), boost)); + break; + case LESS: + queryBuilder // + .range(rb -> rb // + .field(fieldName) // + .lt(JsonData.of(value)) // + .boost(boost)); // + break; + case LESS_EQUAL: + queryBuilder // + .range(rb -> rb // + .field(fieldName) // + .lte(JsonData.of(value)) // + .boost(boost)); // + break; + case GREATER: + queryBuilder // + .range(rb -> rb // + .field(fieldName) // + .gt(JsonData.of(value)) // + .boost(boost)); // + break; + case GREATER_EQUAL: + queryBuilder // + .range(rb -> rb // + .field(fieldName) // + .gte(JsonData.of(value)) // + .boost(boost)); // + break; + case BETWEEN: + Object[] ranges = (Object[]) value; + queryBuilder // + .range(rb -> { + rb.field(fieldName); + if (ranges[0] != null) { + rb.gte(JsonData.of(ranges[0])); + } + + if (ranges[1] != null) { + rb.lte(JsonData.of(ranges[1])); + } + rb.boost(boost); // + return rb; + }); // + + break; + case FUZZY: + queryBuilder // + .fuzzy(fb -> fb // + .field(fieldName) // + .value(FieldValue.of(searchText)) // + .boost(boost)); // + break; + case MATCHES: + queryBuilder.match(matchQuery(fieldName, value.toString(), Operator.Or, boost)); + break; + case MATCHES_ALL: + queryBuilder.match(matchQuery(fieldName, value.toString(), Operator.And, boost)); + + break; + case IN: + if (value instanceof Iterable iterable) { + if (isKeywordField) { + queryBuilder.bool(bb -> bb // + .must(mb -> mb // + .terms(tb -> tb // + .field(fieldName) // + .terms(tsb -> tsb // + .value(toFieldValueList(iterable))))) // + .boost(boost)); // + } else { + queryBuilder // + .queryString(qsb -> qsb // + .fields(fieldName) // + .query(orQueryString(iterable)) // + .boost(boost)); // + } + } else { + throw new CriteriaQueryException("value for " + fieldName + " is not an Iterable"); + } + break; + case NOT_IN: + if (value instanceof Iterable iterable) { + if (isKeywordField) { + queryBuilder.bool(bb -> bb // + .mustNot(mnb -> mnb // + .terms(tb -> tb // + .field(fieldName) // + .terms(tsb -> tsb // + .value(toFieldValueList(iterable))))) // + .boost(boost)); // + } else { + queryBuilder // + .queryString(qsb -> qsb // + .fields(fieldName) // + .query("NOT(" + orQueryString(iterable) + ')') // + .boost(boost)); // + } + } else { + throw new CriteriaQueryException("value for " + fieldName + " is not an Iterable"); + } + break; + case REGEXP: + queryBuilder // + .regexp(rb -> rb // + .field(fieldName) // + .value(value.toString()) // + .boost(boost)); // + break; + default: + throw new CriteriaQueryException("Could not build query for " + entry); + } + + return queryBuilder; + } + + private static List toFieldValueList(Iterable iterable) { + List list = new ArrayList<>(); + for (Object item : iterable) { + list.add(item != null ? FieldValue.of(item.toString()) : null); + } + return list; + } + + private static String orQueryString(Iterable iterable) { + StringBuilder sb = new StringBuilder(); + + for (Object item : iterable) { + + if (item != null) { + + if (sb.length() > 0) { + sb.append(' '); + } + sb.append('"'); + sb.append(escape(item.toString())); + sb.append('"'); + } + } + + return sb.toString(); + } + + /** + * Returns a String where those characters that TextParser expects to be escaped are escaped by a preceding + * \. Copied from Apachae 2 licensed org.apache.lucene.queryparser.flexible.standard.QueryParserUtil + * class + */ + public static String escape(String s) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + // These characters are part of the query syntax and must be escaped + if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')' || c == ':' || c == '^' || c == '[' + || c == ']' || c == '\"' || c == '{' || c == '}' || c == '~' || c == '*' || c == '?' || c == '|' || c == '&' + || c == '/') { + sb.append('\\'); + } + sb.append(c); + } + return sb.toString(); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/DocumentAdapters.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/DocumentAdapters.java new file mode 100644 index 0000000..6a917d3 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/DocumentAdapters.java @@ -0,0 +1,239 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch.core.GetResponse; +import org.opensearch.client.opensearch.core.MgetResponse; +import org.opensearch.client.opensearch.core.explain.ExplanationDetail; +import org.opensearch.client.opensearch.core.get.GetResult; +import org.opensearch.client.opensearch.core.search.CompletionSuggestOption; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.client.opensearch.core.search.NestedIdentity; +import org.springframework.data.elasticsearch.core.MultiGetItem; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.document.Explanation; +import org.springframework.data.elasticsearch.core.document.NestedMetaData; +import org.springframework.data.elasticsearch.core.document.SearchDocument; +import org.springframework.data.elasticsearch.core.document.SearchDocumentAdapter; +import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Utility class to adapt different Elasticsearch responses to a + * {@link org.springframework.data.elasticsearch.core.document.Document} + * + * @author Peter-Josef Meisch + * @author Haibo Liu + * @since 4.4 + */ +final class DocumentAdapters { + + private static final Log LOGGER = LogFactory.getLog(DocumentAdapters.class); + + private DocumentAdapters() {} + + /** + * Creates a {@link SearchDocument} from a {@link Hit} returned by the Elasticsearch client. + * + * @param hit the hit object + * @param jsonpMapper to map JsonData objects + * @return the created {@link SearchDocument} + */ + @SuppressWarnings("unchecked") + public static SearchDocument from(Hit hit, JsonpMapper jsonpMapper) { + + Assert.notNull(hit, "hit must not be null"); + + Map> highlightFields = hit.highlight(); + + Map innerHits = new LinkedHashMap<>(); + hit.innerHits().forEach((name, innerHitsResult) -> { + // noinspection ReturnOfNull + innerHits.put(name, SearchDocumentResponseBuilder.from(innerHitsResult.hits(), null, null, null, null, + searchDocument -> null, jsonpMapper)); + }); + + NestedMetaData nestedMetaData = from(hit.nested()); + + Explanation explanation = from(hit.explanation()); + + List matchedQueries = hit.matchedQueries(); + + Function, EntityAsMap> fromFields = fields -> { + StringBuilder sb = new StringBuilder("{"); + final boolean[] firstField = { true }; + hit.fields().forEach((key, jsonData) -> { + if (!firstField[0]) { + sb.append(','); + } + sb.append('"').append(key).append("\":") // + .append(jsonData.toJson(jsonpMapper).toString()); + firstField[0] = false; + }); + sb.append('}'); + return new EntityAsMap().fromJson(sb.toString()); + }; + + EntityAsMap hitFieldsAsMap = fromFields.apply(hit.fields()); + + Map> documentFields = new LinkedHashMap<>(); + hitFieldsAsMap.forEach((key, value) -> { + if (value instanceof List) { + // noinspection unchecked + documentFields.put(key, (List) value); + } else { + documentFields.put(key, Collections.singletonList(value)); + } + }); + + Document document; + Object source = hit.source(); + if (source == null) { + document = Document.from(hitFieldsAsMap); + } else { + if (source instanceof EntityAsMap entityAsMap) { + document = Document.from(entityAsMap); + } else if (source instanceof JsonData jsonData) { + document = Document.from(jsonData.to(EntityAsMap.class)); + } else { + + if (LOGGER.isWarnEnabled()) { + LOGGER.warn(String.format("Cannot map from type " + source.getClass().getName())); + } + document = Document.create(); + } + } + document.setIndex(hit.index()); + document.setId(hit.id()); + + if (hit.version() != null) { + document.setVersion(hit.version()); + } + document.setSeqNo(hit.seqNo() != null && hit.seqNo() >= 0 ? hit.seqNo() : -2); // -2 was the default value in the + // old client + document.setPrimaryTerm(hit.primaryTerm() != null && hit.primaryTerm() > 0 ? hit.primaryTerm() : 0); + + float score = hit.score() != null ? hit.score().floatValue() : Float.NaN; + return new SearchDocumentAdapter(document, score, hit.sort().stream().toArray(), + documentFields, highlightFields, innerHits, nestedMetaData, explanation, matchedQueries, hit.routing()); + } + + public static SearchDocument from(CompletionSuggestOption completionSuggestOption) { + + Document document = completionSuggestOption.source() != null ? Document.from(completionSuggestOption.source()) + : Document.create(); + document.setIndex(completionSuggestOption.index()); + + if (completionSuggestOption.id() != null) { + document.setId(completionSuggestOption.id()); + } + + float score = (float) completionSuggestOption.score(); + return new SearchDocumentAdapter(document, score, new Object[] {}, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), null, null, null, completionSuggestOption.routing()); + } + + @Nullable + private static Explanation from(@Nullable org.opensearch.client.opensearch.core.explain.Explanation explanation) { + + if (explanation == null) { + return null; + } + List details = explanation.details().stream().map(DocumentAdapters::from).collect(Collectors.toList()); + return new Explanation(true, (double) explanation.value(), explanation.description(), details); + } + + private static Explanation from(ExplanationDetail explanationDetail) { + + List details = explanationDetail.details().stream().map(DocumentAdapters::from) + .collect(Collectors.toList()); + return new Explanation(null, (double) explanationDetail.value(), explanationDetail.description(), details); + } + + @Nullable + private static NestedMetaData from(@Nullable NestedIdentity nestedIdentity) { + + if (nestedIdentity == null) { + return null; + } + + NestedMetaData child = from(nestedIdentity.nested()); + return NestedMetaData.of(nestedIdentity.field(), nestedIdentity.offset(), child); + } + + /** + * Creates a {@link Document} from a {@link GetResponse} where the found document is contained as {@link EntityAsMap}. + * + * @param getResponse the response instance + * @return the Document + */ + @Nullable + public static Document from(GetResult getResponse) { + + Assert.notNull(getResponse, "getResponse must not be null"); + + if (!getResponse.found()) { + return null; + } + + Document document = getResponse.source() != null ? Document.from(getResponse.source()) : Document.create(); + document.setIndex(getResponse.index()); + document.setId(getResponse.id()); + + if (getResponse.version() != null) { + document.setVersion(getResponse.version()); + } + + if (getResponse.seqNo() != null) { + document.setSeqNo(getResponse.seqNo()); + } + + if (getResponse.primaryTerm() != null) { + document.setPrimaryTerm(getResponse.primaryTerm()); + } + + return document; + } + + /** + * Creates a list of {@link MultiGetItem}s from a {@link MgetResponse} where the data is contained as + * {@link EntityAsMap} instances. + * + * @param mgetResponse the response instance + * @return list of multiget items + */ + public static List> from(MgetResponse mgetResponse) { + + Assert.notNull(mgetResponse, "mgetResponse must not be null"); + + return mgetResponse.docs().stream() // + .map(itemResponse -> MultiGetItem.of( // + itemResponse.isFailure() ? null : from(itemResponse.result()), // + ResponseConverter.getFailure(itemResponse))) + .collect(Collectors.toList()); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/EntityAsMap.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/EntityAsMap.java new file mode 100644 index 0000000..5af5011 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/EntityAsMap.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.springframework.data.elasticsearch.support.DefaultStringObjectMap; + +/** + * A Map<String,Object> to represent any entity as it's returned from OpenSearch and before it is converted to a + * {@link org.springframework.data.elasticsearch.core.document.Document}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class EntityAsMap extends DefaultStringObjectMap {} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/HighlightQueryBuilder.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/HighlightQueryBuilder.java new file mode 100644 index 0000000..db5fe56 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/HighlightQueryBuilder.java @@ -0,0 +1,228 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.TypeUtils.*; + +import java.util.Arrays; +import java.util.stream.Collectors; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty; +import org.springframework.data.elasticsearch.core.query.highlight.Highlight; +import org.springframework.data.elasticsearch.core.query.highlight.HighlightField; +import org.springframework.data.elasticsearch.core.query.highlight.HighlightFieldParameters; +import org.springframework.data.elasticsearch.core.query.highlight.HighlightParameters; +import org.springframework.data.mapping.context.MappingContext; +import org.springframework.lang.Nullable; +import org.springframework.util.StringUtils; + +/** + * Converts the {@link Highlight} annotation from a method to an OpenSearchClient + * {@link org.opensearch.client.opensearch.core.search.Highlight}. + * + * @author Peter-Josef Meisch + * @author Haibo Liu + * @since 4.4 + */ +class HighlightQueryBuilder { + private final MappingContext, ElasticsearchPersistentProperty> mappingContext; + + HighlightQueryBuilder( + MappingContext, ElasticsearchPersistentProperty> mappingContext) { + this.mappingContext = mappingContext; + } + + public org.opensearch.client.opensearch.core.search.Highlight getHighlight(Highlight highlight, + @Nullable Class type) { + + org.opensearch.client.opensearch.core.search.Highlight.Builder highlightBuilder = new org.opensearch.client.opensearch.core.search.Highlight.Builder(); + + // in the old implementation we could use one addParameters method, but in the new Elasticsearch client + // the builder for highlight and highlightfield share no code + addParameters(highlight.getParameters(), highlightBuilder, type); + + for (HighlightField highlightField : highlight.getFields()) { + String mappedName = mapFieldName(highlightField.getName(), type); + highlightBuilder.fields(mappedName, hf -> { + addParameters(highlightField.getParameters(), hf, type); + return hf; + }); + } + + return highlightBuilder.build(); + } + + /* + * the builder for highlight and highlight fields don't share code, so we have these two methods here that basically are almost copies + */ + private void addParameters(HighlightParameters parameters, + org.opensearch.client.opensearch.core.search.Highlight.Builder builder, @Nullable Class type) { + + if (StringUtils.hasLength(parameters.getBoundaryChars())) { + builder.boundaryChars(parameters.getBoundaryChars()); + } + + if (parameters.getBoundaryMaxScan() > -1) { + builder.boundaryMaxScan(parameters.getBoundaryMaxScan()); + } + + if (StringUtils.hasLength(parameters.getBoundaryScanner())) { + builder.boundaryScanner(boundaryScanner(parameters.getBoundaryScanner())); + } + + if (StringUtils.hasLength(parameters.getBoundaryScannerLocale())) { + builder.boundaryScannerLocale(parameters.getBoundaryScannerLocale()); + } + + if (StringUtils.hasLength(parameters.getFragmenter())) { + builder.fragmenter(highlighterFragmenter(parameters.getFragmenter())); + } + + if (parameters.getFragmentSize() > -1) { + builder.fragmentSize(parameters.getFragmentSize()); + } + + if (parameters.getNoMatchSize() > -1) { + builder.noMatchSize(parameters.getNoMatchSize()); + } + + if (parameters.getNumberOfFragments() > -1) { + builder.numberOfFragments(parameters.getNumberOfFragments()); + } + + if (StringUtils.hasLength(parameters.getOrder())) { + builder.order(highlighterOrder(parameters.getOrder())); + } + + if (parameters.getPreTags().length > 0) { + builder.preTags(Arrays.asList(parameters.getPreTags())); + } + + if (parameters.getPostTags().length > 0) { + builder.postTags(Arrays.asList(parameters.getPostTags())); + } + + if (!parameters.getRequireFieldMatch()) { // default is true + builder.requireFieldMatch(false); + } + + if (StringUtils.hasLength(parameters.getType())) { + builder.type(highlighterType(parameters.getType())); + } + + if (StringUtils.hasLength(parameters.getEncoder())) { + builder.encoder(highlighterEncoder(parameters.getEncoder())); + } + + if (StringUtils.hasLength(parameters.getTagsSchema())) { + builder.tagsSchema(highlighterTagsSchema(parameters.getTagsSchema())); + } + } + + /* + * the builder for highlight and highlight fields don't share code, so we have these two methods here that basically are almost copies + */ + private void addParameters(HighlightFieldParameters parameters, + org.opensearch.client.opensearch.core.search.HighlightField.Builder builder, Class type) { + + if (StringUtils.hasLength(parameters.getBoundaryChars())) { + builder.boundaryChars(parameters.getBoundaryChars()); + } + + if (parameters.getBoundaryMaxScan() > -1) { + builder.boundaryMaxScan(parameters.getBoundaryMaxScan()); + } + + if (StringUtils.hasLength(parameters.getBoundaryScanner())) { + builder.boundaryScanner(boundaryScanner(parameters.getBoundaryScanner())); + } + + if (StringUtils.hasLength(parameters.getBoundaryScannerLocale())) { + builder.boundaryScannerLocale(parameters.getBoundaryScannerLocale()); + } + + if (parameters.getForceSource()) { // default is false + builder.forceSource(parameters.getForceSource()); + } + + if (StringUtils.hasLength(parameters.getFragmenter())) { + builder.fragmenter(highlighterFragmenter(parameters.getFragmenter())); + } + + if (parameters.getFragmentSize() > -1) { + builder.fragmentSize(parameters.getFragmentSize()); + } + + if (parameters.getNoMatchSize() > -1) { + builder.noMatchSize(parameters.getNoMatchSize()); + } + + if (parameters.getNumberOfFragments() > -1) { + builder.numberOfFragments(parameters.getNumberOfFragments()); + } + + if (StringUtils.hasLength(parameters.getOrder())) { + builder.order(highlighterOrder(parameters.getOrder())); + } + + if (parameters.getPhraseLimit() > -1) { + builder.phraseLimit(parameters.getPhraseLimit()); + } + + if (parameters.getPreTags().length > 0) { + builder.preTags(Arrays.asList(parameters.getPreTags())); + } + + if (parameters.getPostTags().length > 0) { + builder.postTags(Arrays.asList(parameters.getPostTags())); + } + + if (!parameters.getRequireFieldMatch()) { // default is true + builder.requireFieldMatch(false); + } + + if (StringUtils.hasLength(parameters.getType())) { + builder.type(highlighterType(parameters.getType())); + } + + if ((parameters).getFragmentOffset() > -1) { + builder.fragmentOffset(parameters.getFragmentOffset()); + } + + if (parameters.getMatchedFields().length > 0) { + builder.matchedFields(Arrays.stream(parameters.getMatchedFields()).map(fieldName -> mapFieldName(fieldName, type)) // + .collect(Collectors.toList())); + } + } + + private String mapFieldName(String fieldName, @Nullable Class type) { + + if (type != null) { + ElasticsearchPersistentEntity persistentEntity = mappingContext.getPersistentEntity(type); + + if (persistentEntity != null) { + ElasticsearchPersistentProperty persistentProperty = persistentEntity.getPersistentProperty(fieldName); + + if (persistentProperty != null) { + return persistentProperty.getFieldName(); + } + } + } + + return fieldName; + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/IndicesTemplate.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/IndicesTemplate.java new file mode 100644 index 0000000..b703a17 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/IndicesTemplate.java @@ -0,0 +1,447 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.springframework.util.StringUtils.*; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.opensearch.client.opensearch.indices.*; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.endpoints.BooleanResponse; +import org.springframework.core.annotation.AnnotatedElementUtils; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.data.elasticsearch.UncategorizedElasticsearchException; +import org.springframework.data.elasticsearch.annotations.Mapping; +import org.springframework.data.elasticsearch.core.IndexInformation; +import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.ResourceUtil; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.index.*; +import org.springframework.data.elasticsearch.core.index.DeleteIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.DeleteTemplateRequest; +import org.springframework.data.elasticsearch.core.index.ExistsIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.ExistsTemplateRequest; +import org.springframework.data.elasticsearch.core.index.GetIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.GetTemplateRequest; +import org.springframework.data.elasticsearch.core.index.PutIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.PutTemplateRequest; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Implementation of the {@link IndexOperations} interface using en {@link OpenSearchIndicesClient}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class IndicesTemplate extends ChildTemplate + implements IndexOperations { + + // we need a cluster client as well because ES has put some methods from the indices API into the cluster client + // (component templates) + private final ClusterTemplate clusterTemplate; + protected final ElasticsearchConverter elasticsearchConverter; + @Nullable protected final Class boundClass; + @Nullable protected final IndexCoordinates boundIndex; + + public IndicesTemplate(OpenSearchIndicesClient client, ClusterTemplate clusterTemplate, + ElasticsearchConverter elasticsearchConverter, Class boundClass) { + super(client, elasticsearchConverter); + + Assert.notNull(clusterTemplate, "cluster must not be null"); + Assert.notNull(elasticsearchConverter, "elasticsearchConverter must not be null"); + Assert.notNull(boundClass, "boundClass may not be null"); + + this.clusterTemplate = clusterTemplate; + this.elasticsearchConverter = elasticsearchConverter; + this.boundClass = boundClass; + this.boundIndex = null; + + } + + public IndicesTemplate(OpenSearchIndicesClient client, ClusterTemplate clusterTemplate, + ElasticsearchConverter elasticsearchConverter, IndexCoordinates boundIndex) { + super(client, elasticsearchConverter); + + Assert.notNull(clusterTemplate, "cluster must not be null"); + Assert.notNull(elasticsearchConverter, "elasticsearchConverter must not be null"); + Assert.notNull(boundIndex, "boundIndex must not be null"); + + this.clusterTemplate = clusterTemplate; + this.elasticsearchConverter = elasticsearchConverter; + this.boundClass = null; + this.boundIndex = boundIndex; + + } + + protected Class checkForBoundClass() { + if (boundClass == null) { + throw new InvalidDataAccessApiUsageException("IndexOperations are not bound"); + } + return boundClass; + } + + @Override + public boolean create() { + + Settings settings = boundClass != null ? createSettings(boundClass) : new Settings(); + return doCreate(getIndexCoordinates(), settings, null); + } + + @Override + public boolean create(Map settings) { + + Assert.notNull(settings, "settings must not be null"); + + return doCreate(getIndexCoordinates(), settings, null); + } + + @Override + public boolean create(Map settings, Document mapping) { + + Assert.notNull(settings, "settings must not be null"); + Assert.notNull(mapping, "mapping must not be null"); + + return doCreate(getIndexCoordinates(), settings, mapping); + } + + @Override + public boolean createWithMapping() { + return doCreate(getIndexCoordinates(), createSettings(), createMapping()); + } + + protected boolean doCreate(IndexCoordinates indexCoordinates, Map settings, + @Nullable Document mapping) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + Assert.notNull(settings, "settings must not be null"); + + CreateIndexRequest createIndexRequest = requestConverter.indicesCreateRequest(indexCoordinates, settings, mapping); + CreateIndexResponse createIndexResponse = execute(client -> client.create(createIndexRequest)); + return Boolean.TRUE.equals(createIndexResponse.acknowledged()); + } + + @Override + public boolean delete() { + return doDelete(getIndexCoordinates()); + } + + private boolean doDelete(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + if (doExists(indexCoordinates)) { + DeleteIndexRequest deleteIndexRequest = requestConverter.indicesDeleteRequest(indexCoordinates); + DeleteIndexResponse deleteIndexResponse = execute(client -> client.delete(deleteIndexRequest)); + return deleteIndexResponse.acknowledged(); + } + + return false; + } + + @Override + public boolean exists() { + return doExists(getIndexCoordinates()); + } + + private boolean doExists(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + ExistsRequest existsRequest = requestConverter.indicesExistsRequest(indexCoordinates); + BooleanResponse existsResponse = execute(client -> client.exists(existsRequest)); + return existsResponse.value(); + } + + @Override + public void refresh() { + + RefreshRequest refreshRequest = requestConverter.indicesRefreshRequest(getIndexCoordinates()); + execute(client -> client.refresh(refreshRequest)); + } + + @Override + public Document createMapping() { + return createMapping(checkForBoundClass()); + } + + @Override + public Document createMapping(Class clazz) { + + Assert.notNull(clazz, "clazz must not be null"); + + // load mapping specified in Mapping annotation if present + Mapping mappingAnnotation = AnnotatedElementUtils.findMergedAnnotation(clazz, Mapping.class); + + if (mappingAnnotation != null) { + String mappingPath = mappingAnnotation.mappingPath(); + + if (hasText(mappingPath)) { + String mappings = ResourceUtil.readFileFromClasspath(mappingPath); + + if (hasText(mappings)) { + return Document.parse(mappings); + } + } + } + + // build mapping from field annotations + try { + String mapping = new MappingBuilder(elasticsearchConverter).buildPropertyMapping(clazz); + return Document.parse(mapping); + } catch (Exception e) { + throw new UncategorizedElasticsearchException("Failed to build mapping for " + clazz.getSimpleName(), e); + } + } + + @Override + public boolean putMapping(Document mapping) { + + Assert.notNull(mapping, "mapping must not be null"); + + PutMappingRequest putMappingRequest = requestConverter.indicesPutMappingRequest(getIndexCoordinates(), mapping); + PutMappingResponse putMappingResponse = execute(client -> client.putMapping(putMappingRequest)); + return putMappingResponse.acknowledged(); + } + + @Override + public Map getMapping() { + + IndexCoordinates indexCoordinates = getIndexCoordinates(); + GetMappingRequest getMappingRequest = requestConverter.indicesGetMappingRequest(indexCoordinates); + GetMappingResponse getMappingResponse = execute(client -> client.getMapping(getMappingRequest)); + + Document mappingResponse = responseConverter.indicesGetMapping(getMappingResponse, indexCoordinates); + return mappingResponse; + } + + @Override + public Settings createSettings() { + return createSettings(checkForBoundClass()); + } + + @Override + public Settings createSettings(Class clazz) { + + Assert.notNull(clazz, "clazz must not be null"); + + ElasticsearchPersistentEntity persistentEntity = getRequiredPersistentEntity(clazz); + String settingPath = persistentEntity.settingPath(); + return hasText(settingPath) // + ? Settings.parse(ResourceUtil.readFileFromClasspath(settingPath)) // + : persistentEntity.getDefaultSettings(); + + } + + @Override + public Settings getSettings() { + return getSettings(false); + } + + @Override + public Settings getSettings(boolean includeDefaults) { + + GetIndicesSettingsRequest getIndicesSettingsRequest = requestConverter + .indicesGetSettingsRequest(getIndexCoordinates(), includeDefaults); + GetIndicesSettingsResponse getIndicesSettingsResponse = execute( + client -> client.getSettings(getIndicesSettingsRequest)); + return responseConverter.indicesGetSettings(getIndicesSettingsResponse, getIndexCoordinates().getIndexName()); + } + + @Override + public boolean alias(AliasActions aliasActions) { + + Assert.notNull(aliasActions, "aliasActions must not be null"); + + UpdateAliasesRequest updateAliasesRequest = requestConverter.indicesUpdateAliasesRequest(aliasActions); + UpdateAliasesResponse updateAliasesResponse = execute(client -> client.updateAliases(updateAliasesRequest)); + return updateAliasesResponse.acknowledged(); + } + + @Override + public Map> getAliases(String... aliasNames) { + + Assert.notNull(aliasNames, "aliasNames must not be null"); + + GetAliasRequest getAliasRequest = requestConverter.indicesGetAliasRequest(aliasNames, null); + var getAliasResponse = execute(client -> client.getAlias(getAliasRequest)); + return responseConverter.indicesGetAliasData(getAliasResponse); + } + + @Override + public Map> getAliasesForIndex(String... indexNames) { + + Assert.notNull(indexNames, "indexNames must not be null"); + + GetAliasRequest getAliasRequest = requestConverter.indicesGetAliasRequest(null, indexNames); + var getAliasResponse = execute(client -> client.getAlias(getAliasRequest)); + return responseConverter.indicesGetAliasData(getAliasResponse); + } + + @Override + public boolean putTemplate(PutTemplateRequest putTemplateRequest) { + + Assert.notNull(putTemplateRequest, "putTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.PutTemplateRequest putTemplateRequestES = requestConverter + .indicesPutTemplateRequest(putTemplateRequest); + return execute(client -> client.putTemplate(putTemplateRequestES)).acknowledged(); + } + + @Override + public TemplateData getTemplate(GetTemplateRequest getTemplateRequest) { + + Assert.notNull(getTemplateRequest, "getTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.GetTemplateRequest getTemplateRequestES = requestConverter + .indicesGetTemplateRequest(getTemplateRequest); + GetTemplateResponse getTemplateResponse = execute(client -> client.getTemplate(getTemplateRequestES)); + + return responseConverter.indicesGetTemplateData(getTemplateResponse, getTemplateRequest.getTemplateName()); + } + + @Override + public boolean existsTemplate(ExistsTemplateRequest existsTemplateRequest) { + + Assert.notNull(existsTemplateRequest, "existsTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.ExistsTemplateRequest existsTemplateRequestSO = requestConverter + .indicesExistsTemplateRequest(existsTemplateRequest); + return execute(client -> client.existsTemplate(existsTemplateRequestSO)).value(); + } + + @Override + public boolean deleteTemplate(DeleteTemplateRequest deleteTemplateRequest) { + + Assert.notNull(deleteTemplateRequest, "deleteTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.DeleteTemplateRequest deleteTemplateRequestES = requestConverter + .indicesDeleteTemplateRequest(deleteTemplateRequest); + return execute(client -> client.deleteTemplate(deleteTemplateRequestES)).acknowledged(); + } + + @Override + public boolean putIndexTemplate(PutIndexTemplateRequest putIndexTemplateRequest) { + + org.opensearch.client.opensearch.indices.PutIndexTemplateRequest putIndexTemplateRequestES = requestConverter + .indicesPutIndexTemplateRequest(putIndexTemplateRequest); + + return execute(client -> client.putIndexTemplate(putIndexTemplateRequestES)).acknowledged(); + } + + @Override + public boolean existsIndexTemplate(ExistsIndexTemplateRequest existsIndexTemplateRequest) { + + Assert.notNull(existsIndexTemplateRequest, "existsIndexTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.ExistsIndexTemplateRequest existsTemplateRequestES = requestConverter + .indicesExistsIndexTemplateRequest(existsIndexTemplateRequest); + return execute(client -> client.existsIndexTemplate(existsTemplateRequestES)).value(); + } + + @Override + public List getIndexTemplate(GetIndexTemplateRequest getIndexTemplateRequest) { + + Assert.notNull(getIndexTemplateRequest, "getIndexTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.GetIndexTemplateRequest getIndexTemplateRequestES = requestConverter + .indicesGetIndexTemplateRequest(getIndexTemplateRequest); + var getIndexTemplateResponse = execute(client -> client.getIndexTemplate(getIndexTemplateRequestES)); + return responseConverter.getIndexTemplates(getIndexTemplateResponse); + } + + @Override + public boolean deleteIndexTemplate(DeleteIndexTemplateRequest deleteIndexTemplateRequest) { + + Assert.notNull(deleteIndexTemplateRequest, "deleteIndexTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.DeleteIndexTemplateRequest deleteIndexTemplateRequestES = requestConverter + .indicesDeleteIndexTemplateRequest(deleteIndexTemplateRequest); + return execute(client -> client.deleteIndexTemplate(deleteIndexTemplateRequestES)).acknowledged(); + } + + @Override + public boolean putComponentTemplate(PutComponentTemplateRequest putComponentTemplateRequest) { + + Assert.notNull(putComponentTemplateRequest, "putComponentTemplateRequest must not be null"); + + org.opensearch.client.opensearch.cluster.PutComponentTemplateRequest putComponentTemplateRequestES = requestConverter + .clusterPutComponentTemplateRequest(putComponentTemplateRequest); + // the new Elasticsearch client has this call in the cluster index + return clusterTemplate.execute(client -> client.putComponentTemplate(putComponentTemplateRequestES)).acknowledged(); + } + + @Override + public boolean existsComponentTemplate(ExistsComponentTemplateRequest existsComponentTemplateRequest) { + + Assert.notNull(existsComponentTemplateRequest, "existsComponentTemplateRequest must not be null"); + + org.opensearch.client.opensearch.cluster.ExistsComponentTemplateRequest existsComponentTemplateRequestES = requestConverter + .clusterExistsComponentTemplateRequest(existsComponentTemplateRequest); + return clusterTemplate.execute(client -> client.existsComponentTemplate(existsComponentTemplateRequestES)).value(); + } + + @Override + public List getComponentTemplate(GetComponentTemplateRequest getComponentTemplateRequest) { + + org.opensearch.client.opensearch.cluster.GetComponentTemplateRequest getComponentTemplateRequestES = requestConverter + .clusterGetComponentTemplateRequest(getComponentTemplateRequest); + var response = clusterTemplate.execute(client -> client.getComponentTemplate(getComponentTemplateRequestES)); + return responseConverter.clusterGetComponentTemplates(response); + } + + @Override + public boolean deleteComponentTemplate(DeleteComponentTemplateRequest deleteComponentTemplateRequest) { + + Assert.notNull(deleteComponentTemplateRequest, "deleteComponentTemplateRequest must not be null"); + + org.opensearch.client.opensearch.cluster.DeleteComponentTemplateRequest deleteComponentTemplateRequestES = requestConverter + .clusterDeleteComponentTemplateRequest(deleteComponentTemplateRequest); + return clusterTemplate.execute(client -> client.deleteComponentTemplate(deleteComponentTemplateRequestES)) + .acknowledged(); + } + + @Override + public List getInformation(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + GetIndexRequest getIndexRequest = requestConverter.indicesGetIndexRequest(indexCoordinates); + GetIndexResponse getIndexResponse = execute(client -> client.get(getIndexRequest)); + return responseConverter.indicesGetIndexInformations(getIndexResponse); + } + + // region Helper functions + ElasticsearchPersistentEntity getRequiredPersistentEntity(Class clazz) { + return elasticsearchConverter.getMappingContext().getRequiredPersistentEntity(clazz); + } + + @Override + public IndexCoordinates getIndexCoordinates() { + return (boundClass != null) ? getIndexCoordinatesFor(boundClass) : Objects.requireNonNull(boundIndex); + } + + public IndexCoordinates getIndexCoordinatesFor(Class clazz) { + return getRequiredPersistentEntity(clazz).getIndexCoordinates(); + } + // endregion +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonUtils.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonUtils.java new file mode 100644 index 0000000..02d7f18 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonUtils.java @@ -0,0 +1,68 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import jakarta.json.stream.JsonGenerator; +import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonpMapper; +import org.springframework.lang.Nullable; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +final class JsonUtils { + + private static final Log LOGGER = LogFactory.getLog(JsonUtils.class); + + private JsonUtils() {} + + public static String toJson(Object object, JsonpMapper mapper) { + + // noinspection SpellCheckingInspection + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); + mapper.serialize(object, generator); + generator.close(); + String json = "{}"; + try { + json = baos.toString("UTF-8"); + } catch (UnsupportedEncodingException e) { + LOGGER.warn("could not read json", e); + } + + return json; + } + + @Nullable + public static String queryToJson(@Nullable org.opensearch.client.opensearch._types.query_dsl.Query query, JsonpMapper mapper) { + + if (query == null) { + return null; + } + + var baos = new ByteArrayOutputStream(); + var generator = mapper.jsonProvider().createGenerator(baos); + query.serialize(generator, mapper); + generator.close(); + return baos.toString(StandardCharsets.UTF_8); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonpUtils.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonpUtils.java new file mode 100644 index 0000000..907a6a1 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonpUtils.java @@ -0,0 +1,70 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import jakarta.json.JsonException; +import jakarta.json.spi.JsonProvider; +import jakarta.json.stream.JsonGenerator; +import jakarta.json.stream.JsonParser; +import java.io.StringReader; +import org.opensearch.client.json.JsonpDeserializer; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.JsonpMapperBase; +import org.opensearch.client.json.JsonpSerializable; +import org.springframework.data.elasticsearch.core.document.Document; + +final class JsonpUtils { + static final JsonProvider DEFAULT_PROVIDER = provider(); + + static final JsonpMapper DEFAULT_JSONP_MAPPER = new JsonpMapperBase() { + @Override + public JsonProvider jsonProvider() { + return DEFAULT_PROVIDER; + } + + @Override + public void serialize(T value, JsonGenerator generator) { + if (value instanceof JsonpSerializable) { + ((JsonpSerializable) value).serialize(generator, this); + return; + } + + throw new JsonException( + "Cannot find a serializer for type " + value.getClass().getName() + + ". Consider using a full-featured JsonpMapper" + ); + } + + @Override + protected JsonpDeserializer getDefaultDeserializer(Class clazz) { + throw new JsonException( + "Cannot find a default deserializer for type " + clazz.getName() + + ". Consider using a full-featured JsonpMapper"); + } + }; + + private JsonpUtils() {} + + static JsonProvider provider() { + return JsonProvider.provider(); + } + + static T fromJson(Document document, JsonpDeserializer deserializer) { + try (JsonParser parser = DEFAULT_JSONP_MAPPER.jsonProvider().createParser(new StringReader(document.toJson()))) { + return deserializer.deserialize(parser, DEFAULT_JSONP_MAPPER); + } + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQuery.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQuery.java new file mode 100644 index 0000000..4e8bd9d --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQuery.java @@ -0,0 +1,133 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.query_dsl.KnnQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch.core.search.FieldCollapse; +import org.opensearch.client.opensearch.core.search.Suggester; +import org.springframework.data.elasticsearch.core.query.BaseQuery; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * A {@link org.springframework.data.elasticsearch.core.query.Query} implementation using query builders from the new + * OpenSearch Client library. + * + * @author Peter-Josef Meisch + * @author Sascha Woo + * @since 4.4 + */ +public class NativeQuery extends BaseQuery { + + @Nullable private final Query query; + @Nullable private org.springframework.data.elasticsearch.core.query.Query springDataQuery; + @Nullable private Query filter; + // note: the new client does not have pipeline aggs, these are just set up as normal aggs + private final Map aggregations = new LinkedHashMap<>(); + @Nullable private Suggester suggester; + @Nullable private FieldCollapse fieldCollapse; + private List sortOptions = Collections.emptyList(); + + private Map searchExtensions = Collections.emptyMap(); + @Nullable private KnnQuery knnQuery; + + public NativeQuery(NativeQueryBuilder builder) { + super(builder); + this.query = builder.getQuery(); + this.filter = builder.getFilter(); + this.aggregations.putAll(builder.getAggregations()); + this.suggester = builder.getSuggester(); + this.fieldCollapse = builder.getFieldCollapse(); + this.sortOptions = builder.getSortOptions(); + this.searchExtensions = builder.getSearchExtensions(); + + if (builder.getSpringDataQuery() != null) { + Assert.isTrue(!NativeQuery.class.isAssignableFrom(builder.getSpringDataQuery().getClass()), + "Cannot add an NativeQuery in a NativeQuery"); + } + this.springDataQuery = builder.getSpringDataQuery(); + this.knnQuery = builder.getKnnQuery(); + } + + public NativeQuery(@Nullable Query query) { + this.query = query; + } + + public static NativeQueryBuilder builder() { + return new NativeQueryBuilder(); + } + + @Nullable + public Query getQuery() { + return query; + } + + @Nullable + public Query getFilter() { + return filter; + } + + public Map getAggregations() { + return aggregations; + } + + @Nullable + public Suggester getSuggester() { + return suggester; + } + + @Nullable + public FieldCollapse getFieldCollapse() { + return fieldCollapse; + } + + public List getSortOptions() { + return sortOptions; + } + + public Map getSearchExtensions() { + return searchExtensions; + } + + /** + * @see NativeQueryBuilder#withQuery(org.springframework.data.elasticsearch.core.query.Query) + * @since 5.1 + */ + public void setSpringDataQuery(@Nullable org.springframework.data.elasticsearch.core.query.Query springDataQuery) { + this.springDataQuery = springDataQuery; + } + + /** + * @since 5.1 + */ + @Nullable + public KnnQuery getKnnQuery() { + return knnQuery; + } + + @Nullable + public org.springframework.data.elasticsearch.core.query.Query getSpringDataQuery() { + return springDataQuery; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQueryBuilder.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQueryBuilder.java new file mode 100644 index 0000000..ec54b05 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQueryBuilder.java @@ -0,0 +1,214 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.query_dsl.KnnQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch.core.search.FieldCollapse; +import org.opensearch.client.opensearch.core.search.Suggester; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * @author Peter-Josef Meisch + * @author Sascha Woo + * @since 4.4 + */ +public class NativeQueryBuilder extends BaseQueryBuilder { + + @Nullable private Query query; + @Nullable private Query filter; + private final Map aggregations = new LinkedHashMap<>(); + @Nullable private Suggester suggester; + @Nullable private FieldCollapse fieldCollapse; + private List sortOptions = new ArrayList<>(); + private Map searchExtensions = new LinkedHashMap<>(); + + @Nullable private org.springframework.data.elasticsearch.core.query.Query springDataQuery; + @Nullable private KnnQuery knnQuery; + + public NativeQueryBuilder() {} + + @Nullable + public Query getQuery() { + return query; + } + + @Nullable + public Query getFilter() { + return this.filter; + } + + public Map getAggregations() { + return aggregations; + } + + @Nullable + public Suggester getSuggester() { + return suggester; + } + + @Nullable + public FieldCollapse getFieldCollapse() { + return fieldCollapse; + } + + public List getSortOptions() { + return sortOptions; + } + + public Map getSearchExtensions() { + return this.searchExtensions; + } + + @Nullable + public KnnQuery getKnnQuery() { + return knnQuery; + } + + @Nullable + public org.springframework.data.elasticsearch.core.query.Query getSpringDataQuery() { + return springDataQuery; + } + + public NativeQueryBuilder withQuery(Query query) { + + Assert.notNull(query, "query must not be null"); + + this.query = query; + return this; + } + + public NativeQueryBuilder withQuery(Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + + return withQuery(fn.apply(new Query.Builder()).build()); + } + + public NativeQueryBuilder withFilter(@Nullable Query filter) { + this.filter = filter; + return this; + } + + public NativeQueryBuilder withFilter(Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + + return withFilter(fn.apply(new Query.Builder()).build()); + } + + public NativeQueryBuilder withAggregation(String name, Aggregation aggregation) { + + Assert.notNull(name, "name must not be null"); + Assert.notNull(aggregation, "aggregation must not be null"); + + this.aggregations.put(name, aggregation); + return this; + } + + public NativeQueryBuilder withSuggester(@Nullable Suggester suggester) { + this.suggester = suggester; + return this; + } + + public NativeQueryBuilder withFieldCollapse(@Nullable FieldCollapse fieldCollapse) { + this.fieldCollapse = fieldCollapse; + return this; + } + + public NativeQueryBuilder withSort(List values) { + + Assert.notEmpty(values, "values must not be empty"); + + sortOptions.clear(); + sortOptions.addAll(values); + + return this; + } + + public NativeQueryBuilder withSort(SortOptions value, SortOptions... values) { + + Assert.notNull(value, "value must not be null"); + sortOptions.add(value); + if (values.length > 0) { + sortOptions.addAll(Arrays.asList(values)); + } + + return this; + } + + public NativeQueryBuilder withSort(Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + withSort(fn.apply(new SortOptions.Builder()).build()); + + return this; + } + + public NativeQueryBuilder withSearchExtension(String key, JsonData value) { + + Assert.notNull(key, "key must not be null"); + Assert.notNull(value, "value must not be null"); + + searchExtensions.put(key, value); + return this; + } + + public NativeQueryBuilder withSearchExtensions(Map searchExtensions) { + + Assert.notNull(searchExtensions, "searchExtensions must not be null"); + + this.searchExtensions.putAll(searchExtensions); + return this; + } + + /** + * Allows to use a {@link org.springframework.data.elasticsearch.core.query.Query} within a NativeQuery. Cannot be + * used together with {@link #withQuery(Query)} that sets an Elasticsearch query. Passing in a {@link NativeQuery} + * will result in an exception when {@link #build()} is called. + * + * @since 5.1 + */ + public NativeQueryBuilder withQuery(org.springframework.data.elasticsearch.core.query.Query query) { + this.springDataQuery = query; + return this; + } + + /** + * @since 5.1 + */ + public NativeQueryBuilder withKnnQuery(KnnQuery knnQuery) { + this.knnQuery = knnQuery; + return this; + } + + public NativeQuery build() { + Assert.isTrue(query == null || springDataQuery == null, "Cannot have both a native query and a Spring Data query"); + return new NativeQuery(this); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregation.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregation.java new file mode 100644 index 0000000..5d0dd82 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregation.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.springframework.data.elasticsearch.core.AggregationContainer; + +/** + * {@link AggregationContainer} for a {@link Aggregation} that holds OpenEearch data. + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class OpenSearchAggregation implements AggregationContainer { + + private final Aggregation aggregation; + + public OpenSearchAggregation(Aggregation aggregation) { + this.aggregation = aggregation; + } + + @Override + public Aggregation aggregation() { + return aggregation; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregations.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregations.java new file mode 100644 index 0000000..42a0d56 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregations.java @@ -0,0 +1,76 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.opensearch.client.opensearch._types.aggregations.Aggregate; +import org.springframework.data.elasticsearch.core.AggregationsContainer; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * AggregationsContainer implementation for the OpenSearch aggregations. + * + * @author Peter-Josef Meisch + * @author Sascha Woo + * @since 4.4 + */ +public class OpenSearchAggregations implements AggregationsContainer> { + + private final List aggregations; + private final Map aggregationsAsMap; + + public OpenSearchAggregations(Map aggregations) { + + Assert.notNull(aggregations, "aggregations must not be null"); + + aggregationsAsMap = new HashMap<>(); + aggregations.forEach((name, aggregate) -> aggregationsAsMap // + .put(name, new OpenSearchAggregation(new Aggregation(name, aggregate)))); + + this.aggregations = new ArrayList<>(aggregationsAsMap.values()); + } + + @Override + public List aggregations() { + return aggregations; + } + + /** + * @return the {@link OpenSearchAggregation}s keyed by aggregation name. + */ + public Map aggregationsAsMap() { + return aggregationsAsMap; + } + + /** + * Returns the aggregation that is associated with the specified name. + * + * @param name the name of the aggregation + * @return the aggregation or {@literal null} if not found + */ + @Nullable + public OpenSearchAggregation get(String name) { + + Assert.notNull(name, "name must not be null"); + + return aggregationsAsMap.get(name); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientBeanDefinitionParser.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientBeanDefinitionParser.java new file mode 100644 index 0000000..d3ab87c --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientBeanDefinitionParser.java @@ -0,0 +1,47 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.springframework.beans.factory.support.AbstractBeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.xml.AbstractBeanDefinitionParser; +import org.springframework.beans.factory.xml.ParserContext; +import org.w3c.dom.Element; + +/** + * @author Peter-Josef Meisch + * @since 5.0 + */ +public class OpenSearchClientBeanDefinitionParser extends AbstractBeanDefinitionParser { + + @Override + protected AbstractBeanDefinition parseInternal(Element element, ParserContext parserContext) { + BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(OpenSearchClientFactoryBean.class); + setConfigurations(element, builder); + return getSourcedBeanDefinition(builder, element, parserContext); + } + + private void setConfigurations(Element element, BeanDefinitionBuilder builder) { + builder.addPropertyValue("hosts", element.getAttribute("hosts")); + } + + private AbstractBeanDefinition getSourcedBeanDefinition(BeanDefinitionBuilder builder, Element source, + ParserContext context) { + AbstractBeanDefinition definition = builder.getBeanDefinition(); + definition.setSource(context.extractSource(source)); + return definition; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientFactoryBean.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientFactoryBean.java new file mode 100644 index 0000000..23f331f --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientFactoryBean.java @@ -0,0 +1,96 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.FactoryBeanNotInitializedException; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * OpenSearchClientFactoryBean + * + * @author Peter-Josef Meisch + * @since 5.0 + */ +public class OpenSearchClientFactoryBean + implements FactoryBean, InitializingBean, DisposableBean { + + private static final Log LOGGER = LogFactory.getLog(OpenSearchClientFactoryBean.class); + + private @Nullable AutoCloseableOpenSearchClient client; + private String hosts = "http://localhost:9200"; + static final String COMMA = ","; + + @Override + public void destroy() { + try { + LOGGER.info("Closing elasticSearch client"); + if (client != null) { + client.close(); + } + } catch (final Exception e) { + LOGGER.error("Error closing ElasticSearch client: ", e); + } + } + + @Override + public void afterPropertiesSet() throws Exception { + buildClient(); + } + + @Override + public OpenSearchClient getObject() { + + if (client == null) { + throw new FactoryBeanNotInitializedException(); + } + + return client; + } + + @Override + public Class getObjectType() { + return OpenSearchClient.class; + } + + @Override + public boolean isSingleton() { + return false; + } + + protected void buildClient() throws Exception { + + Assert.hasText(hosts, "[Assertion Failed] At least one host must be set."); + + var clientConfiguration = ClientConfiguration.builder().connectedTo(hosts).build(); + client = (AutoCloseableOpenSearchClient) OpenSearchClients.createImperative(clientConfiguration); + } + + public void setHosts(String hosts) { + this.hosts = hosts; + } + + public String getHosts() { + return this.hosts; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClients.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClients.java new file mode 100644 index 0000000..e146147 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClients.java @@ -0,0 +1,402 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.net.InetSocketAddress; +import java.security.KeyManagementException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import javax.net.ssl.SSLContext; +import javax.net.ssl.X509TrustManager; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.message.BasicHeader; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.protocol.HttpContext; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.TransportOptions; +import org.opensearch.client.transport.Version; +import org.opensearch.client.transport.rest_client.RestClientOptions; +import org.opensearch.client.transport.rest_client.RestClientTransport; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.support.HttpHeaders; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Utility class to create the different OpenSearch clients + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +@SuppressWarnings("unused") +public final class OpenSearchClients { + public static final String IMPERATIVE_CLIENT = "imperative"; + + /** + * Name of whose value can be used to correlate log messages for this request. + */ + private static final String X_SPRING_DATA_OPENSEARCH_CLIENT = "X-SpringDataOpenSearch-Client"; + private static final JsonpMapper DEFAULT_JSONP_MAPPER = new JacksonJsonpMapper(); + + + // region imperative client + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param clientConfiguration configuration options, must not be {@literal null}. + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(ClientConfiguration clientConfiguration) { + return createImperative(getRestClient(clientConfiguration), null, DEFAULT_JSONP_MAPPER); + } + + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param clientConfiguration configuration options, must not be {@literal null}. + * @param transportOptions options to be added to each request. + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(ClientConfiguration clientConfiguration, + TransportOptions transportOptions) { + return createImperative(getRestClient(clientConfiguration), transportOptions, DEFAULT_JSONP_MAPPER); + } + + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param restClient the RestClient to use + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(RestClient restClient) { + return createImperative(restClient, null, DEFAULT_JSONP_MAPPER); + } + + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param restClient the RestClient to use + * @param transportOptions options to be added to each request. + * @param jsonpMapper the mapper for the transport to use + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(RestClient restClient, @Nullable TransportOptions transportOptions, + JsonpMapper jsonpMapper) { + + Assert.notNull(restClient, "restClient must not be null"); + + OpenSearchTransport transport = getOpenSearchTransport(restClient, IMPERATIVE_CLIENT, transportOptions, + jsonpMapper); + + return createImperative(transport); + } + + /** + * Creates a new {@link OpenSearchClient} that uses the given {@link OpenSearchTransport}. + * + * @param transport the transport to use + * @return the {@link OpenSearchClient} + */ + public static AutoCloseableOpenSearchClient createImperative(OpenSearchTransport transport) { + + Assert.notNull(transport, "transport must not be null"); + + return new AutoCloseableOpenSearchClient(transport); + } + // endregion + + // region low level RestClient + private static RestClientOptions.Builder getRestClientOptionsBuilder(@Nullable TransportOptions transportOptions) { + + if (transportOptions instanceof RestClientOptions restClientOptions) { + return restClientOptions.toBuilder(); + } + + var builder = new RestClientOptions.Builder(RequestOptions.DEFAULT.toBuilder()); + + if (transportOptions != null) { + transportOptions.headers().forEach(header -> builder.addHeader(header.getKey(), header.getValue())); + transportOptions.queryParameters().forEach(builder::setParameter); + builder.onWarnings(transportOptions.onWarnings()); + } + + return builder; + } + + /** + * Creates a low level {@link RestClient} for the given configuration. + * + * @param clientConfiguration must not be {@literal null} + * @return the {@link RestClient} + */ + public static RestClient getRestClient(ClientConfiguration clientConfiguration) { + return getRestClientBuilder(clientConfiguration).build(); + } + + private static RestClientBuilder getRestClientBuilder(ClientConfiguration clientConfiguration) { + HttpHost[] httpHosts = formattedHosts(clientConfiguration.getEndpoints(), clientConfiguration.useSsl()).stream() + .map(HttpHost::create).toArray(HttpHost[]::new); + RestClientBuilder builder = RestClient.builder(httpHosts); + + if (clientConfiguration.getPathPrefix() != null) { + builder.setPathPrefix(clientConfiguration.getPathPrefix()); + } + + HttpHeaders headers = clientConfiguration.getDefaultHeaders(); + + if (!headers.isEmpty()) { + builder.setDefaultHeaders(toHeaderArray(headers)); + } + + builder.setHttpClientConfigCallback(clientBuilder -> { + if (clientConfiguration.getCaFingerprint().isPresent()) { + clientBuilder + .setSSLContext(sslContextFromCaFingerprint(clientConfiguration.getCaFingerprint().get())); + } + clientConfiguration.getSslContext().ifPresent(clientBuilder::setSSLContext); + clientConfiguration.getHostNameVerifier().ifPresent(clientBuilder::setSSLHostnameVerifier); + clientBuilder.addInterceptorLast(new CustomHeaderInjector(clientConfiguration.getHeadersSupplier())); + + RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); + Duration connectTimeout = clientConfiguration.getConnectTimeout(); + + if (!connectTimeout.isNegative()) { + requestConfigBuilder.setConnectTimeout(Math.toIntExact(connectTimeout.toMillis())); + } + + Duration socketTimeout = clientConfiguration.getSocketTimeout(); + + if (!socketTimeout.isNegative()) { + requestConfigBuilder.setSocketTimeout(Math.toIntExact(socketTimeout.toMillis())); + requestConfigBuilder.setConnectionRequestTimeout(Math.toIntExact(socketTimeout.toMillis())); + } + + clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); + + clientConfiguration.getProxy().map(HttpHost::create).ifPresent(clientBuilder::setProxy); + + for (ClientConfiguration.ClientConfigurationCallback clientConfigurer : clientConfiguration + .getClientConfigurers()) { + if (clientConfigurer instanceof OpenSearchHttpClientConfigurationCallback restClientConfigurationCallback) { + clientBuilder = restClientConfigurationCallback.configure(clientBuilder); + } + } + + return clientBuilder; + }); + + for (ClientConfiguration.ClientConfigurationCallback clientConfigurationCallback : clientConfiguration + .getClientConfigurers()) { + if (clientConfigurationCallback instanceof OpenSearchRestClientConfigurationCallback configurationCallback) { + builder = configurationCallback.configure(builder); + } + } + return builder; + } + // endregion + + // region OpenSearch transport + /** + * Creates an {@link OpenSearchTransport} that will use the given client that additionally is customized with a + * header to contain the clientType + * + * @param restClient the client to use + * @param clientType the client type to pass in each request as header + * @param transportOptions options for the transport + * @param jsonpMapper mapper for the transport + * @return OpenSearchTransport + */ + public static OpenSearchTransport getOpenSearchTransport(RestClient restClient, String clientType, + @Nullable TransportOptions transportOptions, JsonpMapper jsonpMapper) { + + Assert.notNull(restClient, "restClient must not be null"); + Assert.notNull(clientType, "clientType must not be null"); + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + TransportOptions.Builder transportOptionsBuilder = transportOptions != null ? transportOptions.toBuilder() + : new RestClientOptions(RequestOptions.DEFAULT).toBuilder(); + + RestClientOptions.Builder restClientOptionsBuilder = getRestClientOptionsBuilder(transportOptions); + + ContentType jsonContentType = Version.VERSION == null ? ContentType.APPLICATION_JSON + : ContentType.create("application/vnd.opensearch+json", + new BasicNameValuePair("compatible-with", String.valueOf(Version.VERSION.major()))); + + Consumer setHeaderIfNotPresent = header -> { + if (restClientOptionsBuilder.build().headers().stream() // + .noneMatch((h) -> h.getKey().equalsIgnoreCase(header))) { + // need to add the compatibility header, this is only done automatically when not passing in custom options. + // code copied from RestClientTransport as it is not available outside the package + restClientOptionsBuilder.addHeader(header, jsonContentType.toString()); + } + }; + + setHeaderIfNotPresent.accept("Content-Type"); + setHeaderIfNotPresent.accept("Accept"); + + restClientOptionsBuilder.addHeader(X_SPRING_DATA_OPENSEARCH_CLIENT, clientType); + + return new RestClientTransport(restClient, jsonpMapper, restClientOptionsBuilder.build()); + } + // endregion + + private static List formattedHosts(List hosts, boolean useSsl) { + return hosts.stream().map(it -> (useSsl ? "https" : "http") + "://" + it.getHostString() + ':' + it.getPort()) + .collect(Collectors.toList()); + } + + private static org.apache.http.Header[] toHeaderArray(HttpHeaders headers) { + return headers.entrySet().stream() // + .flatMap(entry -> entry.getValue().stream() // + .map(value -> new BasicHeader(entry.getKey(), value))) // + .toArray(org.apache.http.Header[]::new); + } + + /** + * Interceptor to inject custom supplied headers. + * + * @since 4.4 + */ + private record CustomHeaderInjector(Supplier headersSupplier) implements HttpRequestInterceptor { + + @Override + public void process(HttpRequest request, HttpContext context) { + HttpHeaders httpHeaders = headersSupplier.get(); + + if (httpHeaders != null && !httpHeaders.isEmpty()) { + Arrays.stream(toHeaderArray(httpHeaders)).forEach(request::addHeader); + } + } + } + + /** + * {@link org.springframework.data.elasticsearch.client.ClientConfiguration.ClientConfigurationCallback} to configure + * the OpenSearch RestClient's Http client with a {@link HttpAsyncClientBuilder} + * + * @since 4.4 + */ + public interface OpenSearchHttpClientConfigurationCallback + extends ClientConfiguration.ClientConfigurationCallback { + + static OpenSearchHttpClientConfigurationCallback from( + Function httpClientBuilderCallback) { + + Assert.notNull(httpClientBuilderCallback, "httpClientBuilderCallback must not be null"); + + return httpClientBuilderCallback::apply; + } + } + + /** + * {@link org.springframework.data.elasticsearch.client.ClientConfiguration.ClientConfigurationCallback} to configure + * the RestClient client with a {@link RestClientBuilder} + * + * @since 5.0 + */ + public interface OpenSearchRestClientConfigurationCallback + extends ClientConfiguration.ClientConfigurationCallback { + + static OpenSearchRestClientConfigurationCallback from( + Function restClientBuilderCallback) { + + Assert.notNull(restClientBuilderCallback, "restClientBuilderCallback must not be null"); + + return restClientBuilderCallback::apply; + } + } + + /** + * Copy / paste of co.elastic.clients.transport.TransportUtils#sslContextFromCaFingerprint (licensed under ASFv2), since + * OpenSearch Java client does not support such SSL configuration at the moment. + */ + private static SSLContext sslContextFromCaFingerprint(String fingerPrint) { + + fingerPrint = fingerPrint.replace(":", ""); + int len = fingerPrint.length(); + byte[] fpBytes = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + fpBytes[i / 2] = (byte) ( + (Character.digit(fingerPrint.charAt(i), 16) << 4) + + Character.digit(fingerPrint.charAt(i+1), 16) + ); + } + + try { + X509TrustManager tm = new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + throw new CertificateException("This is a client-side only trust manager"); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + + // The CA root is the last element of the chain + X509Certificate anchor = chain[chain.length - 1]; + + byte[] bytes; + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + md.update(anchor.getEncoded()); + bytes = md.digest(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + + if (Arrays.equals(fpBytes, bytes)) { + return; + } + + throw new CertificateException("Untrusted certificate: " + anchor.getSubjectX500Principal()); + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + }; + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new X509TrustManager[] { tm }, null); + return sslContext; + + } catch (NoSuchAlgorithmException | KeyManagementException e) { + // Exceptions that should normally not occur + throw new RuntimeException(e); + } + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchConfiguration.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchConfiguration.java new file mode 100644 index 0000000..20a72e5 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchConfiguration.java @@ -0,0 +1,129 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestClient; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.TransportOptions; +import org.opensearch.client.transport.rest_client.RestClientOptions; +import org.springframework.context.annotation.Bean; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.config.ElasticsearchConfigurationSupport; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.util.Assert; + +/** + * Base class for a @{@link org.springframework.context.annotation.Configuration} class to set up the OpenSearch + * connection using the OpenSearch Client. This class exposes different parts of the setup as Spring beans. Deriving + * classes must provide the {@link ClientConfiguration} to use. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public abstract class OpenSearchConfiguration extends ElasticsearchConfigurationSupport { + + /** + * Must be implemented by deriving classes to provide the {@link ClientConfiguration}. + * + * @return configuration, must not be {@literal null} + */ + @Bean(name = "elasticsearchClientConfiguration") + public abstract ClientConfiguration clientConfiguration(); + + /** + * Provides the underlying low level OpenSearch RestClient. + * + * @param clientConfiguration configuration for the client, must not be {@literal null} + * @return RestClient + */ + @Bean + public RestClient elasticsearchRestClient(ClientConfiguration clientConfiguration) { + + Assert.notNull(clientConfiguration, "clientConfiguration must not be null"); + + return OpenSearchClients.getRestClient(clientConfiguration); + } + + /** + * Provides the OpenSearch transport to be used. The default implementation uses the {@link RestClient} bean and + * the {@link JsonpMapper} bean provided in this class. + * + * @return the {@link OpenSearchTransport} + * @since 5.2 + */ + @Bean + public OpenSearchTransport elasticsearchTransport(RestClient restClient, JsonpMapper jsonpMapper) { + + Assert.notNull(restClient, "restClient must not be null"); + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + return OpenSearchClients.getOpenSearchTransport(restClient, OpenSearchClients.IMPERATIVE_CLIENT, + transportOptions(), jsonpMapper); + } + + /** + * Provides the {@link OpenSearchClient} to be used. + * + * @param transport the {@link OpenSearchTransport} to use + * @return OpenSearchClient instance + */ + @Bean + public OpenSearchClient elasticsearchClient(OpenSearchTransport transport) { + + Assert.notNull(transport, "transport must not be null"); + + return OpenSearchClients.createImperative(transport); + } + + /** + * Creates a {@link ElasticsearchOperations} implementation using an + * {@link org.opensearch.client.opensearch.OpenSearchClient}. + * + * @return never {@literal null}. + */ + @Bean(name = { "elasticsearchOperations", "elasticsearchTemplate", "opensearchOperations", "opensearchTemplate" }) + public ElasticsearchOperations elasticsearchOperations(ElasticsearchConverter elasticsearchConverter, + OpenSearchClient elasticsearchClient) { + + OpenSearchTemplate template = new OpenSearchTemplate(elasticsearchClient, elasticsearchConverter); + template.setRefreshPolicy(refreshPolicy()); + + return template; + } + + /** + * Provides the JsonpMapper bean that is used in the {@link #elasticsearchTransport(RestClient, JsonpMapper)} method. + * + * @return the {@link JsonpMapper} to use + * @since 5.2 + */ + @Bean + public JsonpMapper jsonpMapper() { + return new JacksonJsonpMapper(); + } + + /** + * @return the options that should be added to every request. Must not be {@literal null} + */ + public TransportOptions transportOptions() { + return new RestClientOptions(RequestOptions.DEFAULT); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchExceptionTranslator.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchExceptionTranslator.java new file mode 100644 index 0000000..54f0bd5 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchExceptionTranslator.java @@ -0,0 +1,136 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.opensearch.client.ResponseException; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.ErrorResponse; +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.springframework.dao.DataAccessException; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.dao.support.PersistenceExceptionTranslator; +import org.springframework.data.elasticsearch.NoSuchIndexException; +import org.springframework.data.elasticsearch.ResourceNotFoundException; +import org.springframework.data.elasticsearch.UncategorizedElasticsearchException; +import org.springframework.data.elasticsearch.VersionConflictException; + +/** + * Simple {@link PersistenceExceptionTranslator} for OpenSearch. Convert the given runtime exception to an + * appropriate exception from the {@code org.springframework.dao} hierarchy. Return {@literal null} if no translation is + * appropriate: any other exception may have resulted from user code, and should not be translated. + * + * @author Peter-Josef Meisch + * @author Junghoon Ban + * @since 4.4 + */ +public class OpenSearchExceptionTranslator implements PersistenceExceptionTranslator { + + private final JsonpMapper jsonpMapper; + + public OpenSearchExceptionTranslator(JsonpMapper jsonpMapper) { + this.jsonpMapper = jsonpMapper; + } + + /** + * translates an Exception if possible. Exceptions that are no {@link RuntimeException}s are wrapped in a + * RuntimeException + * + * @param throwable the Exception to map + * @return the potentially translated RuntimeException. + */ + public RuntimeException translateException(Throwable throwable) { + + RuntimeException runtimeException = throwable instanceof RuntimeException ex ? ex + : new RuntimeException(throwable.getMessage(), throwable); + RuntimeException potentiallyTranslatedException = translateExceptionIfPossible(runtimeException); + + return potentiallyTranslatedException != null ? potentiallyTranslatedException : runtimeException; + } + + @Override + public DataAccessException translateExceptionIfPossible(RuntimeException ex) { + + checkForConflictException(ex); + + if (ex instanceof OpenSearchException openSearchException) { + + ErrorResponse response = openSearchException.response(); + var errorType = response.error().type(); + var errorReason = response.error().reason() != null ? response.error().reason() : "undefined reason"; + + if (response.status() == 404) { + + if ("index_not_found_exception".equals(errorType)) { + // noinspection RegExpRedundantEscape + Pattern pattern = Pattern.compile(".*no such index \\[(.*)\\]"); + String index = ""; + Matcher matcher = pattern.matcher(errorReason); + if (matcher.matches()) { + index = matcher.group(1); + } + return new NoSuchIndexException(index); + } + + return new ResourceNotFoundException(errorReason); + } + + if (response.status() == 409) { + + } + String body = JsonUtils.toJson(response, jsonpMapper); + + if (errorType != null && errorType.contains("validation_exception")) { + return new DataIntegrityViolationException(errorReason); + } + + return new UncategorizedElasticsearchException(ex.getMessage(), response.status(), body, ex); + } + + Throwable cause = ex.getCause(); + if (cause instanceof IOException) { + return new DataAccessResourceFailureException(ex.getMessage(), ex); + } + + return null; + } + + private void checkForConflictException(Throwable exception) { + Integer status = null; + String message = null; + + if (exception instanceof ResponseException responseException) { + status = responseException.getResponse().getStatusLine().getStatusCode(); + message = responseException.getMessage(); + } else if (exception.getCause() != null) { + checkForConflictException(exception.getCause()); + } + + if (status != null && message != null) { + if (status == 409 && message.contains("type\":\"version_conflict_engine_exception")) + if (message.contains("version conflict, required seqNo")) { + throw new OptimisticLockingFailureException("Cannot index a document due to seq_no+primary_term conflict", + exception); + } else if (message.contains("version conflict, current version [")) { + throw new VersionConflictException("Version conflict", exception); + } + } + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchTemplate.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchTemplate.java new file mode 100644 index 0000000..819cb9a --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchTemplate.java @@ -0,0 +1,723 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.TypeUtils.*; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch.core.*; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; +import org.opensearch.client.opensearch.core.msearch.MultiSearchResponseItem; +import org.opensearch.client.opensearch.core.pit.DeletePitRequest; +import org.opensearch.client.opensearch.core.search.SearchResult; +import org.opensearch.client.transport.Version; +import org.springframework.data.elasticsearch.BulkFailureException; +import org.springframework.data.elasticsearch.client.UnsupportedBackendOperation; +import org.springframework.data.elasticsearch.core.AbstractElasticsearchTemplate; +import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.IndexedObjectInformation; +import org.springframework.data.elasticsearch.core.MultiGetItem; +import org.springframework.data.elasticsearch.core.SearchHits; +import org.springframework.data.elasticsearch.core.SearchScrollHits; +import org.springframework.data.elasticsearch.core.cluster.ClusterOperations; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.data.elasticsearch.core.query.BulkOptions; +import org.springframework.data.elasticsearch.core.query.ByQueryResponse; +import org.springframework.data.elasticsearch.core.query.IndexQuery; +import org.springframework.data.elasticsearch.core.query.MoreLikeThisQuery; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.SearchTemplateQuery; +import org.springframework.data.elasticsearch.core.query.UpdateQuery; +import org.springframework.data.elasticsearch.core.query.UpdateResponse; +import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; +import org.springframework.data.elasticsearch.core.reindex.ReindexResponse; +import org.springframework.data.elasticsearch.core.script.Script; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Implementation of {@link org.springframework.data.elasticsearch.core.ElasticsearchOperations} using the new + * OpenSearch client. + * + * @author Peter-Josef Meisch + * @author Hamid Rahimi + * @author Illia Ulianov + * @author Haibo Liu + * @since 4.4 + */ +public class OpenSearchTemplate extends AbstractElasticsearchTemplate { + + private static final Log LOGGER = LogFactory.getLog(OpenSearchTemplate.class); + + private final OpenSearchClient client; + private final RequestConverter requestConverter; + private final ResponseConverter responseConverter; + private final JsonpMapper jsonpMapper; + private final OpenSearchExceptionTranslator exceptionTranslator; + + // region _initialization + public OpenSearchTemplate(OpenSearchClient client) { + + Assert.notNull(client, "client must not be null"); + + this.client = client; + this.jsonpMapper = client._transport().jsonpMapper(); + requestConverter = new RequestConverter(elasticsearchConverter, jsonpMapper); + responseConverter = new ResponseConverter(jsonpMapper); + exceptionTranslator = new OpenSearchExceptionTranslator(jsonpMapper); + } + + public OpenSearchTemplate(OpenSearchClient client, ElasticsearchConverter elasticsearchConverter) { + super(elasticsearchConverter); + + Assert.notNull(client, "client must not be null"); + + this.client = client; + this.jsonpMapper = client._transport().jsonpMapper(); + requestConverter = new RequestConverter(elasticsearchConverter, jsonpMapper); + responseConverter = new ResponseConverter(jsonpMapper); + exceptionTranslator = new OpenSearchExceptionTranslator(jsonpMapper); + } + + @Override + protected AbstractElasticsearchTemplate doCopy() { + return new OpenSearchTemplate(client, elasticsearchConverter); + } + // endregion + + // region child templates + @Override + public IndexOperations indexOps(Class clazz) { + return new IndicesTemplate(client.indices(), getClusterTemplate(), elasticsearchConverter, clazz); + } + + @Override + public IndexOperations indexOps(IndexCoordinates index) { + return new IndicesTemplate(client.indices(), getClusterTemplate(), elasticsearchConverter, index); + } + + @Override + public ClusterOperations cluster() { + return getClusterTemplate(); + } + + private ClusterTemplate getClusterTemplate() { + return new ClusterTemplate(client.cluster(), elasticsearchConverter); + } + // endregion + + // region document operations + @Override + @Nullable + public T get(String id, Class clazz, IndexCoordinates index) { + + GetRequest getRequest = requestConverter.documentGetRequest(elasticsearchConverter.convertId(id), + routingResolver.getRouting(), index); + GetResponse getResponse = execute(client -> client.get(getRequest, EntityAsMap.class)); + + ReadDocumentCallback callback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + return callback.doWith(DocumentAdapters.from(getResponse)); + } + + @Override + public List> multiGet(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + + MgetRequest request = requestConverter.documentMgetRequest(query, clazz, index); + MgetResponse result = execute(client -> client.mget(request, EntityAsMap.class)); + + ReadDocumentCallback callback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + + return DocumentAdapters.from(result).stream() // + .map(multiGetItem -> MultiGetItem.of( // + multiGetItem.isFailed() ? null : callback.doWith(multiGetItem.getItem()), multiGetItem.getFailure())) // + .collect(Collectors.toList()); + } + + @Override + public void bulkUpdate(List queries, BulkOptions bulkOptions, IndexCoordinates index) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(bulkOptions, "bulkOptions must not be null"); + Assert.notNull(index, "index must not be null"); + + doBulkOperation(queries, bulkOptions, index); + } + + @Override + public ByQueryResponse delete(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + + DeleteByQueryRequest request = requestConverter.documentDeleteByQueryRequest(query, routingResolver.getRouting(), + clazz, index, getRefreshPolicy()); + + DeleteByQueryResponse response = execute(client -> client.deleteByQuery(request)); + + return responseConverter.byQueryResponse(response); + } + + @Override + public UpdateResponse update(UpdateQuery updateQuery, IndexCoordinates index) { + + UpdateRequest request = requestConverter.documentUpdateRequest(updateQuery, index, getRefreshPolicy(), + routingResolver.getRouting()); + org.opensearch.client.opensearch.core.UpdateResponse response = execute( + client -> client.update(request, Document.class)); + return UpdateResponse.of(result(response.result())); + } + + @Override + public ByQueryResponse updateByQuery(UpdateQuery updateQuery, IndexCoordinates index) { + + Assert.notNull(updateQuery, "updateQuery must not be null"); + Assert.notNull(index, "index must not be null"); + + UpdateByQueryRequest request = requestConverter.documentUpdateByQueryRequest(updateQuery, index, + getRefreshPolicy()); + + UpdateByQueryResponse byQueryResponse = execute(client -> client.updateByQuery(request)); + return responseConverter.byQueryResponse(byQueryResponse); + } + + @Override + public String doIndex(IndexQuery query, IndexCoordinates indexCoordinates) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + IndexRequest indexRequest = requestConverter.documentIndexRequest(query, indexCoordinates, refreshPolicy); + + IndexResponse indexResponse = execute(client -> client.index(indexRequest)); + + Object queryObject = query.getObject(); + + if (queryObject != null) { + query.setObject(entityOperations.updateIndexedObject( + queryObject, + new IndexedObjectInformation( + indexResponse.id(), + indexResponse.index(), + indexResponse.seqNo(), + indexResponse.primaryTerm(), + indexResponse.version()), + elasticsearchConverter, + routingResolver)); + } + + return indexResponse.id(); + } + + @Override + protected boolean doExists(String id, IndexCoordinates index) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(index, "index must not be null"); + + ExistsRequest request = requestConverter.documentExistsRequest(id, routingResolver.getRouting(), index); + + return execute(client -> client.exists(request)).value(); + } + + @Override + protected String doDelete(String id, @Nullable String routing, IndexCoordinates index) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(index, "index must not be null"); + + DeleteRequest request = requestConverter.documentDeleteRequest(elasticsearchConverter.convertId(id), routing, index, + getRefreshPolicy()); + return execute(client -> client.delete(request)).id(); + } + + @Override + public ReindexResponse reindex(ReindexRequest reindexRequest) { + + Assert.notNull(reindexRequest, "reindexRequest must not be null"); + + org.opensearch.client.opensearch.core.ReindexRequest reindexRequestES = requestConverter.reindex(reindexRequest, + true); + org.opensearch.client.opensearch.core.ReindexResponse reindexResponse = execute( + client -> client.reindex(reindexRequestES)); + return responseConverter.reindexResponse(reindexResponse); + } + + @Override + public String submitReindex(ReindexRequest reindexRequest) { + + org.opensearch.client.opensearch.core.ReindexRequest reindexRequestES = requestConverter.reindex(reindexRequest, + false); + org.opensearch.client.opensearch.core.ReindexResponse reindexResponse = execute( + client -> client.reindex(reindexRequestES)); + + if (reindexResponse.task() == null) { + throw new UnsupportedBackendOperation("OpenSearchClient did not return a task id on submit request"); + } + + return reindexResponse.task(); + } + + @Override + public List doBulkOperation(List queries, BulkOptions bulkOptions, + IndexCoordinates index) { + + BulkRequest bulkRequest = requestConverter.documentBulkRequest(queries, bulkOptions, index, refreshPolicy); + BulkResponse bulkResponse = execute(client -> client.bulk(bulkRequest)); + List indexedObjectInformationList = checkForBulkOperationFailure(bulkResponse); + updateIndexedObjectsWithQueries(queries, indexedObjectInformationList); + return indexedObjectInformationList; + } + + // endregion + + @Override + public String getClusterVersion() { + return execute(client -> client.info().version().number()); + } + + @Override + public String getVendor() { + return "Elasticsearch"; + } + + @Override + public String getRuntimeLibraryVersion() { + return Version.VERSION != null ? Version.VERSION.toString() : "0.0.0.?"; + } + + // region search operations + @Override + public long count(Query query, @Nullable Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(index, "index must not be null"); + + SearchRequest searchRequest = requestConverter.searchRequest(query, routingResolver.getRouting(), clazz, index, + true); + + SearchResponse searchResponse = execute(client -> client.search(searchRequest, EntityAsMap.class)); + + return searchResponse.hits().total().value(); + } + + @Override + public SearchHits search(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + Assert.notNull(index, "index must not be null"); + + if (query instanceof SearchTemplateQuery searchTemplateQuery) { + return doSearch(searchTemplateQuery, clazz, index); + } else { + return doSearch(query, clazz, index); + } + } + + protected SearchHits doSearch(Query query, Class clazz, IndexCoordinates index) { + SearchRequest searchRequest = requestConverter.searchRequest(query, routingResolver.getRouting(), clazz, index, + false); + SearchResponse searchResponse = execute(client -> client.search(searchRequest, EntityAsMap.class)); + + // noinspection DuplicatedCode + ReadDocumentCallback readDocumentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + SearchDocumentResponse.EntityCreator entityCreator = getEntityCreator(readDocumentCallback); + SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, index); + + return callback.doWith(SearchDocumentResponseBuilder.from(searchResponse, entityCreator, jsonpMapper)); + } + + protected SearchHits doSearch(SearchTemplateQuery query, Class clazz, IndexCoordinates index) { + var searchTemplateRequest = requestConverter.searchTemplate(query, routingResolver.getRouting(), index); + var searchTemplateResponse = execute(client -> client.searchTemplate(searchTemplateRequest, EntityAsMap.class)); + + // noinspection DuplicatedCode + ReadDocumentCallback readDocumentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + SearchDocumentResponse.EntityCreator entityCreator = getEntityCreator(readDocumentCallback); + SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, index); + + return callback.doWith(SearchDocumentResponseBuilder.from(searchTemplateResponse, entityCreator, jsonpMapper)); + } + + @Override + protected SearchHits doSearch(MoreLikeThisQuery query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + Assert.notNull(index, "index must not be null"); + + return search(NativeQuery.builder() // + .withQuery(q -> q.moreLikeThis(requestConverter.moreLikeThisQuery(query, index)))// + .withPageable(query.getPageable()) // + .build(), clazz, index); + } + + @Override + public SearchScrollHits searchScrollStart(long scrollTimeInMillis, Query query, Class clazz, + IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(query.getPageable(), "pageable of query must not be null."); + + SearchRequest request = requestConverter.searchRequest(query, routingResolver.getRouting(), clazz, index, false, + scrollTimeInMillis); + SearchResponse response = execute(client -> client.search(request, EntityAsMap.class)); + + return getSearchScrollHits(clazz, index, response); + } + + @Override + public SearchScrollHits searchScrollContinue(String scrollId, long scrollTimeInMillis, Class clazz, + IndexCoordinates index) { + + Assert.notNull(scrollId, "scrollId must not be null"); + + ScrollRequest request = ScrollRequest + .of(sr -> sr.scrollId(scrollId).scroll(Time.of(t -> t.time(scrollTimeInMillis + "ms")))); + ScrollResponse response = execute(client -> client.scroll(request, EntityAsMap.class)); + + return getSearchScrollHits(clazz, index, response); + } + + private SearchScrollHits getSearchScrollHits(Class clazz, IndexCoordinates index, + SearchResult response) { + ReadDocumentCallback documentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + SearchDocumentResponseCallback> callback = new ReadSearchScrollDocumentResponseCallback<>(clazz, + index); + + return callback + .doWith(SearchDocumentResponseBuilder.from(response, getEntityCreator(documentCallback), jsonpMapper)); + } + + @Override + public void searchScrollClear(List scrollIds) { + + Assert.notNull(scrollIds, "scrollIds must not be null"); + + if (!scrollIds.isEmpty()) { + ClearScrollRequest request = ClearScrollRequest.of(csr -> csr.scrollId(scrollIds)); + execute(client -> client.clearScroll(request)); + } + } + + @Override + public List> multiSearch(List queries, Class clazz, IndexCoordinates index) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + + int size = queries.size(); + // noinspection unchecked + return multiSearch(queries, Collections.nCopies(size, clazz), Collections.nCopies(size, index)) + .stream().map(searchHits -> (SearchHits) searchHits) + .collect(Collectors.toList()); + } + + @Override + public List> multiSearch(List queries, List> classes) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(classes, "classes must not be null"); + Assert.isTrue(queries.size() == classes.size(), "queries and classes must have the same size"); + + return multiSearch(queries, classes, classes.stream().map(this::getIndexCoordinatesFor).toList()); + } + + @Override + public List> multiSearch(List queries, List> classes, + IndexCoordinates index) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(classes, "classes must not be null"); + Assert.notNull(index, "index must not be null"); + Assert.isTrue(queries.size() == classes.size(), "queries and classes must have the same size"); + + return multiSearch(queries, classes, Collections.nCopies(queries.size(), index)); + } + + @Override + public List> multiSearch(List queries, List> classes, + List indexes) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(classes, "classes must not be null"); + Assert.notNull(indexes, "indexes must not be null"); + Assert.isTrue(queries.size() == classes.size() && queries.size() == indexes.size(), + "queries, classes and indexes must have the same size"); + + List multiSearchQueryParameters = new ArrayList<>(queries.size()); + Iterator> it = classes.iterator(); + Iterator indexesIt = indexes.iterator(); + + Assert.isTrue(!queries.isEmpty(), "queries should have at least 1 query"); + boolean isSearchTemplateQuery = queries.get(0) instanceof SearchTemplateQuery; + + for (Query query : queries) { + Assert.isTrue((query instanceof SearchTemplateQuery) == isSearchTemplateQuery, + "SearchTemplateQuery can't be mixed with other types of query in multiple search"); + + Class clazz = it.next(); + IndexCoordinates index = indexesIt.next(); + multiSearchQueryParameters.add(new MultiSearchQueryParameter(query, clazz, index)); + } + + return multiSearch(multiSearchQueryParameters, isSearchTemplateQuery); + } + + private List> multiSearch(List multiSearchQueryParameters, + boolean isSearchTemplateQuery) { + return isSearchTemplateQuery ? + doMultiTemplateSearch(multiSearchQueryParameters.stream() + .map(p -> new MultiSearchTemplateQueryParameter((SearchTemplateQuery) p.query, p.clazz, p.index)) + .toList()) + : doMultiSearch(multiSearchQueryParameters); + } + + private List> doMultiTemplateSearch(List mSearchTemplateQueryParameters) { + MsearchTemplateRequest request = requestConverter.searchMsearchTemplateRequest(mSearchTemplateQueryParameters, + routingResolver.getRouting()); + + MsearchTemplateResponse response = execute(client -> client.msearchTemplate(request, EntityAsMap.class)); + List> responseItems = response.responses(); + + Assert.isTrue(mSearchTemplateQueryParameters.size() == responseItems.size(), + "number of response items does not match number of requests"); + + int size = mSearchTemplateQueryParameters.size(); + List> classes = mSearchTemplateQueryParameters + .stream().map(MultiSearchTemplateQueryParameter::clazz).collect(Collectors.toList()); + List indices = mSearchTemplateQueryParameters + .stream().map(MultiSearchTemplateQueryParameter::index).collect(Collectors.toList()); + + return getSearchHitsFromMsearchResponse(size, classes, indices, responseItems); + } + + private List> doMultiSearch(List multiSearchQueryParameters) { + + MsearchRequest request = requestConverter.searchMsearchRequest(multiSearchQueryParameters, + routingResolver.getRouting()); + + MsearchResponse msearchResponse = execute(client -> client.msearch(request, EntityAsMap.class)); + List> responseItems = msearchResponse.responses(); + + Assert.isTrue(multiSearchQueryParameters.size() == responseItems.size(), + "number of response items does not match number of requests"); + + int size = multiSearchQueryParameters.size(); + List> classes = multiSearchQueryParameters + .stream().map(MultiSearchQueryParameter::clazz).collect(Collectors.toList()); + List indices = multiSearchQueryParameters + .stream().map(MultiSearchQueryParameter::index).collect(Collectors.toList()); + + return getSearchHitsFromMsearchResponse(size, classes, indices, responseItems); + } + + /** + * {@link MsearchResponse} and {@link MsearchTemplateResponse} share the same {@link MultiSearchResponseItem} + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private List> getSearchHitsFromMsearchResponse(int size, List> classes, + List indices, List> responseItems) { + List> searchHitsList = new ArrayList<>(size); + Iterator> clazzIter = classes.iterator(); + Iterator indexIter = indices.iterator(); + Iterator> responseIterator = responseItems.iterator(); + + while (clazzIter.hasNext() && indexIter.hasNext()) { + MultiSearchResponseItem responseItem = responseIterator.next(); + + if (responseItem.isResult()) { + + Class clazz = clazzIter.next(); + IndexCoordinates index = indexIter.next(); + ReadDocumentCallback documentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, + index); + SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, + index); + + SearchHits searchHits = callback.doWith( + SearchDocumentResponseBuilder.from(responseItem.result(), getEntityCreator(documentCallback), jsonpMapper)); + + searchHitsList.add(searchHits); + } else { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn(String.format("multisearch response contains failure: %s", + responseItem.failure().error().reason())); + } + } + } + + return searchHitsList; + } + + /** + * value class combining the information needed for a single query in a multisearch request. + */ + record MultiSearchQueryParameter(Query query, Class clazz, IndexCoordinates index) { + } + + /** + * value class combining the information needed for a single query in a template multisearch request. + */ + record MultiSearchTemplateQueryParameter(SearchTemplateQuery query, Class clazz, IndexCoordinates index) { + } + + @Override + public String openPointInTime(IndexCoordinates index, Duration keepAlive, Boolean ignoreUnavailable) { + + Assert.notNull(index, "index must not be null"); + Assert.notNull(keepAlive, "keepAlive must not be null"); + Assert.notNull(ignoreUnavailable, "ignoreUnavailable must not be null"); + + var request = requestConverter.searchOpenPointInTimeRequest(index, keepAlive, ignoreUnavailable); + return execute(client -> client.createPit(request)).pitId(); + } + + @Override + public Boolean closePointInTime(String pit) { + + Assert.notNull(pit, "pit must not be null"); + + DeletePitRequest request = requestConverter.searchClosePointInTime(pit); + var response = execute(client -> client.deletePit(request)); + return !response.pits().isEmpty(); + } + + // endregion + + // region script methods + @Override + public boolean putScript(Script script) { + + Assert.notNull(script, "script must not be null"); + + var request = requestConverter.scriptPut(script); + return execute(client -> client.putScript(request)).acknowledged(); + } + + @Nullable + @Override + public Script getScript(String name) { + + Assert.notNull(name, "name must not be null"); + + var request = requestConverter.scriptGet(name); + return responseConverter.scriptResponse(execute(client -> client.getScript(request))); + } + + public boolean deleteScript(String name) { + + Assert.notNull(name, "name must not be null"); + + DeleteScriptRequest request = requestConverter.scriptDelete(name); + return execute(client -> client.deleteScript(request)).acknowledged(); + } + // endregion + + // region client callback + /** + * Callback interface to be used with {@link #execute(OpenSearchTemplate.ClientCallback)} for operating directly on + * the {@link OpenSearchClient}. + */ + @FunctionalInterface + public interface ClientCallback { + T doWithClient(OpenSearchClient client) throws IOException; + } + + /** + * Execute a callback with the {@link OpenSearchClient} and provide exception translation. + * + * @param callback the callback to execute, must not be {@literal null} + * @param the type returned from the callback + * @return the callback result + */ + public T execute(OpenSearchTemplate.ClientCallback callback) { + + Assert.notNull(callback, "callback must not be null"); + + try { + return callback.doWithClient(client); + } catch (IOException | RuntimeException e) { + throw exceptionTranslator.translateException(e); + } + } + // endregion + + // region helper methods + @Override + public Query matchAllQuery() { + return NativeQuery.builder().withQuery(qb -> qb.matchAll(mab -> mab)).build(); + } + + @Override + public Query idsQuery(List ids) { + return NativeQuery.builder().withQuery(qb -> qb.ids(iq -> iq.values(ids))).build(); + } + + @Override + public BaseQueryBuilder queryBuilderWithIds(List ids) { + return NativeQuery.builder().withIds(ids); + } + + /** + * extract the list of {@link IndexedObjectInformation} from a {@link BulkResponse}. + * + * @param bulkResponse the response to evaluate + * @return the list of the {@link IndexedObjectInformation}s + */ + protected List checkForBulkOperationFailure(BulkResponse bulkResponse) { + + if (bulkResponse.errors()) { + Map failedDocuments = new HashMap<>(); + for (BulkResponseItem item : bulkResponse.items()) { + + if (item.error() != null) { + failedDocuments.put(item.id(), new BulkFailureException.FailureDetails(item.status(), item.error().reason())); + } + } + throw new BulkFailureException( + "Bulk operation has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages [" + + failedDocuments + ']', + failedDocuments); + } + + return bulkResponse.items().stream().map( + item -> new IndexedObjectInformation(item.id(), item.index(), item.seqNo(), item.primaryTerm(), item.version())) + .collect(Collectors.toList()); + + } + // endregion + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Queries.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Queries.java new file mode 100644 index 0000000..a720d1c --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Queries.java @@ -0,0 +1,194 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.function.Function; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.LatLonGeoLocation; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.query_dsl.IdsQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchAllQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.Operator; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryStringQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; +import org.opensearch.client.opensearch._types.query_dsl.WildcardQuery; +import org.opensearch.client.opensearch._types.query_dsl.WrapperQuery; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.data.elasticsearch.core.geo.GeoPoint; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Utility class simplifying the creation of some more complex queries and type. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public final class Queries { + + private Queries() {} + + public static IdsQuery idsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + return IdsQuery.of(i -> i.values(ids)); + } + + public static Query idsQueryAsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + Function> builder = b -> b.ids(idsQuery(ids)); + + return builder.apply(new Query.Builder()).build(); + } + + public static MatchQuery matchQuery(String fieldName, String query, @Nullable Operator operator, + @Nullable Float boost) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(query, "query must not be null"); + + return MatchQuery.of(mb -> mb.field(fieldName).query(FieldValue.of(query)).operator(operator).boost(boost)); + } + + public static Query matchQueryAsQuery(String fieldName, String query, @Nullable Operator operator, + @Nullable Float boost) { + + Function> builder = b -> b.match(matchQuery(fieldName, query, operator, boost)); + + return builder.apply(new Query.Builder()).build(); + } + + public static MatchAllQuery matchAllQuery() { + + return MatchAllQuery.of(b -> b); + } + + public static Query matchAllQueryAsQuery() { + + Function> builder = b -> b.matchAll(matchAllQuery()); + + return builder.apply(new Query.Builder()).build(); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Float boost) { + return queryStringQuery(fieldName, query, null, null, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, Operator defaultOperator, + @Nullable Float boost) { + return queryStringQuery(fieldName, query, null, defaultOperator, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Boolean analyzeWildcard, + @Nullable Float boost) { + return queryStringQuery(fieldName, query, analyzeWildcard, null, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Boolean analyzeWildcard, + @Nullable Operator defaultOperator, @Nullable Float boost) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(query, "query must not be null"); + + return QueryStringQuery.of(qs -> qs.fields(fieldName).query(query).analyzeWildcard(analyzeWildcard) + .defaultOperator(defaultOperator).boost(boost)); + } + + public static TermQuery termQuery(String fieldName, String value) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(value, "value must not be null"); + + return TermQuery.of(t -> t.field(fieldName).value(FieldValue.of(value))); + } + + public static Query termQueryAsQuery(String fieldName, String value) { + + Function> builder = q -> q.term(termQuery(fieldName, value)); + return builder.apply(new Query.Builder()).build(); + } + + public static WildcardQuery wildcardQuery(String field, String value) { + + Assert.notNull(field, "field must not be null"); + Assert.notNull(value, "value must not be null"); + + return WildcardQuery.of(w -> w.field(field).wildcard(value)); + } + + public static Query wildcardQueryAsQuery(String field, String value) { + Function> builder = q -> q.wildcard(wildcardQuery(field, value)); + return builder.apply(new Query.Builder()).build(); + } + + public static Query wrapperQueryAsQuery(String query) { + + Function> builder = q -> q.wrapper(wrapperQuery(query)); + + return builder.apply(new Query.Builder()).build(); + } + + public static WrapperQuery wrapperQuery(String query) { + + Assert.notNull(query, "query must not be null"); + + String encodedValue = Base64.getEncoder().encodeToString(query.getBytes(StandardCharsets.UTF_8)); + + return WrapperQuery.of(wq -> wq.query(encodedValue)); + } + + public static LatLonGeoLocation latLon(GeoPoint geoPoint) { + + Assert.notNull(geoPoint, "geoPoint must not be null"); + + return latLon(geoPoint.getLat(), geoPoint.getLon()); + } + + public static LatLonGeoLocation latLon(double lat, double lon) { + return LatLonGeoLocation.of(_0 -> _0.lat(lat).lon(lon)); + } + + public static org.springframework.data.elasticsearch.core.query.Query getTermsAggsQuery(String aggsName, + String aggsField) { + return NativeQuery.builder() // + .withQuery(Queries.matchAllQueryAsQuery()) // + .withAggregation(aggsName, Aggregation.of(a -> a // + .terms(ta -> ta.field(aggsField)))) // + .withMaxResults(0) // + .build(); + } + + public static org.springframework.data.elasticsearch.core.query.Query queryWithIds(String... ids) { + return NativeQuery.builder().withIds(ids).build(); + } + + public static BaseQueryBuilder getBuilderWithMatchAllQuery() { + return NativeQuery.builder().withQuery(matchAllQueryAsQuery()); + } + + public static BaseQueryBuilder getBuilderWithTermQuery(String field, String value) { + return NativeQuery.builder().withQuery(termQueryAsQuery(field, value)); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/QueryBuilders.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/QueryBuilders.java new file mode 100644 index 0000000..d5ca174 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/QueryBuilders.java @@ -0,0 +1,172 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.function.Function; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.LatLonGeoLocation; +import org.opensearch.client.opensearch._types.query_dsl.IdsQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchAllQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.Operator; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryStringQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; +import org.opensearch.client.opensearch._types.query_dsl.WildcardQuery; +import org.opensearch.client.opensearch._types.query_dsl.WrapperQuery; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.data.elasticsearch.core.geo.GeoPoint; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Utility class simplifying the creation of some more complex queries and type. + * + * @author Peter-Josef Meisch + * @since 4.4 + * @deprecated since 5.1, use {@link Queries} instead. + */ +@Deprecated(forRemoval = true) +public final class QueryBuilders { + + private QueryBuilders() {} + + public static IdsQuery idsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + return IdsQuery.of(i -> i.values(ids)); + } + + public static Query idsQueryAsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + Function> builder = b -> b.ids(idsQuery(ids)); + + return builder.apply(new Query.Builder()).build(); + } + + public static MatchQuery matchQuery(String fieldName, String query, @Nullable Operator operator, + @Nullable Float boost) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(query, "query must not be null"); + + return MatchQuery.of(mb -> mb.field(fieldName).query(FieldValue.of(query)).operator(operator).boost(boost)); + } + + public static Query matchQueryAsQuery(String fieldName, String query, @Nullable Operator operator, + @Nullable Float boost) { + + Function> builder = b -> b.match(matchQuery(fieldName, query, operator, boost)); + + return builder.apply(new Query.Builder()).build(); + } + + public static MatchAllQuery matchAllQuery() { + + return MatchAllQuery.of(b -> b); + } + + public static Query matchAllQueryAsQuery() { + + Function> builder = b -> b.matchAll(matchAllQuery()); + + return builder.apply(new Query.Builder()).build(); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Float boost) { + return queryStringQuery(fieldName, query, null, null, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, Operator defaultOperator, + @Nullable Float boost) { + return queryStringQuery(fieldName, query, null, defaultOperator, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Boolean analyzeWildcard, + @Nullable Float boost) { + return queryStringQuery(fieldName, query, analyzeWildcard, null, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Boolean analyzeWildcard, + @Nullable Operator defaultOperator, @Nullable Float boost) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(query, "query must not be null"); + + return QueryStringQuery.of(qs -> qs.fields(fieldName).query(query).analyzeWildcard(analyzeWildcard) + .defaultOperator(defaultOperator).boost(boost)); + } + + public static TermQuery termQuery(String fieldName, String value) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(value, "value must not be null"); + + return TermQuery.of(t -> t.field(fieldName).value(FieldValue.of(value))); + } + + public static Query termQueryAsQuery(String fieldName, String value) { + + Function> builder = q -> q.term(termQuery(fieldName, value)); + return builder.apply(new Query.Builder()).build(); + } + + public static WildcardQuery wildcardQuery(String field, String value) { + + Assert.notNull(field, "field must not be null"); + Assert.notNull(value, "value must not be null"); + + return WildcardQuery.of(w -> w.field(field).wildcard(value)); + } + + public static Query wildcardQueryAsQuery(String field, String value) { + Function> builder = q -> q.wildcard(wildcardQuery(field, value)); + return builder.apply(new Query.Builder()).build(); + } + + public static Query wrapperQueryAsQuery(String query) { + + Function> builder = q -> q.wrapper(wrapperQuery(query)); + + return builder.apply(new Query.Builder()).build(); + } + + public static WrapperQuery wrapperQuery(String query) { + + Assert.notNull(query, "query must not be null"); + + String encodedValue = Base64.getEncoder().encodeToString(query.getBytes(StandardCharsets.UTF_8)); + + return WrapperQuery.of(wq -> wq.query(encodedValue)); + } + + public static LatLonGeoLocation latLon(GeoPoint geoPoint) { + + Assert.notNull(geoPoint, "geoPoint must not be null"); + + return latLon(geoPoint.getLat(), geoPoint.getLon()); + } + + public static LatLonGeoLocation latLon(double lat, double lon) { + return LatLonGeoLocation.of(_0 -> _0.lat(lat).lon(lon)); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/RequestConverter.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/RequestConverter.java new file mode 100644 index 0000000..f0c711b --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/RequestConverter.java @@ -0,0 +1,1951 @@ +/* + * Copyright 2021-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.TypeUtils.*; +import static org.springframework.util.CollectionUtils.*; + +import jakarta.json.stream.JsonParser; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.json.JsonpDeserializer; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.Conflicts; +import org.opensearch.client.opensearch._types.ExpandWildcard; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.InlineScript; +import org.opensearch.client.opensearch._types.NestedSortValue; +import org.opensearch.client.opensearch._types.OpType; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.SortOrder; +import org.opensearch.client.opensearch._types.VersionType; +import org.opensearch.client.opensearch._types.WaitForActiveShardOptions; +import org.opensearch.client.opensearch._types.mapping.FieldType; +import org.opensearch.client.opensearch._types.query_dsl.FieldAndFormat; +import org.opensearch.client.opensearch._types.query_dsl.Like; +import org.opensearch.client.opensearch.cluster.DeleteComponentTemplateRequest; +import org.opensearch.client.opensearch.cluster.ExistsComponentTemplateRequest; +import org.opensearch.client.opensearch.cluster.GetComponentTemplateRequest; +import org.opensearch.client.opensearch.cluster.HealthRequest; +import org.opensearch.client.opensearch.cluster.PutComponentTemplateRequest; +import org.opensearch.client.opensearch.core.*; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.CreateOperation; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.client.opensearch.core.bulk.UpdateOperation; +import org.opensearch.client.opensearch.core.mget.MultiGetOperation; +import org.opensearch.client.opensearch.core.msearch.MultisearchBody; +import org.opensearch.client.opensearch.core.msearch.MultisearchHeader; +import org.opensearch.client.opensearch.core.pit.CreatePitRequest; +import org.opensearch.client.opensearch.core.pit.DeletePitRequest; +import org.opensearch.client.opensearch.core.search.Highlight; +import org.opensearch.client.opensearch.core.search.Pit; +import org.opensearch.client.opensearch.core.search.Rescore; +import org.opensearch.client.opensearch.core.search.SourceConfig; +import org.opensearch.client.opensearch.indices.*; +import org.opensearch.client.opensearch.indices.ExistsIndexTemplateRequest; +import org.opensearch.client.opensearch.indices.ExistsRequest; +import org.opensearch.client.opensearch.indices.update_aliases.Action; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.data.domain.Sort; +import org.springframework.data.elasticsearch.core.RefreshPolicy; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.index.*; +import org.springframework.data.elasticsearch.core.index.DeleteIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.DeleteTemplateRequest; +import org.springframework.data.elasticsearch.core.index.ExistsTemplateRequest; +import org.springframework.data.elasticsearch.core.index.GetIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.GetTemplateRequest; +import org.springframework.data.elasticsearch.core.index.PutIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.PutTemplateRequest; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.*; +import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; +import org.springframework.data.elasticsearch.core.reindex.Remote; +import org.springframework.data.elasticsearch.core.script.Script; +import org.springframework.data.elasticsearch.support.DefaultStringObjectMap; +import org.springframework.lang.NonNull; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * Class to create OpenSearch request and request builders. + * + * @author Peter-Josef Meisch + * @author Sascha Woo + * @author cdalxndr + * @author scoobyzhang + * @author Haibo Liu + * @since 4.4 + */ +@SuppressWarnings("ClassCanBeRecord") +class RequestConverter { + + private static final Log LOGGER = LogFactory.getLog(RequestConverter.class); + + // the default max result window size of Elasticsearch + public static final Integer INDEX_MAX_RESULT_WINDOW = 10_000; + + protected final JsonpMapper jsonpMapper; + protected final ElasticsearchConverter elasticsearchConverter; + + public RequestConverter(ElasticsearchConverter elasticsearchConverter, JsonpMapper jsonpMapper) { + this.elasticsearchConverter = elasticsearchConverter; + + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + this.jsonpMapper = jsonpMapper; + } + + // region Cluster client + public org.opensearch.client.opensearch.cluster.HealthRequest clusterHealthRequest() { + return new HealthRequest.Builder().build(); + } + + public org.opensearch.client.opensearch.cluster.PutComponentTemplateRequest clusterPutComponentTemplateRequest( + org.springframework.data.elasticsearch.core.index.PutComponentTemplateRequest putComponentTemplateRequest) { + + Assert.notNull(putComponentTemplateRequest, "putComponentTemplateRequest must not be null"); + + return PutComponentTemplateRequest.of(b -> b // + .name(putComponentTemplateRequest.name()) // + .create(putComponentTemplateRequest.create()) // + .version(putComponentTemplateRequest.version()) // + .masterTimeout(time(putComponentTemplateRequest.masterTimeout())) // + .template(isb -> { + var componentTemplateData = putComponentTemplateRequest.template(); + isb // + .mappings(typeMapping(componentTemplateData.mapping())) // + .settings(indexSettings(componentTemplateData.settings())); + // same code schema, but different Elasticsearch builder types + // noinspection DuplicatedCode + var aliasActions = componentTemplateData.aliasActions(); + if (aliasActions != null) { + aliasActions.getActions().forEach(aliasAction -> { + if (aliasAction instanceof AliasAction.Add add) { + var parameters = add.getParameters(); + // noinspection DuplicatedCode + String[] parametersAliases = parameters.getAliases(); + if (parametersAliases != null) { + for (String aliasName : parametersAliases) { + isb.aliases(aliasName, aliasBuilder -> buildAlias(parameters, aliasBuilder)); + } + } + } + }); + } + return isb; + })); + } + + private Alias.Builder buildAlias(AliasActionParameters parameters, Alias.Builder aliasBuilder) { + + // noinspection DuplicatedCode + if (parameters.getRouting() != null) { + aliasBuilder.routing(parameters.getRouting()); + } + + if (parameters.getIndexRouting() != null) { + aliasBuilder.indexRouting(parameters.getIndexRouting()); + } + + if (parameters.getSearchRouting() != null) { + aliasBuilder.searchRouting(parameters.getSearchRouting()); + } + + if (parameters.getHidden() != null) { + aliasBuilder.isHidden(parameters.getHidden()); + } + + if (parameters.getWriteIndex() != null) { + aliasBuilder.isWriteIndex(parameters.getWriteIndex()); + } + + Query filterQuery = parameters.getFilterQuery(); + + if (filterQuery != null) { + org.opensearch.client.opensearch._types.query_dsl.Query esQuery = getQuery(filterQuery, null); + + if (esQuery != null) { + aliasBuilder.filter(esQuery); + } + } + return aliasBuilder; + } + + public ExistsComponentTemplateRequest clusterExistsComponentTemplateRequest( + org.springframework.data.elasticsearch.core.index.ExistsComponentTemplateRequest existsComponentTemplateRequest) { + + Assert.notNull(existsComponentTemplateRequest, "existsComponentTemplateRequest must not be null"); + + return ExistsComponentTemplateRequest.of(b -> b.name(existsComponentTemplateRequest.templateName())); + } + + public GetComponentTemplateRequest clusterGetComponentTemplateRequest( + org.springframework.data.elasticsearch.core.index.GetComponentTemplateRequest getComponentTemplateRequest) { + + Assert.notNull(getComponentTemplateRequest, "getComponentTemplateRequest must not be null"); + + return GetComponentTemplateRequest.of(b -> b.name(getComponentTemplateRequest.templateName())); + } + + public DeleteComponentTemplateRequest clusterDeleteComponentTemplateRequest( + org.springframework.data.elasticsearch.core.index.DeleteComponentTemplateRequest deleteComponentTemplateRequest) { + return DeleteComponentTemplateRequest.of(b -> b.name(deleteComponentTemplateRequest.templateName())); + } + // endregion + + // region Indices client + public ExistsRequest indicesExistsRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new ExistsRequest.Builder().index(Arrays.asList(indexCoordinates.getIndexNames())).build(); + } + + public CreateIndexRequest indicesCreateRequest(IndexCoordinates indexCoordinates, Map settings, + @Nullable Document mapping) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + Assert.notNull(settings, "settings must not be null"); + + // note: the new client does not support the index.storeType anymore + return new CreateIndexRequest.Builder() // + .index(indexCoordinates.getIndexName()) // + .settings(indexSettings(settings)) // + .mappings(typeMapping(mapping)) // + .build(); + } + + public RefreshRequest indicesRefreshRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new RefreshRequest.Builder().index(Arrays.asList(indexCoordinates.getIndexNames())).build(); + } + + public DeleteIndexRequest indicesDeleteRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new DeleteIndexRequest.Builder().index(Arrays.asList(indexCoordinates.getIndexNames())).build(); + } + + public UpdateAliasesRequest indicesUpdateAliasesRequest(AliasActions aliasActions) { + + Assert.notNull(aliasActions, "aliasActions must not be null"); + + UpdateAliasesRequest.Builder updateAliasRequestBuilder = new UpdateAliasesRequest.Builder(); + + List actions = new ArrayList<>(); + aliasActions.getActions().forEach(aliasAction -> { + + var actionBuilder = getBuilder(aliasAction); + + if (aliasAction instanceof AliasAction.Remove remove) { + AliasActionParameters parameters = remove.getParameters(); + actionBuilder.remove(removeActionBuilder -> { + removeActionBuilder.indices(Arrays.asList(parameters.getIndices())); + + if (parameters.getAliases() != null) { + removeActionBuilder.aliases(Arrays.asList(parameters.getAliases())); + } + + return removeActionBuilder; + }); + } + + if (aliasAction instanceof AliasAction.RemoveIndex removeIndex) { + AliasActionParameters parameters = removeIndex.getParameters(); + actionBuilder.removeIndex( + removeIndexActionBuilder -> removeIndexActionBuilder.indices(Arrays.asList(parameters.getIndices()))); + } + + actions.add(actionBuilder.build()); + }); + + updateAliasRequestBuilder.actions(actions); + + return updateAliasRequestBuilder.build(); + } + + @NonNull + private Action.Builder getBuilder(AliasAction aliasAction) { + Action.Builder actionBuilder = new Action.Builder(); + + if (aliasAction instanceof AliasAction.Add add) { + AliasActionParameters parameters = add.getParameters(); + actionBuilder.add(addActionBuilder -> { + addActionBuilder // + .indices(Arrays.asList(parameters.getIndices())) // + .isHidden(parameters.getHidden()) // + .isWriteIndex(parameters.getWriteIndex()) // + .routing(parameters.getRouting()) // + .indexRouting(parameters.getIndexRouting()) // + .searchRouting(parameters.getSearchRouting()); // + + if (parameters.getAliases() != null) { + addActionBuilder.aliases(Arrays.asList(parameters.getAliases())); + } + + Query filterQuery = parameters.getFilterQuery(); + + if (filterQuery != null) { + elasticsearchConverter.updateQuery(filterQuery, parameters.getFilterQueryClass()); + org.opensearch.client.opensearch._types.query_dsl.Query esQuery = getQuery(filterQuery, null); + if (esQuery != null) { + addActionBuilder.filter(esQuery); + } + } + return addActionBuilder; + }); + } + return actionBuilder; + } + + public PutMappingRequest indicesPutMappingRequest(IndexCoordinates indexCoordinates, Document mapping) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + Assert.notNull(mapping, "mapping must not be null"); + + final PutMappingRequest request = JsonpUtils.fromJson(mapping, PutMappingRequest._DESERIALIZER); + + return new PutMappingRequest.Builder() + .fieldNames(request.fieldNames()) + .meta(request.meta()) + .routing(request.routing()) + .source(request.source()) + .allowNoIndices(request.allowNoIndices()) + .dateDetection(request.dateDetection()) + .dynamic(request.dynamic()) + .dynamicDateFormats(request.dynamicDateFormats()) + .dynamicTemplates(request.dynamicTemplates()) + .expandWildcards(request.expandWildcards()) + .ignoreUnavailable(request.ignoreUnavailable()) + .masterTimeout(request.masterTimeout()) + .clusterManagerTimeout(request.clusterManagerTimeout()) + .numericDetection(request.numericDetection()) + .properties(request.properties()) + .timeout(request.timeout()) + .writeIndexOnly(request.writeIndexOnly()) + .index(Arrays.asList(indexCoordinates.getIndexNames())) + .build(); + } + + public GetMappingRequest indicesGetMappingRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new GetMappingRequest.Builder().index(List.of(indexCoordinates.getIndexNames())).build(); + } + + public GetIndicesSettingsRequest indicesGetSettingsRequest(IndexCoordinates indexCoordinates, + boolean includeDefaults) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new GetIndicesSettingsRequest.Builder() // + .index(Arrays.asList(indexCoordinates.getIndexNames())) // + .includeDefaults(includeDefaults) // + .build(); + } + + public GetIndexRequest indicesGetIndexRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new GetIndexRequest.Builder() // + .index(Arrays.asList(indexCoordinates.getIndexNames())) // + .includeDefaults(true) // + .build(); // + } + + public GetAliasRequest indicesGetAliasRequest(@Nullable String[] aliasNames, @Nullable String[] indexNames) { + GetAliasRequest.Builder builder = new GetAliasRequest.Builder(); + + if (aliasNames != null) { + builder.name(Arrays.asList(aliasNames)); + } + + if (indexNames != null) { + builder.index(Arrays.asList(indexNames)); + } + + return builder.build(); + } + + public org.opensearch.client.opensearch.indices.PutTemplateRequest indicesPutTemplateRequest( + PutTemplateRequest putTemplateRequest) { + + Assert.notNull(putTemplateRequest, "putTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.PutTemplateRequest.Builder builder = new org.opensearch.client.opensearch.indices.PutTemplateRequest.Builder(); + + builder.name(putTemplateRequest.getName()).indexPatterns(Arrays.asList(putTemplateRequest.getIndexPatterns())) + .order(putTemplateRequest.getOrder()); + + if (putTemplateRequest.getSettings() != null) { + Map settings = getTemplateParams(putTemplateRequest.getSettings().entrySet()); + builder.settings(settings); + } + + if (putTemplateRequest.getMappings() != null) { + builder.mappings(typeMapping(putTemplateRequest.getMappings())); + } + + if (putTemplateRequest.getVersion() != null) { + builder.version(Long.valueOf(putTemplateRequest.getVersion())); + } + AliasActions aliasActions = putTemplateRequest.getAliasActions(); + + if (aliasActions != null) { + aliasActions.getActions().forEach(aliasAction -> { + AliasActionParameters parameters = aliasAction.getParameters(); + // noinspection DuplicatedCode + String[] parametersAliases = parameters.getAliases(); + + if (parametersAliases != null) { + for (String aliasName : parametersAliases) { + builder.aliases(aliasName, aliasBuilder -> buildAlias(parameters, aliasBuilder)); + } + } + }); + } + + return builder.build(); + } + + public org.opensearch.client.opensearch.indices.PutIndexTemplateRequest indicesPutIndexTemplateRequest( + PutIndexTemplateRequest putIndexTemplateRequest) { + + Assert.notNull(putIndexTemplateRequest, "putIndexTemplateRequest must not be null"); + + var builder = new org.opensearch.client.opensearch.indices.PutIndexTemplateRequest.Builder() + .name(putIndexTemplateRequest.name()) // + .indexPatterns(Arrays.asList(putIndexTemplateRequest.indexPatterns())) // + .template(t -> { + t // + .settings(indexSettings(putIndexTemplateRequest.settings())) // + .mappings(typeMapping(putIndexTemplateRequest.mapping())); + + // same code schema, but different Elasticsearch builder types + // noinspection DuplicatedCode + var aliasActions = putIndexTemplateRequest.aliasActions(); + if (aliasActions != null) { + aliasActions.getActions().forEach(aliasAction -> { + if (aliasAction instanceof AliasAction.Add add) { + var parameters = add.getParameters(); + // noinspection DuplicatedCode + String[] parametersAliases = parameters.getAliases(); + if (parametersAliases != null) { + for (String aliasName : parametersAliases) { + t.aliases(aliasName, aliasBuilder -> buildAlias(parameters, aliasBuilder)); + } + } + } + }); + } + return t; + }); + + if (!putIndexTemplateRequest.composedOf().isEmpty()) { + builder.composedOf(putIndexTemplateRequest.composedOf()); + } + + return builder.build(); + } + + public ExistsIndexTemplateRequest indicesExistsIndexTemplateRequest( + org.springframework.data.elasticsearch.core.index.ExistsIndexTemplateRequest existsIndexTemplateRequest) { + + Assert.notNull(existsIndexTemplateRequest, "existsIndexTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.ExistsIndexTemplateRequest + .of(b -> b.name(existsIndexTemplateRequest.templateName())); + } + + public org.opensearch.client.opensearch.indices.ExistsTemplateRequest indicesExistsTemplateRequest( + ExistsTemplateRequest existsTemplateRequest) { + + Assert.notNull(existsTemplateRequest, "existsTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.ExistsTemplateRequest + .of(etr -> etr.name(existsTemplateRequest.getTemplateName())); + } + + public org.opensearch.client.opensearch.indices.GetIndexTemplateRequest indicesGetIndexTemplateRequest( + GetIndexTemplateRequest getIndexTemplateRequest) { + + Assert.notNull(getIndexTemplateRequest, "getIndexTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.GetIndexTemplateRequest + .of(gitr -> gitr.name(getIndexTemplateRequest.templateName())); + } + + public org.opensearch.client.opensearch.indices.DeleteIndexTemplateRequest indicesDeleteIndexTemplateRequest( + DeleteIndexTemplateRequest deleteIndexTemplateRequest) { + + Assert.notNull(deleteIndexTemplateRequest, "deleteIndexTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.DeleteIndexTemplateRequest + .of(ditr -> ditr.name(deleteIndexTemplateRequest.templateName())); + } + + public org.opensearch.client.opensearch.indices.DeleteTemplateRequest indicesDeleteTemplateRequest( + DeleteTemplateRequest existsTemplateRequest) { + + Assert.notNull(existsTemplateRequest, "existsTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.DeleteTemplateRequest + .of(dtr -> dtr.name(existsTemplateRequest.getTemplateName())); + } + + public org.opensearch.client.opensearch.indices.GetTemplateRequest indicesGetTemplateRequest( + GetTemplateRequest getTemplateRequest) { + + Assert.notNull(getTemplateRequest, "getTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.GetTemplateRequest + .of(gtr -> gtr.name(getTemplateRequest.getTemplateName()).flatSettings(true)); + } + + // endregion + + // region documents + /* + * the methods documentIndexRequest, bulkIndexOperation and bulkCreateOperation have nearly + * identical code, but the client builders do not have a common accessible base or some reusable parts + * so the code needs to be duplicated. + */ + + public IndexRequest documentIndexRequest(IndexQuery query, IndexCoordinates indexCoordinates, + @Nullable RefreshPolicy refreshPolicy) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + IndexRequest.Builder builder = new IndexRequest.Builder<>(); + + builder.index(query.getIndexName() != null ? query.getIndexName() : indexCoordinates.getIndexName()); + + Object queryObject = query.getObject(); + + if (queryObject != null) { + String id = StringUtils.hasText(query.getId()) ? query.getId() : getPersistentEntityId(queryObject); + builder // + .id(id) // + .document(elasticsearchConverter.mapObject(queryObject)); + } else if (query.getSource() != null) { + builder // + .id(query.getId()) // + .document(new DefaultStringObjectMap<>().fromJson(query.getSource())); + } else { + throw new InvalidDataAccessApiUsageException( + "object or source is null, failed to index the document [id: " + query.getId() + ']'); + } + + if (query.getVersion() != null) { + VersionType versionType = retrieveVersionTypeFromPersistentEntity( + queryObject != null ? queryObject.getClass() : null); + builder.version(query.getVersion()).versionType(versionType); + } + + builder // + .ifSeqNo(query.getSeqNo()) // + .ifPrimaryTerm(query.getPrimaryTerm()) // + .routing(query.getRouting()); // + + if (query.getOpType() != null) { + switch (query.getOpType()) { + case INDEX -> builder.opType(OpType.Index); + case CREATE -> builder.opType(OpType.Create); + } + } + + builder.refresh(refresh(refreshPolicy)); + + return builder.build(); + } + /* + * the methods documentIndexRequest, bulkIndexOperation and bulkCreateOperation have nearly + * identical code, but the client builders do not have a common accessible base or some reusable parts + * so the code needs to be duplicated. + */ + + @SuppressWarnings("DuplicatedCode") + private IndexOperation bulkIndexOperation(IndexQuery query, IndexCoordinates indexCoordinates, + @Nullable RefreshPolicy refreshPolicy) { + + IndexOperation.Builder builder = new IndexOperation.Builder<>(); + + builder.index(query.getIndexName() != null ? query.getIndexName() : indexCoordinates.getIndexName()); + + Object queryObject = query.getObject(); + + if (queryObject != null) { + String id = StringUtils.hasText(query.getId()) ? query.getId() : getPersistentEntityId(queryObject); + builder // + .id(id) // + .document(elasticsearchConverter.mapObject(queryObject)); + } else if (query.getSource() != null) { + builder.document(new DefaultStringObjectMap<>().fromJson(query.getSource())); + } else { + throw new InvalidDataAccessApiUsageException( + "object or source is null, failed to index the document [id: " + query.getId() + ']'); + } + + if (query.getVersion() != null) { + VersionType versionType = retrieveVersionTypeFromPersistentEntity( + queryObject != null ? queryObject.getClass() : null); + builder.version(query.getVersion()).versionType(versionType); + } + + builder // + .ifSeqNo(query.getSeqNo()) // + .ifPrimaryTerm(query.getPrimaryTerm()) // + .routing(query.getRouting()); // + + return builder.build(); + } + /* + * the methods documentIndexRequest, bulkIndexOperation and bulkCreateOperation have nearly + * identical code, but the client builders do not have a common accessible base or some reusable parts + * so the code needs to be duplicated. + */ + + @SuppressWarnings("DuplicatedCode") + private CreateOperation bulkCreateOperation(IndexQuery query, IndexCoordinates indexCoordinates, + @Nullable RefreshPolicy refreshPolicy) { + + CreateOperation.Builder builder = new CreateOperation.Builder<>(); + + builder.index(query.getIndexName() != null ? query.getIndexName() : indexCoordinates.getIndexName()); + + Object queryObject = query.getObject(); + + if (queryObject != null) { + String id = StringUtils.hasText(query.getId()) ? query.getId() : getPersistentEntityId(queryObject); + builder // + .id(id) // + .document(elasticsearchConverter.mapObject(queryObject)); + } else if (query.getSource() != null) { + builder.document(new DefaultStringObjectMap<>().fromJson(query.getSource())); + } else { + throw new InvalidDataAccessApiUsageException( + "object or source is null, failed to index the document [id: " + query.getId() + ']'); + } + + if (query.getVersion() != null) { + VersionType versionType = retrieveVersionTypeFromPersistentEntity( + queryObject != null ? queryObject.getClass() : null); + builder.version(query.getVersion()).versionType(versionType); + } + + builder // + .ifSeqNo(query.getSeqNo()) // + .ifPrimaryTerm(query.getPrimaryTerm()) // + .routing(query.getRouting()); // + + return builder.build(); + } + + private UpdateOperation bulkUpdateOperation(UpdateQuery query, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy) { + + UpdateOperation.Builder uob = new UpdateOperation.Builder<>(); + String indexName = query.getIndexName() != null ? query.getIndexName() : index.getIndexName(); + + uob.index(indexName).id(query.getId()); + uob.script(getScript(query.getScriptData())) // + .document(query.getDocument()) // + .upsert(query.getUpsert()) // + .scriptedUpsert(query.getScriptedUpsert()) // + .docAsUpsert(query.getDocAsUpsert()) // + ; + + if (query.getFetchSource() != null) { + uob.source(sc -> sc.fetch(query.getFetchSource())); + } + + if (query.getFetchSourceIncludes() != null || query.getFetchSourceExcludes() != null) { + List includes = query.getFetchSourceIncludes() != null ? query.getFetchSourceIncludes() + : Collections.emptyList(); + List excludes = query.getFetchSourceExcludes() != null ? query.getFetchSourceExcludes() + : Collections.emptyList(); + uob.source(sc -> sc.filter(sf -> sf.includes(includes).excludes(excludes))); + } + + uob // + .routing(query.getRouting()) // + .ifSeqNo(query.getIfSeqNo() != null ? Long.valueOf(query.getIfSeqNo()) : null) // + .ifPrimaryTerm(query.getIfPrimaryTerm() != null ? Long.valueOf(query.getIfPrimaryTerm()) : null) // + .retryOnConflict(query.getRetryOnConflict()) // + ; + + // no refresh, timeout, waitForActiveShards on UpdateOperation or UpdateAction + + return uob.build(); + } + + @Nullable + private org.opensearch.client.opensearch._types.Script getScript(@Nullable ScriptData scriptData) { + + if (scriptData == null) { + return null; + } + + Map params = new HashMap<>(); + + if (scriptData.params() != null) { + scriptData.params().forEach((key, value) -> params.put(key, JsonData.of(value, jsonpMapper))); + } + return org.opensearch.client.opensearch._types.Script.of(sb -> { + if (scriptData.type() == ScriptType.INLINE) { + sb.inline(is -> is // + .lang(scriptData.language()) // + .source(scriptData.script()) // + .params(params)); // + } else if (scriptData.type() == ScriptType.STORED) { + sb.stored(ss -> ss // + .id(scriptData.script()) // + .params(params) // + ); + } + return sb; + }); + } + + public BulkRequest documentBulkRequest(List queries, BulkOptions bulkOptions, IndexCoordinates indexCoordinates, + @Nullable RefreshPolicy refreshPolicy) { + + BulkRequest.Builder builder = new BulkRequest.Builder(); + + if (bulkOptions.getTimeout() != null) { + builder.timeout(tb -> tb.time(Long.valueOf(bulkOptions.getTimeout().toMillis()).toString() + "ms")); + } + + builder.refresh(refresh(refreshPolicy)); + if (bulkOptions.getRefreshPolicy() != null) { + builder.refresh(refresh(bulkOptions.getRefreshPolicy())); + } + + if (bulkOptions.getWaitForActiveShards() != null) { + builder.waitForActiveShards(wasb -> wasb.count(bulkOptions.getWaitForActiveShards().value())); + } + + if (bulkOptions.getPipeline() != null) { + builder.pipeline(bulkOptions.getPipeline()); + } + + if (bulkOptions.getRoutingId() != null) { + builder.routing(bulkOptions.getRoutingId()); + } + + List operations = queries.stream().map(query -> { + BulkOperation.Builder ob = new BulkOperation.Builder(); + if (query instanceof IndexQuery indexQuery) { + + if (indexQuery.getOpType() == IndexQuery.OpType.CREATE) { + ob.create(bulkCreateOperation(indexQuery, indexCoordinates, refreshPolicy)); + } else { + ob.index(bulkIndexOperation(indexQuery, indexCoordinates, refreshPolicy)); + } + } else if (query instanceof UpdateQuery updateQuery) { + ob.update(bulkUpdateOperation(updateQuery, indexCoordinates, refreshPolicy)); + } + return ob.build(); + }).collect(Collectors.toList()); + + builder.operations(operations); + + return builder.build(); + } + + public GetRequest documentGetRequest(String id, @Nullable String routing, IndexCoordinates indexCoordinates) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return GetRequest.of(grb -> grb // + .index(indexCoordinates.getIndexName()) // + .id(id) // + .routing(routing)); + } + + public org.opensearch.client.opensearch.core.ExistsRequest documentExistsRequest(String id, @Nullable String routing, + IndexCoordinates indexCoordinates) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return org.opensearch.client.opensearch.core.ExistsRequest.of(erb -> erb + .index(indexCoordinates.getIndexName()) + .id(id) + .routing(routing)); + } + + public MgetRequest documentMgetRequest(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + Assert.notNull(index, "index must not be null"); + + if (query.getIdsWithRouting().isEmpty()) { + throw new IllegalArgumentException("query does not contain any ids"); + } + + elasticsearchConverter.updateQuery(query, clazz); // to get the SourceConfig right + + SourceConfig sourceConfig = getSourceConfig(query); + + List multiGetOperations = query.getIdsWithRouting().stream() + .map(idWithRouting -> MultiGetOperation.of(mgo -> mgo // + .index(index.getIndexName()) // + .id(idWithRouting.id()) // + .routing(idWithRouting.routing()) // + .source(sourceConfig))) + .collect(Collectors.toList()); + + return MgetRequest.of(mg -> mg// + .docs(multiGetOperations)); + } + + public org.opensearch.client.opensearch.core.ReindexRequest reindex(ReindexRequest reindexRequest, + boolean waitForCompletion) { + + Assert.notNull(reindexRequest, "reindexRequest must not be null"); + + org.opensearch.client.opensearch.core.ReindexRequest.Builder builder = new org.opensearch.client.opensearch.core.ReindexRequest.Builder(); + builder // + .source(s -> { + ReindexRequest.Source source = reindexRequest.getSource(); + s.index(Arrays.asList(source.getIndexes().getIndexNames())) // + .size(source.getSize()); + + ReindexRequest.Slice slice = source.getSlice(); + if (slice != null) { + s.slice(sl -> sl.id(slice.getId()).max(slice.getMax())); + } + + if (source.getQuery() != null) { + s.query(getQuery(source.getQuery(), null)); + } + + if (source.getRemote() != null) { + Remote remote = source.getRemote(); + + s.remote(rs -> { + StringBuilder sb = new StringBuilder(remote.getScheme()); + sb.append("://"); + sb.append(remote.getHost()); + sb.append(':'); + sb.append(remote.getPort()); + + if (remote.getPathPrefix() != null) { + sb.append(remote.getPathPrefix()); + } + + String socketTimeoutSecs = remote.getSocketTimeout() != null + ? remote.getSocketTimeout().getSeconds() + "s" + : "30s"; + String connectTimeoutSecs = remote.getConnectTimeout() != null + ? remote.getConnectTimeout().getSeconds() + "s" + : "30s"; + return rs // + .host(sb.toString()) // + .username(remote.getUsername()) // + .password(remote.getPassword()) // + .socketTimeout(tv -> tv.time(socketTimeoutSecs)) // + .connectTimeout(tv -> tv.time(connectTimeoutSecs)); + }); + } + + SourceFilter sourceFilter = source.getSourceFilter(); + if (sourceFilter != null && sourceFilter.getIncludes() != null) { + s.sourceFields(Arrays.asList(sourceFilter.getIncludes())); + } + return s; + }) // + .dest(d -> { + ReindexRequest.Dest dest = reindexRequest.getDest(); + return d // + .index(dest.getIndex().getIndexName()) // + .versionType(versionType(dest.getVersionType())) // + .opType(opType(dest.getOpType())); + } // + ); + + if (reindexRequest.getConflicts() != null) { + builder.conflicts(conflicts(reindexRequest.getConflicts())); + } + + ReindexRequest.Script script = reindexRequest.getScript(); + if (script != null) { + builder.script(s -> s.inline(InlineScript.of(i -> i.lang(script.getLang()).source(script.getSource())))); + } + + builder.timeout(time(reindexRequest.getTimeout())) // + .scroll(time(reindexRequest.getScroll())); + + if (reindexRequest.getWaitForActiveShards() != null) { + builder.waitForActiveShards(wfas -> wfas // + .count(waitForActiveShardsCount(reindexRequest.getWaitForActiveShards()))); + } + + builder // + .maxDocs(reindexRequest.getMaxDocs()).waitForCompletion(waitForCompletion) // + .refresh(reindexRequest.getRefresh()) // + .requireAlias(reindexRequest.getRequireAlias()) // + .requestsPerSecond(reindexRequest.getRequestsPerSecond()) // + .slices(reindexRequest.getSlices()); + + return builder.build(); + } + + public DeleteRequest documentDeleteRequest(String id, @Nullable String routing, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(index, "index must not be null"); + + return DeleteRequest.of(r -> { + r.id(id).index(index.getIndexName()); + + if (routing != null) { + r.routing(routing); + } + r.refresh(refresh(refreshPolicy)); + return r; + }); + } + + public DeleteByQueryRequest documentDeleteByQueryRequest(Query query, @Nullable String routing, Class clazz, + IndexCoordinates index, @Nullable RefreshPolicy refreshPolicy) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(index, "index must not be null"); + + return DeleteByQueryRequest.of(b -> { + b.index(Arrays.asList(index.getIndexNames())) // + .query(getQuery(query, clazz))// + .refresh(deleteByQueryRefresh(refreshPolicy)); + + if (query.isLimiting()) { + // noinspection ConstantConditions + b.maxDocs(Long.valueOf(query.getMaxResults())); + } + + b.scroll(time(query.getScrollTime())); + + if (query.getRoute() != null) { + b.routing(query.getRoute()); + } else if (StringUtils.hasText(routing)) { + b.routing(routing); + } + + return b; + }); + } + + public UpdateRequest documentUpdateRequest(UpdateQuery query, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy, @Nullable String routing) { + + String indexName = query.getIndexName() != null ? query.getIndexName() : index.getIndexName(); + return UpdateRequest.of(uqb -> { + uqb.index(indexName).id(query.getId()); + + if (query.getScript() != null) { + Map params = new HashMap<>(); + + if (query.getParams() != null) { + query.getParams().forEach((key, value) -> params.put(key, JsonData.of(value, jsonpMapper))); + } + + uqb.script(sb -> { + if (query.getScriptType() == ScriptType.INLINE) { + sb.inline(is -> is // + .lang(query.getLang()) // + .source(query.getScript()) // + .params(params)); // + } else if (query.getScriptType() == ScriptType.STORED) { + sb.stored(ss -> ss // + .id(query.getScript()) // + .params(params) // + ); + } + return sb; + } + + ); + } + + uqb // + .doc(query.getDocument()) // + .upsert(query.getUpsert()) // + .routing(query.getRouting() != null ? query.getRouting() : routing) // + .scriptedUpsert(query.getScriptedUpsert()) // + .docAsUpsert(query.getDocAsUpsert()) // + .ifSeqNo(query.getIfSeqNo() != null ? Long.valueOf(query.getIfSeqNo()) : null) // + .ifPrimaryTerm(query.getIfPrimaryTerm() != null ? Long.valueOf(query.getIfPrimaryTerm()) : null) // + .refresh(query.getRefreshPolicy() != null ? refresh(query.getRefreshPolicy()) : refresh(refreshPolicy)) // + .retryOnConflict(query.getRetryOnConflict()) // + ; + + if (query.getFetchSource() != null) { + uqb.source(sc -> sc.fetch(query.getFetchSource())); + } + + if (query.getFetchSourceIncludes() != null || query.getFetchSourceExcludes() != null) { + List includes = query.getFetchSourceIncludes() != null ? query.getFetchSourceIncludes() + : Collections.emptyList(); + List excludes = query.getFetchSourceExcludes() != null ? query.getFetchSourceExcludes() + : Collections.emptyList(); + uqb.source(sc -> sc.filter(sf -> sf.includes(includes).excludes(excludes))); + } + + if (query.getTimeout() != null) { + uqb.timeout(tv -> tv.time(query.getTimeout())); + } + + String waitForActiveShards = query.getWaitForActiveShards(); + if (waitForActiveShards != null) { + if ("all".equalsIgnoreCase(waitForActiveShards)) { + uqb.waitForActiveShards(wfa -> wfa.option(WaitForActiveShardOptions.All)); + } else { + int val; + try { + val = Integer.parseInt(waitForActiveShards); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("cannot parse ActiveShardCount[" + waitForActiveShards + ']', e); + } + uqb.waitForActiveShards(wfa -> wfa.count(val)); + } + } + + return uqb; + } // + ); + } + + public UpdateByQueryRequest documentUpdateByQueryRequest(UpdateQuery updateQuery, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy) { + + return UpdateByQueryRequest.of(ub -> { + ub // + .index(Arrays.asList(index.getIndexNames())) // + .refresh(refreshPolicy == RefreshPolicy.IMMEDIATE) // + .routing(updateQuery.getRouting()) // + .script(getScript(updateQuery.getScriptData())) // + .maxDocs(updateQuery.getMaxDocs() != null ? Long.valueOf(updateQuery.getMaxDocs()) : null) // + .pipeline(updateQuery.getPipeline()) // + .requestsPerSecond(toLong(updateQuery.getRequestsPerSecond())) // + .slices(updateQuery.getSlices() != null ? Long.valueOf(updateQuery.getSlices()) : null); + + if (updateQuery.getAbortOnVersionConflict() != null) { + ub.conflicts(updateQuery.getAbortOnVersionConflict() ? Conflicts.Abort : Conflicts.Proceed); + } + + if (updateQuery.getQuery() != null) { + Query queryQuery = updateQuery.getQuery(); + + if (updateQuery.getBatchSize() != null) { + ((BaseQuery) queryQuery).setMaxResults(updateQuery.getBatchSize()); + } + ub.query(getQuery(queryQuery, null)); + + // no indicesOptions available like in old client + + ub.scroll(time(queryQuery.getScrollTime())); + + } + + // no maxRetries available like in old client + // no shouldStoreResult + + if (updateQuery.getRefreshPolicy() != null) { + ub.refresh(updateQuery.getRefreshPolicy() == RefreshPolicy.IMMEDIATE); + } + + if (updateQuery.getTimeout() != null) { + ub.timeout(tb -> tb.time(updateQuery.getTimeout())); + } + + if (updateQuery.getWaitForActiveShards() != null) { + ub.waitForActiveShards(w -> w.count(waitForActiveShardsCount(updateQuery.getWaitForActiveShards()))); + } + + return ub; + }); + } + + // endregion + + // region search + + public SearchRequest searchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, boolean forCount) { + return searchRequest(query, routing, clazz, indexCoordinates, forCount, false, null); + } + + public SearchRequest searchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, boolean forCount, long scrollTimeInMillis) { + return searchRequest(query, routing, clazz, indexCoordinates, forCount, true, scrollTimeInMillis); + } + + public SearchRequest searchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, boolean forCount, boolean forBatchedSearch) { + return searchRequest(query, routing, clazz, indexCoordinates, forCount, forBatchedSearch, null); + } + + public SearchRequest searchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, boolean forCount, boolean forBatchedSearch, + @Nullable Long scrollTimeInMillis) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + elasticsearchConverter.updateQuery(query, clazz); + SearchRequest.Builder builder = new SearchRequest.Builder(); + prepareSearchRequest(query, routing, clazz, indexCoordinates, builder, forCount, forBatchedSearch); + + if (scrollTimeInMillis != null) { + builder.scroll(t -> t.time(scrollTimeInMillis + "ms")); + } + + builder.query(getQuery(query, clazz)); + + if (StringUtils.hasText(query.getRoute())) { + builder.routing(query.getRoute()); + } + if (StringUtils.hasText(routing)) { + builder.routing(routing); + } + + addFilter(query, builder); + + return builder.build(); + } + + public MsearchTemplateRequest searchMsearchTemplateRequest( + List multiSearchTemplateQueryParameters, + @Nullable String routing) { + + // basically the same stuff as in template search + return MsearchTemplateRequest.of(mtrb -> { + multiSearchTemplateQueryParameters.forEach(param -> { + var query = param.query(); + mtrb.searchTemplates(stb -> stb + .header(msearchHeaderBuilder(query, param.index(), routing)) + .body(bb -> { + bb // + .explain(query.getExplain()) // + .id(query.getId()) // + .source(query.getSource()) // + ; + + if (!CollectionUtils.isEmpty(query.getParams())) { + Map params = getTemplateParams(query.getParams().entrySet()); + bb.params(params); + } + + return bb; + }) + ); + }); + return mtrb; + }); + } + + public MsearchRequest searchMsearchRequest( + List multiSearchQueryParameters, @Nullable String routing) { + + // basically the same stuff as in prepareSearchRequest, but the new Elasticsearch has different builders for a + // normal search and msearch + return MsearchRequest.of(mrb -> { + multiSearchQueryParameters.forEach(param -> { + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(param.clazz()); + + var query = param.query(); + mrb.searches(sb -> sb // + .header(msearchHeaderBuilder(query, param.index(), routing)) // + .body(bb -> { + bb // + .query(getQuery(query, param.clazz()))// + .seqNoPrimaryTerm(persistentEntity != null ? persistentEntity.hasSeqNoPrimaryTermProperty() : null) // + .version(true) // + .trackScores(query.getTrackScores()) // + .source(getSourceConfig(query)) // + .timeout(timeStringMs(query.getTimeout())) // + ; + + if (query.getPageable().isPaged()) { + bb // + .from((int) query.getPageable().getOffset()) // + .size(query.getPageable().getPageSize()); + } + + if (!isEmpty(query.getFields())) { + bb.fields(fb -> { + query.getFields().forEach(fb::field); + return fb; + }); + } + + if (!isEmpty(query.getStoredFields())) { + bb.storedFields(query.getStoredFields()); + } + + if (query.isLimiting()) { + bb.size(query.getMaxResults()); + } + + if (query.getMinScore() > 0) { + bb.minScore((double) query.getMinScore()); + } + + if (query.getSort() != null) { + List sortOptions = getSortOptions(query.getSort(), persistentEntity); + + if (!sortOptions.isEmpty()) { + bb.sort(sortOptions); + } + } + + addHighlight(query, bb); + + if (query.getExplain()) { + bb.explain(true); + } + + if (!isEmpty(query.getSearchAfter())) { + bb.searchAfter(query.getSearchAfter() + .stream() + .map(TypeUtils::toFieldValue) + .map(FieldValue::_toJsonString) + .toList()); + } + + query.getRescorerQueries().forEach(rescorerQuery -> bb.rescore(getRescore(rescorerQuery))); + + if (!isEmpty(query.getIndicesBoost())) { + bb.indicesBoost(query.getIndicesBoost().stream() + .map(indexBoost -> Map.of(indexBoost.getIndexName(), (double) indexBoost.getBoost())) + .collect(Collectors.toList())); + } + + query.getScriptedFields().forEach(scriptedField -> bb.scriptFields(scriptedField.getFieldName(), + sf -> sf.script(getScript(scriptedField.getScriptData())))); + + if (query instanceof NativeQuery nativeQuery) { + prepareNativeSearch(nativeQuery, bb); + } + return bb; + } // + ) // + ); + + }); + + return mrb; + }); + } + + /** + * {@link MsearchRequest} and {@link MsearchTemplateRequest} share the same {@link MultisearchHeader} + */ + private Function> msearchHeaderBuilder(Query query, + IndexCoordinates index, @Nullable String routing) { + return h -> { + var searchType = (query instanceof NativeQuery nativeQuery && nativeQuery.getKnnQuery() != null) ? null + : searchType(query.getSearchType()); + + h // + .index(Arrays.asList(index.getIndexNames())) // + .searchType(searchType) // + .requestCache(query.getRequestCache()) // + ; + + if (StringUtils.hasText(query.getRoute())) { + h.routing(query.getRoute()); + } else if (StringUtils.hasText(routing)) { + h.routing(routing); + } + + if (query.getPreference() != null) { + h.preference(query.getPreference()); + } + + return h; + }; + } + + private void prepareSearchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, SearchRequest.Builder builder, boolean forCount, boolean forBatchedSearch) { + + String[] indexNames = indexCoordinates.getIndexNames(); + + Assert.notEmpty(indexNames, "indexCoordinates does not contain entries"); + + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(clazz); + + var searchType = (query instanceof NativeQuery nativeQuery && nativeQuery.getKnnQuery() != null) ? null + : searchType(query.getSearchType()); + + builder // + .version(true) // + .trackScores(query.getTrackScores()) // + .allowNoIndices(query.getAllowNoIndices()) // + .source(getSourceConfig(query)) // + .searchType(searchType) // + .timeout(timeStringMs(query.getTimeout())) // + .requestCache(query.getRequestCache()) // + ; + + var pointInTime = query.getPointInTime(); + if (pointInTime != null) { + builder.pit(new Pit.Builder().id(pointInTime.id()).keepAlive(time(pointInTime.keepAlive()).time()).build()); + } else { + builder // + .index(Arrays.asList(indexNames)) // + ; + + var expandWildcards = query.getExpandWildcards(); + if (expandWildcards != null && !expandWildcards.isEmpty()) { + builder.expandWildcards(expandWildcards(expandWildcards)); + } + + if (query.getRoute() != null) { + builder.routing(query.getRoute()); + } else if (StringUtils.hasText(routing)) { + builder.routing(routing); + } + + if (query.getPreference() != null) { + builder.preference(query.getPreference()); + } + } + + if (persistentEntity != null && persistentEntity.hasSeqNoPrimaryTermProperty()) { + builder.seqNoPrimaryTerm(true); + } + + if (query.getPageable().isPaged()) { + builder // + .from((int) query.getPageable().getOffset()) // + .size(query.getPageable().getPageSize()); + } else { + builder.from(0).size(INDEX_MAX_RESULT_WINDOW); + } + + if (!isEmpty(query.getFields())) { + var fieldAndFormats = query.getFields().stream().map(field -> FieldAndFormat.of(b -> b.field(field))).toList(); + builder.fields(fieldAndFormats); + } + + if (!isEmpty(query.getStoredFields())) { + builder.storedFields(query.getStoredFields()); + } + + if (query.getIndicesOptions() != null) { + addIndicesOptions(builder, query.getIndicesOptions()); + } + + if (query.isLimiting()) { + builder.size(query.getMaxResults()); + } + + if (query.getMinScore() > 0) { + builder.minScore((double) query.getMinScore()); + } + + addHighlight(query, builder); + + query.getScriptedFields().forEach(scriptedField -> builder.scriptFields(scriptedField.getFieldName(), + sf -> sf.script(getScript(scriptedField.getScriptData())))); + + if (query instanceof NativeQuery nativeQuery) { + prepareNativeSearch(nativeQuery, builder); + } + // query.getSort() must be checked after prepareNativeSearch as this already might hav a sort set that must have + // higher priority + if (query.getSort() != null) { + List sortOptions = getSortOptions(query.getSort(), persistentEntity); + + if (!sortOptions.isEmpty()) { + builder.sort(sortOptions); + } + } + + if (query.getTrackTotalHits() != null) { + // logic from the RHLC, choose between -1 and Integer.MAX_VALUE + int value = query.getTrackTotalHits() ? Integer.MAX_VALUE : -1; + builder.trackTotalHits(th -> th.count(value)); + } else if (query.getTrackTotalHitsUpTo() != null) { + builder.trackTotalHits(th -> th.count(query.getTrackTotalHitsUpTo())); + } + + if (query.getExplain()) { + builder.explain(true); + } + + if (!isEmpty(query.getSearchAfter())) { + builder.searchAfter(query.getSearchAfter() + .stream() + .map(TypeUtils::toFieldValue) + .map(FieldValue::_toJsonString) + .toList()); + } + + query.getRescorerQueries().forEach(rescorerQuery -> builder.rescore(getRescore(rescorerQuery))); + + if (forCount) { + builder.size(0) // + .trackTotalHits(th -> th.count(Integer.MAX_VALUE)) // + .source(SourceConfig.of(sc -> sc.fetch(false))); + } else if (forBatchedSearch) { + // request_cache is not allowed on scroll requests. + builder.requestCache(null); + // limit the number of documents in a batch if not already set in a pageable + if (query.getPageable().isUnpaged()) { + builder.size(query.getReactiveBatchSize()); + } + } + + if (!isEmpty(query.getIndicesBoost())) { + builder.indicesBoost(query.getIndicesBoost().stream() + .map(indexBoost -> Map.of(indexBoost.getIndexName(), (double) indexBoost.getBoost())) + .collect(Collectors.toList())); + } + + if (!isEmpty(query.getDocValueFields())) { + builder.docvalueFields(query.getDocValueFields().stream() // + .map(docValueField -> FieldAndFormat.of(b -> b.field(docValueField.field()).format(docValueField.format()))) + .toList()); + } + } + + private void addIndicesOptions(SearchRequest.Builder builder, IndicesOptions indicesOptions) { + + indicesOptions.getOptions().forEach(option -> { + switch (option) { + case ALLOW_NO_INDICES -> builder.allowNoIndices(true); + case IGNORE_UNAVAILABLE -> builder.ignoreUnavailable(true); + case IGNORE_THROTTLED -> builder.ignoreThrottled(true); + // the following ones aren't supported by the builder + case FORBID_ALIASES_TO_MULTIPLE_INDICES, FORBID_CLOSED_INDICES, IGNORE_ALIASES -> { + if (LOGGER.isWarnEnabled()) { + LOGGER + .warn(String.format("indices option %s is not supported by the Elasticsearch client.", option.name())); + } + } + } + }); + + builder.expandWildcards(indicesOptions.getExpandWildcards().stream() + .map(wildcardStates -> switch (wildcardStates) { + case OPEN -> ExpandWildcard.Open; + case CLOSED -> ExpandWildcard.Closed; + case HIDDEN -> ExpandWildcard.Hidden; + case ALL -> ExpandWildcard.All; + case NONE -> ExpandWildcard.None; + }).collect(Collectors.toList())); + } + + private Rescore getRescore(RescorerQuery rescorerQuery) { + + return Rescore.of(r -> r // + .query(rq -> rq // + .query(getQuery(rescorerQuery.getQuery(), null)) // + .scoreMode(scoreMode(rescorerQuery.getScoreMode())) // + .queryWeight(rescorerQuery.getQueryWeight() != null ? Double.valueOf(rescorerQuery.getQueryWeight()) : 1.0) // + .rescoreQueryWeight( + rescorerQuery.getRescoreQueryWeight() != null ? Double.valueOf(rescorerQuery.getRescoreQueryWeight()) + : 1.0) // + + ) // + .windowSize(rescorerQuery.getWindowSize())); + } + + private void addHighlight(Query query, SearchRequest.Builder builder) { + + Highlight highlight = query.getHighlightQuery() + .map(highlightQuery -> new HighlightQueryBuilder(elasticsearchConverter.getMappingContext()) + .getHighlight(highlightQuery.getHighlight(), highlightQuery.getType())) + .orElse(null); + + builder.highlight(highlight); + } + + private void addHighlight(Query query, MultisearchBody.Builder builder) { + + Highlight highlight = query.getHighlightQuery() + .map(highlightQuery -> new HighlightQueryBuilder(elasticsearchConverter.getMappingContext()) + .getHighlight(highlightQuery.getHighlight(), highlightQuery.getType())) + .orElse(null); + + builder.highlight(highlight); + } + + private List getSortOptions(Sort sort, @Nullable ElasticsearchPersistentEntity persistentEntity) { + return sort.stream().map(order -> getSortOptions(order, persistentEntity)).collect(Collectors.toList()); + } + + private SortOptions getSortOptions(Sort.Order order, @Nullable ElasticsearchPersistentEntity persistentEntity) { + SortOrder sortOrder = order.getDirection().isDescending() ? SortOrder.Desc : SortOrder.Asc; + + Order.Mode mode = order.getDirection().isAscending() ? Order.Mode.min : Order.Mode.max; + String unmappedType = null; + String missing = null; + NestedSortValue nestedSortValue = null; + + if (SortOptions.Kind.Score.jsonValue().equals(order.getProperty())) { + return SortOptions.of(so -> so.score(s -> s.order(sortOrder))); + } + + if (order instanceof Order o) { + + if (o.getMode() != null) { + mode = o.getMode(); + } + unmappedType = o.getUnmappedType(); + missing = o.getMissing(); + nestedSortValue = getNestedSort(o.getNested(), persistentEntity); + } + Order.Mode finalMode = mode; + String finalUnmappedType = unmappedType; + var finalNestedSortValue = nestedSortValue; + + ElasticsearchPersistentProperty property = (persistentEntity != null) // + ? persistentEntity.getPersistentProperty(order.getProperty()) // + : null; + String fieldName = property != null ? property.getFieldName() : order.getProperty(); + + if (order instanceof GeoDistanceOrder geoDistanceOrder) { + return getSortOptions(geoDistanceOrder, fieldName, finalMode); + } + + var finalMissing = missing != null ? missing + : (order.getNullHandling() == Sort.NullHandling.NULLS_FIRST) ? "_first" + : ((order.getNullHandling() == Sort.NullHandling.NULLS_LAST) ? "_last" : null); + + return SortOptions.of(so -> so // + .field(f -> { + f.field(fieldName) // + .order(sortOrder) // + .mode(sortMode(finalMode)); + + if (finalUnmappedType != null) { + FieldType fieldType = fieldType(finalUnmappedType); + + if (fieldType != null) { + f.unmappedType(fieldType); + } + } + + if (finalMissing != null) { + f.missing(fv -> fv // + .stringValue(finalMissing)); + } + + if (finalNestedSortValue != null) { + f.nested(finalNestedSortValue); + } + + return f; + })); + } + + @Nullable + private NestedSortValue getNestedSort(@Nullable Order.Nested nested, + @Nullable ElasticsearchPersistentEntity persistentEntity) { + return (nested == null || persistentEntity == null) ? null + : NestedSortValue.of(b -> b // + .path(elasticsearchConverter.updateFieldNames(nested.getPath(), persistentEntity)) // + .maxChildren(nested.getMaxChildren()) // + .nested(getNestedSort(nested.getNested(), persistentEntity)) // + .filter(getQuery(nested.getFilter(), persistentEntity.getType()))); + } + + private static SortOptions getSortOptions(GeoDistanceOrder geoDistanceOrder, String fieldName, Order.Mode finalMode) { + return SortOptions.of(so -> so // + .geoDistance(gd -> gd // + .field(fieldName) // + .location(loc -> loc.latlon(Queries.latLon(geoDistanceOrder.getGeoPoint()))) // + .distanceType(geoDistanceType(geoDistanceOrder.getDistanceType())).mode(sortMode(finalMode)) // + .order(sortOrder(geoDistanceOrder.getDirection())) // + .unit(distanceUnit(geoDistanceOrder.getUnit())) // + .ignoreUnmapped(geoDistanceOrder.getIgnoreUnmapped()))); + } + + @SuppressWarnings("DuplicatedCode") + private void prepareNativeSearch(NativeQuery query, SearchRequest.Builder builder) { + + builder // + .suggest(query.getSuggester()) // + .collapse(query.getFieldCollapse()) // + .sort(query.getSortOptions()) // + ; + + if (query.getKnnQuery() != null) { + builder.query(query.getKnnQuery().toQuery()); + } + + if (!isEmpty(query.getAggregations())) { + builder.aggregations(query.getAggregations()); + } + + if (!isEmpty(query.getSearchExtensions())) { + builder.ext(query.getSearchExtensions()); + } + } + + @SuppressWarnings("DuplicatedCode") + private void prepareNativeSearch(NativeQuery query, MultisearchBody.Builder builder) { + + builder // + .suggest(query.getSuggester()) // + .collapse(query.getFieldCollapse()) // + .sort(query.getSortOptions()); + + if (query.getKnnQuery() != null) { + builder.query(query.getKnnQuery().toQuery()); + } + + if (!isEmpty(query.getAggregations())) { + builder.aggregations(query.getAggregations()); + } + + if (!isEmpty(query.getSearchExtensions())) { + builder.ext(query.getSearchExtensions()); + } + } + + @Nullable + org.opensearch.client.opensearch._types.query_dsl.Query getQuery(@Nullable Query query, + @Nullable Class clazz) { + + if (query == null) { + return null; + } + + elasticsearchConverter.updateQuery(query, clazz); + + org.opensearch.client.opensearch._types.query_dsl.Query esQuery = null; + + if (query instanceof CriteriaQuery) { + esQuery = CriteriaQueryProcessor.createQuery(((CriteriaQuery) query).getCriteria()); + } else if (query instanceof StringQuery) { + esQuery = Queries.wrapperQueryAsQuery(((StringQuery) query).getSource()); + } else if (query instanceof NativeQuery nativeQuery) { + + if (nativeQuery.getQuery() != null) { + esQuery = nativeQuery.getQuery(); + } else if (nativeQuery.getSpringDataQuery() != null) { + esQuery = getQuery(nativeQuery.getSpringDataQuery(), clazz); + } + } else { + throw new IllegalArgumentException("unhandled Query implementation " + query.getClass().getName()); + } + + return esQuery; + } + + private void addFilter(Query query, SearchRequest.Builder builder) { + + if (query instanceof CriteriaQuery) { + CriteriaFilterProcessor.createQuery(((CriteriaQuery) query).getCriteria()).ifPresent(builder::postFilter); + } else // noinspection StatementWithEmptyBody + if (query instanceof StringQuery) { + // no filter for StringQuery + } else if (query instanceof NativeQuery) { + builder.postFilter(((NativeQuery) query).getFilter()); + } else { + throw new IllegalArgumentException("unhandled Query implementation " + query.getClass().getName()); + } + } + + public org.opensearch.client.opensearch._types.query_dsl.MoreLikeThisQuery moreLikeThisQuery(MoreLikeThisQuery query, + IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(index, "index must not be null"); + + return org.opensearch.client.opensearch._types.query_dsl.MoreLikeThisQuery + .of(q -> { + q.like(Like.of(l -> l.document(ld -> ld.index(index.getIndexName()).id(query.getId())))) + .fields(query.getFields()); + + if (query.getMinTermFreq() != null) { + q.minTermFreq(query.getMinTermFreq()); + } + + if (query.getMaxQueryTerms() != null) { + q.maxQueryTerms(query.getMaxQueryTerms()); + } + + if (!isEmpty(query.getStopWords())) { + q.stopWords(query.getStopWords()); + } + + if (query.getMinDocFreq() != null) { + q.minDocFreq(query.getMinDocFreq()); + } + + if (query.getMaxDocFreq() != null) { + q.maxDocFreq(query.getMaxDocFreq()); + } + + if (query.getMinWordLen() != null) { + q.minWordLength(query.getMinWordLen()); + } + + if (query.getMaxWordLen() != null) { + q.maxWordLength(query.getMaxWordLen()); + } + + if (query.getBoostTerms() != null) { + q.boostTerms(Double.valueOf(query.getBoostTerms())); + } + + return q; + }); + } + + public CreatePitRequest searchOpenPointInTimeRequest(IndexCoordinates index, Duration keepAlive, + Boolean allowPartialPitCreation) { + + Assert.notNull(index, "index must not be null"); + Assert.notNull(keepAlive, "keepAlive must not be null"); + Assert.notNull(allowPartialPitCreation, "allowPartialPitCreation must not be null"); + + return CreatePitRequest.of(opit -> opit // + .targetIndexes(Arrays.asList(index.getIndexNames())) // + .allowPartialPitCreation(allowPartialPitCreation) // + .keepAlive(time(keepAlive)) // + ); + } + + public DeletePitRequest searchClosePointInTime(String pit) { + + Assert.notNull(pit, "pit must not be null"); + + return DeletePitRequest.of(cpit -> cpit.pitId(Collections.singletonList(pit))); + } + + public SearchTemplateRequest searchTemplate(SearchTemplateQuery query, @Nullable String routing, + IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + + return SearchTemplateRequest.of(builder -> { + builder // + .allowNoIndices(query.getAllowNoIndices()) // + .explain(query.getExplain()) // + .id(query.getId()) // + .index(Arrays.asList(index.getIndexNames())) // + .preference(query.getPreference()) // + .searchType(searchType(query.getSearchType())) // + .source(query.getSource()) // + ; + + if (query.getRoute() != null) { + builder.routing(query.getRoute()); + } else if (StringUtils.hasText(routing)) { + builder.routing(routing); + } + + var expandWildcards = query.getExpandWildcards(); + if (expandWildcards != null && !expandWildcards.isEmpty()) { + builder.expandWildcards(expandWildcards(expandWildcards)); + } + + if (query.hasScrollTime()) { + builder.scroll(time(query.getScrollTime())); + } + + if (!CollectionUtils.isEmpty(query.getParams())) { + Map params = getTemplateParams(query.getParams().entrySet()); + builder.params(params); + } + + return builder; + }); + } + + @NonNull + private Map getTemplateParams(Set> query) { + Function, String> keyMapper = Map.Entry::getKey; + Function, JsonData> valueMapper = entry -> JsonData.of(entry.getValue(), jsonpMapper); + return query.stream() + .collect(Collectors.toMap(keyMapper, valueMapper)); + } + + // endregion + + public PutScriptRequest scriptPut(Script script) { + + Assert.notNull(script, "script must not be null"); + + return PutScriptRequest.of(b -> b // + .id(script.id()) // + .script(sb -> sb // + .lang(script.language()) // + .source(script.source()))); + } + + public GetScriptRequest scriptGet(String name) { + + Assert.notNull(name, "name must not be null"); + + return GetScriptRequest.of(b -> b.id(name)); + } + + public DeleteScriptRequest scriptDelete(String name) { + + Assert.notNull(name, "name must not be null"); + + return DeleteScriptRequest.of(b -> b.id(name)); + } + // region helper functions + + public T fromJson(String json, JsonpDeserializer deserializer) { + + Assert.notNull(json, "json must not be null"); + Assert.notNull(deserializer, "deserializer must not be null"); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)); + return fromJson(byteArrayInputStream, deserializer); + } + + public T fromJson(ByteArrayInputStream byteArrayInputStream, JsonpDeserializer deserializer) { + + Assert.notNull(byteArrayInputStream, "byteArrayInputStream must not be null"); + Assert.notNull(deserializer, "deserializer must not be null"); + + JsonParser parser = jsonpMapper.jsonProvider().createParser(byteArrayInputStream); + return deserializer.deserialize(parser, jsonpMapper); + } + + @Nullable + private ElasticsearchPersistentEntity getPersistentEntity(Object entity) { + return elasticsearchConverter.getMappingContext().getPersistentEntity(entity.getClass()); + } + + @Nullable + private ElasticsearchPersistentEntity getPersistentEntity(@Nullable Class clazz) { + return clazz != null ? elasticsearchConverter.getMappingContext().getPersistentEntity(clazz) : null; + } + + @Nullable + private String getPersistentEntityId(Object entity) { + + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(entity); + + if (persistentEntity != null) { + Object identifier = persistentEntity // + .getIdentifierAccessor(entity).getIdentifier(); + + if (identifier != null) { + return identifier.toString(); + } + } + + return null; + } + + private VersionType retrieveVersionTypeFromPersistentEntity(@Nullable Class clazz) { + + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(clazz); + + VersionType versionType = null; + + if (persistentEntity != null) { + org.springframework.data.elasticsearch.annotations.Document.VersionType entityVersionType = persistentEntity + .getVersionType(); + + if (entityVersionType != null) { + versionType = switch (entityVersionType) { + case INTERNAL -> VersionType.Internal; + case EXTERNAL -> VersionType.External; + case EXTERNAL_GTE -> VersionType.ExternalGte; + case FORCE -> VersionType.Force; + }; + } + } + + return versionType != null ? versionType : VersionType.External; + } + + @Nullable + private SourceConfig getSourceConfig(Query query) { + + if (query.getSourceFilter() != null) { + return SourceConfig.of(s -> s // + .filter(sfb -> { + SourceFilter sourceFilter = query.getSourceFilter(); + String[] includes = sourceFilter.getIncludes(); + String[] excludes = sourceFilter.getExcludes(); + + if (includes != null) { + sfb.includes(Arrays.asList(includes)); + } + + if (excludes != null) { + sfb.excludes(Arrays.asList(excludes)); + } + + return sfb; + })); + } else { + return null; + } + } + + @Nullable + static Boolean deleteByQueryRefresh(@Nullable RefreshPolicy refreshPolicy) { + + if (refreshPolicy == null) { + return null; + } + + return switch (refreshPolicy) { + case IMMEDIATE -> true; + case WAIT_UNTIL -> null; + case NONE -> false; + }; + } + + // endregion +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ResponseConverter.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ResponseConverter.java new file mode 100644 index 0000000..c747fff --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ResponseConverter.java @@ -0,0 +1,567 @@ +/* + * Copyright 2021-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.JsonUtils.toJson; +import static org.opensearch.data.client.osc.TypeUtils.removePrefixFromJson; +import static org.opensearch.data.client.osc.TypeUtils.typeMapping; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.BulkIndexByScrollFailure; +import org.opensearch.client.opensearch._types.ErrorCause; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch.cluster.ComponentTemplateSummary; +import org.opensearch.client.opensearch.cluster.GetComponentTemplateResponse; +import org.opensearch.client.opensearch.cluster.HealthResponse; +import org.opensearch.client.opensearch.core.DeleteByQueryResponse; +import org.opensearch.client.opensearch.core.GetScriptResponse; +import org.opensearch.client.opensearch.core.UpdateByQueryResponse; +import org.opensearch.client.opensearch.core.mget.MultiGetError; +import org.opensearch.client.opensearch.core.mget.MultiGetResponseItem; +import org.opensearch.client.opensearch.indices.*; +import org.opensearch.client.opensearch.indices.get_index_template.IndexTemplateItem; +import org.opensearch.client.opensearch.indices.get_index_template.IndexTemplateSummary; +import org.opensearch.client.opensearch.indices.get_mapping.IndexMappingRecord; +import org.springframework.data.elasticsearch.ElasticsearchErrorCause; +import org.springframework.data.elasticsearch.core.IndexInformation; +import org.springframework.data.elasticsearch.core.MultiGetItem; +import org.springframework.data.elasticsearch.core.cluster.ClusterHealth; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.index.*; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.ByQueryResponse; +import org.springframework.data.elasticsearch.core.query.StringQuery; +import org.springframework.data.elasticsearch.core.reindex.ReindexResponse; +import org.springframework.data.elasticsearch.core.script.Script; +import org.springframework.data.elasticsearch.support.DefaultStringObjectMap; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Class to convert Elasticsearch responses into Spring Data Elasticsearch classes. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +class ResponseConverter { + + private static final Log LOGGER = LogFactory.getLog(ResponseConverter.class); + + private final JsonpMapper jsonpMapper; + + public ResponseConverter(JsonpMapper jsonpMapper) { + this.jsonpMapper = jsonpMapper; + } + + // region cluster client + public ClusterHealth clusterHealth(HealthResponse healthResponse) { + + Assert.notNull(healthResponse, "healthResponse must not be null"); + + return ClusterHealth.builder() // + .withActivePrimaryShards(healthResponse.activePrimaryShards()) // + .withActiveShards(healthResponse.activeShards()) // + .withActiveShardsPercent(Double.parseDouble(healthResponse.activeShardsPercentAsNumber()))// + .withClusterName(healthResponse.clusterName()) // + .withDelayedUnassignedShards(healthResponse.delayedUnassignedShards()) // + .withInitializingShards(healthResponse.initializingShards()) // + .withNumberOfDataNodes(healthResponse.numberOfDataNodes()) // + .withNumberOfInFlightFetch(healthResponse.numberOfInFlightFetch()) // + .withNumberOfNodes(healthResponse.numberOfNodes()) // + .withNumberOfPendingTasks(healthResponse.numberOfPendingTasks()) // + .withRelocatingShards(healthResponse.relocatingShards()) // + .withStatus(healthResponse.status().toString()) // + .withTaskMaxWaitingTimeMillis(Long.parseLong(healthResponse.taskMaxWaitingInQueueMillis())) // + .withTimedOut(healthResponse.timedOut()) // + .withUnassignedShards(healthResponse.unassignedShards()) // + .build(); // + } + + public List clusterGetComponentTemplates( + GetComponentTemplateResponse getComponentTemplateResponse) { + + Assert.notNull(getComponentTemplateResponse, "getComponentTemplateResponse must not be null"); + + var componentTemplates = new ArrayList(); + getComponentTemplateResponse.componentTemplates().forEach(componentTemplate -> { + componentTemplates.add(clusterGetComponentTemplate(componentTemplate)); + }); + + return componentTemplates; + } + + private TemplateResponse clusterGetComponentTemplate( + org.opensearch.client.opensearch.cluster.ComponentTemplate componentTemplate) { + var componentTemplateNode = componentTemplate.componentTemplate(); + var componentTemplateSummary = componentTemplateNode.template(); + return TemplateResponse.builder() // + .withName(componentTemplate.name()) // + .withVersion(componentTemplateNode.version()) // + .withTemplateData(clusterGetComponentTemplateData(componentTemplateSummary)) // + .build(); + } + + private TemplateResponseData clusterGetComponentTemplateData(ComponentTemplateSummary componentTemplateSummary) { + + var mapping = typeMapping(componentTemplateSummary.mappings()); + var settings = new Settings(); + componentTemplateSummary.settings().forEach((key, indexSettings) -> { + settings.put(key, Settings.parse(removePrefixFromJson(indexSettings.toString()))); + }); + + Function, String> keyMapper = Map.Entry::getKey; + Function, AliasData> valueMapper = entry -> indicesGetAliasData( + entry.getKey(), entry.getValue()); + + Map aliases = componentTemplateSummary.aliases().entrySet().stream() + .collect(Collectors.toMap(keyMapper, valueMapper)); + + return TemplateResponseData.builder() // + .withMapping(mapping) // + .withSettings(settings) // + .withAliases(aliases) // + .build(); + } + + // endregion + + // region indices client + public Settings indicesGetSettings(GetIndicesSettingsResponse getIndicesSettingsResponse, String indexName) { + + Assert.notNull(getIndicesSettingsResponse, "getIndicesSettingsResponse must not be null"); + Assert.notNull(indexName, "indexName must not be null"); + + Settings settings = new Settings(); + IndexState indexState = getIndicesSettingsResponse.get(indexName); + + if (indexState != null) { + + Function indexSettingsToSettings = indexSettings -> { + Settings parsedSettings = Settings.parse(toJson(indexSettings, jsonpMapper)); + return (indexSettings.index() != null) ? parsedSettings : new Settings().append("index", parsedSettings); + }; + + if (indexState.defaults() != null) { + Settings defaultSettings = indexSettingsToSettings.apply(indexState.defaults()); + settings.merge(defaultSettings); + } + + if (indexState.settings() != null) { + Settings nonDefaultSettings = indexSettingsToSettings.apply(indexState.settings()); + settings.merge(nonDefaultSettings); + } + } + + return settings; + } + + public Document indicesGetMapping(GetMappingResponse getMappingResponse, IndexCoordinates indexCoordinates) { + + Assert.notNull(getMappingResponse, "getMappingResponse must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + Map mappings = getMappingResponse.result(); + + if (mappings == null || mappings.size() == 0) { + return Document.create(); + } + + IndexMappingRecord indexMappingRecord = mappings.get(indexCoordinates.getIndexName()); + + // this can happen when the mapping was requested with an alias + if (indexMappingRecord == null) { + + if (mappings.size() != 1) { + LOGGER.warn(String.format("no mapping returned for index %s", indexCoordinates.getIndexName())); + return Document.create(); + } + String index = mappings.keySet().iterator().next(); + indexMappingRecord = mappings.get(index); + } + + return Document.parse(toJson(indexMappingRecord.mappings(), jsonpMapper)); + } + + public List indicesGetIndexInformations(GetIndexResponse getIndexResponse) { + + Assert.notNull(getIndexResponse, "getIndexResponse must not be null"); + + List indexInformationList = new ArrayList<>(); + + getIndexResponse.result().forEach((indexName, indexState) -> { + Settings settings = indexState.settings() != null ? Settings.parse(toJson(indexState.settings(), jsonpMapper)) + : new Settings(); + Document mappings = indexState.mappings() != null ? Document.parse(toJson(indexState.mappings(), jsonpMapper)) + : Document.create(); + + List aliasDataList = new ArrayList<>(); + indexState.aliases().forEach((aliasName, alias) -> aliasDataList.add(indicesGetAliasData(aliasName, alias))); + + indexInformationList.add(IndexInformation.of(indexName, settings, mappings, aliasDataList)); + + }); + return indexInformationList; + } + + public Map> indicesGetAliasData(GetAliasResponse getAliasResponse) { + + Assert.notNull(getAliasResponse, "getAliasResponse must not be null"); + + Map> aliasDataMap = new HashMap<>(); + getAliasResponse.result().forEach((indexName, alias) -> { + Set aliasDataSet = new HashSet<>(); + alias.aliases() + .forEach((aliasName, aliasDefinition) -> aliasDataSet.add(indicesGetAliasData(aliasName, aliasDefinition))); + aliasDataMap.put(indexName, aliasDataSet); + }); + return aliasDataMap; + } + + private AliasData indicesGetAliasData(String aliasName, Alias alias) { + + Query filter = alias.filter(); + String filterJson = filter != null ? toJson(filter, jsonpMapper) : null; + var filterQuery = filterJson != null ? StringQuery.builder(filterJson).build() : null; + return AliasData.of(aliasName, filterQuery, alias.indexRouting(), alias.searchRouting(), alias.isWriteIndex(), + alias.isHidden()); + } + + private AliasData indicesGetAliasData(String aliasName, AliasDefinition alias) { + Query filter = alias.filter(); + String filterJson = filter != null ? toJson(filter, jsonpMapper) : null; + var filterQuery = filterJson != null ? StringQuery.builder(filterJson).build() : null; + return AliasData.of(aliasName, filterQuery, alias.indexRouting(), alias.searchRouting(), alias.isWriteIndex(), + null); + } + + @Nullable + public TemplateData indicesGetTemplateData(GetTemplateResponse getTemplateResponse, String templateName) { + + Assert.notNull(getTemplateResponse, "getTemplateResponse must not be null"); + Assert.notNull(templateName, "templateName must not be null"); + + TemplateMapping templateMapping = getTemplateResponse.get(templateName); + if (templateMapping != null) { + + Settings settings = new Settings(); + templateMapping.settings().forEach((key, jsonData) -> { + + if (key.contains(".")) { + // returned string contains " quotes + settings.put(key, jsonData.toJson().toString().replaceAll("^\"|\"$", "")); + } else { + settings.put(key, new DefaultStringObjectMap<>().fromJson(jsonData.toJson().toString())); + } + }); + + Function, String> keyMapper = Map.Entry::getKey; + Function, AliasData> valueMapper = entry -> indicesGetAliasData(entry.getKey(), + entry.getValue()); + + Map aliases = templateMapping.aliases().entrySet().stream() + .collect(Collectors.toMap(keyMapper, valueMapper)); + + Document mapping = Document.parse(toJson(templateMapping.mappings(), jsonpMapper)); + + TemplateData.TemplateDataBuilder builder = TemplateData.builder() // + .withIndexPatterns(templateMapping.indexPatterns().toArray(new String[0])) // + .withOrder(templateMapping.order()) // + .withSettings(settings) // + .withMapping(mapping) // + .withAliases(aliases) // + ; + + if (templateMapping.version() != null) { + builder.withVersion(templateMapping.version().intValue()); + } + + return builder.build(); + } + + return null; + } + + public List getIndexTemplates(GetIndexTemplateResponse getIndexTemplateResponse) { + + Assert.notNull(getIndexTemplateResponse, "getIndexTemplateResponse must not be null"); + + var componentTemplates = new ArrayList(); + getIndexTemplateResponse.indexTemplates().forEach(indexTemplateItem -> { + componentTemplates.add(indexGetComponentTemplate(indexTemplateItem)); + }); + + return componentTemplates; + } + + private TemplateResponse indexGetComponentTemplate(IndexTemplateItem indexTemplateItem) { + var indexTemplate = indexTemplateItem.indexTemplate(); + var composedOf = indexTemplate.composedOf(); + var indexTemplateSummary = indexTemplate.template(); + return TemplateResponse.builder() // + .withName(indexTemplateItem.name()) // + .withVersion(indexTemplate.version()) // + .withTemplateData(indexGetComponentTemplateData(indexTemplateSummary, composedOf)) // + .build(); + } + + private TemplateResponseData indexGetComponentTemplateData(IndexTemplateSummary indexTemplateSummary, + List composedOf) { + var mapping = typeMapping(indexTemplateSummary.mappings()); + + Function, Settings> indexSettingsToSettings = indexSettings -> { + + if (indexSettings == null) { + return null; + } + + Settings parsedSettings = Settings.parse(toJson(indexSettings, jsonpMapper)); + return (indexSettings.get("index") != null) ? parsedSettings : new Settings().append("index", parsedSettings); + }; + var settings = indexSettingsToSettings.apply(indexTemplateSummary.settings()); + + Function, String> keyMapper = Map.Entry::getKey; + Function, AliasData> valueMapper = entry -> indicesGetAliasData(entry.getKey(), + entry.getValue()); + + Map aliases1 = indexTemplateSummary.aliases(); + Map aliases = aliases1.entrySet().stream().collect(Collectors.toMap(keyMapper, valueMapper)); + + return TemplateResponseData.builder() // + .withMapping(mapping) // + .withSettings(settings) // + .withAliases(aliases) // + .withComposedOf(composedOf) // + .build(); + } + + // endregion + + // region document operations + public ReindexResponse reindexResponse(org.opensearch.client.opensearch.core.ReindexResponse reindexResponse) { + + Assert.notNull(reindexResponse, "reindexResponse must not be null"); + + List failures = reindexResponse.failures() // + .stream() // + .map(this::reindexResponseFailureOf) // + .collect(Collectors.toList()); + + // noinspection ConstantConditions + return ReindexResponse.builder() // + .withTook(timeToLong(reindexResponse.took())) // + .withTimedOut(reindexResponse.timedOut()) // + .withTotal(reindexResponse.total()) // + .withCreated(reindexResponse.created()) // + .withUpdated(reindexResponse.updated()) // + .withDeleted(reindexResponse.deleted()) // + .withBatches(reindexResponse.batches()) // + .withVersionConflicts(reindexResponse.versionConflicts()) // + .withNoops(reindexResponse.noops()) // + .withBulkRetries(reindexResponse.retries().bulk()) // + .withSearchRetries(reindexResponse.retries().search()) // + .withThrottledMillis(timeToLong(reindexResponse.throttledMillis())) // + .withRequestsPerSecond(reindexResponse.requestsPerSecond()) // + .withThrottledUntilMillis(timeToLong(reindexResponse.throttledUntilMillis())) // + .withFailures(failures) // + .build(); + } + + private ReindexResponse.Failure reindexResponseFailureOf(BulkIndexByScrollFailure failure) { + return ReindexResponse.Failure.builder() // + .withIndex(failure.index()) // + .withType(failure.type()) // + .withId(failure.id()) // + .withStatus(failure.status())// + .withErrorCause(toErrorCause(failure.cause())) // + // seqno, term, aborted are not available in the new client + .build(); + } + + private ByQueryResponse.Failure byQueryResponseFailureOf(BulkIndexByScrollFailure failure) { + return ByQueryResponse.Failure.builder() // + .withIndex(failure.index()) // + .withType(failure.type()) // + .withId(failure.id()) // + .withStatus(failure.status())// + .withErrorCause(toErrorCause(failure.cause())).build(); + } + + @Nullable + public static MultiGetItem.Failure getFailure(MultiGetResponseItem itemResponse) { + + MultiGetError responseFailure = itemResponse.isFailure() ? itemResponse.failure() : null; + + return responseFailure != null + ? MultiGetItem.Failure.of(responseFailure.index(), null, responseFailure.id(), null, + toErrorCause(responseFailure.error())) + : null; + } + + public ByQueryResponse byQueryResponse(DeleteByQueryResponse response) { + // the code for the methods taking a DeleteByQueryResponse or a UpdateByQueryResponse is duplicated because the + // Elasticsearch responses do not share a common class + // noinspection DuplicatedCode + List failures = response.failures().stream().map(this::byQueryResponseFailureOf) + .collect(Collectors.toList()); + + ByQueryResponse.ByQueryResponseBuilder builder = ByQueryResponse.builder(); + + if (response.took() != null) { + builder.withTook(response.took()); + } + + if (response.timedOut() != null) { + builder.withTimedOut(response.timedOut()); + } + + if (response.total() != null) { + builder.withTotal(response.total()); + } + + if (response.deleted() != null) { + builder.withDeleted(response.deleted()); + } + + if (response.batches() != null) { + builder.withBatches(Math.toIntExact(response.batches())); + } + + if (response.versionConflicts() != null) { + builder.withVersionConflicts(response.versionConflicts()); + } + + if (response.noops() != null) { + builder.withNoops(response.noops()); + } + + if (response.retries() != null) { + builder.withBulkRetries(response.retries().bulk()); + builder.withSearchRetries(response.retries().search()); + } + + builder.withFailures(failures); + + return builder.build(); + } + + public ByQueryResponse byQueryResponse(UpdateByQueryResponse response) { + // the code for the methods taking a DeleteByQueryResponse or a UpdateByQueryResponse is duplicated because the + // Elasticsearch responses do not share a common class + // noinspection DuplicatedCode + List failures = response.failures().stream().map(this::byQueryResponseFailureOf) + .collect(Collectors.toList()); + + ByQueryResponse.ByQueryResponseBuilder builder = ByQueryResponse.builder(); + + if (response.took() != null) { + builder.withTook(response.took()); + } + + if (response.timedOut() != null) { + builder.withTimedOut(response.timedOut()); + } + + if (response.total() != null) { + builder.withTotal(response.total()); + } + + if (response.deleted() != null) { + builder.withDeleted(response.deleted()); + } + + if (response.batches() != null) { + builder.withBatches(Math.toIntExact(response.batches())); + } + + if (response.versionConflicts() != null) { + builder.withVersionConflicts(response.versionConflicts()); + } + + if (response.noops() != null) { + builder.withNoops(response.noops()); + } + + if (response.retries() != null) { + builder.withBulkRetries(response.retries().bulk()); + builder.withSearchRetries(response.retries().search()); + } + + builder.withFailures(failures); + + return builder.build(); + } + + // endregion + + // region script API + @Nullable + public Script scriptResponse(GetScriptResponse response) { + + Assert.notNull(response, "response must not be null"); + + return response.found() // + ? Script.builder() // + .withId(response.id()) // + .withLanguage(response.script().lang()) // + .withSource(response.script().source()).build() // + : null; + } + // endregion + + // region helper functions + + private long timeToLong(String time) { + + if (time == null) { + return 0L; + } else { + return Long.parseLong(time); + } + } + + private long timeToLong(Time time) { + + if (time.isTime()) { + return Long.parseLong(time.time()); + } else { + return time.offset(); + } + } + + @Nullable + static ElasticsearchErrorCause toErrorCause(@Nullable ErrorCause errorCause) { + + if (errorCause != null) { + return new ElasticsearchErrorCause( // + errorCause.type(), // + errorCause.reason(), // + errorCause.stackTrace(), // + toErrorCause(errorCause.causedBy()), // + errorCause.rootCause().stream().map(ResponseConverter::toErrorCause).collect(Collectors.toList()), // + errorCause.suppressed().stream().map(ResponseConverter::toErrorCause).collect(Collectors.toList())); + } else { + return null; + } + } + // endregion +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilder.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilder.java new file mode 100644 index 0000000..96d3590 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilder.java @@ -0,0 +1,264 @@ +/* + * Copyright 2021-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.aggregations.Aggregate; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.SearchTemplateResponse; +import org.opensearch.client.opensearch.core.search.CompletionSuggest; +import org.opensearch.client.opensearch.core.search.CompletionSuggestOption; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.client.opensearch.core.search.HitsMetadata; +import org.opensearch.client.opensearch.core.search.SearchResult; +import org.opensearch.client.opensearch.core.search.TotalHits; +import org.springframework.data.elasticsearch.core.TotalHitsRelation; +import org.springframework.data.elasticsearch.core.document.SearchDocument; +import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; +import org.springframework.data.elasticsearch.core.suggest.response.CompletionSuggestion; +import org.springframework.data.elasticsearch.core.suggest.response.PhraseSuggestion; +import org.springframework.data.elasticsearch.core.suggest.response.Suggest; +import org.springframework.data.elasticsearch.core.suggest.response.TermSuggestion; +import org.springframework.data.elasticsearch.support.ScoreDoc; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +/** + * Factory class to create {@link SearchDocumentResponse} instances. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +class SearchDocumentResponseBuilder { + + private static final Log LOGGER = LogFactory.getLog(SearchDocumentResponseBuilder.class); + + /** + * creates a SearchDocumentResponse from the {@link SearchResponse} + * + * @param responseBody the Elasticsearch response body + * @param entityCreator function to create an entity from a {@link SearchDocument} + * @param jsonpMapper to map JsonData objects + * @return the SearchDocumentResponse + */ + public static SearchDocumentResponse from(SearchResult responseBody, + SearchDocumentResponse.EntityCreator entityCreator, JsonpMapper jsonpMapper) { + + Assert.notNull(responseBody, "responseBody must not be null"); + Assert.notNull(entityCreator, "entityCreator must not be null"); + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + HitsMetadata hitsMetadata = responseBody.hits(); + String scrollId = responseBody.scrollId(); + Map aggregations = responseBody.aggregations(); + Map>> suggest = responseBody.suggest(); + var pointInTimeId = responseBody.pitId(); + + return from(hitsMetadata, scrollId, pointInTimeId, aggregations, suggest, entityCreator, jsonpMapper); + } + + /** + * creates a SearchDocumentResponse from the {@link SearchTemplateResponse} + * + * @param response the Elasticsearch response body + * @param entityCreator function to create an entity from a {@link SearchDocument} + * @param jsonpMapper to map JsonData objects + * @return the SearchDocumentResponse + * @since 5.1 + */ + public static SearchDocumentResponse from(SearchTemplateResponse response, + SearchDocumentResponse.EntityCreator entityCreator, JsonpMapper jsonpMapper) { + + Assert.notNull(response, "response must not be null"); + Assert.notNull(entityCreator, "entityCreator must not be null"); + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + var hitsMetadata = response.hits(); + var scrollId = response.scrollId(); + var aggregations = response.aggregations(); + var suggest = response.suggest(); + var pointInTimeId = response.pitId(); + + return from(hitsMetadata, scrollId, pointInTimeId, aggregations, suggest, entityCreator, jsonpMapper); + } + + /** + * creates a {@link SearchDocumentResponseBuilder} from {@link HitsMetadata} with the given scrollId aggregations and + * suggestES + * + * @param entity type + * @param hitsMetadata the {@link HitsMetadata} to process + * @param scrollId scrollId + * @param aggregations aggregations + * @param suggestES the suggestion response from Elasticsearch + * @param entityCreator function to create an entity from a {@link SearchDocument}, needed in mapping the suggest data + * @param jsonpMapper to map JsonData objects + * @return the {@link SearchDocumentResponse} + */ + public static SearchDocumentResponse from(HitsMetadata hitsMetadata, + @Nullable String scrollId, @Nullable String pointInTimeId, @Nullable Map aggregations, + Map>> suggestES, SearchDocumentResponse.EntityCreator entityCreator, + JsonpMapper jsonpMapper) { + + Assert.notNull(hitsMetadata, "hitsMetadata must not be null"); + + long totalHits; + String totalHitsRelation; + + TotalHits responseTotalHits = hitsMetadata.total(); + if (responseTotalHits != null) { + totalHits = responseTotalHits.value(); + totalHitsRelation = switch (responseTotalHits.relation().jsonValue()) { + case "eq" -> TotalHitsRelation.EQUAL_TO.name(); + case "gte" -> TotalHitsRelation.GREATER_THAN_OR_EQUAL_TO.name(); + default -> TotalHitsRelation.OFF.name(); + }; + } else { + totalHits = hitsMetadata.hits().size(); + totalHitsRelation = "OFF"; + } + + float maxScore = hitsMetadata.maxScore() != null ? hitsMetadata.maxScore().floatValue() : Float.NaN; + + List searchDocuments = new ArrayList<>(); + for (Hit hit : hitsMetadata.hits()) { + searchDocuments.add(DocumentAdapters.from(hit, jsonpMapper)); + } + + OpenSearchAggregations aggregationsContainer = aggregations != null ? new OpenSearchAggregations(aggregations) + : null; + + Suggest suggest = suggestFrom(suggestES, entityCreator); + + return new SearchDocumentResponse(totalHits, totalHitsRelation, maxScore, scrollId, pointInTimeId, searchDocuments, + aggregationsContainer, suggest); + } + + @Nullable + private static Suggest suggestFrom(Map>> suggestES, + SearchDocumentResponse.EntityCreator entityCreator) { + + if (CollectionUtils.isEmpty(suggestES)) { + return null; + } + + List>> suggestions = new ArrayList<>(); + + suggestES.forEach((name, suggestionsES) -> { + + if (!suggestionsES.isEmpty()) { + // take the type from the first entry + switch (suggestionsES.get(0)._kind()) { + case Term -> { + suggestions.add(getTermSuggestion(name, suggestionsES)); + break; + } + case Phrase -> { + suggestions.add(getPhraseSuggestion(name, suggestionsES)); + break; + } + case Completion -> { + suggestions.add(getCompletionSuggestion(name, suggestionsES, entityCreator)); + break; + } + default -> {} + } + } + }); + + // todo: hasScoreDocs checks if any one + boolean hasScoreDocs = false; + + return new Suggest(suggestions, hasScoreDocs); + } + + private static TermSuggestion getTermSuggestion(String name, List> suggestionsES) { + + List entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + var termSuggest = suggestionES.term(); + var termSuggestOptions = termSuggest.options(); + List options = new ArrayList<>(); + termSuggestOptions.forEach(optionES -> options.add(new TermSuggestion.Entry.Option(optionES.text(), null, + optionES.score(), null, Math.toIntExact(optionES.freq())))); + entries.add(new TermSuggestion.Entry(termSuggest.text(), termSuggest.offset(), termSuggest.length(), options)); + }); + return new TermSuggestion(name, suggestionsES.size(), entries, null); + } + + private static PhraseSuggestion getPhraseSuggestion(String name, List> suggestionsES) { + + List entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + var phraseSuggest = suggestionES.phrase(); + var phraseSuggestOptions = phraseSuggest.options(); + List options = new ArrayList<>(); + phraseSuggestOptions.forEach(optionES -> options.add(new PhraseSuggestion.Entry.Option(optionES.text(), + optionES.highlighted(), optionES.score(), optionES.collateMatch()))); + entries.add(new PhraseSuggestion.Entry(phraseSuggest.text(), phraseSuggest.offset(), phraseSuggest.length(), + options, null)); + }); + return new PhraseSuggestion(name, suggestionsES.size(), entries); + } + + private static CompletionSuggestion getCompletionSuggestion(String name, + List> suggestionsES, SearchDocumentResponse.EntityCreator entityCreator) { + List> entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + CompletionSuggest completionSuggest = suggestionES.completion(); + List> options = new ArrayList<>(); + List> optionsES = completionSuggest.options(); + optionsES.forEach(optionES -> { + SearchDocument searchDocument = (optionES.source() != null) ? DocumentAdapters.from(optionES) : null; + T hitEntity = null; + + if (searchDocument != null) { + try { + hitEntity = entityCreator.apply(searchDocument).get(); + } catch (Exception e) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn("Error creating entity from SearchDocument: " + e.getMessage()); + } + } + } + + Map> contexts = new HashMap<>(); + optionES.contexts().forEach((key, contextList) -> contexts.put(key, + contextList.stream().map(context -> context._get().toString()).collect(Collectors.toSet()))); + + // response from the new client does not have a doc and shardindex as the ScoreDoc from the old client responses + + options.add(new CompletionSuggestion.Entry.Option<>(optionES.text(), null, optionES.score(), + optionES.collateMatch() != null ? optionES.collateMatch() : false, contexts, + new ScoreDoc(optionES.score(), null, null), searchDocument, + hitEntity)); + }); + + entries.add(new CompletionSuggestion.Entry<>(completionSuggest.text(), completionSuggest.offset(), + completionSuggest.length(), options)); + }); + return new CompletionSuggestion<>(name, suggestionsES.size(), entries); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/TypeUtils.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/TypeUtils.java new file mode 100644 index 0000000..41c61f4 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/TypeUtils.java @@ -0,0 +1,480 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.time.Duration; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.opensearch.client.opensearch._types.*; +import org.opensearch.client.opensearch._types.mapping.FieldType; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.core.search.BoundaryScanner; +import org.opensearch.client.opensearch.core.search.BuiltinHighlighterType; +import org.opensearch.client.opensearch.core.search.HighlighterEncoder; +import org.opensearch.client.opensearch.core.search.HighlighterFragmenter; +import org.opensearch.client.opensearch.core.search.HighlighterOrder; +import org.opensearch.client.opensearch.core.search.HighlighterTagsSchema; +import org.opensearch.client.opensearch.core.search.HighlighterType; +import org.opensearch.client.opensearch.core.search.ScoreMode; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.springframework.data.domain.Sort; +import org.springframework.data.elasticsearch.core.RefreshPolicy; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.query.GeoDistanceOrder; +import org.springframework.data.elasticsearch.core.query.IndexQuery; +import org.springframework.data.elasticsearch.core.query.IndicesOptions; +import org.springframework.data.elasticsearch.core.query.Order; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.RescorerQuery; +import org.springframework.data.elasticsearch.core.query.UpdateResponse; +import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; +import org.springframework.lang.Nullable; + +/** + * Utility to handle new OpenSearch client type values. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +final class TypeUtils { + + @Nullable + static BoundaryScanner boundaryScanner(@Nullable String value) { + + if (value != null) { + return switch (value.toLowerCase()) { + case "chars" -> BoundaryScanner.Chars; + case "sentence" -> BoundaryScanner.Sentence; + case "word" -> BoundaryScanner.Word; + default -> null; + }; + } + return null; + } + + static Conflicts conflicts(ReindexRequest.Conflicts conflicts) { + return switch (conflicts) { + case ABORT -> Conflicts.Abort; + case PROCEED -> Conflicts.Proceed; + }; + } + + @Nullable + static DistanceUnit distanceUnit(String unit) { + + return switch (unit.toLowerCase()) { + case "in", "inch" -> DistanceUnit.Inches; + case "yd", "yards" -> DistanceUnit.Yards; + case "ft", "feet" -> DistanceUnit.Feet; + case "km", "kilometers" -> DistanceUnit.Kilometers; + case "nm", "nmi" -> DistanceUnit.NauticMiles; + case "mm", "millimeters" -> DistanceUnit.Millimeters; + case "cm", "centimeters" -> DistanceUnit.Centimeters; + case "mi", "miles" -> DistanceUnit.Miles; + case "m", "meters" -> DistanceUnit.Meters; + default -> null; + }; + } + + @Nullable + static FieldType fieldType(String type) { + + for (FieldType fieldType : FieldType.values()) { + + if (fieldType.jsonValue().equals(type)) { + return fieldType; + } + } + return null; + } + + @Nullable + static String toString(@Nullable FieldValue fieldValue) { + + if (fieldValue == null) { + return null; + } + + switch (fieldValue._kind()) { + case Double -> { + return String.valueOf(fieldValue.doubleValue()); + } + case Long -> { + return String.valueOf(fieldValue.longValue()); + } + case Boolean -> { + return String.valueOf(fieldValue.booleanValue()); + } + case String -> { + return fieldValue.stringValue(); + } + case Null -> { + return null; + } + + default -> throw new IllegalStateException("Unexpected value: " + fieldValue._kind()); + } + } + + @Nullable + static Object toObject(@Nullable FieldValue fieldValue) { + + if (fieldValue == null) { + return null; + } + + switch (fieldValue._kind()) { + case Double -> { + return Double.valueOf(fieldValue.doubleValue()); + } + case Long -> { + return Long.valueOf(fieldValue.longValue()); + } + case Boolean -> { + return Boolean.valueOf(fieldValue.booleanValue()); + } + case String -> { + return fieldValue.stringValue(); + } + case Null -> { + return null; + } + + default -> throw new IllegalStateException("Unexpected value: " + fieldValue._kind()); + } + } + + @Nullable + static FieldValue toFieldValue(@Nullable Object fieldValue) { + + if (fieldValue == null) { + return FieldValue.NULL; + } + + if (fieldValue instanceof Boolean b) { + return b ? FieldValue.TRUE : FieldValue.FALSE; + } + + if (fieldValue instanceof String s) { + return FieldValue.of(s); + } + + if (fieldValue instanceof Long l) { + return FieldValue.of(l); + } + + if (fieldValue instanceof Integer i) { + return FieldValue.of((long) i); + } + + if (fieldValue instanceof Double d) { + return FieldValue.of(d); + } + + if (fieldValue instanceof Float f) { + return FieldValue.of((double) f); + } + + throw new IllegalStateException("Unexpected value: " + fieldValue); + } + + @Nullable + static GeoDistanceType geoDistanceType(GeoDistanceOrder.DistanceType distanceType) { + + return switch (distanceType) { + case arc -> GeoDistanceType.Arc; + case plane -> GeoDistanceType.Plane; + }; + + } + + @Nullable + static SortOrder sortOrder(@Nullable Sort.Direction direction) { + + if (direction == null) { + return null; + } + + return switch (direction) { + case ASC -> SortOrder.Asc; + case DESC -> SortOrder.Desc; + }; + + } + + @Nullable + static HighlighterFragmenter highlighterFragmenter(@Nullable String value) { + + if (value != null) { + return switch (value.toLowerCase()) { + case "simple" -> HighlighterFragmenter.Simple; + case "span" -> HighlighterFragmenter.Span; + default -> null; + }; + } + + return null; + } + + @Nullable + static HighlighterOrder highlighterOrder(@Nullable String value) { + + if (value != null) { + if ("score".equals(value.toLowerCase())) { + return HighlighterOrder.Score; + } + } + + return null; + } + + @Nullable + static HighlighterType highlighterType(@Nullable String value) { + + if (value != null) { + return switch (value.toLowerCase()) { + case "unified" -> HighlighterType.of(fn -> fn.builtin(BuiltinHighlighterType.Unified)); + case "plain" -> HighlighterType.of(fn -> fn.builtin(BuiltinHighlighterType.Plain)); + case "fvh" -> HighlighterType.of(fn -> fn.builtin(BuiltinHighlighterType.FastVector)); + default -> null; + }; + } + + return null; + } + + @Nullable + static HighlighterEncoder highlighterEncoder(@Nullable String value) { + + if (value != null) { + return switch (value.toLowerCase()) { + case "default" -> HighlighterEncoder.Default; + case "html" -> HighlighterEncoder.Html; + default -> null; + }; + } + + return null; + } + + @Nullable + static HighlighterTagsSchema highlighterTagsSchema(@Nullable String value) { + + if (value != null) { + if ("styled".equals(value.toLowerCase())) { + return HighlighterTagsSchema.Styled; + } + } + + return null; + } + + @Nullable + static OpType opType(@Nullable IndexQuery.OpType opType) { + + if (opType != null) { + return switch (opType) { + case INDEX -> OpType.Index; + case CREATE -> OpType.Create; + }; + } + return null; + } + + static Refresh refresh(@Nullable RefreshPolicy refreshPolicy) { + + if (refreshPolicy == null) { + return Refresh.False; + } + + return switch (refreshPolicy) { + case IMMEDIATE -> Refresh.True; + case WAIT_UNTIL -> Refresh.WaitFor; + case NONE -> Refresh.False; + }; + } + + @Nullable + static UpdateResponse.Result result(@Nullable Result result) { + + if (result == null) { + return null; + } + + return switch (result) { + case Created -> UpdateResponse.Result.CREATED; + case Updated -> UpdateResponse.Result.UPDATED; + case Deleted -> UpdateResponse.Result.DELETED; + case NotFound -> UpdateResponse.Result.NOT_FOUND; + case NoOp -> UpdateResponse.Result.NOOP; + }; + + } + + @Nullable + static ScoreMode scoreMode(@Nullable RescorerQuery.ScoreMode scoreMode) { + + if (scoreMode == null) { + return null; + } + + return switch (scoreMode) { + case Default -> null; + case Avg -> ScoreMode.Avg; + case Max -> ScoreMode.Max; + case Min -> ScoreMode.Min; + case Total -> ScoreMode.Total; + case Multiply -> ScoreMode.Multiply; + }; + + } + + @Nullable + static SearchType searchType(@Nullable Query.SearchType searchType) { + + if (searchType == null) { + return null; + } + + return switch (searchType) { + case QUERY_THEN_FETCH -> SearchType.QueryThenFetch; + case DFS_QUERY_THEN_FETCH -> SearchType.DfsQueryThenFetch; + }; + + } + + @Nullable + static SortMode sortMode(Order.Mode mode) { + + return switch (mode) { + case min -> SortMode.Min; + case max -> SortMode.Max; + case median -> SortMode.Median; + case avg -> SortMode.Avg; + }; + + } + + @Nullable + static Time time(@Nullable Duration duration) { + + if (duration == null) { + return null; + } + + return Time.of(t -> t.time(duration.toMillis() + "ms")); + } + + @Nullable + static String timeStringMs(@Nullable Duration duration) { + + if (duration == null) { + return null; + } + + return duration.toMillis() + "ms"; + } + + @Nullable + static VersionType versionType( + @Nullable org.springframework.data.elasticsearch.annotations.Document.VersionType versionType) { + + if (versionType != null) { + return switch (versionType) { + case INTERNAL -> VersionType.Internal; + case EXTERNAL -> VersionType.External; + case EXTERNAL_GTE -> VersionType.ExternalGte; + case FORCE -> VersionType.Force; + }; + } + + return null; + } + + static Integer waitForActiveShardsCount(@Nullable String value) { + // values taken from the RHLC implementation + if (value == null) { + return -2; + } else if ("all".equals(value.toUpperCase())) { + return -1; + } else { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Illegale value for waitForActiveShards" + value); + } + } + } + + /** + * Converts a Long to a Float, returning null if the input is null. + * + * @param value the long value + * @return a FLoat with the given value + * @since 5.0 + */ + @Nullable + static Float toFloat(@Nullable Long value) { + return value != null ? Float.valueOf(value) : null; + } + + /** + * Converts a Float to a Long, returning null if the input is null. + * + * @param value the float value + * @return a LOng with the given value + * @since 5.0 + */ + @Nullable + static Long toLong(@Nullable Float value) { + return value != null ? value.longValue() : null; + } + + /** + * @sice 5.1 + */ + @Nullable + public static List expandWildcards(@Nullable EnumSet wildcardStates) { + return (wildcardStates != null && !wildcardStates.isEmpty()) ? wildcardStates.stream() + .map(wildcardState -> ExpandWildcard.valueOf(wildcardState.name().toLowerCase())).collect(Collectors.toList()) + : null; + } + + @Nullable + static TypeMapping typeMapping(@Nullable Document mapping) { + if (mapping != null) { + return JsonpUtils.fromJson(mapping, TypeMapping._DESERIALIZER); + } + return null; + } + + @Nullable + static Document typeMapping(@Nullable TypeMapping typeMapping) { + return (typeMapping != null) ? Document.parse(removePrefixFromJson(typeMapping.toString())) : null; + } + + public static String removePrefixFromJson(String jsonWithPrefix) { + return jsonWithPrefix.substring(jsonWithPrefix.indexOf("{")); + } + + @Nullable + static IndexSettings indexSettings(@Nullable Map settings) { + return settings != null ? JsonpUtils.fromJson(Document.from(settings), IndexSettings._DESERIALIZER) + : null; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/package-info.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/package-info.java new file mode 100644 index 0000000..0f392d1 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/package-info.java @@ -0,0 +1,23 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains classes that use the new OpenSearch client library (org.opensearch.client:opensearch-java) + * to access OpenSearch. + */ +@org.springframework.lang.NonNullApi +@org.springframework.lang.NonNullFields +package org.opensearch.data.client.osc;