diff --git a/CHANGELOG.md b/CHANGELOG.md index 47a26e09349f8..e6dae6a61202e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Addition of Doc values on the GeoShape Field - Addition of GeoShape ValueSource level code interfaces for accessing the DocValues. - Addition of Missing Value feature in the GeoShape Aggregations. - +- Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589)) ### Dependencies - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814) - Bump `com.google.guava:guava` from 30.1.1-jre to 32.0.0-jre (#7565, #7811, #7807, #7808) diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index 6b00709f08bf9..7ab6f80b65ca2 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -31,7 +31,7 @@ apply plugin: 'opensearch.yaml-rest-test' apply plugin: 'opensearch.internal-cluster-test' opensearchplugin { - description 'Plugin for geospatial features in OpenSearch. Registering the geo_shape and aggregations GeoBounds on Geo_Shape and Geo_Point' + description 'Plugin for geospatial features in OpenSearch. Registering the geo_shape and aggregations on GeoShape and GeoPoint' classname 'org.opensearch.geo.GeoModulePlugin' } diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java index 31ff2ef4689bd..b17f4804d4d50 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java @@ -8,6 +8,8 @@ package org.opensearch.geo; +import org.opensearch.geometry.utils.StandardValidator; +import org.opensearch.geometry.utils.WellKnownText; import org.opensearch.index.mapper.GeoShapeFieldMapper; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; @@ -24,6 +26,8 @@ public abstract class GeoModulePluginIntegTestCase extends OpenSearchIntegTestCa protected static final double GEOHASH_TOLERANCE = 1E-5D; + protected static final WellKnownText WKT = new WellKnownText(true, new StandardValidator(true)); + /** * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this * geo plugin is not getting packaged in a zip, we need to load it before the tests run. diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java new file mode 100644 index 0000000000000..4df30211e2804 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/AbstractGeoBucketAggregationIntegTest.java @@ -0,0 +1,254 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket; + +import com.carrotsearch.hppc.ObjectIntHashMap; +import com.carrotsearch.hppc.ObjectIntMap; +import org.opensearch.Version; +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.geo.tests.common.RandomGeoGeometryGenerator; +import org.opensearch.geometry.Geometry; +import org.opensearch.geometry.Rectangle; +import org.opensearch.test.VersionUtils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; + +/** + * This is the base class for all the Bucket Aggregation related integration tests. Use this class to add common + * methods which can be used across different bucket aggregations. If there is any common code that can be used + * across other integration test too then this is not the class. Use {@link GeoModulePluginIntegTestCase} + */ +public abstract class AbstractGeoBucketAggregationIntegTest extends GeoModulePluginIntegTestCase { + + protected static final int MAX_PRECISION_FOR_GEO_SHAPES_AGG_TESTING = 4; + + protected static final int NUM_DOCS = 100; + + protected static final String GEO_SHAPE_INDEX_NAME = "geoshape_index"; + + protected static Rectangle boundingRectangleForGeoShapesAgg; + + protected static ObjectIntMap expectedDocsCountForGeoShapes; + + protected static ObjectIntMap expectedDocCountsForSingleGeoPoint; + + protected static ObjectIntMap multiValuedExpectedDocCountsGeoPoint; + + protected static final String GEO_SHAPE_FIELD_NAME = "location_geo_shape"; + + protected static final String GEO_POINT_FIELD_NAME = "location"; + + protected static final String KEYWORD_FIELD_NAME = "city"; + + protected static String smallestGeoHash = null; + + protected final Version version = VersionUtils.randomIndexCompatibleVersion(random()); + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + /** + * Prepares a GeoShape index for testing the GeoShape bucket aggregations. Different bucket aggregations can use + * different techniques for creating buckets. Override the method + * {@link AbstractGeoBucketAggregationIntegTest#generateBucketsForGeometry} in the test class for creating the + * buckets which will then be used for verifications. + * + * @param random {@link Random} + * @throws Exception thrown during index creation. + */ + protected void prepareGeoShapeIndexForAggregations(final Random random) throws Exception { + expectedDocsCountForGeoShapes = new ObjectIntHashMap<>(); + final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); + final List geoshapes = new ArrayList<>(); + assertAcked(prepareCreate(GEO_SHAPE_INDEX_NAME).setSettings(settings).setMapping(GEO_SHAPE_FIELD_NAME, "type" + "=geo_shape")); + boolean isShapeIntersectingBB = false; + for (int i = 0; i < NUM_DOCS;) { + final Geometry geometry = RandomGeoGeometryGenerator.randomGeometry(random); + final GeoShapeDocValue geometryDocValue = GeoShapeDocValue.createGeometryDocValue(geometry); + // make sure that there is 1 shape is intersecting with the bounding box + if (!isShapeIntersectingBB) { + isShapeIntersectingBB = geometryDocValue.isIntersectingRectangle(boundingRectangleForGeoShapesAgg); + if (!isShapeIntersectingBB && i == NUM_DOCS - 1) { + continue; + } + } + i++; + final Set values = generateBucketsForGeometry(geometry, geometryDocValue); + geoshapes.add(indexGeoShape(GEO_SHAPE_INDEX_NAME, geometry)); + for (final String hash : values) { + expectedDocsCountForGeoShapes.put(hash, expectedDocsCountForGeoShapes.getOrDefault(hash, 0) + 1); + } + } + indexRandom(true, geoshapes); + ensureGreen(GEO_SHAPE_INDEX_NAME); + } + + /** + * Returns a set of buckets for the shape at different precision level. Override this method for different bucket + * aggregations. + * + * @param geometry {@link Geometry} + * @param geoShapeDocValue {@link GeoShapeDocValue} + * @return A {@link Set} of {@link String} which represents the buckets. + */ + protected abstract Set generateBucketsForGeometry(final Geometry geometry, final GeoShapeDocValue geoShapeDocValue); + + /** + * Prepares a GeoPoint index for testing the GeoPoint bucket aggregations. Different bucket aggregations can use + * different techniques for creating buckets. Override the method + * {@link AbstractGeoBucketAggregationIntegTest#generateBucketsForGeoPoint} in the test class for creating the + * buckets which will then be used for verifications. + * + * @param random {@link Random} + * @throws Exception thrown during index creation. + */ + protected void prepareSingleValueGeoPointIndex(final Random random) throws Exception { + expectedDocCountsForSingleGeoPoint = new ObjectIntHashMap<>(); + createIndex("idx_unmapped"); + final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, version) + .put("index.number_of_shards", 4) + .put("index.number_of_replicas", 0) + .build(); + assertAcked( + prepareCreate("idx").setSettings(settings) + .setMapping(GEO_POINT_FIELD_NAME, "type=geo_point", KEYWORD_FIELD_NAME, "type=keyword") + ); + final List cities = new ArrayList<>(); + for (int i = 0; i < NUM_DOCS; i++) { + // generate random point + final GeoPoint geoPoint = RandomGeoGenerator.randomPoint(random); + cities.add(indexGeoPoint("idx", geoPoint.toString(), geoPoint.getLat() + ", " + geoPoint.getLon())); + final Set buckets = generateBucketsForGeoPoint(geoPoint); + for (final String bucket : buckets) { + expectedDocCountsForSingleGeoPoint.put(bucket, expectedDocCountsForSingleGeoPoint.getOrDefault(bucket, 0) + 1); + } + } + indexRandom(true, cities); + ensureGreen("idx_unmapped", "idx"); + } + + protected void prepareMultiValuedGeoPointIndex(final Random random) throws Exception { + multiValuedExpectedDocCountsGeoPoint = new ObjectIntHashMap<>(); + final Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); + final List cities = new ArrayList<>(); + assertAcked( + prepareCreate("multi_valued_idx").setSettings(settings) + .setMapping(GEO_POINT_FIELD_NAME, "type=geo_point", KEYWORD_FIELD_NAME, "type=keyword") + ); + for (int i = 0; i < NUM_DOCS; i++) { + final int numPoints = random.nextInt(4); + final List points = new ArrayList<>(); + final Set buckets = new HashSet<>(); + for (int j = 0; j < numPoints; ++j) { + // generate random point + final GeoPoint geoPoint = RandomGeoGenerator.randomPoint(random); + points.add(geoPoint.getLat() + "," + geoPoint.getLon()); + buckets.addAll(generateBucketsForGeoPoint(geoPoint)); + } + cities.add(indexGeoPoints("multi_valued_idx", Integer.toString(i), points)); + for (final String bucket : buckets) { + multiValuedExpectedDocCountsGeoPoint.put(bucket, multiValuedExpectedDocCountsGeoPoint.getOrDefault(bucket, 0) + 1); + } + } + indexRandom(true, cities); + ensureGreen("multi_valued_idx"); + } + + /** + * Returns a set of buckets for the GeoPoint at different precision level. Override this method for different bucket + * aggregations. + * + * @param geoPoint {@link GeoPoint} + * @return A {@link Set} of {@link String} which represents the buckets. + */ + protected abstract Set generateBucketsForGeoPoint(final GeoPoint geoPoint); + + /** + * Indexes a GeoShape in the provided index. + * @param index {@link String} index name + * @param geometry {@link Geometry} the Geometry to be indexed + * @return {@link IndexRequestBuilder} + * @throws Exception thrown during creation of {@link IndexRequestBuilder} + */ + protected IndexRequestBuilder indexGeoShape(final String index, final Geometry geometry) throws Exception { + XContentBuilder source = jsonBuilder().startObject(); + source = source.field(GEO_SHAPE_FIELD_NAME, WKT.toWKT(geometry)); + source = source.endObject(); + return client().prepareIndex(index).setSource(source); + } + + /** + * Indexes a {@link List} of {@link GeoPoint}s in the provided Index name. + * @param index {@link String} index name + * @param name {@link String} value for the string field in index + * @param latLon {@link List} of {@link String} representing the String representation of GeoPoint + * @return {@link IndexRequestBuilder} + * @throws Exception thrown during indexing. + */ + protected IndexRequestBuilder indexGeoPoints(final String index, final String name, final List latLon) throws Exception { + XContentBuilder source = jsonBuilder().startObject().field(KEYWORD_FIELD_NAME, name); + if (latLon != null) { + source = source.field(GEO_POINT_FIELD_NAME, latLon); + } + source = source.endObject(); + return client().prepareIndex(index).setSource(source); + } + + /** + * Indexes a {@link GeoPoint} in the provided Index name. + * @param index {@link String} index name + * @param name {@link String} value for the string field in index + * @param latLon {@link String} representing the String representation of GeoPoint + * @return {@link IndexRequestBuilder} + * @throws Exception thrown during indexing. + */ + protected IndexRequestBuilder indexGeoPoint(final String index, final String name, final String latLon) throws Exception { + return indexGeoPoints(index, name, List.of(latLon)); + } + + /** + * Generates a Bounding Box of a fixed radius that can be used for shapes aggregations to reduce the size of + * aggregation results. + * @param random {@link Random} + * @return {@link Rectangle} + */ + protected Rectangle getGridAggregationBoundingBox(final Random random) { + final double radius = getRadiusOfBoundingBox(); + assertTrue("The radius of Bounding Box is less than or equal to 0", radius > 0); + return RandomGeoGeometryGenerator.randomRectangle(random, radius); + } + + /** + * Returns a radius for the Bounding box. Test classes can override this method to change the radius of BBox for + * the test cases. If we increase this value, it will lead to creation of a lot of buckets that can lead of + * IndexOutOfBoundsExceptions. + * @return double + */ + protected double getRadiusOfBoundingBox() { + return 5.0; + } + +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridIT.java new file mode 100644 index 0000000000000..6198c4cef3a34 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridIT.java @@ -0,0 +1,164 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket; + +import org.hamcrest.MatcherAssert; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.common.GeoBoundsHelper; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.geometry.Geometry; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class GeoTileGridIT extends AbstractGeoBucketAggregationIntegTest { + + private static final int GEOPOINT_MAX_PRECISION = 17; + + private static final String AGG_NAME = "geotilegrid"; + + @Override + public void setupSuiteScopeCluster() throws Exception { + final Random random = random(); + // Creating a BB for limiting the number buckets generated during aggregation + boundingRectangleForGeoShapesAgg = getGridAggregationBoundingBox(random); + prepareSingleValueGeoPointIndex(random); + prepareMultiValuedGeoPointIndex(random); + prepareGeoShapeIndexForAggregations(random); + ensureSearchable(); + } + + public void testGeoShapes() { + final GeoBoundingBox boundingBox = new GeoBoundingBox( + new GeoPoint(boundingRectangleForGeoShapesAgg.getMaxLat(), boundingRectangleForGeoShapesAgg.getMinLon()), + new GeoPoint(boundingRectangleForGeoShapesAgg.getMinLat(), boundingRectangleForGeoShapesAgg.getMaxLon()) + ); + for (int precision = 1; precision <= MAX_PRECISION_FOR_GEO_SHAPES_AGG_TESTING; precision++) { + final GeoGridAggregationBuilder builder = AggregationBuilders.geotileGrid(AGG_NAME) + .field(GEO_SHAPE_FIELD_NAME) + .precision(precision); + // This makes sure that for only higher precision we are providing the GeoBounding Box. This also ensures + // that we are able to test both bounded and unbounded aggregations + if (precision > 2) { + builder.setGeoBoundingBox(boundingBox); + } + final SearchResponse response = client().prepareSearch(GEO_SHAPE_INDEX_NAME).addAggregation(builder).get(); + final GeoGrid geoGrid = response.getAggregations().get(AGG_NAME); + final List buckets = geoGrid.getBuckets(); + final Object[] propertiesKeys = (Object[]) ((InternalAggregation) geoGrid).getProperty("_key"); + final Object[] propertiesDocCounts = (Object[]) ((InternalAggregation) geoGrid).getProperty("_count"); + for (int i = 0; i < buckets.size(); i++) { + final GeoGrid.Bucket cell = buckets.get(i); + final String geoTile = cell.getKeyAsString(); + + final long bucketCount = cell.getDocCount(); + final int expectedBucketCount = expectedDocsCountForGeoShapes.get(geoTile); + assertNotSame(bucketCount, 0); + assertEquals("Geotile " + geoTile + " has wrong doc count ", expectedBucketCount, bucketCount); + final GeoPoint geoPoint = (GeoPoint) propertiesKeys[i]; + MatcherAssert.assertThat(GeoTileUtils.stringEncode(geoPoint.lon(), geoPoint.lat(), precision), equalTo(geoTile)); + MatcherAssert.assertThat((long) propertiesDocCounts[i], equalTo(bucketCount)); + } + } + } + + public void testSimpleGeoPointsAggregation() { + for (int precision = 1; precision <= GEOPOINT_MAX_PRECISION; precision++) { + SearchResponse response = client().prepareSearch("idx") + .addAggregation(AggregationBuilders.geotileGrid(AGG_NAME).field(GEO_POINT_FIELD_NAME).precision(precision)) + .get(); + + assertSearchResponse(response); + + GeoGrid geoGrid = response.getAggregations().get(AGG_NAME); + List buckets = geoGrid.getBuckets(); + Object[] propertiesKeys = (Object[]) ((InternalAggregation) geoGrid).getProperty("_key"); + Object[] propertiesDocCounts = (Object[]) ((InternalAggregation) geoGrid).getProperty("_count"); + for (int i = 0; i < buckets.size(); i++) { + GeoGrid.Bucket cell = buckets.get(i); + String geoTile = cell.getKeyAsString(); + + long bucketCount = cell.getDocCount(); + int expectedBucketCount = expectedDocCountsForSingleGeoPoint.get(geoTile); + assertNotSame(bucketCount, 0); + assertEquals("GeoTile " + geoTile + " has wrong doc count ", expectedBucketCount, bucketCount); + GeoPoint geoPoint = (GeoPoint) propertiesKeys[i]; + assertThat(GeoTileUtils.stringEncode(geoPoint.lon(), geoPoint.lat(), precision), equalTo(geoTile)); + assertThat((long) propertiesDocCounts[i], equalTo(bucketCount)); + } + } + } + + public void testMultivaluedGeoPointsAggregation() throws Exception { + for (int precision = 1; precision <= GEOPOINT_MAX_PRECISION; precision++) { + SearchResponse response = client().prepareSearch("multi_valued_idx") + .addAggregation(AggregationBuilders.geotileGrid(AGG_NAME).field(GEO_POINT_FIELD_NAME).precision(precision)) + .get(); + + assertSearchResponse(response); + + GeoGrid geoGrid = response.getAggregations().get(AGG_NAME); + for (GeoGrid.Bucket cell : geoGrid.getBuckets()) { + String geohash = cell.getKeyAsString(); + + long bucketCount = cell.getDocCount(); + int expectedBucketCount = multiValuedExpectedDocCountsGeoPoint.get(geohash); + assertNotSame(bucketCount, 0); + assertEquals("Geohash " + geohash + " has wrong doc count ", expectedBucketCount, bucketCount); + } + } + } + + /** + * Returns a set of buckets for the shape at different precision level. Override this method for different bucket + * aggregations. + * + * @param geometry {@link Geometry} + * @param geoShapeDocValue {@link GeoShapeDocValue} + * @return A {@link Set} of {@link String} which represents the buckets. + */ + @Override + protected Set generateBucketsForGeometry(Geometry geometry, GeoShapeDocValue geoShapeDocValue) { + final GeoPoint topLeft = new GeoPoint(); + final GeoPoint bottomRight = new GeoPoint(); + assert geometry != null; + GeoBoundsHelper.updateBoundsForGeometry(geometry, topLeft, bottomRight); + final Set geoTiles = new HashSet<>(); + for (int precision = MAX_PRECISION_FOR_GEO_SHAPES_AGG_TESTING; precision > 0; precision--) { + geoTiles.addAll( + GeoTileUtils.encodeShape(geoShapeDocValue, precision).stream().map(GeoTileUtils::stringEncode).collect(Collectors.toSet()) + ); + } + return geoTiles; + } + + protected Set generateBucketsForGeoPoint(final GeoPoint geoPoint) { + Set buckets = new HashSet<>(); + for (int precision = GEOPOINT_MAX_PRECISION; precision > 0; precision--) { + final String tile = GeoTileUtils.stringEncode(geoPoint.getLon(), geoPoint.getLat(), precision); + buckets.add(tile); + } + return buckets; + } +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java index 3ccb62d40cbe3..03ed2ea6d1e3b 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java @@ -27,8 +27,6 @@ import org.opensearch.geo.tests.common.RandomGeoGeometryGenerator; import org.opensearch.geometry.Geometry; import org.opensearch.geometry.utils.Geohash; -import org.opensearch.geometry.utils.StandardValidator; -import org.opensearch.geometry.utils.WellKnownText; import org.opensearch.search.SearchHit; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; @@ -70,8 +68,6 @@ public abstract class AbstractGeoAggregatorModulePluginTestCase extends GeoModul protected static ObjectIntMap expectedDocCountsForGeoHash = null; protected static ObjectObjectMap expectedCentroidsForGeoHash = null; - protected static final WellKnownText WKT = new WellKnownText(true, new StandardValidator(true)); - @Override public void setupSuiteScopeCluster() throws Exception { createIndex(UNMAPPED_IDX_NAME); diff --git a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java index 77abba7f54677..efee09d01d04e 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java +++ b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java @@ -40,7 +40,6 @@ import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.geo.search.aggregations.metrics.GeoBounds; import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; -import org.opensearch.geo.search.aggregations.metrics.GeoBoundsGeoShapeAggregator; import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; import org.opensearch.index.mapper.GeoShapeFieldMapper; import org.opensearch.index.mapper.Mapper; @@ -48,13 +47,10 @@ import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; -import org.opensearch.search.aggregations.support.CoreValuesSourceType; -import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Consumer; public class GeoModulePlugin extends Plugin implements MapperPlugin, SearchPlugin { @@ -64,7 +60,8 @@ public Map getMappers() { } /** - * Registering {@link GeoBounds} aggregation on GeoPoint field. + * Registering {@link GeoBounds}, {@link InternalGeoHashGrid}, {@link InternalGeoTileGrid} aggregation on GeoPoint and GeoShape + * fields. */ @Override public List getAggregations() { @@ -106,23 +103,4 @@ public List getCompositeAggregations() { ) ); } - - /** - * Registering the GeoBounds Aggregation on the GeoShape Field. This function allows plugins to register new - * aggregations using aggregation names that are already defined in Core, as long as the new aggregations target - * different ValuesSourceTypes. - * - * @return A list of the new registrar functions - */ - @Override - public List> getAggregationExtentions() { - final Consumer geoShapeConsumer = builder -> builder.register( - GeoBoundsAggregationBuilder.REGISTRY_KEY, - CoreValuesSourceType.GEO_SHAPE, - GeoBoundsGeoShapeAggregator::new, - true - ); - return Collections.singletonList(geoShapeConsumer); - } - } diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index d2bf3541b5cce..5ee4a18a4f325 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -43,7 +43,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.geo.search.aggregations.bucket.geogrid.CellIdSource; +import org.opensearch.geo.search.aggregations.bucket.geogrid.cells.CellIdSource; import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.QueryShardContext; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorSupplier.java similarity index 93% rename from modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorSupplier.java index 43ccb8b89545a..0ef1957f88ef6 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorSupplier.java @@ -30,10 +30,9 @@ * GitHub history for details. */ -package org.opensearch.geo.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; -import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregator; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.CardinalityUpperBound; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java index 9631998649272..06fbaac8cdfed 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java @@ -40,7 +40,6 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.geo.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java index 1914c07e831f7..5502e0c418cf4 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java @@ -33,6 +33,9 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.geo.search.aggregations.bucket.geogrid.cells.CellIdSource; +import org.opensearch.geo.search.aggregations.bucket.geogrid.cells.GeoShapeCellIdSource; +import org.opensearch.geo.search.aggregations.bucket.geogrid.util.GeoShapeHashUtil; import org.opensearch.geometry.utils.Geohash; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; @@ -120,6 +123,7 @@ protected Aggregator doCreateInternal( } static void registerAggregators(ValuesSourceRegistry.Builder builder) { + // register GeoPoint Aggregation builder.register( GeoHashGridAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.GEOPOINT, @@ -155,5 +159,41 @@ static void registerAggregators(ValuesSourceRegistry.Builder builder) { }, true ); + // register GeoShape Aggregation + builder.register( + GeoHashGridAggregationBuilder.REGISTRY_KEY, + CoreValuesSourceType.GEO_SHAPE, + ( + name, + factories, + valuesSource, + precision, + geoBoundingBox, + requiredSize, + shardSize, + aggregationContext, + parent, + cardinality, + metadata) -> { + final GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource( + (ValuesSource.GeoShape) valuesSource, + precision, + geoBoundingBox, + GeoShapeHashUtil::encodeShape + ); + return new GeoHashGridAggregator( + name, + factories, + cellIdSource, + requiredSize, + shardSize, + aggregationContext, + parent, + cardinality, + metadata + ); + }, + true + ); } } diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index a07f3b438dc7a..10aa07a6712ee 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -39,7 +39,6 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.geo.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index b830988a3d410..b8e3efbb891df 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -33,6 +33,8 @@ package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.geo.search.aggregations.bucket.geogrid.cells.CellIdSource; +import org.opensearch.geo.search.aggregations.bucket.geogrid.cells.GeoShapeCellIdSource; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; @@ -154,5 +156,42 @@ static void registerAggregators(ValuesSourceRegistry.Builder builder) { }, true ); + + // registers Aggregation on GeoShape + builder.register( + GeoTileGridAggregationBuilder.REGISTRY_KEY, + CoreValuesSourceType.GEO_SHAPE, + ( + name, + factories, + valuesSource, + precision, + geoBoundingBox, + requiredSize, + shardSize, + aggregationContext, + parent, + cardinality, + metadata) -> { + GeoShapeCellIdSource cellIdSource = new GeoShapeCellIdSource( + (ValuesSource.GeoShape) valuesSource, + precision, + geoBoundingBox, + GeoTileUtils::encodeShape + ); + return new GeoTileGridAggregator( + name, + factories, + cellIdSource, + requiredSize, + shardSize, + aggregationContext, + parent, + cardinality, + metadata + ); + }, + true + ); } } diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/BoundedCellValues.java similarity index 96% rename from modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/BoundedCellValues.java index 06d2dcaee3932..588c8bc59c2e0 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/BoundedCellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.geo.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid.cells; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/CellIdSource.java similarity index 98% rename from modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/CellIdSource.java index cec49a867d660..42c4722e065af 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/CellIdSource.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.geo.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid.cells; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/CellValues.java similarity index 97% rename from modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/CellValues.java index d01896c8136fa..0b69040ec977a 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/CellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.geo.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid.cells; import org.opensearch.index.fielddata.AbstractSortingNumericDocValues; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/GeoShapeCellIdSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/GeoShapeCellIdSource.java new file mode 100644 index 0000000000000..0ea4d96c450ec --- /dev/null +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/GeoShapeCellIdSource.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.geogrid.cells; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.index.fielddata.GeoShapeValue; +import org.opensearch.index.fielddata.SortedBinaryDocValues; +import org.opensearch.index.fielddata.SortedNumericDoubleValues; +import org.opensearch.search.aggregations.support.ValuesSource; + +import java.io.IOException; +import java.util.List; + +/** + * ValueSource class which converts the {@link GeoShapeValue} to numeric long values for bucketing. This class uses the + * {@link GeoShapeCellIdSource.GeoShapeLongEncoder} to encode the geo_shape to {@link Long} values which can be iterated + * to do the bucket aggregation. + * + * @opensearch.internal + */ +public class GeoShapeCellIdSource extends ValuesSource.Numeric { + + private final ValuesSource.GeoShape geoShape; + private final int precision; + private final GeoBoundingBox geoBoundingBox; + private final GeoShapeCellIdSource.GeoShapeLongEncoder encoder; + + public GeoShapeCellIdSource( + final ValuesSource.GeoShape geoShape, + final int precision, + final GeoBoundingBox geoBoundingBox, + final GeoShapeCellIdSource.GeoShapeLongEncoder encoder + ) { + this.geoShape = geoShape; + this.geoBoundingBox = geoBoundingBox; + this.precision = precision; + this.encoder = encoder; + } + + /** + * Get the current {@link SortedBinaryDocValues}. + * + * @param context {@link LeafReaderContext} + */ + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + throw new UnsupportedOperationException("The bytesValues operation is not supported on GeoShapeCellIdSource"); + } + + /** + * Whether the underlying data is floating-point or not. + */ + @Override + public boolean isFloatingPoint() { + return false; + } + + /** + * Whether the underlying data is big integer or not. + */ + @Override + public boolean isBigInteger() { + return false; + } + + /** + * Get the current {@link SortedNumericDocValues}. + * + * @param context {@link LeafReaderContext} + */ + @Override + public SortedNumericDocValues longValues(final LeafReaderContext context) { + if (geoBoundingBox.isUnbounded()) { + return new GeoShapeCellValues.UnboundedCellValues(geoShape.getGeoShapeValues(context), precision, encoder); + } + return new GeoShapeCellValues.BoundedCellValues(geoShape.getGeoShapeValues(context), precision, encoder, geoBoundingBox); + } + + /** + * Get the current {@link SortedNumericDoubleValues}. + * + * @param context {@link LeafReaderContext} + */ + @Override + public SortedNumericDoubleValues doubleValues(LeafReaderContext context) { + throw new UnsupportedOperationException("The doubleValues operation is not supported on GeoShapeCellIdSource"); + } + + /** + * Encoder to encode the GeoShapes to the specific long values for the aggregation. + * + * @opensearch.internal + */ + @FunctionalInterface + public interface GeoShapeLongEncoder { + List encode(final GeoShapeDocValue geoShapeDocValue, final int precision); + } +} diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/GeoShapeCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/GeoShapeCellValues.java new file mode 100644 index 0000000000000..4911818cd448f --- /dev/null +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/GeoShapeCellValues.java @@ -0,0 +1,136 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.geogrid.cells; + +import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.index.fielddata.AbstractSortingNumericDocValues; +import org.opensearch.index.fielddata.GeoShapeValue; + +import java.io.IOException; +import java.util.List; + +/** + * Class representing the long-encoded grid-cells belonging to the geoshape-doc-values. Class must encode the values + * as long and then sort them in order to account for the cells correctly. + * + * @opensearch.internal + */ +abstract class GeoShapeCellValues extends AbstractSortingNumericDocValues { + private final GeoShapeValue geoShapeValue; + protected int precision; + protected final GeoShapeCellIdSource.GeoShapeLongEncoder encoder; + + public GeoShapeCellValues(GeoShapeValue geoShapeValue, int precision, GeoShapeCellIdSource.GeoShapeLongEncoder encoder) { + this.geoShapeValue = geoShapeValue; + this.precision = precision; + this.encoder = encoder; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (geoShapeValue.advanceExact(docId)) { + final GeoShapeDocValue geoShapeDocValue = geoShapeValue.nextValue(); + relateShape(geoShapeDocValue); + sort(); + return true; + } + return false; + } + + /** + * This function relates the shape's with the grid, and then put the intersecting grid's info as long, which + * can be iterated in the aggregation. It uses the encoder to find the relation. + * + * @param geoShapeDocValue {@link GeoShapeDocValue} + */ + abstract void relateShape(final GeoShapeDocValue geoShapeDocValue); + + /** + * Provides the {@link GeoShapeCellValues} for the input bounding box. + * @opensearch.internal + */ + static class BoundedCellValues extends GeoShapeCellValues { + + private final GeoBoundingBox geoBoundingBox; + + public BoundedCellValues( + final GeoShapeValue geoShapeValue, + final int precision, + final GeoShapeCellIdSource.GeoShapeLongEncoder encoder, + final GeoBoundingBox boundingBox + ) { + super(geoShapeValue, precision, encoder); + this.geoBoundingBox = boundingBox; + } + + /** + * This function relates the shape's with the grid, and then put the intersecting grid's info as long, which + * can be iterated in the aggregation. It uses the encoder to find the relation. + * + * @param geoShapeDocValue {@link GeoShapeDocValue} + */ + @Override + void relateShape(final GeoShapeDocValue geoShapeDocValue) { + if (intersect(geoShapeDocValue.getBoundingRectangle())) { + // now we know the shape is in the bounding rectangle, we need add them in longValues + // generate all grid that this shape intersects + final List encodedValues = encoder.encode(geoShapeDocValue, precision); + resize(encodedValues.size()); + for (int i = 0; i < encodedValues.size(); i++) { + values[i] = encodedValues.get(i); + } + } + } + + /** + * Validate that shape is intersecting the bounding box provided as input. + * + * @param rectangle {@link GeoShapeDocValue.BoundingRectangle} + * @return true or false + */ + private boolean intersect(final GeoShapeDocValue.BoundingRectangle rectangle) { + return geoBoundingBox.pointInBounds(rectangle.getMaxLongitude(), rectangle.getMaxLatitude()) + || geoBoundingBox.pointInBounds(rectangle.getMaxLongitude(), rectangle.getMinLatitude()) + || geoBoundingBox.pointInBounds(rectangle.getMinLongitude(), rectangle.getMaxLatitude()) + || geoBoundingBox.pointInBounds(rectangle.getMinLongitude(), rectangle.getMinLatitude()); + } + + } + + /** + * Provides the {@link GeoShapeCellValues} for unbounded cells + * @opensearch.internal + */ + static class UnboundedCellValues extends GeoShapeCellValues { + + public UnboundedCellValues( + final GeoShapeValue geoShapeValue, + final int precision, + final GeoShapeCellIdSource.GeoShapeLongEncoder encoder + ) { + super(geoShapeValue, precision, encoder); + } + + /** + * This function relates the shape's with the grid, and then put the intersecting grid's info as long, which + * can be iterated in the aggregation. It uses the encoder to find the relation. + * + * @param geoShapeDocValue {@link GeoShapeDocValue} + */ + @Override + void relateShape(final GeoShapeDocValue geoShapeDocValue) { + final List encodedValues = encoder.encode(geoShapeDocValue, precision); + resize(encodedValues.size()); + for (int i = 0; i < encodedValues.size(); i++) { + values[i] = encodedValues.get(i); + } + } + } +} diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/UnboundedCellValues.java similarity index 96% rename from modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/UnboundedCellValues.java index c628c7bfdc8ec..0a520c7162002 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/UnboundedCellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.geo.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid.cells; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/package-info.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/package-info.java new file mode 100644 index 0000000000000..16a5dd11f6210 --- /dev/null +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/cells/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * A Cells package which provide the different grid cells related functionalities for different aggregations + */ +package org.opensearch.geo.search.aggregations.bucket.geogrid.cells; diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/util/GeoShapeHashUtil.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/util/GeoShapeHashUtil.java new file mode 100644 index 0000000000000..aefb31e623bb5 --- /dev/null +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/util/GeoShapeHashUtil.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.geogrid.util; + +import org.opensearch.common.geo.GeoShapeDocValue; +import org.opensearch.geometry.Rectangle; +import org.opensearch.geometry.utils.Geohash; + +import java.util.ArrayList; +import java.util.List; + +/** + * We have a {@link Geohash} class present at the libs level, not using that because while encoding the shapes we need + * {@link GeoShapeDocValue}. This class provided the utilities encode the shape as GeoHashes + */ +public class GeoShapeHashUtil { + + /** + * The function encodes the shape provided as {@link GeoShapeDocValue} to a {@link List} of {@link Long} values + * (representing the GeoHashes) which are intersecting with the shapes at a given precision. + * + * @param geoShapeDocValue {@link GeoShapeDocValue} + * @param precision int + * @return {@link List} containing encoded {@link Long} values + */ + public static List encodeShape(final GeoShapeDocValue geoShapeDocValue, final int precision) { + final List encodedValues = new ArrayList<>(); + final GeoShapeDocValue.BoundingRectangle boundingRectangle = geoShapeDocValue.getBoundingRectangle(); + long topLeftGeoHash = Geohash.longEncode(boundingRectangle.getMinX(), boundingRectangle.getMaxY(), precision); + long topRightGeoHash = Geohash.longEncode(boundingRectangle.getMaxX(), boundingRectangle.getMaxY(), precision); + long bottomRightGeoHash = Geohash.longEncode(boundingRectangle.getMaxX(), boundingRectangle.getMinY(), precision); + + long currentValue = topLeftGeoHash; + long rightMax = topRightGeoHash; + long tempCurrent = currentValue; + while (true) { + // check if this currentValue intersect with shape. + final Rectangle geohashRectangle = Geohash.toBoundingBox(Geohash.stringEncode(tempCurrent)); + if (geoShapeDocValue.isIntersectingRectangle(geohashRectangle)) { + encodedValues.add(tempCurrent); + } + + // Breaking condition + if (tempCurrent == bottomRightGeoHash) { + break; + } + // now change the iterator => tempCurrent + if (tempCurrent == rightMax) { + // move to next row + tempCurrent = Geohash.longEncode(Geohash.getNeighbor(Geohash.stringEncode(currentValue), precision, 0, -1)); + currentValue = tempCurrent; + // update right max + rightMax = Geohash.longEncode(Geohash.getNeighbor(Geohash.stringEncode(rightMax), precision, 0, -1)); + } else { + // move to next column + tempCurrent = Geohash.longEncode(Geohash.getNeighbor(Geohash.stringEncode(tempCurrent), precision, 1, 0)); + } + } + return encodedValues; + } +} diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java index 149e052b4db7d..780f25ba3d7fb 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java @@ -87,5 +87,6 @@ protected Aggregator doCreateInternal( static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register(GeoBoundsAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.GEOPOINT, GeoBoundsAggregator::new, true); + builder.register(GeoBoundsAggregationBuilder.REGISTRY_KEY, CoreValuesSourceType.GEO_SHAPE, GeoBoundsGeoShapeAggregator::new, true); } } diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java index 2fb403155e2bc..a3def686b282d 100644 --- a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java @@ -64,7 +64,7 @@ public static GeoPoint randomPointIn(Random r, final double minLon, final double } /** Puts latitude in range of -90 to 90. */ - private static double normalizeLatitude(double latitude) { + public static double normalizeLatitude(double latitude) { if (latitude >= -90 && latitude <= 90) { return latitude; // common case, and avoids slight double precision shifting } @@ -73,7 +73,7 @@ private static double normalizeLatitude(double latitude) { } /** Puts longitude in range of -180 to +180. */ - private static double normalizeLongitude(double longitude) { + public static double normalizeLongitude(double longitude) { if (longitude >= -180 && longitude <= 180) { return longitude; // common case, and avoids slight double precision shifting } diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGeometryGenerator.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGeometryGenerator.java index caf15507e08c5..c6f78e846955d 100644 --- a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGeometryGenerator.java +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGeometryGenerator.java @@ -199,6 +199,26 @@ public static Rectangle randomRectangle(final Random r) { return new Rectangle(minX, maxX, maxY, minY); } + /** + * Generates a {@link Rectangle} of a specific radius. The generated rectangle can cross the international date line. + * + * @param r {@link Random} + * @param radius double + * @return {@link Rectangle} + */ + public static Rectangle randomRectangle(final Random r, double radius) { + final double[] centre = new double[2]; + RandomGeoGenerator.randomPointIn(r, -180, -(90 - radius), 180, 90 - radius, centre); + final double centreX = centre[0]; + final double centreY = centre[1]; + return new Rectangle( + RandomGeoGenerator.normalizeLongitude(centreX - radius), + RandomGeoGenerator.normalizeLongitude(centreX + radius), + centreY + radius, + centreY - radius + ); + } + /** * Returns a double array where pt[0] : longitude and pt[1] : latitude * diff --git a/server/src/main/java/org/opensearch/common/geo/GeoShapeDocValue.java b/server/src/main/java/org/opensearch/common/geo/GeoShapeDocValue.java index 9bc28c1f67d47..0c6158598a423 100644 --- a/server/src/main/java/org/opensearch/common/geo/GeoShapeDocValue.java +++ b/server/src/main/java/org/opensearch/common/geo/GeoShapeDocValue.java @@ -10,12 +10,18 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.document.LatLonShapeDocValues; import org.apache.lucene.document.LatLonShapeDocValuesField; +import org.apache.lucene.geo.LatLonGeometry; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.PointValues; import org.apache.lucene.util.BytesRef; import org.opensearch.geometry.Geometry; +import org.opensearch.geometry.GeometryVisitor; +import org.opensearch.geometry.Rectangle; import org.opensearch.index.mapper.GeoShapeIndexer; +import java.io.IOException; import java.util.List; /** @@ -25,6 +31,7 @@ */ public class GeoShapeDocValue extends ShapeDocValue { private static final String FIELD_NAME = "missingField"; + private final LatLonShapeDocValues shapeDocValues; public GeoShapeDocValue(final String fieldName, final BytesRef bytesRef) { this(LatLonShape.createDocValueField(fieldName, bytesRef)); @@ -39,6 +46,7 @@ public GeoShapeDocValue(final LatLonShapeDocValuesField shapeDocValuesField) { shapeDocValuesField.getBoundingBox().minLon, shapeDocValuesField.getBoundingBox().minLat ); + this.shapeDocValues = LatLonShape.createLatLonShapeDocValues(shapeDocValuesField.binaryValue()); } /** @@ -172,4 +180,21 @@ public String toString() { } } + + /** + * Checks if the input {@link Rectangle} is intersecting with the shape represented as {@link GeoShapeDocValue}. + * We could have used the {@link GeometryVisitor} here and added the functionality to check the intersection with + * other {@link Geometry} also, but that will be an overkill for now, if required we can easily create a + * {@link GeometryVisitor} to check the intersection with this Shape represented as {@link GeoShapeDocValue}. + * @return boolean + */ + public boolean isIntersectingRectangle(final Rectangle rectangle) { + final org.apache.lucene.geo.Rectangle luceneRectangle = GeoShapeUtils.toLuceneRectangle(rectangle); + try { + final PointValues.Relation relation = shapeDocValues.relate(LatLonGeometry.create(luceneRectangle)); + return relation != PointValues.Relation.CELL_OUTSIDE_QUERY; + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java index c0b91cd42928d..d37780f9808dc 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java @@ -35,6 +35,7 @@ import org.apache.lucene.util.SloppyMath; import org.opensearch.OpenSearchParseException; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.geo.GeoShapeDocValue; import org.opensearch.common.util.OpenSearchSloppyMath; import org.opensearch.core.xcontent.ObjectParser.ValueType; import org.opensearch.core.xcontent.XContentParser; @@ -42,6 +43,8 @@ import org.opensearch.geometry.Rectangle; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import static org.opensearch.common.geo.GeoUtils.normalizeLat; @@ -249,6 +252,13 @@ public static String stringEncode(long hash) { return "" + res[0] + "/" + res[1] + "/" + res[2]; } + /** + * Encode lon/lat to the geotile based string format which is "zoom/x/y" + */ + public static String stringEncode(double longitude, double latitude, int precision) { + return stringEncode(longEncode(longitude, latitude, precision)); + } + /** * Decode long hash as a GeoPoint (center of the tile) */ @@ -278,6 +288,40 @@ public static Rectangle toBoundingBox(String hash) { return toBoundingBox(hashAsInts[1], hashAsInts[2], hashAsInts[0]); } + /** + * The function encodes the shape provided as {@link GeoShapeDocValue} to a {@link List} of {@link Long} values + * (representing the GeoTiles) which are intersecting with the shapes at a given precision. + * + * @param geoShapeDocValue {@link GeoShapeDocValue} + * @param precision int + * @return {@link List} of {@link Long} + */ + public static List encodeShape(final GeoShapeDocValue geoShapeDocValue, final int precision) { + final GeoShapeDocValue.BoundingRectangle boundingRectangle = geoShapeDocValue.getBoundingRectangle(); + // generate all the grid long values that this shape intersects. + final long totalTilesAtPrecision = 1L << checkPrecisionRange(precision); + int maxXTile = getXTile(boundingRectangle.getMaxX(), totalTilesAtPrecision); + int minXTile = getXTile(boundingRectangle.getMinX(), totalTilesAtPrecision); + // as tuples in tiles are x,y and y(lat) increases from north to south in tiles, so for minYTile we need to + // take maxY and for maxYTile we need to take minY. + int minYTile = getYTile(boundingRectangle.getMaxY(), totalTilesAtPrecision); + int maxYTile = getYTile(boundingRectangle.getMinY(), totalTilesAtPrecision); + final List encodedValues = new ArrayList<>(); + for (int x = minXTile; x <= maxXTile; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + // Convert the precision, x , y to encoded value. + long encodedValue = longEncodeTiles(precision, x, y); + // Convert encoded value to rectangle + final Rectangle tileRectangle = toBoundingBox(encodedValue); + // check to see if the GeoShape is intersecting with the rectangle. + if (geoShapeDocValue.isIntersectingRectangle(tileRectangle)) { + encodedValues.add(encodedValue); + } + } + } + return encodedValues; + } + public static Rectangle toBoundingBox(int xTile, int yTile, int precision) { final double tiles = validateZXY(precision, xTile, yTile); final double minN = Math.PI - (2.0 * Math.PI * (yTile + 1)) / tiles;