Skip to content

Commit

Permalink
expose GeoParquet assets in job results as STAC items
Browse files Browse the repository at this point in the history 
Open-EO/openeo-geopyspark-driver#787
  • Loading branch information
@bossie
bossie committed Jun 11, 2024
1 parent 0b9e657 commit 726f7cf
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 103 deletions.
12 changes: 12 additions & 0 deletions openeo_driver/dummy/dummy_backend.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -708,6 +708,18 @@ def get_result_assets(self, job_id: str, user_id: str) -> Dict[str, dict]:
[2.777548414187305, 51.00377983772219],
[2.70964374625748, 51.00377983772219]]]}
},
"timeseries.parquet": {
"roles": ["data"],
"type": "application/parquet; profile=geo",
"href": "s3://OpenEO-data/batch_jobs/j-2406047c20fc4966ab637d387502728f/timeseries.parquet",
"bands": [Band(name="S2-L2A-EVI_t0"), Band(name="S2-L2A-EVI_t1"), Band(name="S2-L2A-EVI_t2")],
"geometry": {"type": "Polygon",
"coordinates": [[[2.70964374625748, 51.00377983772219],
[2.70964374625748, 51.10589339112414],
[2.777548414187305, 51.10589339112414],
[2.777548414187305, 51.00377983772219],
[2.70964374625748, 51.00377983772219]]]}
},
}

return {
Expand Down
2 changes: 1 addition & 1 deletion openeo_driver/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1121,7 +1121,7 @@ def job_result_item_url(item_id) -> str:
for filename, metadata in result_assets.items():
if ("data" in metadata.get("roles", []) and
any(media_type in metadata.get("type", "") for media_type in
["geotiff", "netcdf", "text/csv"])):
["geotiff", "netcdf", "text/csv", "application/parquet"])):
links.append(
{"rel": "item", "href": job_result_item_url(item_id=filename), "type": stac_item_media_type}
)
Expand Down
226 changes: 124 additions & 102 deletions tests/test_views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1784,7 +1784,6 @@ def test_get_job_results_110(self, api110):
"openeo:status": "finished",
}

# TODO: another test method altogether?
resp = api110.get("/jobs/j-2406047c20fc4966ab637d387502728f/results", headers=self.AUTH_HEADER)

assert resp.assert_status_code(200).json == DictSubSet({
Expand All @@ -1809,6 +1808,17 @@ def test_get_job_results_110(self, api110):
{"name": "S2-L2A-EVI_t2"}
],
},
'timeseries.parquet': {
'href': 'http://oeo.net/openeo/1.1.0/jobs/j-2406047c20fc4966ab637d387502728f/results/assets/timeseries.parquet',
'roles': ['data'],
'title': 'timeseries.parquet',
'type': 'application/parquet; profile=geo',
'eo:bands': [
{"name": "S2-L2A-EVI_t0"},
{"name": "S2-L2A-EVI_t1"},
{"name": "S2-L2A-EVI_t2"}
],
},
},
"links": [
{
Expand All @@ -1831,6 +1841,11 @@ def test_get_job_results_110(self, api110):
"rel": "item",
"type": "application/geo+json",
},
{
"href": "http://oeo.net/openeo/1.1.0/jobs/j-2406047c20fc4966ab637d387502728f/results/items/timeseries.parquet",
"rel": "item",
"type": "application/geo+json",
},
]
})

Expand Down Expand Up @@ -2605,116 +2620,123 @@ def test_get_job_result_item(self, flask_app, api110, backend_config_overrides):
"/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results/items/output.tiff", headers=self.AUTH_HEADER
)

resp_data = resp.assert_status_code(200).json
resp_data = resp.assert_status_code(200).json

assert resp_data == {
"type": "Feature",
"stac_version": "1.0.0",
"stac_extensions": [
"https://stac-extensions.github.io/eo/v1.1.0/schema.json",
"https://stac-extensions.github.io/file/v2.1.0/schema.json",
'https://stac-extensions.github.io/projection/v1.1.0/schema.json',
],
"id": "output.tiff",
"geometry": {"type": "Polygon",
"coordinates": [[[0.0, 50.0], [0.0, 55.0], [5.0, 55.0], [5.0, 50.0], [0.0, 50.0]]]},
"bbox": [0.0, 50.0, 5.0, 55.0],
"epsg": 4326,
"properties": {
"datetime": "1981-04-24T03:00:00Z",
"proj:shape": [300, 600],
"proj:epsg":4326
},
'links': [{
'rel': 'self',
'href': 'http://oeo.net/openeo/1.1.0/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results/items/output.tiff',
'type': 'application/geo+json'
}, {
'rel': 'collection',
'href': 'http://oeo.net/openeo/1.1.0/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results',
'type': 'application/json'
}],
'assets': {
'output.tiff': {
'title': 'output.tiff',
'href': 'http://oeo.net/openeo/1.1.0/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results/assets/TXIuVGVzdA==/f5d336336d36e3e987ba6a34b87cde01/output.tiff?expires=2234',
'type': 'image/tiff; application=geotiff',
'proj:epsg': 4326,
'proj:shape': [300, 600],
'eo:bands': [{'center_wavelength': 1.23, 'name': 'NDVI'}],
'roles': ['data']
}
},
'collection': '53c71345-09b4-46b4-b6b0-03fd6fe1f199'
}
assert resp.headers["Content-Type"] == "application/geo+json"
assert resp_data == {
"type": "Feature",
"stac_version": "1.0.0",
"stac_extensions": [
"https://stac-extensions.github.io/eo/v1.1.0/schema.json",
"https://stac-extensions.github.io/file/v2.1.0/schema.json",
'https://stac-extensions.github.io/projection/v1.1.0/schema.json',
],
"id": "output.tiff",
"geometry": {"type": "Polygon",
"coordinates": [[[0.0, 50.0], [0.0, 55.0], [5.0, 55.0], [5.0, 50.0], [0.0, 50.0]]]},
"bbox": [0.0, 50.0, 5.0, 55.0],
"epsg": 4326,
"properties": {
"datetime": "1981-04-24T03:00:00Z",
"proj:shape": [300, 600],
"proj:epsg":4326
},
'links': [{
'rel': 'self',
'href': 'http://oeo.net/openeo/1.1.0/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results/items/output.tiff',
'type': 'application/geo+json'
}, {
'rel': 'collection',
'href': 'http://oeo.net/openeo/1.1.0/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results',
'type': 'application/json'
}],
'assets': {
'output.tiff': {
'title': 'output.tiff',
'href': 'http://oeo.net/openeo/1.1.0/jobs/53c71345-09b4-46b4-b6b0-03fd6fe1f199/results/assets/TXIuVGVzdA==/f5d336336d36e3e987ba6a34b87cde01/output.tiff?expires=2234',
'type': 'image/tiff; application=geotiff',
'proj:epsg': 4326,
'proj:shape': [300, 600],
'eo:bands': [{'center_wavelength': 1.23, 'name': 'NDVI'}],
'roles': ['data']
}
},
'collection': '53c71345-09b4-46b4-b6b0-03fd6fe1f199'
}
assert resp.headers["Content-Type"] == "application/geo+json"

pystac.validation.stac_validator.JsonSchemaSTACValidator().validate(
stac_dict=resp_data,
stac_object_type=pystac.STACObjectType.ITEM,
stac_version=resp_data.get("stac_version", "0.9.0"),
extensions=resp_data.get("stac_extensions", []),
)
pystac.validation.stac_validator.JsonSchemaSTACValidator().validate(
stac_dict=resp_data,
stac_object_type=pystac.STACObjectType.ITEM,
stac_version=resp_data.get("stac_version", "0.9.0"),
extensions=resp_data.get("stac_extensions", []),
)

# TODO: another test method altogether?
resp = api110.get(
"/jobs/j-2406047c20fc4966ab637d387502728f/results/items/timeseries.csv", headers=self.AUTH_HEADER
)
@pytest.mark.parametrize(["vector_item_id", "vector_asset_media_type"], [
("timeseries.csv", "text/csv"),
("timeseries.parquet", "application/parquet; profile=geo"),
])
def test_get_vector_cube_job_result_item(self, flask_app, api110, backend_config_overrides, vector_item_id,
vector_asset_media_type):
vector_asset_filename = vector_item_id

resp_data = resp.assert_status_code(200).json
with self._fresh_job_registry():
resp = api110.get(f"/jobs/j-2406047c20fc4966ab637d387502728f/results/items/{vector_item_id}",
headers=self.AUTH_HEADER)

resp_data = resp.assert_status_code(200).json

assert resp_data == DictSubSet({
'type': 'Feature',
'stac_version': "1.0.0",
'id': 'timeseries.csv',
'geometry': {"type": "Polygon",
"coordinates": [[[2.70964374625748, 51.00377983772219],
[2.70964374625748, 51.10589339112414],
[2.777548414187305, 51.10589339112414],
[2.777548414187305, 51.00377983772219],
[2.70964374625748, 51.00377983772219]]]},
'bbox': [2.70964374625748, 51.00377983772219, 2.777548414187305, 51.10589339112414],
'properties': {
'datetime': None,
'start_datetime': "2020-01-01T00:00:00Z",
'end_datetime': "2020-03-15T00:00:00Z",
assert resp_data == DictSubSet({
'type': 'Feature',
'stac_version': "1.0.0",
'id': vector_item_id,
'geometry': {"type": "Polygon",
"coordinates": [[[2.70964374625748, 51.00377983772219],
[2.70964374625748, 51.10589339112414],
[2.777548414187305, 51.10589339112414],
[2.777548414187305, 51.00377983772219],
[2.70964374625748, 51.00377983772219]]]},
'bbox': [2.70964374625748, 51.00377983772219, 2.777548414187305, 51.10589339112414],
'properties': {
'datetime': None,
'start_datetime': "2020-01-01T00:00:00Z",
'end_datetime': "2020-03-15T00:00:00Z",
},
'collection': 'j-2406047c20fc4966ab637d387502728f',
'links': [
{
'href': f"http://oeo.net/openeo/1.1.0/jobs/j-2406047c20fc4966ab637d387502728f/results/items/{vector_item_id}",
'rel': 'self',
'type': 'application/geo+json',
},
'collection': 'j-2406047c20fc4966ab637d387502728f',
'links': [
{
'href': "http://oeo.net/openeo/1.1.0/jobs/j-2406047c20fc4966ab637d387502728f/results/items/timeseries.csv",
'rel': 'self',
'type': 'application/geo+json',
},
{
'href': "http://oeo.net/openeo/1.1.0/jobs/j-2406047c20fc4966ab637d387502728f/results",
'rel': 'collection',
'type': 'application/json',
}
],
'assets': {
'timeseries.csv': {
'href': "http://oeo.net/openeo/1.1.0/jobs/j-2406047c20fc4966ab637d387502728f/results/assets/TXIuVGVzdA==/942de26b3135191d65c40a40ae398b1e/timeseries.csv?expires=2234",
'type': "text/csv",
'roles': ["data"],
'title': "timeseries.csv",
'eo:bands': [
{"name": "S2-L2A-EVI_t0"},
{"name": "S2-L2A-EVI_t1"},
{"name": "S2-L2A-EVI_t2"}
],
}
{
'href': "http://oeo.net/openeo/1.1.0/jobs/j-2406047c20fc4966ab637d387502728f/results",
'rel': 'collection',
'type': 'application/json',
}
})
],
'assets': {
vector_asset_filename: {
'href': f"http://oeo.net/openeo/1.1.0/jobs/j-2406047c20fc4966ab637d387502728f/results/assets/{vector_asset_filename}",
'type': vector_asset_media_type,
'roles': ["data"],
'title': vector_asset_filename,
'eo:bands': [
{"name": "S2-L2A-EVI_t0"},
{"name": "S2-L2A-EVI_t1"},
{"name": "S2-L2A-EVI_t2"}
],
}
}
})

assert resp.headers["Content-Type"] == "application/geo+json"
assert resp.headers["Content-Type"] == "application/geo+json"

pystac.validation.stac_validator.JsonSchemaSTACValidator().validate(
stac_dict=resp_data,
stac_object_type=pystac.STACObjectType.ITEM,
stac_version=resp_data.get("stac_version", "0.9.0"),
extensions=resp_data.get("stac_extensions", []),
)
pystac.validation.stac_validator.JsonSchemaSTACValidator().validate(
stac_dict=resp_data,
stac_object_type=pystac.STACObjectType.ITEM,
stac_version=resp_data.get("stac_version", "0.9.0"),
extensions=resp_data.get("stac_extensions", []),
)

@mock.patch("time.time", mock.MagicMock(return_value=1234))
@pytest.mark.parametrize("backend_config_overrides", [{"url_signer": UrlSigner(secret="123&@#", expiration=1000)}])
Expand Down

0 comments on commit 726f7cf

Please sign in to comment.