Skip to content
This repository has been archived by the owner on Nov 23, 2023. It is now read-only.

Commit

Permalink
fix: Fix a bug with overriden pystac collection write method (#786)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MitchellPaff
MitchellPaff authored Jun 10, 2021
1 parent ac60fb4 commit 9f441f9
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 19 deletions.
17 changes: 3 additions & 14 deletions backend/populate_catalog/task.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
from json import dumps
from os.path import join
from typing import TYPE_CHECKING
from urllib.parse import urlparse

import boto3
from pystac import STAC_IO, Catalog, CatalogType, Collection, Item # type: ignore[import]
Expand Down Expand Up @@ -72,29 +70,20 @@ class UnhandledSQSMessageException(Exception):

class GeostoreSTACLayoutStrategy(HrefLayoutStrategy):
def get_catalog_href(self, cat: Catalog, parent_dir: str, is_root: bool) -> str:
original_path = urlparse(cat.get_self_href()).path.rsplit("/", maxsplit=2)
if is_root:
cat_root = parent_dir
else:
cat_root = join(parent_dir, original_path[-2])

return join(cat_root, original_path[-1])
return str(cat.get_self_href())

def get_collection_href(self, col: Collection, parent_dir: str, is_root: bool) -> str:
original_path = urlparse(col.get_self_href()).path.rsplit("/", maxsplit=2)
assert not is_root
return join(parent_dir, *original_path[-2:])
return str(col.get_self_href())

def get_item_href(self, item: Item, parent_dir: str) -> str:
original_path = item.get_self_href().split("/")
return join(parent_dir, original_path[-1])
return str(item.get_self_href())


def handle_dataset(version_metadata_key: str) -> None:
"""Handle writing a new dataset version to the dataset catalog"""
storage_bucket_path = f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}"
dataset_prefix = version_metadata_key.split("/", maxsplit=1)[0]

dataset_catalog = Catalog.from_file(f"{storage_bucket_path}/{dataset_prefix}/{CATALOG_KEY}")

dataset_version_metadata = STAC_IO.read_stac_object(
Expand Down
69 changes: 64 additions & 5 deletions tests/test_populate_catalog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,10 +225,59 @@ def should_update_existing_root_catalog(subtests: SubTests) -> None:

@mark.infrastructure
def should_update_dataset_catalog_with_new_version_catalog(subtests: SubTests) -> None:

collection_filename = f"{any_safe_filename()}.json"
item_filename = f"{any_safe_filename()}.json"
dataset_version = any_dataset_version_id()
filename = f"{any_safe_filename()}.json"
catalog_filename = f"{any_safe_filename()}.json"
with Dataset() as dataset, S3Object(
file_object=json_dict_to_file_object(
{
**deepcopy(MINIMAL_VALID_STAC_ITEM_OBJECT),
STAC_ID_KEY: any_dataset_version_id(),
STAC_LINKS_KEY: [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"./{catalog_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"./{collection_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
],
}
),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{dataset.dataset_prefix}/{dataset_version}/{item_filename}",
), S3Object(
file_object=json_dict_to_file_object(
{
**deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT),
STAC_ID_KEY: dataset_version,
STAC_TITLE_KEY: dataset.title,
STAC_LINKS_KEY: [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"./{catalog_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_ITEM,
STAC_HREF_KEY: f"./{item_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_GEOJSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"./{catalog_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
],
}
),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{dataset.dataset_prefix}/{dataset_version}/{collection_filename}",
), S3Object(
file_object=json_dict_to_file_object(
{
**deepcopy(MINIMAL_VALID_STAC_CATALOG_OBJECT),
Expand All @@ -237,14 +286,19 @@ def should_update_dataset_catalog_with_new_version_catalog(subtests: SubTests) -
STAC_LINKS_KEY: [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"./{filename}",
STAC_HREF_KEY: f"./{catalog_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_CHILD,
STAC_HREF_KEY: f"./{collection_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
],
}
),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{dataset.dataset_prefix}/{dataset_version}/{filename}",
key=f"{dataset.dataset_prefix}/{dataset_version}/{catalog_filename}",
) as dataset_version_metadata, S3Object(
file_object=json_dict_to_file_object(
{
Expand Down Expand Up @@ -305,7 +359,7 @@ def should_update_dataset_catalog_with_new_version_catalog(subtests: SubTests) -
},
{
STAC_REL_KEY: STAC_REL_CHILD,
STAC_HREF_KEY: f"./{dataset_version}/{filename}",
STAC_HREF_KEY: f"./{dataset_version}/{catalog_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
]
Expand All @@ -315,6 +369,11 @@ def should_update_dataset_catalog_with_new_version_catalog(subtests: SubTests) -
STAC_HREF_KEY: f"../../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_CHILD,
STAC_HREF_KEY: f"./{collection_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"../{CATALOG_KEY}",
Expand Down

0 comments on commit 9f441f9

Please sign in to comment.