From ac8e143d3f6b5c7c58b207a1b5887c080cfd4427 Mon Sep 17 00:00:00 2001 From: Thomas Maschler Date: Thu, 15 Feb 2024 16:09:28 -0500 Subject: [PATCH] add tests and enforce type property --- .github/workflows/cicd.yml | 9 +- pyproject.toml | 2 +- stac_pydantic/catalog.py | 22 +++- stac_pydantic/collection.py | 2 +- stac_pydantic/item.py | 32 ++--- .../v1.0.0/example-collection-list.json | 8 +- tests/api/test_landing_page.py | 2 + .../example_stac/example-collection-list.json | 10 +- .../example-collection_version-extension.json | 83 +++++++----- ...xample-landsat8_item-assets-extension.json | 3 +- tests/example_stac/landsat-collection.json | 3 +- tests/test_catalog.py | 22 ++++ tests/test_item.py | 124 ++++++++++++++++++ 13 files changed, 250 insertions(+), 72 deletions(-) create mode 100644 tests/test_catalog.py create mode 100644 tests/test_item.py diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 7634c12..6a7dcf0 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -27,11 +27,14 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install tox pre-commit + python -m pip install '.[lint]' pre-commit install + - name: Lint + run: pre-commit run --all + # Run tox using the version of Python in `PATH` - - name: Run Tox + - name: Test run: tox -e py - name: Upload Results diff --git a/pyproject.toml b/pyproject.toml index 3671898..f504939 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ lint = ["types-requests>=2.31.0.5", "isort>=5.12.0", "flake8>=6.1.0", "Flake8-pyproject>=1.2.3", - "mypy>=1.8.0", + "mypy==1.4.1", "pre-commit>=3.4.0", "tox>=4.11.3"] diff --git a/stac_pydantic/catalog.py b/stac_pydantic/catalog.py index 5604cb4..8b830ad 100644 --- a/stac_pydantic/catalog.py +++ b/stac_pydantic/catalog.py @@ -1,6 +1,6 @@ -from typing import List, Literal, Optional +from typing import Any, List, Literal, Optional -from pydantic import AnyUrl, ConfigDict, Field +from pydantic import AnyUrl, ConfigDict, Field, model_validator from stac_pydantic.links import Links from stac_pydantic.shared import SEMVER_REGEX, StacBaseModel @@ -14,13 +14,23 @@ class _Catalog(StacBaseModel): id: str = Field(..., alias="id", min_length=1) description: str = Field(..., alias="description", min_length=1) - stac_version: str = Field(STAC_VERSION, pattern=SEMVER_REGEX) - links: Links = Links(root=[]) - stac_extensions: Optional[List[AnyUrl]] = [] + stac_version: str = Field(..., pattern=SEMVER_REGEX) + links: Links + stac_extensions: Optional[List[AnyUrl]] = None title: Optional[str] = None type: str model_config = ConfigDict(use_enum_values=True, extra="allow") + @model_validator(mode="before") + @classmethod + def set_default_links(cls, data: Any) -> Any: + if isinstance(data, dict): + if data.get("links") is None: + data["links"] = [] + if data.get("stac_version") is None: + data["stac_version"] = STAC_VERSION + return data + class Catalog(_Catalog): - type: Literal["Catalog"] = "Catalog" + type: Literal["Catalog"] diff --git a/stac_pydantic/collection.py b/stac_pydantic/collection.py index 1704726..4b4cae8 100644 --- a/stac_pydantic/collection.py +++ b/stac_pydantic/collection.py @@ -52,4 +52,4 @@ class Collection(_Catalog): keywords: Optional[List[str]] = None providers: Optional[List[Provider]] = None summaries: Optional[Dict[str, Union[Range, List[Any], Dict[str, Any]]]] = None - type: Literal["Collection"] = "Collection" + type: Literal["Collection"] diff --git a/stac_pydantic/item.py b/stac_pydantic/item.py index 4a4fad4..f002fa8 100644 --- a/stac_pydantic/item.py +++ b/stac_pydantic/item.py @@ -1,25 +1,11 @@ -from datetime import datetime as dt from typing import Any, Dict, List, Optional from ciso8601 import parse_rfc3339 from geojson_pydantic import Feature -from pydantic import ( - AnyUrl, - ConfigDict, - Field, - field_serializer, - model_serializer, - model_validator, -) +from pydantic import AnyUrl, ConfigDict, Field, model_serializer, model_validator from stac_pydantic.links import Links -from stac_pydantic.shared import ( - DATETIME_RFC339, - SEMVER_REGEX, - Asset, - StacBaseModel, - StacCommonMetadata, -) +from stac_pydantic.shared import SEMVER_REGEX, Asset, StacBaseModel, StacCommonMetadata from stac_pydantic.version import STAC_VERSION @@ -60,19 +46,25 @@ class Item(Feature, StacBaseModel): """ id: str = Field(..., alias="id", min_length=1) - stac_version: str = Field(STAC_VERSION, pattern=SEMVER_REGEX) + stac_version: str = Field(..., pattern=SEMVER_REGEX) properties: ItemProperties - assets: Dict[str, Asset] = {} - links: Links = Links(root=[]) + assets: Dict[str, Asset] + links: Links stac_extensions: Optional[List[AnyUrl]] = None collection: Optional[str] = None @model_validator(mode="before") @classmethod - def validate_bbox(cls, data: Any) -> Any: + def validate_defaults(cls, data: Any) -> Any: if isinstance(data, dict): + if data.get("stac_version") is None: + data["stac_version"] = STAC_VERSION if data.get("geometry") and data.get("bbox") is None: raise ValueError("bbox is required if geometry is not null") + if data.get("assets") is None: + data["assets"] = {} + if data.get("links") is None: + data["links"] = [] return data # https://github.com/developmentseed/geojson-pydantic/issues/147 diff --git a/tests/api/examples/v1.0.0/example-collection-list.json b/tests/api/examples/v1.0.0/example-collection-list.json index 9594583..24af86c 100644 --- a/tests/api/examples/v1.0.0/example-collection-list.json +++ b/tests/api/examples/v1.0.0/example-collection-list.json @@ -13,7 +13,8 @@ ], "collections":[ { - "id":"aster-l1t", + "type":"Collection", + "id":"aster-l1t", "description":"The [ASTER](https://terra.nasa.gov/about/terra-instruments/aster) instrument, launched on-board NASA's [Terra](https://terra.nasa.gov/) satellite in 1999, provides multispectral images of the Earth at 15m-90m resolution. ASTER images provide information about land surface temperature, color, elevation, and mineral composition.\n\nThis dataset represents ASTER [L1T](https://lpdaac.usgs.gov/products/ast_l1tv003/) data from 2000-2006. L1T images have been terrain-corrected and rotated to a north-up UTM projection. Images are in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n", "stac_version":"1.0.0", "links":[ @@ -405,6 +406,7 @@ } }, { + "type":"Collection", "id":"landsat-8-c2-l2", "description":"The [Landsat](https://landsat.gsfc.nasa.gov/) program has been imaging the Earth since 1972; it provides a comprehensive, continuous archive of the Earth's surface. [Landsat 8](https://www.usgs.gov/core-science-systems/nli/landsat/landsat-8) is the most recent satellite in the Landsat series. Launched in 2013, Landsat 8 captures data in eleven spectral bands: ten optical/IR bands from the [Operational Land Imager](https://landsat.gsfc.nasa.gov/landsat-8/operational-land-imager) (OLI) instrument, and two thermal bands from the [Thermal Infrared Sensor](https://landsat.gsfc.nasa.gov/landsat-8/thermal-infrared-sensor-tirs) (TIRS) instrument.\n\nThis dataset represents the global archive of Level-2 Landsat 8 data from [Landsat Collection 2](https://www.usgs.gov/core-science-systems/nli/landsat/landsat-collection-2). Because there is some latency before Level-2 data is available, a rolling window of recent Level-1 data is available as well. Images are stored in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n", "stac_version":"1.0.0", @@ -871,6 +873,7 @@ } }, { + "type":"Collection", "id":"sentinel-2-l2a", "description":"The [Sentinel-2](https://sentinel.esa.int/web/sentinel/missions/sentinel-2) program provides global imagery in thirteen spectral bands at 10m-60m resolution and a revisit time of approximately five days. This dataset represents the global Sentinel-2 archive, from 2016 to the present, processed to L2A (bottom-of-atmosphere) using [Sen2Cor](https://step.esa.int/main/snap-supported-plugins/sen2cor/) and converted to [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.", "stac_version":"1.0.0", @@ -1496,7 +1499,8 @@ } }, { - "id":"naip", + "type":"Collection", + "id":"naip", "description":"The [National Agriculture Imagery Program](https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/) (NAIP) provides US-wide, high-resolution aerial imagery, with four spectral bands (R, G, B, IR). NAIP is administered by the [Aerial Field Photography Office](https://www.fsa.usda.gov/programs-and-services/aerial-photography/) (AFPO) within the [US Department of Agriculture](https://www.usda.gov/) (USDA). Data are captured at least once every three years for each state. This dataset represents NAIP data from 2010-present, in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n", "stac_version":"1.0.0", "links":[ diff --git a/tests/api/test_landing_page.py b/tests/api/test_landing_page.py index c9d59d3..08fb765 100644 --- a/tests/api/test_landing_page.py +++ b/tests/api/test_landing_page.py @@ -71,6 +71,7 @@ def test_schema(example_url, schema_url): def test_api_landing_page(): LandingPage( + type="Catalog", id="test-landing-page", description="stac-api landing page", stac_extensions=[ @@ -100,6 +101,7 @@ def test_api_landing_page(): def test_api_landing_page_is_catalog(): landing_page = LandingPage( + type="Catalog", id="test-landing-page", description="stac-api landing page", stac_extensions=[ diff --git a/tests/example_stac/example-collection-list.json b/tests/example_stac/example-collection-list.json index d12357b..4c84ae6 100644 --- a/tests/example_stac/example-collection-list.json +++ b/tests/example_stac/example-collection-list.json @@ -8,6 +8,7 @@ ], "collections":[ { + "type":"Collection", "id":"aster-l1t", "description":"The [ASTER](https://terra.nasa.gov/about/terra-instruments/aster) instrument, launched on-board NASA's [Terra](https://terra.nasa.gov/) satellite in 1999, provides multispectral images of the Earth at 15m-90m resolution. ASTER images provide information about land surface temperature, color, elevation, and mineral composition.\n\nThis dataset represents ASTER [L1T](https://lpdaac.usgs.gov/products/ast_l1tv003/) data from 2000-2006. L1T images have been terrain-corrected and rotated to a north-up UTM projection. Images are in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n", "stac_version":"1.0.0", @@ -400,7 +401,8 @@ } }, { - "id":"landsat-8-c2-l2", + "type":"Collection", + "id":"landsat-8-c2-l2", "description":"The [Landsat](https://landsat.gsfc.nasa.gov/) program has been imaging the Earth since 1972; it provides a comprehensive, continuous archive of the Earth's surface. [Landsat 8](https://www.usgs.gov/core-science-systems/nli/landsat/landsat-8) is the most recent satellite in the Landsat series. Launched in 2013, Landsat 8 captures data in eleven spectral bands: ten optical/IR bands from the [Operational Land Imager](https://landsat.gsfc.nasa.gov/landsat-8/operational-land-imager) (OLI) instrument, and two thermal bands from the [Thermal Infrared Sensor](https://landsat.gsfc.nasa.gov/landsat-8/thermal-infrared-sensor-tirs) (TIRS) instrument.\n\nThis dataset represents the global archive of Level-2 Landsat 8 data from [Landsat Collection 2](https://www.usgs.gov/core-science-systems/nli/landsat/landsat-collection-2). Because there is some latency before Level-2 data is available, a rolling window of recent Level-1 data is available as well. Images are stored in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n", "stac_version":"1.0.0", "links":[ @@ -866,7 +868,8 @@ } }, { - "id":"sentinel-2-l2a", + "type":"Collection", + "id":"sentinel-2-l2a", "description":"The [Sentinel-2](https://sentinel.esa.int/web/sentinel/missions/sentinel-2) program provides global imagery in thirteen spectral bands at 10m-60m resolution and a revisit time of approximately five days. This dataset represents the global Sentinel-2 archive, from 2016 to the present, processed to L2A (bottom-of-atmosphere) using [Sen2Cor](https://step.esa.int/main/snap-supported-plugins/sen2cor/) and converted to [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.", "stac_version":"1.0.0", "links":[ @@ -1491,7 +1494,8 @@ } }, { - "id":"naip", + "type":"Collection", + "id":"naip", "description":"The [National Agriculture Imagery Program](https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/) (NAIP) provides US-wide, high-resolution aerial imagery, with four spectral bands (R, G, B, IR). NAIP is administered by the [Aerial Field Photography Office](https://www.fsa.usda.gov/programs-and-services/aerial-photography/) (AFPO) within the [US Department of Agriculture](https://www.usda.gov/) (USDA). Data are captured at least once every three years for each state. This dataset represents NAIP data from 2010-present, in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\n", "stac_version":"1.0.0", "links":[ diff --git a/tests/example_stac/example-collection_version-extension.json b/tests/example_stac/example-collection_version-extension.json index bf5ba87..f698ca8 100644 --- a/tests/example_stac/example-collection_version-extension.json +++ b/tests/example_stac/example-collection_version-extension.json @@ -1,36 +1,51 @@ { - "stac_version": "1.0.0", - "stac_extensions": ["https://stac-extensions.github.io/version/v1.0.0/schema.json"], - "id": "merraclim", - "title": "MERRAclim", - "description": "A high-resolution global dataset of remotely sensed bioclimatic variables for ecological modelling.", - "license": "CC0-1.0", - "version": "1", - "deprecated": true, - "extent": { - "spatial": { - "bbox": [[-180,-90,180,90]] - }, - "temporal": { - "interval": [["1980-01-01T00:00:00Z","2009-12-31T23:59:59Z"]] - } + "type": "Collection", + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/version/v1.0.0/schema.json" + ], + "id": "merraclim", + "title": "MERRAclim", + "description": "A high-resolution global dataset of remotely sensed bioclimatic variables for ecological modelling.", + "license": "CC0-1.0", + "version": "1", + "deprecated": true, + "extent": { + "spatial": { + "bbox": [ + [ + -180, + -90, + 180, + 90 + ] + ] }, - "links": [ - { - "rel": "self", - "href": "https://datadryad.org/resource/doi:10.5061/dryad.s2v81/v1/collection.json" - }, - { - "rel": "item", - "href": "https://datadryad.org/resource/doi:10.5061/dryad.s2v81/v1/item.json" - }, - { - "rel": "root", - "href": "https://datadryad.org/resource/doi:10.5061/dryad.s2v81/catalog.json" - }, - { - "rel": "latest-version", - "href": "https://datadryad.org/resource/doi:10.5061/dryad.s2v81/v2/collection.json" - } - ] - } + "temporal": { + "interval": [ + [ + "1980-01-01T00:00:00Z", + "2009-12-31T23:59:59Z" + ] + ] + } + }, + "links": [ + { + "rel": "self", + "href": "https://datadryad.org/resource/doi:10.5061/dryad.s2v81/v1/collection.json" + }, + { + "rel": "item", + "href": "https://datadryad.org/resource/doi:10.5061/dryad.s2v81/v1/item.json" + }, + { + "rel": "root", + "href": "https://datadryad.org/resource/doi:10.5061/dryad.s2v81/catalog.json" + }, + { + "rel": "latest-version", + "href": "https://datadryad.org/resource/doi:10.5061/dryad.s2v81/v2/collection.json" + } + ] +} diff --git a/tests/example_stac/example-landsat8_item-assets-extension.json b/tests/example_stac/example-landsat8_item-assets-extension.json index 31304fa..725f732 100644 --- a/tests/example_stac/example-landsat8_item-assets-extension.json +++ b/tests/example_stac/example-landsat8_item-assets-extension.json @@ -1,5 +1,6 @@ { - "id": "landsat-8-l1", + "type": "Collection", + "id": "landsat-8-l1", "title": "Landsat 8 L1", "description": "Landat 8 imagery radiometrically calibrated and orthorectified using gound points and Digital Elevation Model (DEM) data to correct relief displacement.", "keywords": [ diff --git a/tests/example_stac/landsat-collection.json b/tests/example_stac/landsat-collection.json index 91a8018..a5158e8 100644 --- a/tests/example_stac/landsat-collection.json +++ b/tests/example_stac/landsat-collection.json @@ -1,5 +1,6 @@ { - "stac_version": "1.0.0", + "type":"Collection", + "stac_version": "1.0.0", "stac_extensions": [], "id": "landsat-8-l1", "title": "Landsat 8 L1", diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 0000000..b3186c0 --- /dev/null +++ b/tests/test_catalog.py @@ -0,0 +1,22 @@ +from stac_pydantic.catalog import Catalog + + +def test_catalog(): + # Create a valid Catalog instance + catalog = Catalog( + type="Catalog", + id="my-catalog", + description="My STAC catalog", + ) + + catalog_json = catalog.model_dump(mode="json") + + # Make default all values are set + assert catalog_json["id"] == "my-catalog" + assert catalog_json["description"] == "My STAC catalog" + assert catalog_json["stac_version"] == "1.0.0" + assert catalog_json["links"] == [] + assert catalog_json["type"] == "Catalog" + + assert "stac_extensions" not in catalog_json + assert "title" not in catalog_json diff --git a/tests/test_item.py b/tests/test_item.py new file mode 100644 index 0000000..0f2e7a4 --- /dev/null +++ b/tests/test_item.py @@ -0,0 +1,124 @@ +import pytest +from ciso8601 import parse_rfc3339 +from pydantic import ValidationError + +from stac_pydantic.item import Item + + +def test_item(): + + item_data = { + "type": "Feature", + "id": "sample-item", + "stac_version": "1.0.0", + "geometry": {"type": "Point", "coordinates": [125.6, 10.1]}, + "bbox": [125.6, 10.1, 125.6, 10.1], + "properties": {"datetime": "2022-01-01T00:00:00Z"}, + } + item = Item(**item_data) + + item_json = item.model_dump(mode="json") + + # make sure that assets and links are set and parsed correctly + # datetime should be parsed as string and internally handled as datetime + # Collection and stac_extensions should not be parsed if not set + assert item_json["id"] == "sample-item" + assert item_json["stac_version"] == "1.0.0" + assert item_json["properties"]["datetime"] == "2022-01-01T00:00:00Z" + assert item.properties.datetime == parse_rfc3339("2022-01-01T00:00:00Z") + assert item_json["assets"] == {} + assert item_json["links"] == [] + + assert "collection" not in item_json + assert "stac_extensions" not in item_json + + +def test_item_datetime_set_null(): + + item_data = { + "type": "Feature", + "id": "sample-item", + "stac_version": "1.0.0", + "geometry": {"type": "Point", "coordinates": [125.6, 10.1]}, + "bbox": [125.6, 10.1, 125.6, 10.1], + "properties": { + "start_datetime": "2022-01-01T00:00:00Z", + "end_datetime": "2022-12-01T00:00:00Z", + }, + } + item = Item(**item_data) + + item_json = item.model_dump(mode="json") + + # make sure datetime is parsed as null and start_datetime and end_datetime are parsed as strings + assert item_json["properties"]["datetime"] is None + assert item_json["properties"]["start_datetime"] == "2022-01-01T00:00:00Z" + assert item_json["properties"]["end_datetime"] == "2022-12-01T00:00:00Z" + + +def test_item_bbox_missing(): + + item_data = { + "type": "Feature", + "id": "sample-item", + "stac_version": "1.0.0", + "geometry": {"type": "Point", "coordinates": [125.6, 10.1]}, + "properties": {"datetime": "2022-01-01T00:00:00Z"}, + } + + with pytest.raises(ValidationError) as e: + Item(**item_data) + assert e.value.errors() == [ + { + "loc": ("bbox",), + "msg": "bbox is required if geometry is not null", + "type": "value_error", + } + ] + + +@pytest.mark.parametrize("property", ["start_datetime", "end_datetime"]) +def test_item_start_end_datetime_missing(property): + + item_data = { + "type": "Feature", + "id": "sample-item", + "stac_version": "1.0.0", + "geometry": {"type": "Point", "coordinates": [125.6, 10.1]}, + "bbox": [125.6, 10.1, 125.6, 10.1], + "properties": { + property: "2022-12-01T00:00:00Z", + }, + } + + with pytest.raises(ValidationError) as e: + Item(**item_data) + assert e.value.errors() == [ + { + "loc": ("properties",), + "msg": "start_datetime and end_datetime must be specified when datetime is null", + "type": "value_error", + } + ] + + +def test_item_datetime_missing(): + + item_data = { + "type": "Feature", + "id": "sample-item", + "stac_version": "1.0.0", + "geometry": {"type": "Point", "coordinates": [125.6, 10.1]}, + "bbox": [125.6, 10.1, 125.6, 10.1], + "properties": {}, + } + + with pytest.raises(ValidationError) as e: + Item(**item_data) + assert e.value.errors() == [ + { + "loc": ("properties",), + "msg": "start_datetime and end_datetime must be specified when datetime is null", + "type": "value_error", + } + ]