KDP-1761: Upgrade to Pydantic v2

.pre-commit-config.yaml

@@ -53,7 +53,7 @@ repos:
         additional_dependencies: [ 'pep8-naming==0.12.1' ]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.971
+    rev: v1.5.1
     hooks:
       - id: mypy
         language_version: python
@@ -64,5 +64,4 @@ repos:
         # Don't pass it the individual filenames because it is already doing the whole folder.
         pass_filenames: false
         additional_dependencies:
-          - orjson
           - pydantic

README.md

@@ -35,25 +35,27 @@ pip install git+https://github.com/KNMI/covjson-pydantic.git
 ## Usage
 
 ```python
-import datetime
+from datetime import datetime, timezone
+from pydantic import AwareDatetime
 from covjson_pydantic.coverage import Coverage
-from covjson_pydantic.domain import Domain
+from covjson_pydantic.domain import Domain, Axes, ValuesAxis
 from covjson_pydantic.ndarray import NdArray
 
 c = Coverage(
     domain=Domain(
         domainType="PointSeries",
-        axes={
-            "x": {"dataType": "float", "values": [1.23]},
-            "y": {"values": [4.56]},
-            "t": {"dataType": "datetime", "values": [datetime.datetime.now()]}
-        },
+        axes=Axes(
+            x=ValuesAxis[float](values=[1.23]),
+            y=ValuesAxis[float](values=[4.56]),
+            t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)])
+        )
     ),
     ranges={
         "temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])
     }
 )
-print(c.json(exclude_none=True,indent=True))
+
+print(c.model_dump_json(exclude_none=True, indent=4))
 ```
 Will print
 ```json
@@ -64,7 +66,6 @@ Will print
         "domainType": "PointSeries",
         "axes": {
             "x": {
-                "dataType": "float",
                 "values": [
                     1.23
                 ]
@@ -75,9 +76,8 @@ Will print
                 ]
             },
             "t": {
-                "dataType": "datetime",
                 "values": [
-                    "2023-01-19T13:14:47.126631Z"
+                    "2023-09-14T11:54:02.151493Z"
                 ]
             }
         }

example.py

@@ -1,19 +1,23 @@
-import datetime
+from datetime import datetime
+from datetime import timezone
 
 from covjson_pydantic.coverage import Coverage
+from covjson_pydantic.domain import Axes
 from covjson_pydantic.domain import Domain
+from covjson_pydantic.domain import ValuesAxis
 from covjson_pydantic.ndarray import NdArray
+from pydantic import AwareDatetime
 
 c = Coverage(
     domain=Domain(
         domainType="PointSeries",
-        axes={
-            "x": {"dataType": "float", "values": [1.23]},
-            "y": {"values": [4.56]},
-            "t": {"dataType": "datetime", "values": [datetime.datetime.now()]},
-        },
+        axes=Axes(
+            x=ValuesAxis[float](values=[1.23]),
+            y=ValuesAxis[float](values=[4.56]),
+            t=ValuesAxis[AwareDatetime](values=[datetime.now(tz=timezone.utc)]),
+        ),
     ),
     ranges={"temperature": NdArray(axisNames=["x", "y", "t"], shape=[1, 1, 1], values=[42.0])},
 )
 
-print(c.json(exclude_none=True, indent=True))
+print(c.model_dump_json(exclude_none=True, indent=4))

pyproject.toml

@@ -19,8 +19,8 @@ classifiers = [
     "Topic :: Scientific/Engineering :: GIS",
     "Typing :: Typed",
 ]
-version = "0.1.0"
-dependencies = ["pydantic>=1,<2", "orjson>=3"]
+version = "0.2.0"
+dependencies = ["pydantic>=2,<3"]
 
 [project.optional-dependencies]
 test = ["pytest", "pytest-cov"]

src/covjson_pydantic/base_models.py

@@ -1,32 +1,16 @@
-import orjson
 from pydantic import BaseModel as PydanticBaseModel
-from pydantic import Extra
-
-
-def orjson_dumps(v, *, default, indent=None, sort_keys=False):
-    options = orjson.OPT_NON_STR_KEYS | orjson.OPT_UTC_Z | orjson.OPT_NAIVE_UTC
-    if indent:
-        options |= orjson.OPT_INDENT_2
-
-    if sort_keys:
-        options |= orjson.OPT_SORT_KEYS
-
-    # orjson.dumps returns bytes, to match standard json.dumps we need to decode
-    return orjson.dumps(v, default=default, option=options).decode()
+from pydantic import ConfigDict
 
 
 class BaseModel(PydanticBaseModel):
-    class Config:
-        anystr_strip_whitespace = True
-        min_anystr_length = 1
-        extra = Extra.forbid
-        validate_all = True
-        validate_assignment = True
-
-        json_loads = orjson.loads
-        json_dumps = orjson_dumps
+    model_config = ConfigDict(
+        str_strip_whitespace=True,
+        str_min_length=1,
+        extra="forbid",
+        validate_default=True,
+        validate_assignment=True,
+    )
 
 
 class CovJsonBaseModel(BaseModel):
-    class Config:
-        extra = Extra.allow  # allow custom members
+    model_config = ConfigDict(extra="allow")

src/covjson_pydantic/coverage.py

@@ -17,18 +17,18 @@
 
 
 class Coverage(CovJsonBaseModel):
-    id: Optional[str]
+    id: Optional[str] = None
     type: Literal["Coverage"] = "Coverage"
     domain: Domain
-    parameters: Optional[Dict[str, Parameter]]
-    parameterGroups: Optional[List[ParameterGroup]]  # noqa: N815
+    parameters: Optional[Dict[str, Parameter]] = None
+    parameterGroups: Optional[List[ParameterGroup]] = None  # noqa: N815
     ranges: Dict[str, Union[NdArray, TiledNdArray, AnyUrl]]
 
 
 class CoverageCollection(CovJsonBaseModel):
     type: Literal["CoverageCollection"] = "CoverageCollection"
-    domainType: Optional[DomainType]  # noqa: N815
+    domainType: Optional[DomainType] = None  # noqa: N815
     coverages: List[Coverage]
-    parameters: Optional[Dict[str, Parameter]]
-    parameterGroups: Optional[List[ParameterGroup]]  # noqa: N815
-    referencing: Optional[List[ReferenceSystemConnectionObject]]
+    parameters: Optional[Dict[str, Parameter]] = None
+    parameterGroups: Optional[List[ParameterGroup]] = None  # noqa: N815
+    referencing: Optional[List[ReferenceSystemConnectionObject]] = None

src/covjson_pydantic/domain.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from enum import Enum
 from typing import Generic
 from typing import List
@@ -8,10 +7,10 @@
 from typing import TypeVar
 from typing import Union
 
-from pydantic import Extra
+from pydantic import AwareDatetime
+from pydantic import ConfigDict
+from pydantic import model_validator
 from pydantic import PositiveInt
-from pydantic.class_validators import root_validator
-from pydantic.generics import GenericModel
 
 from .base_models import BaseModel
 from .base_models import CovJsonBaseModel
@@ -23,34 +22,30 @@ class CompactAxis(BaseModel):
     stop: float
     num: PositiveInt
 
-    @root_validator(skip_on_failure=True)
-    def single_value_case(cls, values):
-        if values["num"] == 1 and values["start"] != values["stop"]:
+    @model_validator(mode="after")
+    def single_value_case(self):
+        if self.num == 1 and self.start != self.stop:
             raise ValueError("If the value of 'num' is 1, then 'start' and 'stop' MUST have identical values.")
-        return values
+        return self
 
 
 ValuesT = TypeVar("ValuesT")
 
 
-class ValuesAxis(GenericModel, Generic[ValuesT]):
-    dataType: Optional[str]  # noqa: N815
-    coordinates: Optional[List[str]]
+class ValuesAxis(BaseModel, Generic[ValuesT]):
+    dataType: Optional[str] = None  # noqa: N815
+    coordinates: Optional[List[str]] = None
     values: List[ValuesT]
-    bounds: Optional[List[ValuesT]]
-
-    class Config:
-        anystr_strip_whitespace = True
-        min_anystr_length = 1
-        extra = Extra.allow  # allow custom members
-        validate_all = True
-        validate_assignment = True
-
-    @root_validator(skip_on_failure=True)
-    def bounds_length(cls, values):
-        if values["bounds"] is not None and len(values["bounds"]) != 2 * len(values["values"]):
+    bounds: Optional[List[ValuesT]] = None
+    model_config = ConfigDict(
+        str_strip_whitespace=True, str_min_length=1, extra="allow", validate_default=True, validate_assignment=True
+    )
+
+    @model_validator(mode="after")
+    def bounds_length(self):
+        if self.bounds is not None and len(self.bounds) != 2 * len(self.values):
             raise ValueError("If provided, the length of 'bounds' should be twice that of 'values'.")
-        return values
+        return self
 
 
 class DomainType(str, Enum):
@@ -63,30 +58,24 @@ class DomainType(str, Enum):
 
 
 class Axes(BaseModel):
-    x: Optional[Union[ValuesAxis[float], CompactAxis]]
-    y: Optional[Union[ValuesAxis[float], CompactAxis]]
-    z: Optional[Union[ValuesAxis[float], CompactAxis]]
-    t: Optional[ValuesAxis[datetime]]
-    composite: Optional[ValuesAxis[Tuple]]
-
-    @root_validator(skip_on_failure=True)
-    def at_least_one_axes(cls, values):
-        if (
-            values["x"] is None
-            and values["y"] is None
-            and values["z"] is None
-            and values["t"] is None
-            and values["composite"] is None
-        ):
+    x: Optional[Union[ValuesAxis[float], CompactAxis]] = None
+    y: Optional[Union[ValuesAxis[float], CompactAxis]] = None
+    z: Optional[Union[ValuesAxis[float], CompactAxis]] = None
+    t: Optional[ValuesAxis[AwareDatetime]] = None
+    composite: Optional[ValuesAxis[Tuple]] = None
+
+    @model_validator(mode="after")
+    def at_least_one_axes(self):
+        if self.x is None and self.y is None and self.z is None and self.t is None and self.composite is None:
             raise ValueError("At least one axis of x,y,z,t or composite must be given.")
-        return values
+        return self
 
 
 class Domain(CovJsonBaseModel):
     type: Literal["Domain"] = "Domain"
-    domainType: Optional[DomainType]  # noqa: N815
+    domainType: Optional[DomainType] = None  # noqa: N815
     axes: Axes
-    referencing: Optional[List[ReferenceSystemConnectionObject]]
+    referencing: Optional[List[ReferenceSystemConnectionObject]] = None
 
     @staticmethod
     def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes):
@@ -119,10 +108,10 @@ def check_axis(domain_type, axes, required_axes, allowed_axes, single_value_axes
                         f"of a '{domain_type.value}' domain must contain a single value."
                     )
 
-    @root_validator(skip_on_failure=True)
-    def check_domain_consistent(cls, values):
-        domain_type = values.get("domainType")
-        axes = values.get("axes")
+    @model_validator(mode="after")
+    def check_domain_consistent(self):
+        domain_type = self.domainType
+        axes = self.axes
 
         if domain_type == DomainType.grid:
             Domain.check_axis(
@@ -158,4 +147,4 @@ def check_domain_consistent(cls, values):
                 domain_type, axes, required_axes={"composite"}, allowed_axes={"t"}, single_value_axes={"t"}
             )
 
-        return values
+        return self

src/covjson_pydantic/i18n.py

@@ -9,4 +9,10 @@ class LanguageTag(str, Enum):
     undefined = "und"
 
 
-i18n = Dict[LanguageTag, str]
+# TODO: This was throwing warning:
+#   Expected `definition-ref` but got `LanguageTag` - serialized value may not be as expected
+# This may be a bug in Pydantic: https://github.com/pydantic/pydantic/issues/6467
+# or: https://github.com/pydantic/pydantic/issues/6422
+# So, for now, reverted to a less strict type
+# i18n = Dict[LanguageTag, str]
+i18n = Dict[str, str]

src/covjson_pydantic/ndarray.py

@@ -4,8 +4,7 @@
 from typing import Literal
 from typing import Optional
 
-from pydantic import AnyUrl
-from pydantic.class_validators import root_validator
+from pydantic import model_validator
 
 from .base_models import BaseModel
 from .base_models import CovJsonBaseModel
@@ -19,39 +18,35 @@ class DataType(str, Enum):
 class NdArray(CovJsonBaseModel):
     type: Literal["NdArray"] = "NdArray"
     dataType: DataType = DataType.float  # noqa: N815
-    axisNames: Optional[List[str]]  # noqa: N815
-    shape: Optional[List[int]]
+    axisNames: Optional[List[str]] = None  # noqa: N815
+    shape: Optional[List[int]] = None
     values: List[Optional[float]]
 
-    @root_validator(skip_on_failure=True)
-    def check_field_dependencies(cls, values):
-        if len(values["values"]) > 1 and (values.get("axisNames") is None or len(values.get("axisNames")) == 0):
+    @model_validator(mode="after")
+    def check_field_dependencies(self):
+        if len(self.values) > 1 and (self.axisNames is None or len(self.axisNames) == 0):
             raise ValueError("'axisNames' must to be provided if array is not 0D")
 
-        if len(values["values"]) > 1 and (values.get("shape") is None or len(values.get("shape")) == 0):
+        if len(self.values) > 1 and (self.shape is None or len(self.shape) == 0):
             raise ValueError("'shape' must to be provided if array is not 0D")
 
-        if (
-            values.get("axisNames") is not None
-            and values.get("shape") is not None
-            and len(values.get("axisNames")) != len(values.get("shape"))
-        ):
+        if self.axisNames is not None and self.shape is not None and len(self.axisNames) != len(self.shape):
             raise ValueError("'axisNames' and 'shape' should have equal length")
 
-        if values.get("shape") is not None and len(values.get("shape")) >= 1:
-            prod = math.prod(values["shape"])
-            if len(values["values"]) != prod:
+        if self.shape is not None and len(self.shape) >= 1:
+            prod = math.prod(self.shape)
+            if len(self.values) != prod:
                 raise ValueError(
                     "Where 'shape' is present and non-empty, the product of its values MUST equal "
                     "the number of elements in the 'values' array."
                 )
 
-        return values
+        return self
 
 
 class TileSet(BaseModel):
     tileShape: List[Optional[int]]  # noqa: N815
-    urlTemplate: AnyUrl  # noqa: N815
+    urlTemplate: str  # noqa: N815
 
 
 # TODO: Validation of field dependencies