Update to Pydantic v2 and Ruff formatting (#169)

* Update to using Pydantic v2

* Switch from black to ruff for code formatting

* Remove black config from pyproject.toml

.pre-commit-config.yaml

@@ -30,13 +30,7 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
-
-  # Deterministic python formatting:
-  - repo: https://github.com/psf/black
-    rev: 23.11.0
-    hooks:
-      - id: black
-        language_version: python3.10
+      - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.1.0

docs/release_notes.rst

@@ -2,6 +2,12 @@
 PACKAGE_NAME Release Notes
 =======================================================================================
 
+---------------------------------------------------------------------------------------
+1.2.x
+---------------------------------------------------------------------------------------
+
+* Migrate to using Pydantic v2.
+
 .. _release-v1-2-0:
 
 ---------------------------------------------------------------------------------------

pyproject.toml

@@ -18,7 +18,7 @@ requires-python = ">=3.10,<3.13"
 dynamic = ["version"]
 license = {file = "LICENSE.txt"}
 dependencies = [
-    "pydantic>=1.9,<3",
+    "pydantic>=2,<3",
     "coloredlogs>=14.0,<15.1",
     "arelle-release>=2.3,<3",
     "frictionless>=4.4,<5",
@@ -68,7 +68,6 @@ xbrl_extract = "ferc_xbrl_extractor.cli:main"
 
 [project.optional-dependencies]
 dev = [
-    "black>=22.0,<23.12",  # A deterministic code formatter
     "build>=1.0,<1.1",
     "ruff>=0.1,<0.2",  # A very fast linter and autofixer
     "tox>=4.0,<4.12",  # Python test environment manager
@@ -106,11 +105,6 @@ where = ["src"]
 
 [tool.setuptools_scm]
 
-[tool.black]
-line-length = 88
-target-version = ["py310", "py311", "py312"]
-include = "\\.pyi?$"
-
 [tool.ruff]
 select = [
     "A", # flake8-builtins
@@ -156,7 +150,7 @@ ignore = [
     "S101",   # Use of assert
 ]
 
-# Assume Python 3.11
+# Assume Python 3.12
 target-version = "py312"
 line-length = 88
 
@@ -167,10 +161,6 @@ unfixable = ["ISC"]
 "__init__.py" = ["F401"]    # Ignore unused imports
 "tests/*" = ["D"]
 
-[tool.ruff.pep8-naming]
-# Allow Pydantic's `@validator` decorator to trigger class method treatment.
-classmethod-decorators = ["pydantic.validator", "pydantic.root_validator"]
-
 [tool.ruff.isort]
 known-first-party = ["ferc_xbrl_extractor"]
 
@@ -185,6 +175,9 @@ docstring-quotes = "double"
 inline-quotes = "double"
 multiline-quotes = "double"
 
+[tool.ruff.format]
+quote-style = "double"
+
 [tool.doc8]
 max-line-length = 88
 ignore-path = ["docs/_build"]
@@ -194,6 +187,8 @@ testpaths = "./"
 filterwarnings = [
     "ignore:distutils Version classes are deprecated:DeprecationWarning",
     "ignore:Creating a LegacyVersion:DeprecationWarning:pkg_resources[.*]",
+    "ignore:The `update_forward_refs` method is deprecated:pydantic.PydanticDeprecatedSince20:pydantic.main",
+    "ignore:datetime.datetime.utcfromtimestamp\\(\\) is deprecated:DeprecationWarning:dateutil.tz",
 ]
 
 addopts = "--verbose"

src/ferc_xbrl_extractor/arelle_interface.py

@@ -87,7 +87,7 @@ class Metadata(BaseModel):
     name: str
     references: References
     calculations: list[Calculation]
-    balance: Literal["credit", "debit"] | None
+    balance: Literal["credit", "debit"] | None = None
 
     @classmethod
     def from_concept(cls, concept: ModelConcept) -> "Metadata":

src/ferc_xbrl_extractor/instance.py

@@ -13,7 +13,7 @@
 import stringcase
 from lxml import etree  # nosec: B410
 from lxml.etree import _Element as Element  # nosec: B410
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 
 from ferc_xbrl_extractor.helpers import get_logger
 
@@ -70,8 +70,9 @@ class Axis(BaseModel):
     value: str = ""
     dimension_type: DimensionType
 
-    @validator("name", pre=True)
-    def strip_prefix(cls, name: str):  # noqa: N805
+    @field_validator("name", mode="before")
+    @classmethod
+    def strip_prefix(cls, name: str) -> str:
         """Strip XML prefix from name."""
         return name.split(":")[1] if ":" in name else name
 
@@ -203,7 +204,7 @@ class Fact(BaseModel):
 
     name: str
     c_id: str
-    value: str | None
+    value: str | None = None
 
     @classmethod
     def from_xml(cls, elem: Element) -> "Fact":

src/ferc_xbrl_extractor/taxonomy.py

@@ -146,12 +146,12 @@ def get_metadata(
         # If concept is leaf node return metadata
         else:
             if period_type == self.period_type:
-                metadata[self.name] = self.metadata.dict()
+                metadata[self.name] = self.metadata.model_dump()
 
         return metadata
 
 
-Concept.update_forward_refs()
+Concept.model_rebuild()
 
 
 class LinkRole(BaseModel):

src/ferc_xbrl_extractor/xbrl.py

@@ -2,6 +2,7 @@
 import io
 import math
 import re
+import warnings
 from collections import defaultdict, namedtuple
 from collections.abc import Iterable
 from concurrent.futures import ProcessPoolExecutor as Executor
@@ -166,7 +167,14 @@ def process_batch(
             "total_facts": instance.total_facts,
         }
 
-    dfs = {key: pd.concat(df_list) for key, df_list in dfs.items()}
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            action="once",
+            module="ferc_xbrl_extractor.xbrl",
+            category=FutureWarning,
+            message="The behavior of DataFrame concatenation with empty or all-NA entries is deprecated.",
+        )
+        dfs = {key: pd.concat(df_list) for key, df_list in dfs.items()}
 
     return {"dfs": dfs, "metadata": metadata}
 
@@ -242,14 +250,14 @@ def get_fact_tables(
 
     if datapackage_path:
         # Verify that datapackage descriptor is valid before outputting
-        frictionless_package = Package(descriptor=datapackage.dict(by_alias=True))
+        frictionless_package = Package(descriptor=datapackage.model_dump(by_alias=True))
         if not frictionless_package.metadata_valid:
             raise RuntimeError(
                 f"Generated datapackage is invalid - {frictionless_package.metadata_errors}"
             )
 
         # Write to JSON file
         with Path(datapackage_path).open(mode="w") as f:
-            f.write(datapackage.json(by_alias=True))
+            f.write(datapackage.model_dump_json(by_alias=True))
 
     return datapackage.get_fact_tables(filter_tables=filter_tables)

tests/integration/datapackage_test.py

@@ -38,7 +38,7 @@ def test_datapackage_generation(test_dir):
     # test than a normative statement
     assert len(all_tables) == 366
 
-    assert Package(descriptor=datapackage.dict(by_alias=True)).metadata_valid
+    assert Package(descriptor=datapackage.model_dump(by_alias=True)).metadata_valid
 
 
 def _create_schema(instant=True, axes=None):