From 7fc8580f0a54cd912dd76f9cbb3128427ccbc741 Mon Sep 17 00:00:00 2001 From: Zane Selvans Date: Mon, 27 Nov 2023 09:52:39 -0600 Subject: [PATCH] Update to Pydantic v2 and Ruff formatting (#169) * Update to using Pydantic v2 * Switch from black to ruff for code formatting * Remove black config from pyproject.toml --- .pre-commit-config.yaml | 8 +------- docs/release_notes.rst | 6 ++++++ pyproject.toml | 19 +++++++------------ src/ferc_xbrl_extractor/arelle_interface.py | 2 +- src/ferc_xbrl_extractor/instance.py | 9 +++++---- src/ferc_xbrl_extractor/taxonomy.py | 4 ++-- src/ferc_xbrl_extractor/xbrl.py | 14 +++++++++++--- tests/integration/datapackage_test.py | 2 +- 8 files changed, 34 insertions(+), 30 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ab2c78..c898512 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,13 +30,7 @@ repos: hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - - # Deterministic python formatting: - - repo: https://github.com/psf/black - rev: 23.11.0 - hooks: - - id: black - language_version: python3.10 + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.1.0 diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 582e341..5fb22a2 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -2,6 +2,12 @@ PACKAGE_NAME Release Notes ======================================================================================= +--------------------------------------------------------------------------------------- +1.2.x +--------------------------------------------------------------------------------------- + +* Migrate to using Pydantic v2. + .. _release-v1-2-0: --------------------------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 450f2a5..77587a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ requires-python = ">=3.10,<3.13" dynamic = ["version"] license = {file = "LICENSE.txt"} dependencies = [ - "pydantic>=1.9,<3", + "pydantic>=2,<3", "coloredlogs>=14.0,<15.1", "arelle-release>=2.3,<3", "frictionless>=4.4,<5", @@ -68,7 +68,6 @@ xbrl_extract = "ferc_xbrl_extractor.cli:main" [project.optional-dependencies] dev = [ - "black>=22.0,<23.12", # A deterministic code formatter "build>=1.0,<1.1", "ruff>=0.1,<0.2", # A very fast linter and autofixer "tox>=4.0,<4.12", # Python test environment manager @@ -106,11 +105,6 @@ where = ["src"] [tool.setuptools_scm] -[tool.black] -line-length = 88 -target-version = ["py310", "py311", "py312"] -include = "\\.pyi?$" - [tool.ruff] select = [ "A", # flake8-builtins @@ -156,7 +150,7 @@ ignore = [ "S101", # Use of assert ] -# Assume Python 3.11 +# Assume Python 3.12 target-version = "py312" line-length = 88 @@ -167,10 +161,6 @@ unfixable = ["ISC"] "__init__.py" = ["F401"] # Ignore unused imports "tests/*" = ["D"] -[tool.ruff.pep8-naming] -# Allow Pydantic's `@validator` decorator to trigger class method treatment. -classmethod-decorators = ["pydantic.validator", "pydantic.root_validator"] - [tool.ruff.isort] known-first-party = ["ferc_xbrl_extractor"] @@ -185,6 +175,9 @@ docstring-quotes = "double" inline-quotes = "double" multiline-quotes = "double" +[tool.ruff.format] +quote-style = "double" + [tool.doc8] max-line-length = 88 ignore-path = ["docs/_build"] @@ -194,6 +187,8 @@ testpaths = "./" filterwarnings = [ "ignore:distutils Version classes are deprecated:DeprecationWarning", "ignore:Creating a LegacyVersion:DeprecationWarning:pkg_resources[.*]", + "ignore:The `update_forward_refs` method is deprecated:pydantic.PydanticDeprecatedSince20:pydantic.main", + "ignore:datetime.datetime.utcfromtimestamp\\(\\) is deprecated:DeprecationWarning:dateutil.tz", ] addopts = "--verbose" diff --git a/src/ferc_xbrl_extractor/arelle_interface.py b/src/ferc_xbrl_extractor/arelle_interface.py index 97bf470..10549a0 100644 --- a/src/ferc_xbrl_extractor/arelle_interface.py +++ b/src/ferc_xbrl_extractor/arelle_interface.py @@ -87,7 +87,7 @@ class Metadata(BaseModel): name: str references: References calculations: list[Calculation] - balance: Literal["credit", "debit"] | None + balance: Literal["credit", "debit"] | None = None @classmethod def from_concept(cls, concept: ModelConcept) -> "Metadata": diff --git a/src/ferc_xbrl_extractor/instance.py b/src/ferc_xbrl_extractor/instance.py index 41a5e52..a5f002a 100644 --- a/src/ferc_xbrl_extractor/instance.py +++ b/src/ferc_xbrl_extractor/instance.py @@ -13,7 +13,7 @@ import stringcase from lxml import etree # nosec: B410 from lxml.etree import _Element as Element # nosec: B410 -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from ferc_xbrl_extractor.helpers import get_logger @@ -70,8 +70,9 @@ class Axis(BaseModel): value: str = "" dimension_type: DimensionType - @validator("name", pre=True) - def strip_prefix(cls, name: str): # noqa: N805 + @field_validator("name", mode="before") + @classmethod + def strip_prefix(cls, name: str) -> str: """Strip XML prefix from name.""" return name.split(":")[1] if ":" in name else name @@ -203,7 +204,7 @@ class Fact(BaseModel): name: str c_id: str - value: str | None + value: str | None = None @classmethod def from_xml(cls, elem: Element) -> "Fact": diff --git a/src/ferc_xbrl_extractor/taxonomy.py b/src/ferc_xbrl_extractor/taxonomy.py index 100c086..e83dc53 100644 --- a/src/ferc_xbrl_extractor/taxonomy.py +++ b/src/ferc_xbrl_extractor/taxonomy.py @@ -146,12 +146,12 @@ def get_metadata( # If concept is leaf node return metadata else: if period_type == self.period_type: - metadata[self.name] = self.metadata.dict() + metadata[self.name] = self.metadata.model_dump() return metadata -Concept.update_forward_refs() +Concept.model_rebuild() class LinkRole(BaseModel): diff --git a/src/ferc_xbrl_extractor/xbrl.py b/src/ferc_xbrl_extractor/xbrl.py index 841539e..324ebda 100644 --- a/src/ferc_xbrl_extractor/xbrl.py +++ b/src/ferc_xbrl_extractor/xbrl.py @@ -2,6 +2,7 @@ import io import math import re +import warnings from collections import defaultdict, namedtuple from collections.abc import Iterable from concurrent.futures import ProcessPoolExecutor as Executor @@ -166,7 +167,14 @@ def process_batch( "total_facts": instance.total_facts, } - dfs = {key: pd.concat(df_list) for key, df_list in dfs.items()} + with warnings.catch_warnings(): + warnings.filterwarnings( + action="once", + module="ferc_xbrl_extractor.xbrl", + category=FutureWarning, + message="The behavior of DataFrame concatenation with empty or all-NA entries is deprecated.", + ) + dfs = {key: pd.concat(df_list) for key, df_list in dfs.items()} return {"dfs": dfs, "metadata": metadata} @@ -242,7 +250,7 @@ def get_fact_tables( if datapackage_path: # Verify that datapackage descriptor is valid before outputting - frictionless_package = Package(descriptor=datapackage.dict(by_alias=True)) + frictionless_package = Package(descriptor=datapackage.model_dump(by_alias=True)) if not frictionless_package.metadata_valid: raise RuntimeError( f"Generated datapackage is invalid - {frictionless_package.metadata_errors}" @@ -250,6 +258,6 @@ def get_fact_tables( # Write to JSON file with Path(datapackage_path).open(mode="w") as f: - f.write(datapackage.json(by_alias=True)) + f.write(datapackage.model_dump_json(by_alias=True)) return datapackage.get_fact_tables(filter_tables=filter_tables) diff --git a/tests/integration/datapackage_test.py b/tests/integration/datapackage_test.py index 3123017..77919ac 100644 --- a/tests/integration/datapackage_test.py +++ b/tests/integration/datapackage_test.py @@ -38,7 +38,7 @@ def test_datapackage_generation(test_dir): # test than a normative statement assert len(all_tables) == 366 - assert Package(descriptor=datapackage.dict(by_alias=True)).metadata_valid + assert Package(descriptor=datapackage.model_dump(by_alias=True)).metadata_valid def _create_schema(instant=True, axes=None):