diff --git a/README.md b/README.md index 846c273..a453346 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,14 @@ Support for Pydantic settings configuration file loading ## Installation `pip install pydantic-config` +### Optional Dependencies + +Pydantic-Config has the following optional dependencies: + - yaml - `pip install pydantic-config[yaml]` + - toml - `pip install pydantic-config[toml]` + +You can install all the optional dependencies with `pip install pydantic-config[all]` + ## Usage ```toml @@ -13,7 +21,7 @@ description = "Test application description" ``` ```python -from pydantic_config import SettingsModel +from pydantic_config import SettingsModel, SettingsConfig class Settings(SettingsModel): @@ -21,9 +29,10 @@ class Settings(SettingsModel): app_name: str = None description: str = None log_level: str = 'INFO' - - class Config: - config_file = 'config.toml' + + model_config = SettingsConfig( + config_file='config.toml', + ) settings = Settings() @@ -53,7 +62,7 @@ description = "Test application description" ``` ```python -from pydantic_config import SettingsModel +from pydantic_config import SettingsModel, SettingsConfig class Settings(SettingsModel): @@ -61,16 +70,23 @@ class Settings(SettingsModel): app_name: str = 'App Name' description: str = None log_level: str = 'INFO' - - class Config: - config_file = ['config.toml', 'config.json'] # The config.json file will take priority over config.toml - + + model_config = SettingsConfig( + config_file=['config.toml', 'config.json'] # The config.json file will take priority over config.toml + ) settings = Settings() print(settings) # app_id='1' app_name='Python Application' description='Description from JSON file' log_level='WARNING' ``` +## Supported file formats +Currently, the following file formats are supported: + - `.yaml` _Requires `pyyaml` package_ + - `.toml` _Requires `toml` package_ + - `.json` + - `.ini` + ## Merging If your configurations have existing `list` or `dict` variables the contents will be merged by default. To disable @@ -91,15 +107,16 @@ item2 = "value2" ``` ```python -from pydantic_config import SettingsModel +from pydantic_config import SettingsModel, SettingsConfig class Settings(SettingsModel): foo: dict = {} - - class Config: - config_file = ['config.toml', 'config2.toml'] - config_merge: bool = True + + model_config = SettingsConfig( + config_file=['config.toml', 'config2.toml'], + config_merge= True, + ) settings = Settings() diff --git a/environment.yml b/environment.yml index 1f3ecb4..3029696 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: - defaults dependencies: - python=3.10 - - pydantic<2.0.0, >=1.10.0 + - pydantic-settings>=2.0.1 - toml=0.10.2 - python-dotenv=0.21.0 - pyyaml=6.0 diff --git a/pyproject.toml b/pyproject.toml index a0de24f..ad13162 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "pydantic-config" description = "Support for Pydantic settings configuration file loading" -version = "0.1.2" +version = "0.2.0" authors = [{name="Jordan Shaw"}] readme = "README.md" requires-python = ">=3.7" @@ -19,24 +19,41 @@ classifiers = [ ] dependencies = [ - 'pydantic<2.0.0, >=1.10.0', - 'pyyaml>=6.0', - 'python-dotenv>=0.21.0', - 'toml>=0.10.2' + 'pydantic-settings>=2.0.2', + ] [project.optional-dependencies] +yaml = [ + "pyyaml>=5.1" +] +toml = [ + 'toml>=0.10.0' +] +all = [ + 'pyyaml>=5.1', + 'toml>=0.10.0' +] dev = [ - "pytest", 'twine', 'build' + 'pytest', + 'twine', + 'build', + 'pyyaml>=5.1', + 'python-dotenv>=0.15.0', + 'toml>=0.10.0' ] test = [ - "pytest" + 'pytest', + 'pyyaml==5.1', + 'python-dotenv>=0.15.0', + 'toml>=0.10.0' ] [project.urls] "Homepage" = "https://github.com/jordantshaw/pydantic-config" [tool.pytest.ini_options] -addopts = [ - "--import-mode=importlib", +pythonpath = [ + "src" ] + diff --git a/src/pydantic_config/__init__.py b/src/pydantic_config/__init__.py index a565ae2..d77de1d 100644 --- a/src/pydantic_config/__init__.py +++ b/src/pydantic_config/__init__.py @@ -1 +1 @@ -from .main import SettingsModel \ No newline at end of file +from .main import SettingsModel, SettingsConfig diff --git a/src/pydantic_config/main.py b/src/pydantic_config/main.py index d1975de..4ff11a2 100644 --- a/src/pydantic_config/main.py +++ b/src/pydantic_config/main.py @@ -1,106 +1,145 @@ -from copy import deepcopy +import os from pathlib import Path -from typing import Dict, Any, Union, List - -from pydantic import BaseSettings, BaseModel - -from .loaders import ( - ini_file_loader, - toml_file_loader, - yaml_file_loader, - json_file_loader +from typing import Dict, Any, Union, List, Type, Tuple, Mapping + +from pydantic.fields import FieldInfo +from pydantic_settings import BaseSettings, SettingsConfigDict, PydanticBaseSettingsSource +from pydantic_settings.sources import PydanticBaseEnvSettingsSource + +from .merge import deep_merge +from .readers import ( + ini_file_reader, + toml_file_reader, + yaml_file_reader, + json_file_reader ) -class SettingsModel (BaseSettings): - - class Config: - - config_files: List = [] - config_file_encoding: str = None - config_merge: bool = True - config_merge_unique: bool = True - - @classmethod - def customise_sources( - cls, - init_settings, - env_settings, - file_secret_settings, - ): - return ( - init_settings, - env_settings, - file_secret_settings, - config_file_settings, - ) - - -def config_file_settings(settings: BaseSettings) -> Dict[str, Any]: - encoding = getattr(settings.__config__, 'config_file_encoding', None) - files = getattr(settings.__config__, 'config_file', []) - config_merge = getattr(settings.__config__, 'config_merge', True) - config_merge_unique = getattr(settings.__config__, 'config_merge_unique', True) - - if isinstance(files, str): - files = [files] - - config = {} - for file in files: - if config_merge: - config = _deep_merge( - base=config, - nxt=_load_config_file(file, encoding), - unique=config_merge_unique, - ) - else: - config.update(_load_config_file(file, encoding)) - - return config - - -def _deep_merge(base: dict, nxt: dict, unique: bool = True) -> dict: - """ - Merges nested dictionaries. - - Parameters - ---------- - base: dict - The base dictionary to merge into - nxt: dict - The dictionary to merge into base - unique: bool - This determines the behavior when merging list values. Lists are appended together which could result - in duplicate values. To avoid duplicates set this value to True. - - """ - result = deepcopy(base) - - for key, value in nxt.items(): - if isinstance(result.get(key), dict) and isinstance(value, dict): - result[key] = _deep_merge(result.get(key), value) - elif isinstance(result.get(key), list) and isinstance(value, list): - if unique: - result[key] = result.get(key) + [i for i in deepcopy(value) if i not in set(result.get(key))] +ConfigFileType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]] + + +class SettingsError(ValueError): + pass + + +class SettingsConfig(SettingsConfigDict): + config_file: Union[ConfigFileType, None] + config_file_encoding: Union[str, None] + config_merge: bool + config_merge_unique: bool + + +class SettingsModel(BaseSettings): + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return ( + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + ConfigFileSettingsSource(settings_cls), + ) + + +class ConfigFileSettingsSource(PydanticBaseEnvSettingsSource): + """ Settings source class that loads values from one more configuration files. """ + def __init__( + self, + settings_cls: Type[BaseSettings], + case_sensitive: Union[bool, None] = None, + config_file: Union[ConfigFileType, None] = None, + file_encoding: Union[str, None] = None, + config_merge: bool = True, + config_merge_unique: bool = True, + + ) -> None: + super().__init__(settings_cls, case_sensitive) + self.config_file = config_file or self.config.get('config_file', None) + self.file_encoding = file_encoding or self.config.get('file_encoding', None) + self.config_merge = config_merge or self.config.get('config_merge', True) + self.config_merge_unique = config_merge_unique or self.config.get('config_merge_unique', True) + self.config_values = self._load_config_values() + + def get_field_value(self, field: FieldInfo, field_name: str) -> Tuple[Any, str, bool]: + config_val: Any = self.config_values.get(field_name) + return config_val, field_name, False + + def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any: + if value is None: + value = self.config_values.get(field_name if self.case_sensitive else field_name.lower()) + if value: + return value + + return value + + def _lowercase_dict_keys(self, d: Dict): + return {k.lower(): self._lowercase_dict_keys(v) if isinstance(v, dict) else v for k, v in d.items()} + + def _load_config_values(self) -> Dict[str, Any]: + """ Gets the config values from the configuration files """ + config_values = self._read_config_files() + if self.case_sensitive: + return config_values + + return self._lowercase_dict_keys(config_values) + + def _read_config_files(self) -> Dict[str, Any]: + """ Reads config files and merges config values if merging is enabled """ + config_files = self.config_file + if config_files is None: + return {} + + if isinstance(config_files, (str, os.PathLike)): + config_files = [config_files] + + config = {} + for file in config_files: + file_path = Path(file).expanduser() + if not file_path.exists(): + raise OSError(f"Config file `{file}` not found") + + if self.config_merge: + config = deep_merge( + base=config, + nxt=self._read_config_file(file_path), + unique=self.config_merge_unique, + ) else: - result[key] = result.get(key) + deepcopy(value) - else: - result[key] = deepcopy(value) - return result - - -def _load_config_file(file: str, encoding: str) -> Dict[str, Any]: - """ Loads config file with selected encoding """ - file = Path(file) - if not file.exists(): - raise OSError(f"Could not load file from provided config file: {file}") - - file_loaders = { - '.ini': ini_file_loader, - '.toml': toml_file_loader, - '.yaml': yaml_file_loader, - '.json': json_file_loader, - } - - return file_loaders.get(file.suffix)(str(file), encoding) - + config.update(self._read_config_file(file_path)) + return config + + def _read_config_file(self, file: Path) -> Dict[str, Any]: + """ Reads single config file based on file extension """ + file_loaders = { + '.ini': ini_file_reader, + '.toml': toml_file_reader, + '.yaml': yaml_file_reader, + '.json': json_file_reader, + } + + return file_loaders.get(file.suffix)(str(file), self.file_encoding) + + def __call__(self) -> Dict[str, Any]: + data: Dict[str, Any] = super().__call__() + + data_lower_keys: List[str] = [] + if not self.case_sensitive: + data_lower_keys = [x.lower() for x in data.keys()] + + # As `extra` config is allowed in config settings source, We have to + # update data with extra config values from the config files. + for config_name, config_value in self.config_values.items(): + if config_value is not None: + if (data_lower_keys and config_name not in data_lower_keys) or ( + not data_lower_keys and config_name not in data): + data[config_name] = config_value + + return data diff --git a/src/pydantic_config/merge.py b/src/pydantic_config/merge.py new file mode 100644 index 0000000..2be1774 --- /dev/null +++ b/src/pydantic_config/merge.py @@ -0,0 +1,31 @@ +from copy import deepcopy + + +def deep_merge(base: dict, nxt: dict, unique: bool = True) -> dict: + """ + Merges nested dictionaries. + + Parameters + ---------- + base: dict + The base dictionary to merge into + nxt: dict + The dictionary to merge into base + unique: bool + This determines the behavior when merging list values. Lists are appended together which could result + in duplicate values. To avoid duplicates set this value to True. + + """ + result = deepcopy(base) + + for key, value in nxt.items(): + if isinstance(result.get(key), dict) and isinstance(value, dict): + result[key] = deep_merge(result.get(key), value) + elif isinstance(result.get(key), list) and isinstance(value, list): + if unique: + result[key] = result.get(key) + [i for i in deepcopy(value) if i not in set(result.get(key))] + else: + result[key] = result.get(key) + deepcopy(value) + else: + result[key] = deepcopy(value) + return result \ No newline at end of file diff --git a/src/pydantic_config/loaders.py b/src/pydantic_config/readers.py similarity index 52% rename from src/pydantic_config/loaders.py rename to src/pydantic_config/readers.py index d8b7755..13f6598 100644 --- a/src/pydantic_config/loaders.py +++ b/src/pydantic_config/readers.py @@ -4,26 +4,26 @@ import yaml -def json_file_loader(file: str, encoding: str = None): - """ .json file type loader """ +def json_file_reader(file: str, encoding: str = None): + """ .json file type reader """ with open(file, 'r', encoding=encoding) as file: return json.load(file) -def ini_file_loader(file: str, encoding: str = None): - """ .ini file type loader """ +def ini_file_reader(file: str, encoding: str = None): + """ .ini file type reader """ config = configparser.ConfigParser() config.read(file, encoding=encoding) return {k: dict(v) for k, v in config.items()} -def toml_file_loader(file: str, encoding: str = None): - """ .toml file type loader """ +def toml_file_reader(file: str, encoding: str = None): + """ .toml file type reader """ return toml.load(file) -def yaml_file_loader(file: str, encoding: str = None): - """ .yaml file type loader """ +def yaml_file_reader(file: str, encoding: str = None): + """ .yaml file type reader """ with open(file, 'r', encoding=encoding) as file: return yaml.safe_load(file) diff --git a/tests/test_loaders.py b/tests/test_loaders.py index 3ac39e9..1eb958d 100644 --- a/tests/test_loaders.py +++ b/tests/test_loaders.py @@ -1,21 +1,21 @@ -from pydantic_config.loaders import toml_file_loader, ini_file_loader, yaml_file_loader, json_file_loader +from pydantic_config.readers import toml_file_reader, ini_file_reader, yaml_file_reader, json_file_reader def test_toml_load(config_toml_file): - data = toml_file_loader(config_toml_file) + data = toml_file_reader(config_toml_file) assert data == {'app': {'description': 'description from config.toml'}} def test_ini_load(config_ini_file): - data = ini_file_loader(config_ini_file) + data = ini_file_reader(config_ini_file) assert data == {'DEFAULT': {}, 'APP': {'description': 'description from config.ini'}} def test_yaml_load(config_yaml_file): - data = yaml_file_loader(config_yaml_file) + data = yaml_file_reader(config_yaml_file) assert data == {'app': {'description': 'description from config.yaml'}} def test_json_load(config_json_file): - data = json_file_loader(config_json_file) + data = json_file_reader(config_json_file) assert data == {'app': {'description': 'description from config.json'}} diff --git a/tests/test_settings_model.py b/tests/test_settings_model.py index 0c37b46..0102fcc 100644 --- a/tests/test_settings_model.py +++ b/tests/test_settings_model.py @@ -1,7 +1,7 @@ import pytest from pydantic import BaseModel -from pydantic_config import SettingsModel +from pydantic_config import SettingsModel, SettingsConfig def test_config_file(config_toml_file): @@ -12,11 +12,67 @@ class App(BaseModel): class Settings(SettingsModel): app: App - class Config: - config_file = [config_toml_file] + model_config = SettingsConfig( + config_file=[config_toml_file], + ) settings = Settings() - assert settings.dict() == {'app': {'name': 'AppName', 'description': 'description from config.toml'}} + assert settings.model_dump() == {'app': {'name': 'AppName', 'description': 'description from config.toml'}} + + +def test_case_sensitive_config(config_toml_file): + class App(BaseModel): + name: str = 'AppName' + description: str = None + + class Settings(SettingsModel): + APP: App = App() + + model_config = SettingsConfig( + case_sensitive=True, + extra='allow', + config_file=[config_toml_file], + ) + + settings = Settings() + assert settings.model_dump() == { + 'APP': {'name': 'AppName', 'description': None}, + 'app': {'description': 'description from config.toml'} + } + + +def test_case_insensitive_config(config_toml_file): + class App(BaseModel): + Name: str = 'AppName' + description: str = None + + class Settings(SettingsModel): + APP: App = App() + + model_config = SettingsConfig( + case_sensitive=False, + config_file=[config_toml_file], + ) + + settings = Settings() + assert settings.model_dump() == {'APP': {'Name': 'AppName', 'description': 'description from config.toml'}} + + +def test_extra_config(config_toml_file): + + class Settings(SettingsModel): + foo: str = 'bar' + + model_config = SettingsConfig( + extra='allow', + config_file=[config_toml_file], + ) + + settings = Settings() + assert settings.model_dump() == { + 'foo': 'bar', + 'app': {'description': 'description from config.toml'} + } def test_invalid_config_file(): @@ -25,8 +81,9 @@ def test_invalid_config_file(): class Settings(SettingsModel): foo: str = 'bar' - class Config: - config_file = [file] + model_config = SettingsConfig( + config_file=[file] + ) with pytest.raises(OSError) as exc: Settings() @@ -40,9 +97,10 @@ class App(BaseModel): class Settings(SettingsModel): app: App - class Config: - config_file = [config_toml_file, config_yaml_file] + model_config = SettingsConfig( + config_file=[config_toml_file, config_yaml_file] + ) settings = Settings() - assert settings.dict() == {'app': {'name': 'AppName', 'description': 'description from config.yaml'}} + assert settings.model_dump() == {'app': {'name': 'AppName', 'description': 'description from config.yaml'}}