From 60e79d0aef191f80ef5c785583e098fab6ae0346 Mon Sep 17 00:00:00 2001 From: Michael Makukha Date: Fri, 16 Aug 2024 18:42:17 +0300 Subject: [PATCH] Add support for multiple secrets directories (#12) * Add support for multiple secrets directories --- .bumpversion.toml | 2 +- README.md | 39 ++++++---- Taskfile.yml | 6 ++ docs/badge/tests.svg | 8 +-- src/pydantic_file_secrets/__init__.py | 91 +++++++++++++----------- src/pydantic_file_secrets/__version__.py | 2 +- tests/test_secrets_dir_list.py | 35 +++++++++ tox.ini | 2 + 8 files changed, 126 insertions(+), 59 deletions(-) create mode 100644 tests/test_secrets_dir_list.py diff --git a/.bumpversion.toml b/.bumpversion.toml index c90f318..fa4d1f0 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.1.3" +current_version = "0.2.0" allow_dirty = true files = [ {filename = "src/pydantic_file_secrets/__version__.py"}, diff --git a/README.md b/README.md index c463162..450f4a3 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,18 @@ # pydantic-file-secrets 🔑 -> Use file secrets in nested [Pydantic Settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) models. + +> Use file secrets in nested [Pydantic Settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) models, drop-in replacement of `SecretsSettingsSource`. ![GitHub License](https://img.shields.io/github/license/makukha/pydantic-file-secrets) -[![Tests](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/0.1.3/docs/badge/tests.svg)](https://github.com/makukha/pydantic-file-secrets) -[![Coverage](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/0.1.3/docs/badge/coverage.svg)](https://github.com/makukha/pydantic-file-secrets) +[![Tests](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/0.2.0/docs/badge/tests.svg)](https://github.com/makukha/pydantic-file-secrets) +[![Coverage](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/0.2.0/docs/badge/coverage.svg)](https://github.com/makukha/pydantic-file-secrets) [![linting - Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v1.json)](https://github.com/astral-sh/ruff) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) \ -[![pypi](https://img.shields.io/pypi/v/pydantic-file-secrets.svg#0.1.3)](https://pypi.python.org/pypi/pydantic-file-secrets) +[![pypi](https://img.shields.io/pypi/v/pydantic-file-secrets.svg#0.2.0)](https://pypi.python.org/pypi/pydantic-file-secrets) [![versions](https://img.shields.io/pypi/pyversions/pydantic-file-secrets.svg)](https://pypi.org/project/pydantic-file-secrets) [![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json)](https://pydantic.dev) -This project is inspired by discussions in [pydantic-settings issue #154](https://github.com/pydantic/pydantic-settings/issues/154). +This project is inspired by discussions in Pydantic Settings and solves problems in issues [#3](https://github.com/pydantic/pydantic-settings/issues/3), [#30](https://github.com/pydantic/pydantic-settings/issues/30), [#154](https://github.com/pydantic/pydantic-settings/issues/154). This package unties secrets from environment variables config options and implements other long waited features. @@ -23,6 +24,7 @@ This package unties secrets from environment variables config options and implem * Plain or nested directory layout: `/run/secrets/dir__key` or `/run/secrets/dir/key` * Respects `env_prefix`, `env_nested_delimiter` and other [config options](https://github.com/makukha/pydantic-file-secrets?tab=readme-ov-file#configuration-options) * Has `secrets_prefix`, `secrets_nested_delimiter`, [etc.](https://github.com/makukha/pydantic-file-secrets?tab=readme-ov-file#configuration-options) to configure secrets and env vars separately +* Use multiple `secrets_dir` directories * Pure Python thin wrapper over standard `EnvSettingsSource` * No third party dependencies except `pydantic-settings` * 100% test coverage @@ -128,11 +130,23 @@ secret2 ... ``` +### Multiple `secrets_dir` + +When passing `list` to `secrets_dir`, last match wins. + +```python +... + model_config = SettingsConfigDict( + secrets_dir=['/run/configs/', '/run/secrets'], + ) +... +``` + ## Configuration options ### secrets_dir -Path to secrets directory, same as `SecretsSettingsSource.secrets_dir`. +Path to secrets directory. Same as `SecretsSettingsSource.secrets_dir` if `str` or `Path`. If `list`, the last match wins. If `secrets_dir` is passed in both source constructor and model config, values are not merged (constructor takes priority). ### secrets_dir_missing @@ -142,12 +156,16 @@ If `secrets_dir` does not exist, original `SecretsSettingsSource` issues a warni * `'warn'` (default) — print warning, same as `SecretsSettingsSource` * `'error'` — raise `SettingsError` +If multiple `secrets_dir` passed, the same `secrets_dir_missing` action applies to each of them. + ### secrets_dir_max_size Limit the size of `secrets_dir` for security reasons, defaults to 8 MiB. `FileSecretsSettingsSource` is a thin wrapper around [`EnvSettingsSource`](https://docs.pydantic.dev/latest/api/pydantic_settings/#pydantic_settings.EnvSettingsSource), which loads all potential secrets on initialization. This could lead to `MemoryError` if we mount a large file under `secrets_dir`. +If multiple `secrets_dir` passed, the limit applies to each directory independently. + ### secrets_case_sensitive Same as [`case_sensitive`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/#case-sensitivity), but works for secrets only. If not specified, defaults to `case_sensitive`. @@ -158,7 +176,7 @@ Same as [`env_nested_delimiter`](https://docs.pydantic.dev/latest/concepts/pydan ### secrets_nested_subdir -Boolean flag to turn on nested secrets directory mode, `False` by default. If `True`, sets `secrets_nested_delimiter` to [`os.sep`](https://docs.python.org/3/library/os.html#os.sep). Raises `settingsError` if `secrets_nested_delimiter` is already specified. +Boolean flag to turn on nested secrets directory mode, `False` by default. If `True`, sets `secrets_nested_delimiter` to [`os.sep`](https://docs.python.org/3/library/os.html#os.sep). Raises `SettingsError` if `secrets_nested_delimiter` is already specified. ### secrets_prefix @@ -182,12 +200,7 @@ We [ensure](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/main We [test](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/main/tox.ini) all minor Pydantic Settings v2 versions and all minor Python 3 versions supported by Pydantic Settings: -* Python 3.13 + pydantic-settings 2.{0,1,2,3,4} -* Python 3.12 + pydantic-settings 2.{0,1,2,3,4} -* Python 3.11 + pydantic-settings 2.{0,1,2,3,4} -* Python 3.10 + pydantic-settings 2.{0,1,2,3,4} -* Python 3.9 + pydantic-settings 2.{0,1,2,3,4} -* Python 3.8 + pydantic-settings 2.{0,1,2,3,4} +* Python 3.{8,9,10,11,12,13} + pydantic-settings 2.{0,1,2,3,4} ## Roadmap diff --git a/Taskfile.yml b/Taskfile.yml index a80d812..5231fda 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -52,6 +52,12 @@ tasks: cmds: - tox run + test:main: + desc: Run tests in main environment. + deps: [install] + cmds: + - tox run -m main + test:pdb: desc: Run tests and open debugger on errors. deps: [install] diff --git a/docs/badge/tests.svg b/docs/badge/tests.svg index 0b4e38e..b2ac141 100644 --- a/docs/badge/tests.svg +++ b/docs/badge/tests.svg @@ -1,5 +1,5 @@ - - tests: 26 + + tests: 28 @@ -15,7 +15,7 @@ tests - - 26 + + 28 diff --git a/src/pydantic_file_secrets/__init__.py b/src/pydantic_file_secrets/__init__.py index 60a4e69..66d5fa0 100644 --- a/src/pydantic_file_secrets/__init__.py +++ b/src/pydantic_file_secrets/__init__.py @@ -1,3 +1,4 @@ +from functools import reduce import os from pathlib import Path from typing import Any, Literal @@ -20,7 +21,7 @@ class FileSecretsSettingsSource(EnvSettingsSource): def __init__( self, settings_cls: type[BaseSettings], - secrets_dir: str | Path | None = None, + secrets_dir: str | Path | list[str | Path] | None = None, secrets_dir_missing: SecretsDirMissing | None = None, secrets_dir_max_size: int | None = None, secrets_case_sensitive: bool | None = None, @@ -30,7 +31,7 @@ def __init__( ) -> None: # config options conf = settings_cls.model_config - self.secrets_dir: str | None = first_not_none( + self.secrets_dir: str | Path | list[str | Path] | None = first_not_none( secrets_dir, conf.get('secrets_dir'), ) @@ -79,39 +80,14 @@ def __init__( # ensure valid secrets_path if self.secrets_dir is None: - self.secrets_path = None + paths = [] + elif isinstance(self.secrets_dir, list): + paths = self.secrets_dir else: - self.secrets_path: Path = Path(self.secrets_dir).expanduser().resolve() - if not self.secrets_path.exists(): - match self.secrets_dir_missing: - case 'ok': - pass - case 'warn': - warnings.warn(f'directory "{self.secrets_path}" does not exist') - case 'error': - raise SettingsError( - f'directory "{self.secrets_path}" does not exist' - ) - case _: - raise SettingsError( - f'invalid secrets_dir_missing value: ' - f'{self.secrets_dir_missing}' - ) - else: - if not self.secrets_path.is_dir(): - raise SettingsError( - 'secrets_dir must reference a directory, ' - f'not a {path_type_label(self.secrets_path)}' - ) - secrets_dir_size = sum( - f.stat().st_size - for f in self.secrets_path.glob('**/*') - if f.is_file() - ) - if secrets_dir_size > self.secrets_dir_max_size: - raise SettingsError( - f'secrets_dir size is above {self.secrets_dir_max_size} bytes' - ) + paths = [self.secrets_dir] + self.secrets_paths: list[Path] = [Path(p).expanduser().resolve() for p in paths] + for path in self.secrets_paths: + self.validate_secrets_path(path) # construct parent super().__init__( @@ -126,14 +102,13 @@ def __init__( self.env_parse_none_str = None # update manually because of None # update parent members - if self.secrets_path is None: + if not len(self.secrets_paths): self.env_vars = {} else: - secrets = { - str(p.relative_to(self.secrets_path)): p.read_text() - for p in self.secrets_path.glob('**/*') - if p.is_file() - } + secrets = reduce( + lambda d1, d2: d1 | d2, + (self.load_secrets(p) for p in self.secrets_paths), + ) self.env_vars = parse_env_vars( secrets, self.case_sensitive, @@ -141,6 +116,42 @@ def __init__( self.env_parse_none_str, ) + def validate_secrets_path(self, path: Path) -> None: + if not path.exists(): + match self.secrets_dir_missing: + case 'ok': + pass + case 'warn': + warnings.warn(f'directory "{path}" does not exist') + case 'error': + raise SettingsError(f'directory "{path}" does not exist') + case _: + raise SettingsError( + f'invalid secrets_dir_missing value: ' + f'{self.secrets_dir_missing}' + ) + else: + if not path.is_dir(): + raise SettingsError( + f'secrets_dir must reference a directory, ' + f'not a {path_type_label(path)}' + ) + secrets_dir_size = sum( + f.stat().st_size for f in path.glob('**/*') if f.is_file() + ) + if secrets_dir_size > self.secrets_dir_max_size: + raise SettingsError( + f'secrets_dir size is above {self.secrets_dir_max_size} bytes' + ) + + @staticmethod + def load_secrets(path: Path) -> dict[str, str]: + return { + str(p.relative_to(path)): p.read_text() + for p in path.glob('**/*') + if p.is_file() + } + def __repr__(self) -> str: return f'FileSecretsSettingsSource(secrets_dir={self.secrets_dir!r})' diff --git a/src/pydantic_file_secrets/__version__.py b/src/pydantic_file_secrets/__version__.py index 1682803..9f137b9 100644 --- a/src/pydantic_file_secrets/__version__.py +++ b/src/pydantic_file_secrets/__version__.py @@ -1 +1 @@ -__version__ = '0.1.3' +__version__ = '0.2.0' diff --git a/tests/test_secrets_dir_list.py b/tests/test_secrets_dir_list.py new file mode 100644 index 0000000..3c1c77b --- /dev/null +++ b/tests/test_secrets_dir_list.py @@ -0,0 +1,35 @@ +import pytest + + +def test_merge(settings_model, monkeypatch, secrets_dir): + monkeypatch.setenv('DB__USER', 'user') + secrets_dir.add_files( + ('dir1/app_key', 'secret1'), + ('dir1/db___password', 'secret2'), + ('dir2/db___password', 'secret3'), + ) + Settings = settings_model( + model_config=dict( + env_nested_delimiter='__', + secrets_dir=[secrets_dir / 'dir1', secrets_dir / 'dir2'], + secrets_nested_delimiter='___', + ), + ) + conf = Settings() + assert conf.app_key == 'secret1' + assert conf.db.password == 'secret3' # noqa: S105 + + +def test_missing(settings_model, monkeypatch, secrets_dir): + monkeypatch.setenv('DB__USER', 'user') + Settings = settings_model( + model_config=dict( + env_nested_delimiter='__', + secrets_dir=[secrets_dir / 'dir1', secrets_dir / 'dir2'], + secrets_nested_delimiter='___', + ), + ) + with pytest.warns(UserWarning) as warninfo: + Settings() + assert len(warninfo) == 2 + assert all('does not exist' in w.message.args[0] for w in warninfo) diff --git a/tox.ini b/tox.ini index b12d598..7b7fae6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,8 @@ [tox] env_list = py{38,39,310,311,312,3.13.0rc1}-ps{20,21,22,23,24} setenv = VIRTUALENV_DISCOVERY=pyenv +labels = + main = py312-ps24 [testenv] parallel_show_output = true