Add support for multiple secrets directories (#12)

* Add support for multiple secrets directories
makukha · Aug 16, 2024 · 60e79d0 · 60e79d0
1 parent c913509
commit 60e79d0
Show file tree

Hide file tree

Showing 8 changed files with 126 additions and 59 deletions.
diff --git a/.bumpversion.toml b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.1.3"
+current_version = "0.2.0"
 allow_dirty = true
 files = [
   {filename = "src/pydantic_file_secrets/__version__.py"},

diff --git a/README.md b/README.md
@@ -1,17 +1,18 @@
 # pydantic-file-secrets 🔑
-> Use file secrets in nested [Pydantic Settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) models.
+
+> Use file secrets in nested [Pydantic Settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) models, drop-in replacement of `SecretsSettingsSource`.
 
 ![GitHub License](https://img.shields.io/github/license/makukha/pydantic-file-secrets)
-[![Tests](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/0.1.3/docs/badge/tests.svg)](https://github.com/makukha/pydantic-file-secrets)
-[![Coverage](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/0.1.3/docs/badge/coverage.svg)](https://github.com/makukha/pydantic-file-secrets)
+[![Tests](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/0.2.0/docs/badge/tests.svg)](https://github.com/makukha/pydantic-file-secrets)
+[![Coverage](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/0.2.0/docs/badge/coverage.svg)](https://github.com/makukha/pydantic-file-secrets)
 [![linting - Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v1.json)](https://github.com/astral-sh/ruff)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) \
-[![pypi](https://img.shields.io/pypi/v/pydantic-file-secrets.svg#0.1.3)](https://pypi.python.org/pypi/pydantic-file-secrets)
+[![pypi](https://img.shields.io/pypi/v/pydantic-file-secrets.svg#0.2.0)](https://pypi.python.org/pypi/pydantic-file-secrets)
 [![versions](https://img.shields.io/pypi/pyversions/pydantic-file-secrets.svg)](https://pypi.org/project/pydantic-file-secrets)
 [![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json)](https://pydantic.dev)
 
 
-This project is inspired by discussions in [pydantic-settings issue #154](https://github.com/pydantic/pydantic-settings/issues/154).
+This project is inspired by discussions in Pydantic Settings and solves problems in issues [#3](https://github.com/pydantic/pydantic-settings/issues/3), [#30](https://github.com/pydantic/pydantic-settings/issues/30), [#154](https://github.com/pydantic/pydantic-settings/issues/154).
 
 This package unties secrets from environment variables config options and implements other long waited features.
 
@@ -23,6 +24,7 @@ This package unties secrets from environment variables config options and implem
 * Plain or nested directory layout: `/run/secrets/dir__key` or `/run/secrets/dir/key`
 * Respects `env_prefix`, `env_nested_delimiter` and other [config options](https://github.com/makukha/pydantic-file-secrets?tab=readme-ov-file#configuration-options)
 * Has `secrets_prefix`, `secrets_nested_delimiter`, [etc.](https://github.com/makukha/pydantic-file-secrets?tab=readme-ov-file#configuration-options) to configure secrets and env vars separately
+* Use multiple `secrets_dir` directories
 * Pure Python thin wrapper over standard `EnvSettingsSource`
 * No third party dependencies except `pydantic-settings`
 * 100% test coverage
@@ -128,11 +130,23 @@ secret2
 ...
 ```
 
+### Multiple `secrets_dir`
+
+When passing `list` to `secrets_dir`, last match wins.
+
+```python
+...
+    model_config = SettingsConfigDict(
+        secrets_dir=['/run/configs/', '/run/secrets'],
+    )
+...
+```
+
 ## Configuration options
 
 ### secrets_dir
 
-Path to secrets directory, same as `SecretsSettingsSource.secrets_dir`.
+Path to secrets directory. Same as `SecretsSettingsSource.secrets_dir` if `str` or `Path`. If `list`, the last match wins. If `secrets_dir` is passed in both source constructor and model config, values are not merged (constructor takes priority).
 
 ### secrets_dir_missing
 
@@ -142,12 +156,16 @@ If `secrets_dir` does not exist, original `SecretsSettingsSource` issues a warni
 * `'warn'` (default) — print warning, same as `SecretsSettingsSource`
 * `'error'` — raise `SettingsError`
 
+If multiple `secrets_dir` passed, the same `secrets_dir_missing` action applies to each of them.
+
 ### secrets_dir_max_size
 
 Limit the size of `secrets_dir` for security reasons, defaults to 8 MiB.
 
 `FileSecretsSettingsSource` is a thin wrapper around [`EnvSettingsSource`](https://docs.pydantic.dev/latest/api/pydantic_settings/#pydantic_settings.EnvSettingsSource), which loads all potential secrets on initialization. This could lead to `MemoryError` if we mount a large file under `secrets_dir`.
 
+If multiple `secrets_dir` passed, the limit applies to each directory independently.
+
 ### secrets_case_sensitive
 
 Same as [`case_sensitive`](https://docs.pydantic.dev/latest/concepts/pydantic_settings/#case-sensitivity), but works for secrets only. If not specified, defaults to `case_sensitive`.
@@ -158,7 +176,7 @@ Same as [`env_nested_delimiter`](https://docs.pydantic.dev/latest/concepts/pydan
 
 ### secrets_nested_subdir
 
-Boolean flag to turn on nested secrets directory mode, `False` by default. If `True`, sets `secrets_nested_delimiter` to [`os.sep`](https://docs.python.org/3/library/os.html#os.sep). Raises `settingsError` if `secrets_nested_delimiter` is already specified.
+Boolean flag to turn on nested secrets directory mode, `False` by default. If `True`, sets `secrets_nested_delimiter` to [`os.sep`](https://docs.python.org/3/library/os.html#os.sep). Raises `SettingsError` if `secrets_nested_delimiter` is already specified.
 
 ### secrets_prefix
 
@@ -182,12 +200,7 @@ We [ensure](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/main
 
 We [test](https://raw.githubusercontent.com/makukha/pydantic-file-secrets/main/tox.ini) all minor Pydantic Settings v2 versions and all minor Python 3 versions supported by Pydantic Settings:
 
-* Python 3.13 + pydantic-settings 2.{0,1,2,3,4}
-* Python 3.12 + pydantic-settings 2.{0,1,2,3,4}
-* Python 3.11 + pydantic-settings 2.{0,1,2,3,4}
-* Python 3.10 + pydantic-settings 2.{0,1,2,3,4}
-* Python 3.9 + pydantic-settings 2.{0,1,2,3,4}
-* Python 3.8 + pydantic-settings 2.{0,1,2,3,4}
+* Python 3.{8,9,10,11,12,13} + pydantic-settings 2.{0,1,2,3,4}
 
 
 ## Roadmap

diff --git a/Taskfile.yml b/Taskfile.yml
@@ -52,6 +52,12 @@ tasks:
     cmds:
       - tox run
 
+  test:main:
+    desc: Run tests in main environment.
+    deps: [install]
+    cmds:
+      - tox run -m main
+
   test:pdb:
     desc: Run tests and open debugger on errors.
     deps: [install]

diff --git a/docs/badge/tests.svg b/docs/badge/tests.svg
diff --git a/src/pydantic_file_secrets/__init__.py b/src/pydantic_file_secrets/__init__.py
@@ -1,3 +1,4 @@
+from functools import reduce
 import os
 from pathlib import Path
 from typing import Any, Literal
@@ -20,7 +21,7 @@ class FileSecretsSettingsSource(EnvSettingsSource):
     def __init__(
         self,
         settings_cls: type[BaseSettings],
-        secrets_dir: str | Path | None = None,
+        secrets_dir: str | Path | list[str | Path] | None = None,
         secrets_dir_missing: SecretsDirMissing | None = None,
         secrets_dir_max_size: int | None = None,
         secrets_case_sensitive: bool | None = None,
@@ -30,7 +31,7 @@ def __init__(
     ) -> None:
         # config options
         conf = settings_cls.model_config
-        self.secrets_dir: str | None = first_not_none(
+        self.secrets_dir: str | Path | list[str | Path] | None = first_not_none(
             secrets_dir,
             conf.get('secrets_dir'),
         )
@@ -79,39 +80,14 @@ def __init__(
 
         # ensure valid secrets_path
         if self.secrets_dir is None:
-            self.secrets_path = None
+            paths = []
+        elif isinstance(self.secrets_dir, list):
+            paths = self.secrets_dir
         else:
-            self.secrets_path: Path = Path(self.secrets_dir).expanduser().resolve()
-            if not self.secrets_path.exists():
-                match self.secrets_dir_missing:
-                    case 'ok':
-                        pass
-                    case 'warn':
-                        warnings.warn(f'directory "{self.secrets_path}" does not exist')
-                    case 'error':
-                        raise SettingsError(
-                            f'directory "{self.secrets_path}" does not exist'
-                        )
-                    case _:
-                        raise SettingsError(
-                            f'invalid secrets_dir_missing value: '
-                            f'{self.secrets_dir_missing}'
-                        )
-            else:
-                if not self.secrets_path.is_dir():
-                    raise SettingsError(
-                        'secrets_dir must reference a directory, '
-                        f'not a {path_type_label(self.secrets_path)}'
-                    )
-                secrets_dir_size = sum(
-                    f.stat().st_size
-                    for f in self.secrets_path.glob('**/*')
-                    if f.is_file()
-                )
-                if secrets_dir_size > self.secrets_dir_max_size:
-                    raise SettingsError(
-                        f'secrets_dir size is above {self.secrets_dir_max_size} bytes'
-                    )
+            paths = [self.secrets_dir]
+        self.secrets_paths: list[Path] = [Path(p).expanduser().resolve() for p in paths]
+        for path in self.secrets_paths:
+            self.validate_secrets_path(path)
 
         # construct parent
         super().__init__(
@@ -126,21 +102,56 @@ def __init__(
         self.env_parse_none_str = None  # update manually because of None
 
         # update parent members
-        if self.secrets_path is None:
+        if not len(self.secrets_paths):
             self.env_vars = {}
         else:
-            secrets = {
-                str(p.relative_to(self.secrets_path)): p.read_text()
-                for p in self.secrets_path.glob('**/*')
-                if p.is_file()
-            }
+            secrets = reduce(
+                lambda d1, d2: d1 | d2,
+                (self.load_secrets(p) for p in self.secrets_paths),
+            )
             self.env_vars = parse_env_vars(
                 secrets,
                 self.case_sensitive,
                 self.env_ignore_empty,
                 self.env_parse_none_str,
             )
 
+    def validate_secrets_path(self, path: Path) -> None:
+        if not path.exists():
+            match self.secrets_dir_missing:
+                case 'ok':
+                    pass
+                case 'warn':
+                    warnings.warn(f'directory "{path}" does not exist')
+                case 'error':
+                    raise SettingsError(f'directory "{path}" does not exist')
+                case _:
+                    raise SettingsError(
+                        f'invalid secrets_dir_missing value: '
+                        f'{self.secrets_dir_missing}'
+                    )
+        else:
+            if not path.is_dir():
+                raise SettingsError(
+                    f'secrets_dir must reference a directory, '
+                    f'not a {path_type_label(path)}'
+                )
+            secrets_dir_size = sum(
+                f.stat().st_size for f in path.glob('**/*') if f.is_file()
+            )
+            if secrets_dir_size > self.secrets_dir_max_size:
+                raise SettingsError(
+                    f'secrets_dir size is above {self.secrets_dir_max_size} bytes'
+                )
+
+    @staticmethod
+    def load_secrets(path: Path) -> dict[str, str]:
+        return {
+            str(p.relative_to(path)): p.read_text()
+            for p in path.glob('**/*')
+            if p.is_file()
+        }
+
     def __repr__(self) -> str:
         return f'FileSecretsSettingsSource(secrets_dir={self.secrets_dir!r})'
 

diff --git a/src/pydantic_file_secrets/__version__.py b/src/pydantic_file_secrets/__version__.py
@@ -1 +1 @@
-__version__ = '0.1.3'
+__version__ = '0.2.0'
diff --git a/tests/test_secrets_dir_list.py b/tests/test_secrets_dir_list.py
@@ -0,0 +1,35 @@
+import pytest
+
+
+def test_merge(settings_model, monkeypatch, secrets_dir):
+    monkeypatch.setenv('DB__USER', 'user')
+    secrets_dir.add_files(
+        ('dir1/app_key', 'secret1'),
+        ('dir1/db___password', 'secret2'),
+        ('dir2/db___password', 'secret3'),
+    )
+    Settings = settings_model(
+        model_config=dict(
+            env_nested_delimiter='__',
+            secrets_dir=[secrets_dir / 'dir1', secrets_dir / 'dir2'],
+            secrets_nested_delimiter='___',
+        ),
+    )
+    conf = Settings()
+    assert conf.app_key == 'secret1'
+    assert conf.db.password == 'secret3'  # noqa: S105
+
+
+def test_missing(settings_model, monkeypatch, secrets_dir):
+    monkeypatch.setenv('DB__USER', 'user')
+    Settings = settings_model(
+        model_config=dict(
+            env_nested_delimiter='__',
+            secrets_dir=[secrets_dir / 'dir1', secrets_dir / 'dir2'],
+            secrets_nested_delimiter='___',
+        ),
+    )
+    with pytest.warns(UserWarning) as warninfo:
+        Settings()
+    assert len(warninfo) == 2
+    assert all('does not exist' in w.message.args[0] for w in warninfo)
diff --git a/tox.ini b/tox.ini
@@ -1,6 +1,8 @@
 [tox]
 env_list = py{38,39,310,311,312,3.13.0rc1}-ps{20,21,22,23,24}
 setenv = VIRTUALENV_DISCOVERY=pyenv
+labels =
+  main = py312-ps24
 
 [testenv]
 parallel_show_output = true