From 0ef551d7a5965d8c7d9b71870573336d8cc4399a Mon Sep 17 00:00:00 2001 From: Philip Iezzi Date: Mon, 11 Sep 2023 16:08:14 +0200 Subject: [PATCH] fixes for webhook JWT token generation introduce WEBHOOK_USE_JWT env var to make token type better configurable prepare for v0.7.2 release --- .env.docker | 7 ++++--- .env.example | 1 + .env.github | 1 + .env.test | 1 + CHANGELOG.md | 11 +++++++++++ README.md | 31 ++++++++++++++++++++++--------- app/_version.py | 2 +- app/webhook.py | 21 ++++++++++++++------- tests/test_app_webhook.py | 4 +++- 9 files changed, 58 insertions(+), 21 deletions(-) diff --git a/.env.docker b/.env.docker index 55a6027..b2751d0 100644 --- a/.env.docker +++ b/.env.docker @@ -15,6 +15,7 @@ LOG_LEVEL=DEBUG # LOG_MSG_PREFIX=False LOG_CONSOLE=True # SYSLOG=True -# WEBHOOK_ENABLED=True -# WEBHOOK_URL="http://host.docker.internal:8080/api/policyd/{sender}?token={token}" -# WEBHOOK_SECRET="Wk9YZXliVlVtY2pQcFlFUm9KY1U1ZkFFaUpWTk1FU20=" +WEBHOOK_ENABLED=True +WEBHOOK_USE_JWT=False +WEBHOOK_URL="http://host.docker.internal:8080/api/policyd/{sender}?token={token}" +WEBHOOK_SECRET="Wk9YZXliVlVtY2pQcFlFUm9KY1U1ZkFFaUpWTk1FU20=" diff --git a/.env.example b/.env.example index 5bed817..f06c54f 100644 --- a/.env.example +++ b/.env.example @@ -29,5 +29,6 @@ LOG_CONSOLE=True # True or False (default: False) - Output logs to console (stde # SENTRY_ENVIRONMENT=prod # MESSAGE_RETENTION=90 # How many days to keep messages in the database (default: 0, never delete) # WEBHOOK_ENABLED=True # True or False (default: False) - Enable webhook +# WEBHOOK_USE_JWT=True # True or False (default: False) - Use JWT for webhook token authentication # WEBHOOK_URL="https://example.com/api/policyd/{sender}?token={token}" # Webhook URL to trigger sender limit reached notification (default: None) # WEBHOOK_SECRET="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX=" # Webhook secret to generate token for remote API authentication (default: None) diff --git a/.env.github b/.env.github index 34a0a8c..162ba73 100644 --- a/.env.github +++ b/.env.github @@ -16,5 +16,6 @@ LOG_LEVEL=ERROR LOG_CONSOLE=True # SYSLOG=True # WEBHOOK_ENABLED=True +WEBHOOK_USE_JWT=False WEBHOOK_URL="https://example.com/api/policyd/{sender}?token={token}" WEBHOOK_SECRET="Wk9YZXliVlVtY2pQcFlFUm9KY1U1ZkFFaUpWTk1FU20=" diff --git a/.env.test b/.env.test index b967f29..9a97b1a 100644 --- a/.env.test +++ b/.env.test @@ -17,5 +17,6 @@ LOG_CONSOLE=True # SENTRY_DSN=https://**********.ingest.sentry.io/XXXXXXXXXXXXXXXX # SENTRY_ENVIRONMENT=test # WEBHOOK_ENABLED=True +WEBHOOK_USE_JWT=False WEBHOOK_URL="https://example.com/api/policyd/{sender}?token={token}" WEBHOOK_SECRET="Wk9YZXliVlVtY2pQcFlFUm9KY1U1ZkFFaUpWTk1FU20=" diff --git a/CHANGELOG.md b/CHANGELOG.md index a66eb4b..07cb6a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # CHANGELOG +## [v0.7.2](https://github.com/onlime/policyd-rate-guard/releases/tag/v0.7.2) (2023-09-11) + +**Improved:** + +- Webhook authentication token type (Simple hashed token vs. JWT token) can now be configured with new env var `WEBHOOK_USE_JWT` and no longer depends on whether you pass the token as query param (not recommended for JWT tokens) or `Authorization: Bearer` header. +- Webhook JWT token now contains all necessary claims for strict verification: `sub` (Subject), `iss` (Issuer), `iat` (Issued At), `nbf` (Not Before), `exp` (Expiration Time). + +**Fixed:** + +- Webhook JWT token is now correctly encoded using base64 decoded secret (`WEBHOOK_SECRET`) as key. Previously, we forgot to decode it, but always recommended (and still do!) to use a base64 encoded secret. + ## [v0.7.1](https://github.com/onlime/policyd-rate-guard/releases/tag/v0.7.1) (2023-09-11) **Improved:** diff --git a/README.md b/README.md index 101d408..89c626a 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ But let me name some features that make it stand out from other solutions: - A multi-threaded app that uses [DBUtils PooledDB (pooled_db)](https://github.com/WebwareForPython/DBUtils) for **robust and efficient DB connection handling**. - Can be used with any [DB-API 2 (PEP 249)](https://peps.python.org/pep-0249/) conformant database adapter (currently supported: PyMySQL, sqlite3) - A super slick minimal codebase with **only a few dependencies** ([PyMySQL](https://pypi.org/project/pymysql/), [DBUtils](https://webwareforpython.github.io/DBUtils/), [python-dotenv](https://pypi.org/project/python-dotenv/), [yoyo-migrations](https://pypi.org/project/yoyo-migrations/)), using Python virtual environment for easy `pip` install. PyMySQL is a pure-Python MySQL client library, so you won't have any trouble on any future major system upgrades. -- **Supports external API webhooks** with simple token based authentication (passed as query param) or JWT token (passed as `Authorization: Bearer` header). When configured, the webhook is triggered whenever a sender reaches his quota limit for the first time and you can send out notification through your own or any 3rd-party app. +- **Supports external API webhooks** with two variants of authentication tokens: simple hashed token or JWT token. The authentication token can be configured to be passed either as query param or as `Authorization: Bearer` header. When enabled, the webhook is triggered whenever a sender reaches his quota limit for the first time and you can send out notifications through your own or any 3rd-party app. - Provides an Ansible Galaxy role [`onlime.policyd_rate_guard`](https://galaxy.ansible.com/onlime/policyd_rate_guard) for easy installation on a Debian mailserver. - A **well maintained** project, as it is in active use at [Onlime GmbH](https://www.onlime.ch/), a Swiss webhoster with a rock-solid mailserver architecture. @@ -244,8 +244,11 @@ Optional configuration for external service integration: Sentry environment. Suggested values: `dev` or `prod`, but can be any custom string. Defaults to `dev`. - `WEBHOOK_ENABLED` (bool) Enable external API webhook to be called when sender reached his quota limit (first time he's blocked). Possible values: `True` or `False`. Defaults to `False`. +- `WEBHOOK_USE_JWT` (bool) + Use JWT for webhook token authentication, instead of a simple hashed token. This more advanced authentication is recommended only to be used when passing the token as `Authorization: Bearer` header, and not as query param (by using the `{token}` placeholder in your `WEBHOOK_URL`). If not using JWT, the token will be a simple hash from your secret appended to the sender address. + Possible values: `True` or `False`. Defaults to `False`. - `WEBHOOK_URL` - Webhook API URL of the external service that should be called if `WEBHOOK_ENABLED=True`. It supports the following placeholders, which are both optional: `{sender}`, `{token}`. You may provide a URL in the following form: `https://api.example.com/policyd/{sender}?token={token}` (the token will be a simple hash from your secret appended to the sender address), or if you omit the `{token}` in the URL, a signed JWT token will be passed as `Bearer` token in the `Authorization` header, which will also contain the sender in its payload. + Webhook API URL of the external service that should be called if `WEBHOOK_ENABLED=True`. It supports any key of the webhook payload (JSON object) as placeholder, usually the following: `{sender}`, `{token}`. All placeholders are optional. You may provide a URL in the following form: `https://api.example.com/policyd/{sender}?token={token}`, or if you omit the `{token}` in the URL, make sure to enable `WEBHOOK_USE_JWT=True`, so a signed JWT token (including the sender as its `sub` claim) will be passed as `Authorization: Bearer` header. - `WEBHOOK_SECRET` The shared secret to generate the webhook token. Configure this shared secret also on the external API's webhook to verify the token for authentication. Recommended way to generate a secret: `base64.b64encode(secrets.token_bytes(32))` @@ -311,9 +314,11 @@ or with PHP (e.g. using `php artisan tinker` in Laravel, or `php -a` interactive > base64_encode(Str::random(32)) ``` +We always expect your secret in `WEBHOOK_SECRET` to be base64 encoded! + Depending on your external API, PolicydRateGuard supports two different ways of authentication: -**Variant 1) Simple token as query param** +**Variant 1) Simple token** The authentication token can be passed as a query param to your external API webhook. In this case, you need to use the `{token}` placeholder in your `WEBHOOK_URL`, no matter if you use any other (optional) placeholders like `{sender}` or not. The sender will always be part of the JSON data (payload) passed to your webhook anyway. @@ -354,7 +359,7 @@ class AccessApiWebhookPolicyd /** @var App\Models\Mailaccount $mailaccount */ $mailaccount = $request->route('mailaccount'); $token = hash('sha256', config('app.webhooks.secret').$mailaccount->username); - if ($request->query('token') !== $token) { + if (! hash_equals($token, $request->query('token') ?: $request->bearerToken())) { abort(403, 'You are not allowed to access this webhook.'); } return $next($request); @@ -364,19 +369,27 @@ class AccessApiWebhookPolicyd **Variant 2) JWT token in Authorization header** -If your `WEBHOOK_URL` does not contain a `{token}` placeholder, we assume you don't want to pass it as query param, but as JWT token in the `Authorization: Bearer ` header instead. PolicydRateGuard will take care of it and generate a valid JWT token, basically like this: +If you have `WEBHOOK_USE_JWT` enabled in your `.env`, PolicydRateGuard will generate a JWT token instead of the previously mentioned simple hashed token. It's recommended not to put a `{token}` placeholder in your `WEBHOOK_URL`, so that the JWT token will be passed in the `Authorization: Bearer ` header. + +PolicydRateGuard will generate a valid JWT like this: ```python import jwt +from base64 import b64decode from datetime import datetime, timedelta, timezone + +timestamp = datetime.now(tz=timezone.utc) payload = { - 'sub': sender, - 'exp': datetime.now(tz=timezone.utc) + timedelta(seconds=60) + 'sub': sender, # subject + 'iss': 'policyd-rate-guard', # issuer + 'iat': timestamp, # issued at + 'nbf': timestamp, # not before + 'exp': timestamp + timedelta(seconds=60) # expiration time } -return jwt.encode(payload, secret, algorithm='HS256') +return jwt.encode(payload, b64decode(secret), algorithm='HS256') ``` -The token is valid for 60s and contains the `sub` (subject, in our case the `sender`) in its payload. The subject in the JWT token is always the same as the `sender` in the JSON data passed via POST request. +The token is valid for 60s and contains the `sub` ("Subject", in our case the `sender`) in its payload. The subject in the JWT token is always the same as the `sender` in the JSON data passed via POST request. If your external API webhook runs on PHP, we recommend to use the [`lcobucci/jwt`](https://github.com/lcobucci/jwt) library to decode and verify the JWT token. In a Laravel app you can go for a similar implementation as described in Variant 1) and decode the JWT token in your `AccessApiWebhookPolicyd` middleware. diff --git a/app/_version.py b/app/_version.py index 49e0fc1..bc8c296 100644 --- a/app/_version.py +++ b/app/_version.py @@ -1 +1 @@ -__version__ = "0.7.0" +__version__ = "0.7.2" diff --git a/app/webhook.py b/app/webhook.py index 398da5e..407ce42 100644 --- a/app/webhook.py +++ b/app/webhook.py @@ -14,6 +14,7 @@ def call(self) -> None: """Call webhook""" webhook_url = self.conf.get('WEBHOOK_URL') webhook_secret = self.conf.get('WEBHOOK_SECRET') + use_jwt = self.conf.get('WEBHOOK_USE_JWT', False) if webhook_url is None or webhook_secret is None: raise ValueError('WEBHOOK_URL and WEBHOOK_SECRET must be configured') @@ -26,13 +27,14 @@ def call(self) -> None: metadata = self.get_metadata() + # Build token, either simple hash or JWT token + token = self.get_jwt_token(webhook_secret) if use_jwt else self.get_simple_token(webhook_secret) + if '{token}' in webhook_url: - # Variant 1) Simple token as query parameter - token = self.get_simple_token(webhook_secret) + # Variant 1) Pass token (usually simple hash) as query parameter formatted_webhook_url = webhook_url.format(**metadata, token=token) else: - # Variant 2) JWT Token as Authorization header - token = self.get_jwt_token(webhook_secret) + # Variant 2) Pass token (usually JWT token) as Authorization header headers['Authorization'] = f'Bearer {token}' formatted_webhook_url = webhook_url.format(**metadata) @@ -71,9 +73,14 @@ def get_simple_token(self, secret: str) -> str: def get_jwt_token(self, secret: str) -> str: """Build JWT token""" import jwt + from base64 import b64decode from datetime import datetime, timedelta, timezone + timestamp = datetime.now(tz=timezone.utc) payload = { - 'sub': self.message.sender, - 'exp': datetime.now(tz=timezone.utc) + timedelta(seconds=60) + 'sub': self.message.sender, # subject + 'iss': 'policyd-rate-guard', # issuer + 'iat': timestamp, # issued at + 'nbf': timestamp, # not before + 'exp': timestamp + timedelta(seconds=60) # expiration time } - return jwt.encode(payload, secret, algorithm='HS256') + return jwt.encode(payload, b64decode(secret), algorithm='HS256') diff --git a/tests/test_app_webhook.py b/tests/test_app_webhook.py index 777e83e..d3fe32b 100644 --- a/tests/test_app_webhook.py +++ b/tests/test_app_webhook.py @@ -65,8 +65,10 @@ def test_get_simple_token(self) -> None: self.assertEqual(token, '34caa5c52fce98bc56fa3bfd8274a92328f09a6e0b27da2b5d89c1b5c5ed05c5') def test_get_jwt_token(self) -> None: + from base64 import b64decode secret = self.conf.get('WEBHOOK_SECRET') token = self.webhook.get_jwt_token(secret) self.assertEqual(type(token).__name__, 'str') - payload = jwt.decode(token, secret, algorithms=['HS256']) + payload = jwt.decode(token, b64decode(secret), algorithms=['HS256']) self.assertEqual(payload['sub'], 'test@example.com') + self.assertEqual(payload['iss'], 'policyd-rate-guard')