From 53b12688dd2a9d7d8076122d48f088c5c3e9939e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 2 Dec 2020 08:57:51 -0500 Subject: [PATCH 1/7] 1.24.0rc1 --- CHANGES.md | 72 ++++++++++++++++++++++++++++++++++++++++ changelog.d/8565.misc | 1 - changelog.d/8617.feature | 1 - changelog.d/8630.feature | 1 - changelog.d/8731.misc | 1 - changelog.d/8734.doc | 1 - changelog.d/8744.bugfix | 1 - changelog.d/8751.misc | 1 - changelog.d/8754.misc | 1 - changelog.d/8757.misc | 1 - changelog.d/8758.misc | 1 - changelog.d/8759.misc | 1 - changelog.d/8760.misc | 1 - changelog.d/8761.misc | 1 - changelog.d/8765.misc | 1 - changelog.d/8770.misc | 1 - changelog.d/8771.doc | 1 - changelog.d/8772.misc | 1 - changelog.d/8773.misc | 1 - changelog.d/8774.misc | 1 - changelog.d/8776.bugfix | 1 - changelog.d/8777.misc | 1 - changelog.d/8779.doc | 1 - changelog.d/8784.misc | 1 - changelog.d/8785.removal | 1 - changelog.d/8793.doc | 1 - changelog.d/8795.doc | 1 - changelog.d/8798.bugfix | 1 - changelog.d/8799.bugfix | 1 - changelog.d/8800.misc | 1 - changelog.d/8801.feature | 1 - changelog.d/8804.feature | 1 - changelog.d/8806.misc | 1 - changelog.d/8809.misc | 1 - changelog.d/8812.misc | 1 - changelog.d/8815.misc | 1 - changelog.d/8817.bugfix | 1 - changelog.d/8818.doc | 1 - changelog.d/8819.misc | 1 - changelog.d/8820.feature | 1 - changelog.d/8822.doc | 1 - changelog.d/8823.bugfix | 1 - changelog.d/8824.doc | 1 - changelog.d/8833.removal | 1 - changelog.d/8835.bugfix | 1 - changelog.d/8843.feature | 1 - changelog.d/8845.misc | 1 - changelog.d/8847.misc | 1 - changelog.d/8848.bugfix | 1 - changelog.d/8849.misc | 1 - changelog.d/8850.misc | 1 - changelog.d/8851.misc | 1 - changelog.d/8854.misc | 1 - changelog.d/8855.feature | 1 - synapse/__init__.py | 2 +- 55 files changed, 73 insertions(+), 54 deletions(-) delete mode 100644 changelog.d/8565.misc delete mode 100644 changelog.d/8617.feature delete mode 100644 changelog.d/8630.feature delete mode 100644 changelog.d/8731.misc delete mode 100644 changelog.d/8734.doc delete mode 100644 changelog.d/8744.bugfix delete mode 100644 changelog.d/8751.misc delete mode 100644 changelog.d/8754.misc delete mode 100644 changelog.d/8757.misc delete mode 100644 changelog.d/8758.misc delete mode 100644 changelog.d/8759.misc delete mode 100644 changelog.d/8760.misc delete mode 100644 changelog.d/8761.misc delete mode 100644 changelog.d/8765.misc delete mode 100644 changelog.d/8770.misc delete mode 100644 changelog.d/8771.doc delete mode 100644 changelog.d/8772.misc delete mode 100644 changelog.d/8773.misc delete mode 100644 changelog.d/8774.misc delete mode 100644 changelog.d/8776.bugfix delete mode 100644 changelog.d/8777.misc delete mode 100644 changelog.d/8779.doc delete mode 100644 changelog.d/8784.misc delete mode 100644 changelog.d/8785.removal delete mode 100644 changelog.d/8793.doc delete mode 100644 changelog.d/8795.doc delete mode 100644 changelog.d/8798.bugfix delete mode 100644 changelog.d/8799.bugfix delete mode 100644 changelog.d/8800.misc delete mode 100644 changelog.d/8801.feature delete mode 100644 changelog.d/8804.feature delete mode 100644 changelog.d/8806.misc delete mode 100644 changelog.d/8809.misc delete mode 100644 changelog.d/8812.misc delete mode 100644 changelog.d/8815.misc delete mode 100644 changelog.d/8817.bugfix delete mode 100644 changelog.d/8818.doc delete mode 100644 changelog.d/8819.misc delete mode 100644 changelog.d/8820.feature delete mode 100644 changelog.d/8822.doc delete mode 100644 changelog.d/8823.bugfix delete mode 100644 changelog.d/8824.doc delete mode 100644 changelog.d/8833.removal delete mode 100644 changelog.d/8835.bugfix delete mode 100644 changelog.d/8843.feature delete mode 100644 changelog.d/8845.misc delete mode 100644 changelog.d/8847.misc delete mode 100644 changelog.d/8848.bugfix delete mode 100644 changelog.d/8849.misc delete mode 100644 changelog.d/8850.misc delete mode 100644 changelog.d/8851.misc delete mode 100644 changelog.d/8854.misc delete mode 100644 changelog.d/8855.feature diff --git a/CHANGES.md b/CHANGES.md index 4237550818f0..662762c07c7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,75 @@ +Synapse 1.24.0rc1 (2020-12-02) +============================== + +Features +-------- + +- Add admin API for logging in as a user. ([\#8617](https://github.com/matrix-org/synapse/issues/8617)) +- Allow specification of the SAML IdP if the metadata returns multiple IdPs. ([\#8630](https://github.com/matrix-org/synapse/issues/8630)) +- Add support for re-trying generation of a localpart for OpenID Connect mapping providers. ([\#8801](https://github.com/matrix-org/synapse/issues/8801), [\#8855](https://github.com/matrix-org/synapse/issues/8855)) +- Allow Date header through CORS. Contributed by Nicolas Chamo. ([\#8804](https://github.com/matrix-org/synapse/issues/8804)) +- Add a config option, `push.group_by_unread_count`, which controls whether unread message counts in push notifications are defined as "the number of rooms with unread messages" or "total unread messages". ([\#8820](https://github.com/matrix-org/synapse/issues/8820)) +- Add `force_purge` option to delete-room admin api. ([\#8843](https://github.com/matrix-org/synapse/issues/8843)) + + +Bugfixes +-------- + +- Fix a bug where appservices may be sent an excessive amount of read receipts and presence. Broke in v1.22.0. ([\#8744](https://github.com/matrix-org/synapse/issues/8744)) +- Fix a bug in some federation APIs which could lead to unexpected behaviour if different parameters were set in the URI and the request body. ([\#8776](https://github.com/matrix-org/synapse/issues/8776)) +- Fix a bug where synctl could spawn duplicate copies of a worker. Contributed by Waylon Cude. ([\#8798](https://github.com/matrix-org/synapse/issues/8798)) +- Allow per-room profiles to be used for the server notice user. ([\#8799](https://github.com/matrix-org/synapse/issues/8799)) +- Fix bug where logging could break after a call to SIGHUP. ([\#8817](https://github.com/matrix-org/synapse/issues/8817)) +- Fix `register_new_matrix_user` failing with "Bad Request" when trailing slash is included in server URL. Contributed by @angdraug. ([\#8823](https://github.com/matrix-org/synapse/issues/8823)) +- Fix minor long-standing bug in login, where we would offer the `password` login type if a custom auth provider supported it, even if password login was disabled. ([\#8835](https://github.com/matrix-org/synapse/issues/8835)) +- Fix a long-standing bug which caused Synapse to require unspecified parameters during user-interactive authentication. ([\#8848](https://github.com/matrix-org/synapse/issues/8848)) + + +Improved Documentation +---------------------- + +- Clarify the usecase for an msisdn delegate. Contributed by Adrian Wannenmacher. ([\#8734](https://github.com/matrix-org/synapse/issues/8734)) +- Remove extraneous comma from JSON example in User Admin API docs. ([\#8771](https://github.com/matrix-org/synapse/issues/8771)) +- Update `turn-howto.md` with troubleshooting notes. ([\#8779](https://github.com/matrix-org/synapse/issues/8779)) +- Fix the example on how to set the `Content-Type` header in nginx for the Client Well-Known URI. ([\#8793](https://github.com/matrix-org/synapse/issues/8793)) +- Improve the documentation for the admin API to list all media in a room with respect to encrypted events. ([\#8795](https://github.com/matrix-org/synapse/issues/8795)) +- Update the formatting of the `push` section of the homeserver config file to better align with the [code style guidelines](https://github.com/matrix-org/synapse/blob/develop/docs/code_style.md#configuration-file-format). ([\#8818](https://github.com/matrix-org/synapse/issues/8818)) +- Improve documentation how to configure prometheus for workers. ([\#8822](https://github.com/matrix-org/synapse/issues/8822)) +- Update example prometheus console. ([\#8824](https://github.com/matrix-org/synapse/issues/8824)) + + +Deprecations and Removals +------------------------- + +- Remove old `/_matrix/client/*/admin` endpoints which was deprecated since Synapse 1.20.0. ([\#8785](https://github.com/matrix-org/synapse/issues/8785)) +- Disable pretty printing JSON responses for curl. Users who want pretty-printed output should use [jq](https://stedolan.github.io/jq/) in combination with curl. Contributed by @tulir. ([\#8833](https://github.com/matrix-org/synapse/issues/8833)) + + +Internal Changes +---------------- + +- Simplify the way the `HomeServer` object caches its internal attributes. ([\#8565](https://github.com/matrix-org/synapse/issues/8565), [\#8851](https://github.com/matrix-org/synapse/issues/8851)) +- Add an example and documentation for clock skew to the SAML2 sample configuration to allow for clock/time difference between the homserver and IdP. Contributed by @localguru. ([\#8731](https://github.com/matrix-org/synapse/issues/8731)) +- Generalise `RoomMemberHandler._locally_reject_invite` to apply to more flows than just invite. ([\#8751](https://github.com/matrix-org/synapse/issues/8751)) +- Generalise `RoomStore.maybe_store_room_on_invite` to handle other, non-invite membership events. ([\#8754](https://github.com/matrix-org/synapse/issues/8754)) +- Refactor test utilities for injecting HTTP requests. ([\#8757](https://github.com/matrix-org/synapse/issues/8757), [\#8758](https://github.com/matrix-org/synapse/issues/8758), [\#8759](https://github.com/matrix-org/synapse/issues/8759), [\#8760](https://github.com/matrix-org/synapse/issues/8760), [\#8761](https://github.com/matrix-org/synapse/issues/8761), [\#8777](https://github.com/matrix-org/synapse/issues/8777)) +- Consolidate logic between the OpenID Connect and SAML code. ([\#8765](https://github.com/matrix-org/synapse/issues/8765)) +- Use `TYPE_CHECKING` instead of magic `MYPY` variable. ([\#8770](https://github.com/matrix-org/synapse/issues/8770)) +- Add a commandline script to sign arbitrary json objects. ([\#8772](https://github.com/matrix-org/synapse/issues/8772)) +- Minor log line improvements for the SSO mapping code used to generate Matrix IDs from SSO IDs. ([\#8773](https://github.com/matrix-org/synapse/issues/8773)) +- Add additional error checking for OpenID Connect and SAML mapping providers. ([\#8774](https://github.com/matrix-org/synapse/issues/8774), [\#8800](https://github.com/matrix-org/synapse/issues/8800)) +- Fix a bug introduced in v1.20.0 where the user-agent and IP address reported during user registration for CAS, OpenID Connect, and SAML were of the wrong form. ([\#8784](https://github.com/matrix-org/synapse/issues/8784)) +- Add type hints to HTTP abstractions. ([\#8806](https://github.com/matrix-org/synapse/issues/8806), [\#8812](https://github.com/matrix-org/synapse/issues/8812)) +- Remove unnecessary function arguments and add typing to several membership replication classes. ([\#8809](https://github.com/matrix-org/synapse/issues/8809)) +- Optimise the lookup for an invite from another homeserver when trying to reject it. ([\#8815](https://github.com/matrix-org/synapse/issues/8815)) +- Add tests for `password_auth_provider`s. ([\#8819](https://github.com/matrix-org/synapse/issues/8819)) +- Drop redundant database index on `event_json`. ([\#8845](https://github.com/matrix-org/synapse/issues/8845)) +- Simplify `uk.half-shot.msc2778.login.application_service` login handler. ([\#8847](https://github.com/matrix-org/synapse/issues/8847)) +- Refactor `password_auth_provider` support code. ([\#8849](https://github.com/matrix-org/synapse/issues/8849)) +- Add missing `ordering` to background database updates. ([\#8850](https://github.com/matrix-org/synapse/issues/8850)) +- Allow for specifying a room version when creating a room in unit tests via `RestHelper.create_room_as`. ([\#8854](https://github.com/matrix-org/synapse/issues/8854)) + + Synapse 1.23.0 (2020-11-18) =========================== diff --git a/changelog.d/8565.misc b/changelog.d/8565.misc deleted file mode 100644 index 7bef4226186c..000000000000 --- a/changelog.d/8565.misc +++ /dev/null @@ -1 +0,0 @@ -Simplify the way the `HomeServer` object caches its internal attributes. diff --git a/changelog.d/8617.feature b/changelog.d/8617.feature deleted file mode 100644 index 4f1e78850625..000000000000 --- a/changelog.d/8617.feature +++ /dev/null @@ -1 +0,0 @@ -Add admin API for logging in as a user. diff --git a/changelog.d/8630.feature b/changelog.d/8630.feature deleted file mode 100644 index 706051f1316e..000000000000 --- a/changelog.d/8630.feature +++ /dev/null @@ -1 +0,0 @@ -Allow specification of the SAML IdP if the metadata returns multiple IdPs. diff --git a/changelog.d/8731.misc b/changelog.d/8731.misc deleted file mode 100644 index df5882e960a2..000000000000 --- a/changelog.d/8731.misc +++ /dev/null @@ -1 +0,0 @@ -Add an example and documentation for clock skew to the SAML2 sample configuration to allow for clock/time difference between the homserver and IdP. Contributed by @localguru. diff --git a/changelog.d/8734.doc b/changelog.d/8734.doc deleted file mode 100644 index 3bff9021c7a4..000000000000 --- a/changelog.d/8734.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify the usecase for an msisdn delegate. Contributed by Adrian Wannenmacher. diff --git a/changelog.d/8744.bugfix b/changelog.d/8744.bugfix deleted file mode 100644 index f8f9630bd6af..000000000000 --- a/changelog.d/8744.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where appservices may be sent an excessive amount of read receipts and presence. Broke in v1.22.0. diff --git a/changelog.d/8751.misc b/changelog.d/8751.misc deleted file mode 100644 index 204c280c0e7b..000000000000 --- a/changelog.d/8751.misc +++ /dev/null @@ -1 +0,0 @@ -Generalise `RoomMemberHandler._locally_reject_invite` to apply to more flows than just invite. \ No newline at end of file diff --git a/changelog.d/8754.misc b/changelog.d/8754.misc deleted file mode 100644 index 0436bb1be729..000000000000 --- a/changelog.d/8754.misc +++ /dev/null @@ -1 +0,0 @@ -Generalise `RoomStore.maybe_store_room_on_invite` to handle other, non-invite membership events. \ No newline at end of file diff --git a/changelog.d/8757.misc b/changelog.d/8757.misc deleted file mode 100644 index 54502e9b9029..000000000000 --- a/changelog.d/8757.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor test utilities for injecting HTTP requests. diff --git a/changelog.d/8758.misc b/changelog.d/8758.misc deleted file mode 100644 index 54502e9b9029..000000000000 --- a/changelog.d/8758.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor test utilities for injecting HTTP requests. diff --git a/changelog.d/8759.misc b/changelog.d/8759.misc deleted file mode 100644 index 54502e9b9029..000000000000 --- a/changelog.d/8759.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor test utilities for injecting HTTP requests. diff --git a/changelog.d/8760.misc b/changelog.d/8760.misc deleted file mode 100644 index 54502e9b9029..000000000000 --- a/changelog.d/8760.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor test utilities for injecting HTTP requests. diff --git a/changelog.d/8761.misc b/changelog.d/8761.misc deleted file mode 100644 index e6da7d038d17..000000000000 --- a/changelog.d/8761.misc +++ /dev/null @@ -1 +0,0 @@ - Refactor test utilities for injecting HTTP requests. diff --git a/changelog.d/8765.misc b/changelog.d/8765.misc deleted file mode 100644 index 053f9acc9caf..000000000000 --- a/changelog.d/8765.misc +++ /dev/null @@ -1 +0,0 @@ -Consolidate logic between the OpenID Connect and SAML code. diff --git a/changelog.d/8770.misc b/changelog.d/8770.misc deleted file mode 100644 index b5876a82f92f..000000000000 --- a/changelog.d/8770.misc +++ /dev/null @@ -1 +0,0 @@ -Use `TYPE_CHECKING` instead of magic `MYPY` variable. diff --git a/changelog.d/8771.doc b/changelog.d/8771.doc deleted file mode 100644 index 297cf61e9840..000000000000 --- a/changelog.d/8771.doc +++ /dev/null @@ -1 +0,0 @@ -Remove extraneous comma from JSON example in User Admin API docs. \ No newline at end of file diff --git a/changelog.d/8772.misc b/changelog.d/8772.misc deleted file mode 100644 index d74d0a3d5d87..000000000000 --- a/changelog.d/8772.misc +++ /dev/null @@ -1 +0,0 @@ -Add a commandline script to sign arbitrary json objects. diff --git a/changelog.d/8773.misc b/changelog.d/8773.misc deleted file mode 100644 index 62778ba4100d..000000000000 --- a/changelog.d/8773.misc +++ /dev/null @@ -1 +0,0 @@ -Minor log line improvements for the SSO mapping code used to generate Matrix IDs from SSO IDs. diff --git a/changelog.d/8774.misc b/changelog.d/8774.misc deleted file mode 100644 index 57cca8fee51c..000000000000 --- a/changelog.d/8774.misc +++ /dev/null @@ -1 +0,0 @@ -Add additional error checking for OpenID Connect and SAML mapping providers. diff --git a/changelog.d/8776.bugfix b/changelog.d/8776.bugfix deleted file mode 100644 index dd7ebbeb865f..000000000000 --- a/changelog.d/8776.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in some federation APIs which could lead to unexpected behaviour if different parameters were set in the URI and the request body. diff --git a/changelog.d/8777.misc b/changelog.d/8777.misc deleted file mode 100644 index e6da7d038d17..000000000000 --- a/changelog.d/8777.misc +++ /dev/null @@ -1 +0,0 @@ - Refactor test utilities for injecting HTTP requests. diff --git a/changelog.d/8779.doc b/changelog.d/8779.doc deleted file mode 100644 index 3641ae7f9116..000000000000 --- a/changelog.d/8779.doc +++ /dev/null @@ -1 +0,0 @@ -Update `turn-howto.md` with troubleshooting notes. diff --git a/changelog.d/8784.misc b/changelog.d/8784.misc deleted file mode 100644 index 18a4263398bd..000000000000 --- a/changelog.d/8784.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in v1.20.0 where the user-agent and IP address reported during user registration for CAS, OpenID Connect, and SAML were of the wrong form. diff --git a/changelog.d/8785.removal b/changelog.d/8785.removal deleted file mode 100644 index ee8ee32598e1..000000000000 --- a/changelog.d/8785.removal +++ /dev/null @@ -1 +0,0 @@ -Remove old `/_matrix/client/*/admin` endpoints which was deprecated since Synapse 1.20.0. \ No newline at end of file diff --git a/changelog.d/8793.doc b/changelog.d/8793.doc deleted file mode 100644 index f6eee1ea735c..000000000000 --- a/changelog.d/8793.doc +++ /dev/null @@ -1 +0,0 @@ -Fix the example on how to set the `Content-Type` header in nginx for the Client Well-Known URI. diff --git a/changelog.d/8795.doc b/changelog.d/8795.doc deleted file mode 100644 index f97a74efb5f5..000000000000 --- a/changelog.d/8795.doc +++ /dev/null @@ -1 +0,0 @@ -Improve the documentation for the admin API to list all media in a room with respect to encrypted events. diff --git a/changelog.d/8798.bugfix b/changelog.d/8798.bugfix deleted file mode 100644 index 9bdb2b51ea06..000000000000 --- a/changelog.d/8798.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where synctl could spawn duplicate copies of a worker. Contributed by Waylon Cude. diff --git a/changelog.d/8799.bugfix b/changelog.d/8799.bugfix deleted file mode 100644 index a7e6b3556dd9..000000000000 --- a/changelog.d/8799.bugfix +++ /dev/null @@ -1 +0,0 @@ -Allow per-room profiles to be used for the server notice user. diff --git a/changelog.d/8800.misc b/changelog.d/8800.misc deleted file mode 100644 index 57cca8fee51c..000000000000 --- a/changelog.d/8800.misc +++ /dev/null @@ -1 +0,0 @@ -Add additional error checking for OpenID Connect and SAML mapping providers. diff --git a/changelog.d/8801.feature b/changelog.d/8801.feature deleted file mode 100644 index 77f7fe4e5d51..000000000000 --- a/changelog.d/8801.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for re-trying generation of a localpart for OpenID Connect mapping providers. diff --git a/changelog.d/8804.feature b/changelog.d/8804.feature deleted file mode 100644 index a907c8106c2c..000000000000 --- a/changelog.d/8804.feature +++ /dev/null @@ -1 +0,0 @@ -Allow Date header through CORS. Contributed by Nicolas Chamo. diff --git a/changelog.d/8806.misc b/changelog.d/8806.misc deleted file mode 100644 index ee144846a5f1..000000000000 --- a/changelog.d/8806.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to HTTP abstractions. diff --git a/changelog.d/8809.misc b/changelog.d/8809.misc deleted file mode 100644 index bbf83cf18d6a..000000000000 --- a/changelog.d/8809.misc +++ /dev/null @@ -1 +0,0 @@ -Remove unnecessary function arguments and add typing to several membership replication classes. \ No newline at end of file diff --git a/changelog.d/8812.misc b/changelog.d/8812.misc deleted file mode 100644 index ee144846a5f1..000000000000 --- a/changelog.d/8812.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to HTTP abstractions. diff --git a/changelog.d/8815.misc b/changelog.d/8815.misc deleted file mode 100644 index 647edeb56851..000000000000 --- a/changelog.d/8815.misc +++ /dev/null @@ -1 +0,0 @@ -Optimise the lookup for an invite from another homeserver when trying to reject it. \ No newline at end of file diff --git a/changelog.d/8817.bugfix b/changelog.d/8817.bugfix deleted file mode 100644 index e45dbd2ba4f7..000000000000 --- a/changelog.d/8817.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where logging could break after a call to SIGHUP. diff --git a/changelog.d/8818.doc b/changelog.d/8818.doc deleted file mode 100644 index 571b0e3f600a..000000000000 --- a/changelog.d/8818.doc +++ /dev/null @@ -1 +0,0 @@ -Update the formatting of the `push` section of the homeserver config file to better align with the [code style guidelines](https://github.com/matrix-org/synapse/blob/develop/docs/code_style.md#configuration-file-format). \ No newline at end of file diff --git a/changelog.d/8819.misc b/changelog.d/8819.misc deleted file mode 100644 index a5793273a5c9..000000000000 --- a/changelog.d/8819.misc +++ /dev/null @@ -1 +0,0 @@ -Add tests for `password_auth_provider`s. diff --git a/changelog.d/8820.feature b/changelog.d/8820.feature deleted file mode 100644 index 9e35861b1123..000000000000 --- a/changelog.d/8820.feature +++ /dev/null @@ -1 +0,0 @@ -Add a config option, `push.group_by_unread_count`, which controls whether unread message counts in push notifications are defined as "the number of rooms with unread messages" or "total unread messages". diff --git a/changelog.d/8822.doc b/changelog.d/8822.doc deleted file mode 100644 index 429924599062..000000000000 --- a/changelog.d/8822.doc +++ /dev/null @@ -1 +0,0 @@ -Improve documentation how to configure prometheus for workers. \ No newline at end of file diff --git a/changelog.d/8823.bugfix b/changelog.d/8823.bugfix deleted file mode 100644 index 74af1c20b659..000000000000 --- a/changelog.d/8823.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix `register_new_matrix_user` failing with "Bad Request" when trailing slash is included in server URL. Contributed by @angdraug. diff --git a/changelog.d/8824.doc b/changelog.d/8824.doc deleted file mode 100644 index 683b436328e4..000000000000 --- a/changelog.d/8824.doc +++ /dev/null @@ -1 +0,0 @@ -Update example prometheus console. \ No newline at end of file diff --git a/changelog.d/8833.removal b/changelog.d/8833.removal deleted file mode 100644 index 5c2d195f9414..000000000000 --- a/changelog.d/8833.removal +++ /dev/null @@ -1 +0,0 @@ -Disable pretty printing JSON responses for curl. Users who want pretty-printed output should use [jq](https://stedolan.github.io/jq/) in combination with curl. Contributed by @tulir. diff --git a/changelog.d/8835.bugfix b/changelog.d/8835.bugfix deleted file mode 100644 index 446d04aa5512..000000000000 --- a/changelog.d/8835.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix minor long-standing bug in login, where we would offer the `password` login type if a custom auth provider supported it, even if password login was disabled. diff --git a/changelog.d/8843.feature b/changelog.d/8843.feature deleted file mode 100644 index 824d46d5aa5a..000000000000 --- a/changelog.d/8843.feature +++ /dev/null @@ -1 +0,0 @@ -Add `force_purge` option to delete-room admin api. diff --git a/changelog.d/8845.misc b/changelog.d/8845.misc deleted file mode 100644 index 7db1c31520ce..000000000000 --- a/changelog.d/8845.misc +++ /dev/null @@ -1 +0,0 @@ -Drop redundant database index on `event_json`. diff --git a/changelog.d/8847.misc b/changelog.d/8847.misc deleted file mode 100644 index 5028997b0444..000000000000 --- a/changelog.d/8847.misc +++ /dev/null @@ -1 +0,0 @@ -Simplify `uk.half-shot.msc2778.login.application_service` login handler. diff --git a/changelog.d/8848.bugfix b/changelog.d/8848.bugfix deleted file mode 100644 index 499e66f05b53..000000000000 --- a/changelog.d/8848.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug which caused Synapse to require unspecified parameters during user-interactive authentication. diff --git a/changelog.d/8849.misc b/changelog.d/8849.misc deleted file mode 100644 index 3dd496ce615f..000000000000 --- a/changelog.d/8849.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `password_auth_provider` support code. diff --git a/changelog.d/8850.misc b/changelog.d/8850.misc deleted file mode 100644 index 4b54b8dd871a..000000000000 --- a/changelog.d/8850.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing `ordering` to background database updates. diff --git a/changelog.d/8851.misc b/changelog.d/8851.misc deleted file mode 100644 index 7bef4226186c..000000000000 --- a/changelog.d/8851.misc +++ /dev/null @@ -1 +0,0 @@ -Simplify the way the `HomeServer` object caches its internal attributes. diff --git a/changelog.d/8854.misc b/changelog.d/8854.misc deleted file mode 100644 index 5895df2d5c99..000000000000 --- a/changelog.d/8854.misc +++ /dev/null @@ -1 +0,0 @@ -Allow for specifying a room version when creating a room in unit tests via `RestHelper.create_room_as`. \ No newline at end of file diff --git a/changelog.d/8855.feature b/changelog.d/8855.feature deleted file mode 100644 index 77f7fe4e5d51..000000000000 --- a/changelog.d/8855.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for re-trying generation of a localpart for OpenID Connect mapping providers. diff --git a/synapse/__init__.py b/synapse/__init__.py index 65c1f5aa3f6c..d33a99f2306d 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -48,7 +48,7 @@ except ImportError: pass -__version__ = "1.23.0" +__version__ = "1.24.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From c67af840aa015647f6e237935e02eb163151aa5c Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 2 Dec 2020 09:03:12 -0500 Subject: [PATCH 2/7] Minor fixes to changelog. --- CHANGES.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 662762c07c7f..ebc44c4b362d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,7 +7,7 @@ Features - Add admin API for logging in as a user. ([\#8617](https://github.com/matrix-org/synapse/issues/8617)) - Allow specification of the SAML IdP if the metadata returns multiple IdPs. ([\#8630](https://github.com/matrix-org/synapse/issues/8630)) - Add support for re-trying generation of a localpart for OpenID Connect mapping providers. ([\#8801](https://github.com/matrix-org/synapse/issues/8801), [\#8855](https://github.com/matrix-org/synapse/issues/8855)) -- Allow Date header through CORS. Contributed by Nicolas Chamo. ([\#8804](https://github.com/matrix-org/synapse/issues/8804)) +- Allow the `Date` header through CORS. Contributed by Nicolas Chamo. ([\#8804](https://github.com/matrix-org/synapse/issues/8804)) - Add a config option, `push.group_by_unread_count`, which controls whether unread message counts in push notifications are defined as "the number of rooms with unread messages" or "total unread messages". ([\#8820](https://github.com/matrix-org/synapse/issues/8820)) - Add `force_purge` option to delete-room admin api. ([\#8843](https://github.com/matrix-org/synapse/issues/8843)) @@ -19,10 +19,11 @@ Bugfixes - Fix a bug in some federation APIs which could lead to unexpected behaviour if different parameters were set in the URI and the request body. ([\#8776](https://github.com/matrix-org/synapse/issues/8776)) - Fix a bug where synctl could spawn duplicate copies of a worker. Contributed by Waylon Cude. ([\#8798](https://github.com/matrix-org/synapse/issues/8798)) - Allow per-room profiles to be used for the server notice user. ([\#8799](https://github.com/matrix-org/synapse/issues/8799)) -- Fix bug where logging could break after a call to SIGHUP. ([\#8817](https://github.com/matrix-org/synapse/issues/8817)) +- Fix a bug where logging could break after a call to SIGHUP. ([\#8817](https://github.com/matrix-org/synapse/issues/8817)) - Fix `register_new_matrix_user` failing with "Bad Request" when trailing slash is included in server URL. Contributed by @angdraug. ([\#8823](https://github.com/matrix-org/synapse/issues/8823)) -- Fix minor long-standing bug in login, where we would offer the `password` login type if a custom auth provider supported it, even if password login was disabled. ([\#8835](https://github.com/matrix-org/synapse/issues/8835)) +- Fix a minor long-standing bug in login, where we would offer the `password` login type if a custom auth provider supported it, even if password login was disabled. ([\#8835](https://github.com/matrix-org/synapse/issues/8835)) - Fix a long-standing bug which caused Synapse to require unspecified parameters during user-interactive authentication. ([\#8848](https://github.com/matrix-org/synapse/issues/8848)) +- Fix a bug introduced in v1.20.0 where the user-agent and IP address reported during user registration for CAS, OpenID Connect, and SAML were of the wrong form. ([\#8784](https://github.com/matrix-org/synapse/issues/8784)) Improved Documentation @@ -41,7 +42,7 @@ Improved Documentation Deprecations and Removals ------------------------- -- Remove old `/_matrix/client/*/admin` endpoints which was deprecated since Synapse 1.20.0. ([\#8785](https://github.com/matrix-org/synapse/issues/8785)) +- Remove old `/_matrix/client/*/admin` endpoints which were deprecated since Synapse 1.20.0. ([\#8785](https://github.com/matrix-org/synapse/issues/8785)) - Disable pretty printing JSON responses for curl. Users who want pretty-printed output should use [jq](https://stedolan.github.io/jq/) in combination with curl. Contributed by @tulir. ([\#8833](https://github.com/matrix-org/synapse/issues/8833)) @@ -58,7 +59,6 @@ Internal Changes - Add a commandline script to sign arbitrary json objects. ([\#8772](https://github.com/matrix-org/synapse/issues/8772)) - Minor log line improvements for the SSO mapping code used to generate Matrix IDs from SSO IDs. ([\#8773](https://github.com/matrix-org/synapse/issues/8773)) - Add additional error checking for OpenID Connect and SAML mapping providers. ([\#8774](https://github.com/matrix-org/synapse/issues/8774), [\#8800](https://github.com/matrix-org/synapse/issues/8800)) -- Fix a bug introduced in v1.20.0 where the user-agent and IP address reported during user registration for CAS, OpenID Connect, and SAML were of the wrong form. ([\#8784](https://github.com/matrix-org/synapse/issues/8784)) - Add type hints to HTTP abstractions. ([\#8806](https://github.com/matrix-org/synapse/issues/8806), [\#8812](https://github.com/matrix-org/synapse/issues/8812)) - Remove unnecessary function arguments and add typing to several membership replication classes. ([\#8809](https://github.com/matrix-org/synapse/issues/8809)) - Optimise the lookup for an invite from another homeserver when trying to reject it. ([\#8815](https://github.com/matrix-org/synapse/issues/8815)) From e41720d85f792b4bdc1ffb430b0a04967ad813c2 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 2 Dec 2020 09:17:42 -0500 Subject: [PATCH 3/7] Minor changes to the CHANGES doc. --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ebc44c4b362d..e7aaebb1f30a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,7 +29,7 @@ Bugfixes Improved Documentation ---------------------- -- Clarify the usecase for an msisdn delegate. Contributed by Adrian Wannenmacher. ([\#8734](https://github.com/matrix-org/synapse/issues/8734)) +- Clarify the usecase for a msisdn delegate. Contributed by Adrian Wannenmacher. ([\#8734](https://github.com/matrix-org/synapse/issues/8734)) - Remove extraneous comma from JSON example in User Admin API docs. ([\#8771](https://github.com/matrix-org/synapse/issues/8771)) - Update `turn-howto.md` with troubleshooting notes. ([\#8779](https://github.com/matrix-org/synapse/issues/8779)) - Fix the example on how to set the `Content-Type` header in nginx for the Client Well-Known URI. ([\#8793](https://github.com/matrix-org/synapse/issues/8793)) From c4675e1b24f06a72c323c8131eab4998b4e71af1 Mon Sep 17 00:00:00 2001 From: David Florness Date: Wed, 2 Dec 2020 10:01:15 -0500 Subject: [PATCH 4/7] Add additional validation for the admin register endpoint. (#8837) Raise a proper 400 error if the `mac` field is missing. --- changelog.d/8837.bugfix | 1 + synapse/rest/admin/users.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/8837.bugfix diff --git a/changelog.d/8837.bugfix b/changelog.d/8837.bugfix new file mode 100644 index 000000000000..b2977d0c312e --- /dev/null +++ b/changelog.d/8837.bugfix @@ -0,0 +1 @@ +Fix a long standing bug in the register admin endpoint (`/_synapse/admin/v1/register`) when the `mac` field was not provided. The endpoint now properly returns a 400 error. Contributed by @edwargix. diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index b0ff5e1ead22..90940ff18549 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -420,6 +420,9 @@ async def on_POST(self, request): if user_type is not None and user_type not in UserTypes.ALL_USER_TYPES: raise SynapseError(400, "Invalid user type") + if "mac" not in body: + raise SynapseError(400, "mac must be specified", errcode=Codes.BAD_JSON) + got_mac = body["mac"] want_mac_builder = hmac.new( From 0fed46ebe5abc524f10708ce1d5849e53dbab8af Mon Sep 17 00:00:00 2001 From: Johanna Dorothea Reichmann Date: Wed, 2 Dec 2020 16:18:41 +0100 Subject: [PATCH 5/7] Add missing prometheus rules for persisted events (#8802) The official dashboard uses data from these rules, but they were never added to the synapse-v2.rules. They are mentioned in this issue: https://github.com/matrix-org/synapse/issues/7917#issuecomment-661330409, but never got added to the rules. Adding them results in all graphs in the "Event persist rate" section to function as intended. Signed-off-by: Johanna Dorothea Reichmann --- changelog.d/8802.doc | 1 + contrib/prometheus/synapse-v2.rules | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 changelog.d/8802.doc diff --git a/changelog.d/8802.doc b/changelog.d/8802.doc new file mode 100644 index 000000000000..580c4281f8cb --- /dev/null +++ b/changelog.d/8802.doc @@ -0,0 +1 @@ +Fix the "Event persist rate" section of the included grafana dashboard by adding missing prometheus rules. diff --git a/contrib/prometheus/synapse-v2.rules b/contrib/prometheus/synapse-v2.rules index 6ccca2daaf22..7e405bf7f0fe 100644 --- a/contrib/prometheus/synapse-v2.rules +++ b/contrib/prometheus/synapse-v2.rules @@ -58,3 +58,21 @@ groups: labels: type: "PDU" expr: 'synapse_federation_transaction_queue_pending_pdus + 0' + + - record: synapse_storage_events_persisted_by_source_type + expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_type="remote"}) + labels: + type: remote + - record: synapse_storage_events_persisted_by_source_type + expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity="*client*",origin_type="local"}) + labels: + type: local + - record: synapse_storage_events_persisted_by_source_type + expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity!="*client*",origin_type="local"}) + labels: + type: bridges + - record: synapse_storage_events_persisted_by_event_type + expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep) + - record: synapse_storage_events_persisted_by_origin + expr: sum without(type) (synapse_storage_events_persisted_events_sep) + From c5b6abd53d1549c7a7cc30c644dd8921bfc10ea2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 2 Dec 2020 15:22:37 +0000 Subject: [PATCH 6/7] Correctly handle unpersisted events when calculating auth chain difference. (#8827) We do state res with unpersisted events when calculating the new current state of the room, so that should be the only thing impacted. I don't think this is tooooo big of a deal as: 1. the next time a state event happens in the room the current state should correct itself; 2. in the common case all the unpersisted events' auth events will be pulled in by other state, so will still return the correct result (or one which is sufficiently close to not affect the result); and 3. we mostly use the state at an event to do important operations, which isn't affected by this. --- changelog.d/8827.bugfix | 1 + synapse/state/v2.py | 87 ++++++++++++++++- tests/state/test_v2.py | 128 ++++++++++++++++++++++++- tests/storage/test_event_federation.py | 5 + 4 files changed, 216 insertions(+), 5 deletions(-) create mode 100644 changelog.d/8827.bugfix diff --git a/changelog.d/8827.bugfix b/changelog.d/8827.bugfix new file mode 100644 index 000000000000..18195680d39a --- /dev/null +++ b/changelog.d/8827.bugfix @@ -0,0 +1 @@ +Fix bug where we might not correctly calculate the current state for rooms with multiple extremities. diff --git a/synapse/state/v2.py b/synapse/state/v2.py index f57df0d72859..ffc504ce771e 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -38,7 +38,7 @@ from synapse.api.errors import AuthError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import EventBase -from synapse.types import MutableStateMap, StateMap +from synapse.types import Collection, MutableStateMap, StateMap from synapse.util import Clock logger = logging.getLogger(__name__) @@ -252,9 +252,88 @@ async def _get_auth_chain_difference( Set of event IDs """ - difference = await state_res_store.get_auth_chain_difference( - [set(state_set.values()) for state_set in state_sets] - ) + # The `StateResolutionStore.get_auth_chain_difference` function assumes that + # all events passed to it (and their auth chains) have been persisted + # previously. This is not the case for any events in the `event_map`, and so + # we need to manually handle those events. + # + # We do this by: + # 1. calculating the auth chain difference for the state sets based on the + # events in `event_map` alone + # 2. replacing any events in the state_sets that are also in `event_map` + # with their auth events (recursively), and then calling + # `store.get_auth_chain_difference` as normal + # 3. adding the results of 1 and 2 together. + + # Map from event ID in `event_map` to their auth event IDs, and their auth + # event IDs if they appear in the `event_map`. This is the intersection of + # the event's auth chain with the events in the `event_map` *plus* their + # auth event IDs. + events_to_auth_chain = {} # type: Dict[str, Set[str]] + for event in event_map.values(): + chain = {event.event_id} + events_to_auth_chain[event.event_id] = chain + + to_search = [event] + while to_search: + for auth_id in to_search.pop().auth_event_ids(): + chain.add(auth_id) + auth_event = event_map.get(auth_id) + if auth_event: + to_search.append(auth_event) + + # We now a) calculate the auth chain difference for the unpersisted events + # and b) work out the state sets to pass to the store. + # + # Note: If the `event_map` is empty (which is the common case), we can do a + # much simpler calculation. + if event_map: + # The list of state sets to pass to the store, where each state set is a set + # of the event ids making up the state. This is similar to `state_sets`, + # except that (a) we only have event ids, not the complete + # ((type, state_key)->event_id) mappings; and (b) we have stripped out + # unpersisted events and replaced them with the persisted events in + # their auth chain. + state_sets_ids = [] # type: List[Set[str]] + + # For each state set, the unpersisted event IDs reachable (by their auth + # chain) from the events in that set. + unpersisted_set_ids = [] # type: List[Set[str]] + + for state_set in state_sets: + set_ids = set() # type: Set[str] + state_sets_ids.append(set_ids) + + unpersisted_ids = set() # type: Set[str] + unpersisted_set_ids.append(unpersisted_ids) + + for event_id in state_set.values(): + event_chain = events_to_auth_chain.get(event_id) + if event_chain is not None: + # We have an event in `event_map`. We add all the auth + # events that it references (that aren't also in `event_map`). + set_ids.update(e for e in event_chain if e not in event_map) + + # We also add the full chain of unpersisted event IDs + # referenced by this state set, so that we can work out the + # auth chain difference of the unpersisted events. + unpersisted_ids.update(e for e in event_chain if e in event_map) + else: + set_ids.add(event_id) + + # The auth chain difference of the unpersisted events of the state sets + # is calculated by taking the difference between the union and + # intersections. + union = unpersisted_set_ids[0].union(*unpersisted_set_ids[1:]) + intersection = unpersisted_set_ids[0].intersection(*unpersisted_set_ids[1:]) + + difference_from_event_map = union - intersection # type: Collection[str] + else: + difference_from_event_map = () + state_sets_ids = [set(state_set.values()) for state_set in state_sets] + + difference = await state_res_store.get_auth_chain_difference(state_sets_ids) + difference.update(difference_from_event_map) return difference diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index ad9bbef9d219..f5c6db900d9d 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -24,7 +24,11 @@ from synapse.api.room_versions import RoomVersions from synapse.event_auth import auth_types_for_event from synapse.events import make_event_from_dict -from synapse.state.v2 import lexicographical_topological_sort, resolve_events_with_store +from synapse.state.v2 import ( + _get_auth_chain_difference, + lexicographical_topological_sort, + resolve_events_with_store, +) from synapse.types import EventID from tests import unittest @@ -587,6 +591,128 @@ def test_event_map_none(self): self.assert_dict(self.expected_combined_state, state) +class AuthChainDifferenceTestCase(unittest.TestCase): + """We test that `_get_auth_chain_difference` correctly handles unpersisted + events. + """ + + def test_simple(self): + # Test getting the auth difference for a simple chain with a single + # unpersisted event: + # + # Unpersisted | Persisted + # | + # C -|-> B -> A + + a = FakeEvent( + id="A", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([], []) + + b = FakeEvent( + id="B", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([a.event_id], []) + + c = FakeEvent( + id="C", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([b.event_id], []) + + persisted_events = {a.event_id: a, b.event_id: b} + unpersited_events = {c.event_id: c} + + state_sets = [{"a": a.event_id, "b": b.event_id}, {"c": c.event_id}] + + store = TestStateResolutionStore(persisted_events) + + diff_d = _get_auth_chain_difference(state_sets, unpersited_events, store) + difference = self.successResultOf(defer.ensureDeferred(diff_d)) + + self.assertEqual(difference, {c.event_id}) + + def test_multiple_unpersisted_chain(self): + # Test getting the auth difference for a simple chain with multiple + # unpersisted events: + # + # Unpersisted | Persisted + # | + # D -> C -|-> B -> A + + a = FakeEvent( + id="A", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([], []) + + b = FakeEvent( + id="B", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([a.event_id], []) + + c = FakeEvent( + id="C", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([b.event_id], []) + + d = FakeEvent( + id="D", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([c.event_id], []) + + persisted_events = {a.event_id: a, b.event_id: b} + unpersited_events = {c.event_id: c, d.event_id: d} + + state_sets = [ + {"a": a.event_id, "b": b.event_id}, + {"c": c.event_id, "d": d.event_id}, + ] + + store = TestStateResolutionStore(persisted_events) + + diff_d = _get_auth_chain_difference(state_sets, unpersited_events, store) + difference = self.successResultOf(defer.ensureDeferred(diff_d)) + + self.assertEqual(difference, {d.event_id, c.event_id}) + + def test_unpersisted_events_different_sets(self): + # Test getting the auth difference for with multiple unpersisted events + # in different branches: + # + # Unpersisted | Persisted + # | + # D --> C -|-> B -> A + # E ----^ -|---^ + # | + + a = FakeEvent( + id="A", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([], []) + + b = FakeEvent( + id="B", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([a.event_id], []) + + c = FakeEvent( + id="C", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([b.event_id], []) + + d = FakeEvent( + id="D", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([c.event_id], []) + + e = FakeEvent( + id="E", sender=ALICE, type=EventTypes.Member, state_key="", content={}, + ).to_event([c.event_id, b.event_id], []) + + persisted_events = {a.event_id: a, b.event_id: b} + unpersited_events = {c.event_id: c, d.event_id: d, e.event_id: e} + + state_sets = [ + {"a": a.event_id, "b": b.event_id, "e": e.event_id}, + {"c": c.event_id, "d": d.event_id}, + ] + + store = TestStateResolutionStore(persisted_events) + + diff_d = _get_auth_chain_difference(state_sets, unpersited_events, store) + difference = self.successResultOf(defer.ensureDeferred(diff_d)) + + self.assertEqual(difference, {d.event_id, e.event_id}) + + def pairwise(iterable): "s -> (s0,s1), (s1,s2), (s2, s3), ..." a, b = itertools.tee(iterable) diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index d4c3b867e350..71c21d8c7500 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -216,6 +216,11 @@ def insert_event(txn, event_id, stream_ordering): ) self.assertSetEqual(difference, {"a", "b", "c"}) + difference = self.get_success( + self.store.get_auth_chain_difference([{"a", "c"}, {"b", "c"}]) + ) + self.assertSetEqual(difference, {"a", "b"}) + difference = self.get_success( self.store.get_auth_chain_difference([{"a"}, {"b"}, {"d"}]) ) From 30fba6210834a4ecd91badf0c8f3eb278b72e746 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 2 Dec 2020 11:09:24 -0500 Subject: [PATCH 7/7] Apply an IP range blacklist to push and key revocation requests. (#8821) Replaces the `federation_ip_range_blacklist` configuration setting with an `ip_range_blacklist` setting with wider scope. It now applies to: * Federation * Identity servers * Push notifications * Checking key validitity for third-party invite events The old `federation_ip_range_blacklist` setting is still honored if present, but with reduced scope (it only applies to federation and identity servers). --- changelog.d/8821.bugfix | 1 + docs/sample_config.yaml | 14 +++--- synapse/app/generic_worker.py | 1 - synapse/config/federation.py | 40 ++++++++++------ synapse/crypto/keyring.py | 4 +- synapse/federation/federation_server.py | 1 - synapse/federation/transport/client.py | 2 +- synapse/handlers/federation.py | 2 +- synapse/handlers/identity.py | 6 +-- synapse/http/client.py | 46 +++++++++++++------ .../federation/matrix_federation_agent.py | 16 +++++-- synapse/http/matrixfederationclient.py | 26 ++++------- synapse/push/httppusher.py | 2 +- synapse/rest/media/v1/media_repository.py | 2 +- synapse/server.py | 36 ++++++++++++--- tests/api/test_filtering.py | 4 +- tests/app/test_frontend_proxy.py | 2 +- tests/app/test_openid_listener.py | 4 +- tests/crypto/test_keyring.py | 6 ++- tests/handlers/test_device.py | 4 +- tests/handlers/test_directory.py | 2 +- tests/handlers/test_federation.py | 2 +- tests/handlers/test_presence.py | 2 +- tests/handlers/test_profile.py | 2 +- tests/handlers/test_typing.py | 6 +-- .../test_matrix_federation_agent.py | 3 ++ tests/push/test_http.py | 4 +- tests/replication/_base.py | 4 +- .../test_federation_sender_shard.py | 10 ++-- tests/replication/test_pusher_shard.py | 6 +-- tests/rest/admin/test_admin.py | 2 +- tests/rest/client/v1/test_presence.py | 2 +- tests/rest/client/v1/test_profile.py | 2 +- tests/rest/client/v1/test_rooms.py | 2 +- tests/rest/client/v1/test_typing.py | 2 +- tests/rest/key/v2/test_remote_key_resource.py | 4 +- tests/rest/media/v1/test_media_storage.py | 2 +- tests/storage/test_e2e_room_keys.py | 2 +- tests/storage/test_purge.py | 2 +- tests/storage/test_redaction.py | 2 +- tests/storage/test_roommember.py | 2 +- tests/test_federation.py | 2 +- tests/test_server.py | 5 +- 43 files changed, 176 insertions(+), 115 deletions(-) create mode 100644 changelog.d/8821.bugfix diff --git a/changelog.d/8821.bugfix b/changelog.d/8821.bugfix new file mode 100644 index 000000000000..8ddfbf31ced5 --- /dev/null +++ b/changelog.d/8821.bugfix @@ -0,0 +1 @@ +Apply the `federation_ip_range_blacklist` to push and key revocation requests. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 394eb9a3ffab..6dbccf593201 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -642,17 +642,19 @@ acme: # - nyc.example.com # - syd.example.com -# Prevent federation requests from being sent to the following -# blacklist IP address CIDR ranges. If this option is not specified, or -# specified with an empty list, no ip range blacklist will be enforced. +# Prevent outgoing requests from being sent to the following blacklisted IP address +# CIDR ranges. If this option is not specified, or specified with an empty list, +# no IP range blacklist will be enforced. # -# As of Synapse v1.4.0 this option also affects any outbound requests to identity -# servers provided by user input. +# The blacklist applies to the outbound requests for federation, identity servers, +# push servers, and for checking key validitity for third-party invite events. # # (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly # listed here, since they correspond to unroutable addresses.) # -federation_ip_range_blacklist: +# This option replaces federation_ip_range_blacklist in Synapse v1.24.0. +# +ip_range_blacklist: - '127.0.0.0/8' - '10.0.0.0/8' - '172.16.0.0/12' diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 1b511890aa36..aa12c74358a5 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -266,7 +266,6 @@ def __init__(self, hs): super().__init__(hs) self.hs = hs self.is_mine_id = hs.is_mine_id - self.http_client = hs.get_simple_http_client() self._presence_enabled = hs.config.use_presence diff --git a/synapse/config/federation.py b/synapse/config/federation.py index ffd8fca54e05..27ccf61c3c13 100644 --- a/synapse/config/federation.py +++ b/synapse/config/federation.py @@ -36,22 +36,30 @@ def read_config(self, config, **kwargs): for domain in federation_domain_whitelist: self.federation_domain_whitelist[domain] = True - self.federation_ip_range_blacklist = config.get( - "federation_ip_range_blacklist", [] - ) + ip_range_blacklist = config.get("ip_range_blacklist", []) # Attempt to create an IPSet from the given ranges try: - self.federation_ip_range_blacklist = IPSet( - self.federation_ip_range_blacklist - ) - - # Always blacklist 0.0.0.0, :: - self.federation_ip_range_blacklist.update(["0.0.0.0", "::"]) + self.ip_range_blacklist = IPSet(ip_range_blacklist) + except Exception as e: + raise ConfigError("Invalid range(s) provided in ip_range_blacklist: %s" % e) + # Always blacklist 0.0.0.0, :: + self.ip_range_blacklist.update(["0.0.0.0", "::"]) + + # The federation_ip_range_blacklist is used for backwards-compatibility + # and only applies to federation and identity servers. If it is not given, + # default to ip_range_blacklist. + federation_ip_range_blacklist = config.get( + "federation_ip_range_blacklist", ip_range_blacklist + ) + try: + self.federation_ip_range_blacklist = IPSet(federation_ip_range_blacklist) except Exception as e: raise ConfigError( "Invalid range(s) provided in federation_ip_range_blacklist: %s" % e ) + # Always blacklist 0.0.0.0, :: + self.federation_ip_range_blacklist.update(["0.0.0.0", "::"]) federation_metrics_domains = config.get("federation_metrics_domains") or [] validate_config( @@ -76,17 +84,19 @@ def generate_config_section(self, config_dir_path, server_name, **kwargs): # - nyc.example.com # - syd.example.com - # Prevent federation requests from being sent to the following - # blacklist IP address CIDR ranges. If this option is not specified, or - # specified with an empty list, no ip range blacklist will be enforced. + # Prevent outgoing requests from being sent to the following blacklisted IP address + # CIDR ranges. If this option is not specified, or specified with an empty list, + # no IP range blacklist will be enforced. # - # As of Synapse v1.4.0 this option also affects any outbound requests to identity - # servers provided by user input. + # The blacklist applies to the outbound requests for federation, identity servers, + # push servers, and for checking key validitity for third-party invite events. # # (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly # listed here, since they correspond to unroutable addresses.) # - federation_ip_range_blacklist: + # This option replaces federation_ip_range_blacklist in Synapse v1.24.0. + # + ip_range_blacklist: - '127.0.0.0/8' - '10.0.0.0/8' - '172.16.0.0/12' diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index c04ad77cf9ec..f23eacc0d708 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -578,7 +578,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher): def __init__(self, hs): super().__init__(hs) self.clock = hs.get_clock() - self.client = hs.get_http_client() + self.client = hs.get_federation_http_client() self.key_servers = self.config.key_servers async def get_keys(self, keys_to_fetch): @@ -748,7 +748,7 @@ class ServerKeyFetcher(BaseV2KeyFetcher): def __init__(self, hs): super().__init__(hs) self.clock = hs.get_clock() - self.client = hs.get_http_client() + self.client = hs.get_federation_http_client() async def get_keys(self, keys_to_fetch): """ diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 4b6ab470d059..35e345ce7077 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -845,7 +845,6 @@ class FederationHandlerRegistry: def __init__(self, hs: "HomeServer"): self.config = hs.config - self.http_client = hs.get_simple_http_client() self.clock = hs.get_clock() self._instance_name = hs.get_instance_name() diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 17a10f622eb9..abe9168c7866 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -35,7 +35,7 @@ class TransportLayerClient: def __init__(self, hs): self.server_name = hs.hostname - self.client = hs.get_http_client() + self.client = hs.get_federation_http_client() @log_function def get_room_state_ids(self, destination, room_id, event_id): diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index b9799090f784..df82e60b3360 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -140,7 +140,7 @@ def __init__(self, hs: "HomeServer"): self._message_handler = hs.get_message_handler() self._server_notices_mxid = hs.config.server_notices_mxid self.config = hs.config - self.http_client = hs.get_simple_http_client() + self.http_client = hs.get_proxied_blacklisted_http_client() self._instance_name = hs.get_instance_name() self._replication = hs.get_replication_data_handler() diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 9b3c6b4551eb..7301c2471068 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -46,13 +46,13 @@ class IdentityHandler(BaseHandler): def __init__(self, hs): super().__init__(hs) + # An HTTP client for contacting trusted URLs. self.http_client = SimpleHttpClient(hs) - # We create a blacklisting instance of SimpleHttpClient for contacting identity - # servers specified by clients + # An HTTP client for contacting identity servers specified by clients. self.blacklisting_http_client = SimpleHttpClient( hs, ip_blacklist=hs.config.federation_ip_range_blacklist ) - self.federation_http_client = hs.get_http_client() + self.federation_http_client = hs.get_federation_http_client() self.hs = hs async def threepid_from_creds( diff --git a/synapse/http/client.py b/synapse/http/client.py index e5b13593f24d..df7730078ff1 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -125,7 +125,7 @@ def _scheduler(x): return _scheduler -class IPBlacklistingResolver: +class _IPBlacklistingResolver: """ A proxy for reactor.nameResolver which only produces non-blacklisted IP addresses, preventing DNS rebinding attacks on URL preview. @@ -199,6 +199,35 @@ def resolutionComplete() -> None: return r +@implementer(IReactorPluggableNameResolver) +class BlacklistingReactorWrapper: + """ + A Reactor wrapper which will prevent DNS resolution to blacklisted IP + addresses, to prevent DNS rebinding. + """ + + def __init__( + self, + reactor: IReactorPluggableNameResolver, + ip_whitelist: Optional[IPSet], + ip_blacklist: IPSet, + ): + self._reactor = reactor + + # We need to use a DNS resolver which filters out blacklisted IP + # addresses, to prevent DNS rebinding. + self._nameResolver = _IPBlacklistingResolver( + self._reactor, ip_whitelist, ip_blacklist + ) + + def __getattr__(self, attr: str) -> Any: + # Passthrough to the real reactor except for the DNS resolver. + if attr == "nameResolver": + return self._nameResolver + else: + return getattr(self._reactor, attr) + + class BlacklistingAgentWrapper(Agent): """ An Agent wrapper which will prevent access to IP addresses being accessed @@ -292,22 +321,11 @@ def __init__( self.user_agent = self.user_agent.encode("ascii") if self._ip_blacklist: - real_reactor = hs.get_reactor() # If we have an IP blacklist, we need to use a DNS resolver which # filters out blacklisted IP addresses, to prevent DNS rebinding. - nameResolver = IPBlacklistingResolver( - real_reactor, self._ip_whitelist, self._ip_blacklist + self.reactor = BlacklistingReactorWrapper( + hs.get_reactor(), self._ip_whitelist, self._ip_blacklist ) - - @implementer(IReactorPluggableNameResolver) - class Reactor: - def __getattr__(_self, attr): - if attr == "nameResolver": - return nameResolver - else: - return getattr(real_reactor, attr) - - self.reactor = Reactor() else: self.reactor = hs.get_reactor() diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index e77f9587d047..3b756a7dc23d 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -16,7 +16,7 @@ import urllib.parse from typing import List, Optional -from netaddr import AddrFormatError, IPAddress +from netaddr import AddrFormatError, IPAddress, IPSet from zope.interface import implementer from twisted.internet import defer @@ -31,6 +31,7 @@ from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer from synapse.crypto.context_factory import FederationPolicyForHTTPS +from synapse.http.client import BlacklistingAgentWrapper from synapse.http.federation.srv_resolver import Server, SrvResolver from synapse.http.federation.well_known_resolver import WellKnownResolver from synapse.logging.context import make_deferred_yieldable, run_in_background @@ -70,6 +71,7 @@ def __init__( reactor: IReactorCore, tls_client_options_factory: Optional[FederationPolicyForHTTPS], user_agent: bytes, + ip_blacklist: IPSet, _srv_resolver: Optional[SrvResolver] = None, _well_known_resolver: Optional[WellKnownResolver] = None, ): @@ -90,12 +92,18 @@ def __init__( self.user_agent = user_agent if _well_known_resolver is None: + # Note that the name resolver has already been wrapped in a + # IPBlacklistingResolver by MatrixFederationHttpClient. _well_known_resolver = WellKnownResolver( self._reactor, - agent=Agent( + agent=BlacklistingAgentWrapper( + Agent( + self._reactor, + pool=self._pool, + contextFactory=tls_client_options_factory, + ), self._reactor, - pool=self._pool, - contextFactory=tls_client_options_factory, + ip_blacklist=ip_blacklist, ), user_agent=self.user_agent, ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 4e27f93b7ae4..c962994727f6 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -26,11 +26,10 @@ from canonicaljson import encode_canonical_json from prometheus_client import Counter from signedjson.sign import sign_json -from zope.interface import implementer from twisted.internet import defer from twisted.internet.error import DNSLookupError -from twisted.internet.interfaces import IReactorPluggableNameResolver, IReactorTime +from twisted.internet.interfaces import IReactorTime from twisted.internet.task import _EPSILON, Cooperator from twisted.web.http_headers import Headers from twisted.web.iweb import IBodyProducer, IResponse @@ -45,7 +44,7 @@ from synapse.http import QuieterFileBodyProducer from synapse.http.client import ( BlacklistingAgentWrapper, - IPBlacklistingResolver, + BlacklistingReactorWrapper, encode_query_args, readBodyToFile, ) @@ -221,31 +220,22 @@ def __init__(self, hs, tls_client_options_factory): self.signing_key = hs.signing_key self.server_name = hs.hostname - real_reactor = hs.get_reactor() - # We need to use a DNS resolver which filters out blacklisted IP # addresses, to prevent DNS rebinding. - nameResolver = IPBlacklistingResolver( - real_reactor, None, hs.config.federation_ip_range_blacklist + self.reactor = BlacklistingReactorWrapper( + hs.get_reactor(), None, hs.config.federation_ip_range_blacklist ) - @implementer(IReactorPluggableNameResolver) - class Reactor: - def __getattr__(_self, attr): - if attr == "nameResolver": - return nameResolver - else: - return getattr(real_reactor, attr) - - self.reactor = Reactor() - user_agent = hs.version_string if hs.config.user_agent_suffix: user_agent = "%s %s" % (user_agent, hs.config.user_agent_suffix) user_agent = user_agent.encode("ascii") self.agent = MatrixFederationAgent( - self.reactor, tls_client_options_factory, user_agent + self.reactor, + tls_client_options_factory, + user_agent, + hs.config.federation_ip_range_blacklist, ) # Use a BlacklistingAgentWrapper to prevent circumventing the IP diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index eff0975b6aed..0e845212a9a0 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -100,7 +100,7 @@ def __init__(self, hs, pusherdict): if "url" not in self.data: raise PusherConfigException("'url' required in data for HTTP pusher") self.url = self.data["url"] - self.http_client = hs.get_proxied_http_client() + self.http_client = hs.get_proxied_blacklisted_http_client() self.data_minus_url = {} self.data_minus_url.update(self.data) del self.data_minus_url["url"] diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 9cac74ebd87a..83beb02b0574 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -66,7 +66,7 @@ class MediaRepository: def __init__(self, hs): self.hs = hs self.auth = hs.get_auth() - self.client = hs.get_http_client() + self.client = hs.get_federation_http_client() self.clock = hs.get_clock() self.server_name = hs.hostname self.store = hs.get_datastore() diff --git a/synapse/server.py b/synapse/server.py index b017e3489faf..9af759626ef1 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -350,16 +350,45 @@ def get_http_client_context_factory(self) -> IPolicyForHTTPS: @cache_in_self def get_simple_http_client(self) -> SimpleHttpClient: + """ + An HTTP client with no special configuration. + """ return SimpleHttpClient(self) @cache_in_self def get_proxied_http_client(self) -> SimpleHttpClient: + """ + An HTTP client that uses configured HTTP(S) proxies. + """ + return SimpleHttpClient( + self, + http_proxy=os.getenvb(b"http_proxy"), + https_proxy=os.getenvb(b"HTTPS_PROXY"), + ) + + @cache_in_self + def get_proxied_blacklisted_http_client(self) -> SimpleHttpClient: + """ + An HTTP client that uses configured HTTP(S) proxies and blacklists IPs + based on the IP range blacklist. + """ return SimpleHttpClient( self, + ip_blacklist=self.config.ip_range_blacklist, http_proxy=os.getenvb(b"http_proxy"), https_proxy=os.getenvb(b"HTTPS_PROXY"), ) + @cache_in_self + def get_federation_http_client(self) -> MatrixFederationHttpClient: + """ + An HTTP client for federation. + """ + tls_client_options_factory = context_factory.FederationPolicyForHTTPS( + self.config + ) + return MatrixFederationHttpClient(self, tls_client_options_factory) + @cache_in_self def get_room_creation_handler(self) -> RoomCreationHandler: return RoomCreationHandler(self) @@ -514,13 +543,6 @@ def get_filtering(self) -> Filtering: def get_pusherpool(self) -> PusherPool: return PusherPool(self) - @cache_in_self - def get_http_client(self) -> MatrixFederationHttpClient: - tls_client_options_factory = context_factory.FederationPolicyForHTTPS( - self.config - ) - return MatrixFederationHttpClient(self, tls_client_options_factory) - @cache_in_self def get_media_repository_resource(self) -> MediaRepositoryResource: # build the media repo resource. This indirects through the HomeServer diff --git a/tests/api/test_filtering.py b/tests/api/test_filtering.py index c98ae759743d..bcf1a8010e7b 100644 --- a/tests/api/test_filtering.py +++ b/tests/api/test_filtering.py @@ -50,7 +50,9 @@ def setUp(self): self.mock_http_client.put_json = DeferredMockCallable() hs = yield setup_test_homeserver( - self.addCleanup, http_client=self.mock_http_client, keyring=Mock(), + self.addCleanup, + federation_http_client=self.mock_http_client, + keyring=Mock(), ) self.filtering = hs.get_filtering() diff --git a/tests/app/test_frontend_proxy.py b/tests/app/test_frontend_proxy.py index 40abe9d72d3f..43fef5d64a4f 100644 --- a/tests/app/test_frontend_proxy.py +++ b/tests/app/test_frontend_proxy.py @@ -23,7 +23,7 @@ class FrontendProxyTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( - http_client=None, homeserver_to_use=GenericWorkerServer + federation_http_client=None, homeserver_to_use=GenericWorkerServer ) return hs diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index ea3be95cf18f..b260ab734d7a 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -27,7 +27,7 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( - http_client=None, homeserver_to_use=GenericWorkerServer + federation_http_client=None, homeserver_to_use=GenericWorkerServer ) return hs @@ -84,7 +84,7 @@ def test_openid_listener(self, names, expectation): class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( - http_client=None, homeserver_to_use=SynapseHomeServer + federation_http_client=None, homeserver_to_use=SynapseHomeServer ) return hs diff --git a/tests/crypto/test_keyring.py b/tests/crypto/test_keyring.py index 697916a0194b..d146f2254fb1 100644 --- a/tests/crypto/test_keyring.py +++ b/tests/crypto/test_keyring.py @@ -315,7 +315,7 @@ async def get_keys2(keys_to_fetch): class ServerKeyFetcherTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): self.http_client = Mock() - hs = self.setup_test_homeserver(http_client=self.http_client) + hs = self.setup_test_homeserver(federation_http_client=self.http_client) return hs def test_get_keys_from_server(self): @@ -395,7 +395,9 @@ def make_homeserver(self, reactor, clock): } ] - return self.setup_test_homeserver(http_client=self.http_client, config=config) + return self.setup_test_homeserver( + federation_http_client=self.http_client, config=config + ) def build_perspectives_response( self, server_name: str, signing_key: SigningKey, valid_until_ts: int, diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 875aaec2c693..5dfeccfeb6e9 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -27,7 +27,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): - hs = self.setup_test_homeserver("server", http_client=None) + hs = self.setup_test_homeserver("server", federation_http_client=None) self.handler = hs.get_device_handler() self.store = hs.get_datastore() return hs @@ -229,7 +229,7 @@ def _record_user( class DehydrationTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): - hs = self.setup_test_homeserver("server", http_client=None) + hs = self.setup_test_homeserver("server", federation_http_client=None) self.handler = hs.get_device_handler() self.registration = hs.get_registration_handler() self.auth = hs.get_auth() diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index ee6ef5e6fa0d..2f1f2a55171f 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -42,7 +42,7 @@ def register_query_handler(query_type, handler): self.mock_registry.register_query_handler = register_query_handler hs = self.setup_test_homeserver( - http_client=None, + federation_http_client=None, resource_for_federation=Mock(), federation_client=self.mock_federation, federation_registry=self.mock_registry, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index bf866dacf3f2..d0452e1490cd 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -37,7 +37,7 @@ class FederationTestCase(unittest.HomeserverTestCase): ] def make_homeserver(self, reactor, clock): - hs = self.setup_test_homeserver(http_client=None) + hs = self.setup_test_homeserver(federation_http_client=None) self.handler = hs.get_federation_handler() self.store = hs.get_datastore() return hs diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 8ed67640f8e7..0794b32c9c69 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -463,7 +463,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( - "server", http_client=None, federation_sender=Mock() + "server", federation_http_client=None, federation_sender=Mock() ) return hs diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index a69fa28b4153..ea2bcf26557f 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -44,7 +44,7 @@ def register_query_handler(query_type, handler): hs = yield setup_test_homeserver( self.addCleanup, - http_client=None, + federation_http_client=None, resource_for_federation=Mock(), federation_client=self.mock_federation, federation_server=Mock(), diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index abbdf2d5249f..36086ca83694 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -70,7 +70,7 @@ def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( notifier=Mock(), - http_client=mock_federation_client, + federation_http_client=mock_federation_client, keyring=mock_keyring, replication_streams={}, ) @@ -192,7 +192,7 @@ def test_started_typing_remote_send(self): ) ) - put_json = self.hs.get_http_client().put_json + put_json = self.hs.get_federation_http_client().put_json put_json.assert_called_once_with( "farm", path="/_matrix/federation/v1/send/1000000", @@ -270,7 +270,7 @@ def test_stopped_typing(self): self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])]) - put_json = self.hs.get_http_client().put_json + put_json = self.hs.get_federation_http_client().put_json put_json.assert_called_once_with( "farm", path="/_matrix/federation/v1/send/1000000", diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 8b5ad4574f5c..626acdcaa333 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -17,6 +17,7 @@ from mock import Mock import treq +from netaddr import IPSet from service_identity import VerificationError from zope.interface import implementer @@ -103,6 +104,7 @@ def setUp(self): reactor=self.reactor, tls_client_options_factory=self.tls_factory, user_agent="test-agent", # Note that this is unused since _well_known_resolver is provided. + ip_blacklist=IPSet(), _srv_resolver=self.mock_resolver, _well_known_resolver=self.well_known_resolver, ) @@ -736,6 +738,7 @@ def test_get_well_known_unsigned_cert(self): reactor=self.reactor, tls_client_options_factory=tls_factory, user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. + ip_blacklist=IPSet(), _srv_resolver=self.mock_resolver, _well_known_resolver=WellKnownResolver( self.reactor, diff --git a/tests/push/test_http.py b/tests/push/test_http.py index f1184303090f..e8cea39c8374 100644 --- a/tests/push/test_http.py +++ b/tests/push/test_http.py @@ -49,7 +49,9 @@ def post_json_get_json(url, body): config = self.default_config() config["start_pushers"] = True - hs = self.setup_test_homeserver(config=config, proxied_http_client=m) + hs = self.setup_test_homeserver( + config=config, proxied_blacklisted_http_client=m + ) return hs diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 295c5d58a68d..728de282773d 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -67,7 +67,7 @@ def prepare(self, reactor, clock, hs): # Make a new HomeServer object for the worker self.reactor.lookups["testserv"] = "1.2.3.4" self.worker_hs = self.setup_test_homeserver( - http_client=None, + federation_http_client=None, homeserver_to_use=GenericWorkerServer, config=self._get_worker_hs_config(), reactor=self.reactor, @@ -264,7 +264,7 @@ def make_worker_hs( worker_app: Type of worker, e.g. `synapse.app.federation_sender`. extra_config: Any extra config to use for this instances. **kwargs: Options that get passed to `self.setup_test_homeserver`, - useful to e.g. pass some mocks for things like `http_client` + useful to e.g. pass some mocks for things like `federation_http_client` Returns: The new worker HomeServer instance. diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 779745ae9da3..fffdb742c8ea 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -50,7 +50,7 @@ def test_send_event_single_sender(self): self.make_worker_hs( "synapse.app.federation_sender", {"send_federation": True}, - http_client=mock_client, + federation_http_client=mock_client, ) user = self.register_user("user", "pass") @@ -81,7 +81,7 @@ def test_send_event_sharded(self): "worker_name": "sender1", "federation_sender_instances": ["sender1", "sender2"], }, - http_client=mock_client1, + federation_http_client=mock_client1, ) mock_client2 = Mock(spec=["put_json"]) @@ -93,7 +93,7 @@ def test_send_event_sharded(self): "worker_name": "sender2", "federation_sender_instances": ["sender1", "sender2"], }, - http_client=mock_client2, + federation_http_client=mock_client2, ) user = self.register_user("user2", "pass") @@ -144,7 +144,7 @@ def test_send_typing_sharded(self): "worker_name": "sender1", "federation_sender_instances": ["sender1", "sender2"], }, - http_client=mock_client1, + federation_http_client=mock_client1, ) mock_client2 = Mock(spec=["put_json"]) @@ -156,7 +156,7 @@ def test_send_typing_sharded(self): "worker_name": "sender2", "federation_sender_instances": ["sender1", "sender2"], }, - http_client=mock_client2, + federation_http_client=mock_client2, ) user = self.register_user("user3", "pass") diff --git a/tests/replication/test_pusher_shard.py b/tests/replication/test_pusher_shard.py index 67c27a089fab..f894bcd6e7f5 100644 --- a/tests/replication/test_pusher_shard.py +++ b/tests/replication/test_pusher_shard.py @@ -98,7 +98,7 @@ def test_send_push_single_worker(self): self.make_worker_hs( "synapse.app.pusher", {"start_pushers": True}, - proxied_http_client=http_client_mock, + proxied_blacklisted_http_client=http_client_mock, ) event_id = self._create_pusher_and_send_msg("user") @@ -133,7 +133,7 @@ def test_send_push_multiple_workers(self): "worker_name": "pusher1", "pusher_instances": ["pusher1", "pusher2"], }, - proxied_http_client=http_client_mock1, + proxied_blacklisted_http_client=http_client_mock1, ) http_client_mock2 = Mock(spec_set=["post_json_get_json"]) @@ -148,7 +148,7 @@ def test_send_push_multiple_workers(self): "worker_name": "pusher2", "pusher_instances": ["pusher1", "pusher2"], }, - proxied_http_client=http_client_mock2, + proxied_blacklisted_http_client=http_client_mock2, ) # We choose a user name that we know should go to pusher1. diff --git a/tests/rest/admin/test_admin.py b/tests/rest/admin/test_admin.py index 4f76f8f7682c..67d887839567 100644 --- a/tests/rest/admin/test_admin.py +++ b/tests/rest/admin/test_admin.py @@ -210,7 +210,7 @@ def write_to(r): } config["media_storage_providers"] = [provider_config] - hs = self.setup_test_homeserver(config=config, http_client=client) + hs = self.setup_test_homeserver(config=config, federation_http_client=client) return hs diff --git a/tests/rest/client/v1/test_presence.py b/tests/rest/client/v1/test_presence.py index 5d5c24d01c51..11cd8efe21d1 100644 --- a/tests/rest/client/v1/test_presence.py +++ b/tests/rest/client/v1/test_presence.py @@ -38,7 +38,7 @@ def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( "red", - http_client=None, + federation_http_client=None, federation_client=Mock(), presence_handler=presence_handler, ) diff --git a/tests/rest/client/v1/test_profile.py b/tests/rest/client/v1/test_profile.py index 383a9eafacc9..2a3b483eafd3 100644 --- a/tests/rest/client/v1/test_profile.py +++ b/tests/rest/client/v1/test_profile.py @@ -63,7 +63,7 @@ def setUp(self): hs = yield setup_test_homeserver( self.addCleanup, "test", - http_client=None, + federation_http_client=None, resource_for_client=self.mock_resource, federation=Mock(), federation_client=Mock(), diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 49f1073c8842..e67de41c180a 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -45,7 +45,7 @@ class RoomBase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): self.hs = self.setup_test_homeserver( - "red", http_client=None, federation_client=Mock(), + "red", federation_http_client=None, federation_client=Mock(), ) self.hs.get_federation_handler = Mock() diff --git a/tests/rest/client/v1/test_typing.py b/tests/rest/client/v1/test_typing.py index bbd30f594bff..ae0207366b4c 100644 --- a/tests/rest/client/v1/test_typing.py +++ b/tests/rest/client/v1/test_typing.py @@ -39,7 +39,7 @@ class RoomTypingTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( - "red", http_client=None, federation_client=Mock(), + "red", federation_http_client=None, federation_client=Mock(), ) self.event_source = hs.get_event_sources().sources["typing"] diff --git a/tests/rest/key/v2/test_remote_key_resource.py b/tests/rest/key/v2/test_remote_key_resource.py index fbcf8d5b8639..5e90d656f7c5 100644 --- a/tests/rest/key/v2/test_remote_key_resource.py +++ b/tests/rest/key/v2/test_remote_key_resource.py @@ -39,7 +39,7 @@ class BaseRemoteKeyResourceTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): self.http_client = Mock() - return self.setup_test_homeserver(http_client=self.http_client) + return self.setup_test_homeserver(federation_http_client=self.http_client) def create_test_resource(self): return create_resource_tree( @@ -172,7 +172,7 @@ def prepare(self, reactor, clock, homeserver): } ] self.hs2 = self.setup_test_homeserver( - http_client=self.http_client2, config=config + federation_http_client=self.http_client2, config=config ) # wire up outbound POST /key/v2/query requests from hs2 so that they diff --git a/tests/rest/media/v1/test_media_storage.py b/tests/rest/media/v1/test_media_storage.py index 2a3b2a8f27fa..4c749f1a61a7 100644 --- a/tests/rest/media/v1/test_media_storage.py +++ b/tests/rest/media/v1/test_media_storage.py @@ -214,7 +214,7 @@ def write_to(r): } config["media_storage_providers"] = [provider_config] - hs = self.setup_test_homeserver(config=config, http_client=client) + hs = self.setup_test_homeserver(config=config, federation_http_client=client) return hs diff --git a/tests/storage/test_e2e_room_keys.py b/tests/storage/test_e2e_room_keys.py index 35dafbb9047b..3d7760d5d9cf 100644 --- a/tests/storage/test_e2e_room_keys.py +++ b/tests/storage/test_e2e_room_keys.py @@ -26,7 +26,7 @@ class E2eRoomKeysHandlerTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): - hs = self.setup_test_homeserver("server", http_client=None) + hs = self.setup_test_homeserver("server", federation_http_client=None) self.store = hs.get_datastore() return hs diff --git a/tests/storage/test_purge.py b/tests/storage/test_purge.py index cc1f3c53c51c..a06ad2c03e9b 100644 --- a/tests/storage/test_purge.py +++ b/tests/storage/test_purge.py @@ -27,7 +27,7 @@ class PurgeTests(HomeserverTestCase): servlets = [room.register_servlets] def make_homeserver(self, reactor, clock): - hs = self.setup_test_homeserver("server", http_client=None) + hs = self.setup_test_homeserver("server", federation_http_client=None) return hs def prepare(self, reactor, clock, hs): diff --git a/tests/storage/test_redaction.py b/tests/storage/test_redaction.py index d4f9e809db36..fd0add5db3d8 100644 --- a/tests/storage/test_redaction.py +++ b/tests/storage/test_redaction.py @@ -34,7 +34,7 @@ def make_homeserver(self, reactor, clock): config = self.default_config() config["redaction_retention_period"] = "30d" return self.setup_test_homeserver( - resource_for_federation=Mock(), http_client=None, config=config + resource_for_federation=Mock(), federation_http_client=None, config=config ) def prepare(self, reactor, clock, hs): diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index ff972daeaa68..5ba1db233229 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -36,7 +36,7 @@ class RoomMemberStoreTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): hs = self.setup_test_homeserver( - resource_for_federation=Mock(), http_client=None + resource_for_federation=Mock(), federation_http_client=None ) return hs diff --git a/tests/test_federation.py b/tests/test_federation.py index 1ce4ea3a0142..fa45f8b3b7e9 100644 --- a/tests/test_federation.py +++ b/tests/test_federation.py @@ -37,7 +37,7 @@ def setUp(self): self.hs_clock = Clock(self.reactor) self.homeserver = setup_test_homeserver( self.addCleanup, - http_client=self.http_client, + federation_http_client=self.http_client, clock=self.hs_clock, reactor=self.reactor, ) diff --git a/tests/test_server.py b/tests/test_server.py index c387a85f2e41..6b2d2f0401fd 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -38,7 +38,10 @@ def setUp(self): self.reactor = ThreadedMemoryReactorClock() self.hs_clock = Clock(self.reactor) self.homeserver = setup_test_homeserver( - self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.reactor + self.addCleanup, + federation_http_client=None, + clock=self.hs_clock, + reactor=self.reactor, ) def test_handler_for_request(self):