Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Introduce update_user_directory_on #11450

Closed
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
9a3d3b6
shift to worker_to_update_user_directory
ShadowJonathan Nov 29, 2021
5f6bce7
linter
ShadowJonathan Nov 29, 2021
f24f151
news
ShadowJonathan Nov 29, 2021
0f05a6d
update nonsensical name in default_config() to be more inspired
ShadowJonathan Nov 29, 2021
300b51e
add updated sample config
ShadowJonathan Nov 29, 2021
28e255c
update sample config the sequel
ShadowJonathan Nov 29, 2021
47134d1
fix mypy issue
ShadowJonathan Nov 29, 2021
4d518bd
fix sytest
ShadowJonathan Nov 29, 2021
b67fe88
debug sytest failures
ShadowJonathan Nov 29, 2021
7ede21e
debug sytest failures mk 2
ShadowJonathan Nov 29, 2021
434d108
add backwards compatibility
ShadowJonathan Dec 6, 2021
f2490b3
update config with false behaviour
ShadowJonathan Dec 6, 2021
4593e15
whoops
ShadowJonathan Dec 6, 2021
db6625e
add special case for user directory workers
ShadowJonathan Dec 7, 2021
6f6b83b
the lint, it sees all, knows all
ShadowJonathan Dec 7, 2021
7808e86
apply review feedback
ShadowJonathan Dec 8, 2021
2d2c904
update name, move to workers.py
ShadowJonathan Dec 9, 2021
171020d
Merge branch 'develop' into fix-update-user-dir
ShadowJonathan Dec 9, 2021
ec9f035
i sort, you sort, we sort
ShadowJonathan Dec 9, 2021
99b6f60
apply review feedback
ShadowJonathan Jan 15, 2022
db78c2d
change when deprecation message shows up
ShadowJonathan Jan 15, 2022
f54146d
fix backwards compatibility case
ShadowJonathan Jan 15, 2022
cec3310
mighty linter
ShadowJonathan Jan 15, 2022
1e9b096
Apply review feedback
ShadowJonathan Jan 30, 2022
4eb9416
fix regression
ShadowJonathan Jan 30, 2022
dca6e22
apply review feedback
ShadowJonathan Feb 1, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions synapse/app/generic_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,11 @@

logger = logging.getLogger("synapse.app.generic_worker")

DIRECTORY_UPDATE_WARNING = """
The user directory worker is not allowed to perform user directory updates per the config.
Copy link
Member

@richvdh richvdh Jan 31, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

right, I see the confusion here now. "not allowed to" sounds like I've tried to enable something invalid. Maybe s/not allowed to/not configured to/.

But I don't really understand why this is a thing we need to warn about. Why is it a problem if the updates are done on the main process?

Note that the current wording sounds very much like an error, indicating something that definitely won't work.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe s/not allowed to/not configured to/.

👍

Why is it a problem if the updates are done on the main process?

That's not the problem, the problem is is that the configuration does not appoint the local worker to do the updates.

The config can now appoint any worker to do this job by its name, so the job of the user directory worker isn't that unique anymore, but it's uniqueness is still supported by synapse.

Note that the current wording sounds very much like an error

That's because it is, the local worker has a "type" of being a designated user update worker, while the configuration does not allow it to do just that. This is more or less a warning that "your config, as-is, is contradictory"

The previous version exited the process with 1 when hitting this point, should that be re-introduced?

Please add or edit ``update_user_directory_on: "{0}"`` in the config'
"""


class KeyUploadServlet(RestServlet):
"""An implementation of the `KeyUploadServlet` that responds to read only
Expand Down Expand Up @@ -459,12 +464,7 @@ def start(config_options: List[str]) -> None:

if config.worker.worker_app == "synapse.app.user_dir":
if not config.worker.update_user_directory:
sys.stderr.write(
"\nThe user directory worker is not allowed to perform user directory updates per the config."
"\nPlease add or edit "
f'``update_user_directory_on: "{config.worker.worker_name}"`` in the config\n'
)
sys.exit(1)
logger.warning(DIRECTORY_UPDATE_WARNING.format(config.worker.worker_name))
ShadowJonathan marked this conversation as resolved.
Show resolved Hide resolved

synapse.events.USE_FROZEN_DICTS = config.server.use_frozen_dicts
synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage
Expand Down
59 changes: 24 additions & 35 deletions synapse/config/workers.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

import argparse
import logging
from typing import List, Optional, Union
from typing import List, Union

import attr

Expand Down Expand Up @@ -44,10 +44,9 @@
"""

USER_UPDATE_DIRECTORY_DEPRECATION_WARNING = """
Synapse now uses 'update_user_directory_on' over 'update_user_directory',
you have set 'update_user_directory', and while synapse will work in a backwards
compatible manner, it is suggested to change this value to use
'update_user_directory_on' instead.
The 'update_user_directory' configuration setting is now deprecated,
and replaced with 'update_user_directory_on'. See the sample configuration
file for details of 'update_user_directory_on'.
"""


Expand Down Expand Up @@ -302,46 +301,36 @@ def read_config(self, config, **kwargs):
# No effort is made to ensure only a single instance of these tasks is
# running.
background_tasks_instance = config.get("run_background_tasks_on") or "master"
self.run_background_tasks = (
self.worker_name is None and background_tasks_instance == "master"
) or self.worker_name == background_tasks_instance
self.run_background_tasks = self.instance_name == background_tasks_instance
ShadowJonathan marked this conversation as resolved.
Show resolved Hide resolved

# update_user_directory_on controls which worker is responsible for updating the user directory.
# None means that the main process should handle this, so does the absence of the config key.
# None means that the main process should handle this, as does the absence of the config key.
#
# Note that due to backwards compatibility, we must also test for update_user_directory, and apply it if
# update_user_directory_on is not defined at the same time.
self.update_user_directory: bool

uud_result: bool
is_master_process = self.worker_name is None
if "update_user_directory" in config:
# Backwards compatibility case
logger.warning(USER_UPDATE_DIRECTORY_DEPRECATION_WARNING)

if "update_user_directory_on" in config:
update_user_directory_on: Optional[str] = config["update_user_directory_on"]

if update_user_directory_on is None:
uud_result = is_master_process
if config["update_user_directory"]:
# Main process should update
update_user_directory_on = "master"
else:
uud_result = self.worker_name == update_user_directory_on

# user directory worker should update
update_user_directory_on = (
self.instance_name
if self.worker_app == "synapse.app.user_dir"
else ""
)
else:
if "update_user_directory" in config:
# Backwards compatibility case
logger.warning(USER_UPDATE_DIRECTORY_DEPRECATION_WARNING)

update_user_directory: bool = config["update_user_directory"]

if update_user_directory:
# Main process should update
uud_result = is_master_process
else:
# user directory worker should update
uud_result = self.worker_app == "synapse.app.user_dir"
else:
# Both values are undefined, assume None for update_user_directory_on
uud_result = is_master_process
update_user_directory_on = (
config.get("update_user_directory_on") or "master"
)

self.update_user_directory = uud_result
self.update_user_directory: bool = (
ShadowJonathan marked this conversation as resolved.
Show resolved Hide resolved
self.instance_name == update_user_directory_on
)

def generate_config_section(self, config_dir_path, server_name, **kwargs):
return """\
Expand Down
1 change: 1 addition & 0 deletions tests/handlers/test_user_directory.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):

def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer:
config = self.default_config()
# Always process user directory updates on this main homeserver process.
ShadowJonathan marked this conversation as resolved.
Show resolved Hide resolved
config["update_user_directory_on"] = None
ShadowJonathan marked this conversation as resolved.
Show resolved Hide resolved

self.appservice = ApplicationService(
Expand Down
2 changes: 1 addition & 1 deletion tests/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def default_config(name, parse=False):
"default_room_version": DEFAULT_ROOM_VERSION,
# disable user directory updates, because they get done in the
# background, which upsets the test runner.
"update_user_directory_on": "",
"update_user_directory_on": "(non-existent worker)",
"caches": {"global_factor": 1},
"listeners": [{"port": 0, "type": "http"}],
}
Expand Down