Add metrics for sliding sync processing time (#17593)

This should let us see how quickly we actually process things in
practice.

changelog.d/17593.misc

@@ -0,0 +1 @@
+Add histogram metrics for sliding sync processing time.

synapse/handlers/sliding_sync.py

@@ -45,6 +45,7 @@
 
 import attr
 from immutabledict import immutabledict
+from prometheus_client import Histogram
 from typing_extensions import assert_never
 
 from synapse.api.constants import (
@@ -104,6 +105,13 @@
 logger = logging.getLogger(__name__)
 
 
+sync_processing_time = Histogram(
+    "synapse_sliding_sync_processing_time",
+    "Time taken to generate a sliding sync response, ignoring wait times.",
+    ["initial"],
+)
+
+
 class Sentinel(enum.Enum):
     # defining a sentinel in this way allows mypy to correctly handle the
     # type of a dictionary lookup and subsequent type narrowing.
@@ -571,6 +579,8 @@ async def current_sync_for_user(
             from_token: The point in the stream to sync from. Token of the end of the
                 previous batch. May be `None` if this is the initial sync request.
         """
+        start_time_s = self.clock.time()
+
         user_id = sync_config.user.to_string()
         app_service = self.store.get_app_service_by_user_id(user_id)
         if app_service:
@@ -934,6 +944,11 @@ async def handle_room(room_id: str) -> None:
         set_tag(SynapseTags.RESULT_PREFIX + "result", bool(sliding_sync_result))
         set_tag(SynapseTags.FUNC_ARG_PREFIX + "sync_config.user", user_id)
 
+        end_time_s = self.clock.time()
+        sync_processing_time.labels(from_token is not None).observe(
+            end_time_s - start_time_s
+        )
+
         return sliding_sync_result
 
     @trace