Support UDS listen address for tokio-console

doc/developer/guide-tokio-console.md

@@ -1,32 +1,45 @@
 # Developer guide: `tokio-console`
 
-This guide details how to run `tokio-console` with `materialized`
+This guide details how to run `tokio-console` with Materialize.
 
 ## Overview
 
-First, install `tokio-console`:
+First, install `tokio-console`. We require support for Unix domain sockets, [which is still pending
+upstream][uds-pr], so we need to install from a fork for now.
 
-```
-cargo install tokio-console
+```text
+cargo install tokio-console --git https://github.com/MaterializeInc/console.git
 ```
 
 Then run `environmentd`:
 
-```
+```text
 ./bin/environmentd --tokio-console
 ```
 
-(note that this may slow down `environmentd` a lot, as it increases the amount of tracing by a lot,
-and may inadvertently turn on debug logging for `rdkafka`)
+(Note that this may slow down `environmentd` a lot, as it increases the amount of tracing by a lot,
+and may inadvertently turn on debug logging for `rdkafka`.)
 
-Then, in a different tmux pane/terminal, run:
+In the output of the above command, take note of the `tokio-console` listen addresses for the
+different processes, e.g.:
 
+```text
+environmentd: [...]  INFO mz_ore::tracing: starting tokio console on http://127.0.0.1:6669
+compute-cluster-u1-replica-1: [...]  INFO mz_ore::tracing: starting tokio console on /var/folders/30/[...]3ee9
+compute-cluster-s1-replica-2: [...]  INFO mz_ore::tracing: starting tokio console on /var/folders/30/[...]f5b6
+compute-cluster-s2-replica-3: [...]  INFO mz_ore::tracing: starting tokio console on /var/folders/30/[...]7af2
 ```
-tokio-console
+
+Then, in a different tmux pane/terminal, run the `tokio-console` command with the listen address of
+your process of interest:
+
+```text
+tokio-console <listen-address>
 ```
 
 This [README] has some docs on how to navigate the ui.
 
+[uds-pr]: https://github.com/tokio-rs/console/pull/388
 [README]: https://github.com/tokio-rs/console/tree/main/tokio-console
 
 ### Notes on compilation times:

src/orchestrator-tracing/src/lib.rs

@@ -12,8 +12,6 @@
 use std::collections::HashMap;
 use std::ffi::OsString;
 use std::fmt;
-#[cfg(feature = "tokio-console")]
-use std::net::SocketAddr;
 use std::str::FromStr;
 use std::sync::Arc;
 #[cfg(feature = "tokio-console")]
@@ -35,6 +33,8 @@ use mz_orchestrator::{
 };
 use mz_ore::cli::{DefaultTrue, KeyValueArg};
 #[cfg(feature = "tokio-console")]
+use mz_ore::netio::SocketAddr;
+#[cfg(feature = "tokio-console")]
 use mz_ore::tracing::TokioConsoleConfig;
 use mz_ore::tracing::{
     OpenTelemetryConfig, SentryConfig, StderrLogConfig, StderrLogFormat, TracingConfig,
@@ -227,13 +227,13 @@ impl From<&TracingCliArgs> for TracingConfig {
                 }
             }),
             #[cfg(feature = "tokio-console")]
-            tokio_console: args
-                .tokio_console_listen_addr
-                .map(|listen_addr| TokioConsoleConfig {
+            tokio_console: args.tokio_console_listen_addr.clone().map(|listen_addr| {
+                TokioConsoleConfig {
                     listen_addr,
                     publish_interval: args.tokio_console_publish_interval,
                     retention: args.tokio_console_retention,
-                }),
+                }
+            }),
             sentry: args.sentry_dsn.clone().map(|dsn| SentryConfig {
                 dsn,
                 tags: args

src/ore/Cargo.toml

@@ -56,7 +56,7 @@ hyper = { version = "0.14.23", features = ["http1", "server"], optional = true }
 hyper-tls = { version = "0.5.0", optional = true }
 opentelemetry = { git = "https://github.com/MaterializeInc/opentelemetry-rust.git", features = ["rt-tokio", "trace"], optional = true }
 opentelemetry-otlp = { git = "https://github.com/MaterializeInc/opentelemetry-rust.git", optional = true }
-console-subscriber = { version = "0.1.8", optional = true }
+console-subscriber = { git = "https://github.com/MaterializeInc/console.git", optional = true }
 sentry-tracing = { version = "0.29.0", optional = true }
 
 [dev-dependencies]

src/ore/src/tracing/mod.rs

@@ -22,8 +22,6 @@
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::io;
-#[cfg(feature = "tokio-console")]
-use std::net::SocketAddr;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -50,6 +48,9 @@ use tracing_subscriber::registry::LookupSpan;
 use tracing_subscriber::util::SubscriberInitExt;
 use tracing_subscriber::{reload, Registry};
 
+#[cfg(feature = "tokio-console")]
+use crate::netio::SocketAddr;
+
 /// Application tracing configuration.
 ///
 /// See the [`configure`] function for details.
@@ -330,12 +331,17 @@ where
 
     #[cfg(feature = "tokio-console")]
     let tokio_console_layer = if let Some(console_config) = config.tokio_console.clone() {
-        let layer = ConsoleLayer::builder()
-            .server_addr(console_config.listen_addr)
+        let builder = ConsoleLayer::builder()
             .publish_interval(console_config.publish_interval)
-            .retention(console_config.retention)
-            .spawn();
-        Some(layer)
+            .retention(console_config.retention);
+        let builder = match console_config.listen_addr {
+            SocketAddr::Inet(addr) => builder.server_addr(addr),
+            SocketAddr::Unix(addr) => {
+                let path = addr.as_pathname().unwrap().as_ref();
+                builder.server_addr(path)
+            }
+        };
+        Some(builder.spawn())
     } else {
         None
     };
@@ -373,10 +379,11 @@ where
 
     #[cfg(feature = "tokio-console")]
     if let Some(console_config) = config.tokio_console {
-        tracing::info!(
-            "starting tokio console on http://{}",
-            console_config.listen_addr
-        );
+        let endpoint = match console_config.listen_addr {
+            SocketAddr::Inet(addr) => format!("http://{addr}"),
+            SocketAddr::Unix(addr) => format!("file://localhost{addr}"),
+        };
+        tracing::info!("starting tokio console on {endpoint}");
     }
 
     Ok((