Increase timeout of intproxy connection to a ludicrous amount (avoids intproxy issues when debugging with breakpoints).

changelog.d/2652.changed.md

@@ -0,0 +1 @@
+Increase timeout of layer-intproxy socket connection to a ludicrous amount.

mirrord-schema.json

@@ -1301,6 +1301,15 @@
             "null"
           ]
         },
+        "socket_timeout": {
+          "description": "<!--${internal}-->\n\nSometimes the cpu is too busy with other tasks and the internal proxy sockets end up timing out. It's set at a ridiculous high value to prevent this from happening when a user hits a breakpoint while debugging, and stays stopped for a while, which sometimes results in mirrord not working when they resume.\n\n```json { \"internal_proxy\": { \"socket_timeout\": 31536000 } } ```",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
+        },
         "start_idle_timeout": {
           "title": "internal_proxy.start_idle_timeout {#internal_proxy-start_idle_timeout}",
           "description": "How much time to wait for the first connection to the proxy in seconds.\n\nCommon cases would be running with dlv or any other debugger, which sets a breakpoint on process execution, delaying the layer startup and connection to proxy.\n\n```json { \"internal_proxy\": { \"start_idle_timeout\": 60 } } ```",

mirrord/config/src/internal_proxy.rs

@@ -84,6 +84,23 @@ pub struct InternalProxyConfig {
     #[config(default = 5)]
     pub idle_timeout: u64,
 
+    /// <!--${internal}-->
+    ///
+    /// Sometimes the cpu is too busy with other tasks and the internal proxy sockets end
+    /// up timing out. It's set at a ridiculous high value to prevent this from happening
+    /// when a user hits a breakpoint while debugging, and stays stopped for a while, which
+    /// sometimes results in mirrord not working when they resume.
+    ///
+    /// ```json
+    /// {
+    ///   "internal_proxy": {
+    ///     "socket_timeout": 31536000
+    ///   }
+    /// }
+    /// ```
+    #[config(default = 31536000)]
+    pub socket_timeout: u64,
+
     /// ### internal_proxy.log_level {#internal_proxy-log_level}
     ///
     /// Set the log level for the internal proxy.

mirrord/layer/src/lib.rs

@@ -162,9 +162,9 @@ static EXECUTABLE_ARGS: OnceLock<ExecuteArgs> = OnceLock::new();
 /// Executable path we're loaded to
 static EXECUTABLE_PATH: OnceLock<String> = OnceLock::new();
 
-/// Proxy Connection timeout
-/// Set to 10 seconds as most agent operations timeout after 5 seconds
-const PROXY_CONNECTION_TIMEOUT: Duration = Duration::from_secs(10);
+/// Read/write timeout for layer<->intproxy TCP sockets.
+/// Can be configured in the [`LayerConfig`].
+static PROXY_CONNECTION_TIMEOUT: OnceLock<Duration> = OnceLock::new();
 
 /// Loads mirrord configuration and does some patching (SIP, dotnet, etc)
 fn layer_pre_initialization() -> Result<(), LayerError> {
@@ -240,7 +240,8 @@ fn load_only_layer_start(config: &LayerConfig) {
                 .expect("EXECUTABLE_ARGS MUST BE SET")
                 .to_process_info(config),
         ),
-        PROXY_CONNECTION_TIMEOUT,
+        *PROXY_CONNECTION_TIMEOUT
+            .get_or_init(|| Duration::from_secs(config.internal_proxy.socket_timeout)),
     )
     .expect("failed to initialize proxy connection");
 
@@ -337,6 +338,9 @@ fn layer_start(mut config: LayerConfig) {
 
     init_tracing();
 
+    let proxy_connection_timeout = *PROXY_CONNECTION_TIMEOUT
+        .get_or_init(|| Duration::from_secs(config.internal_proxy.socket_timeout));
+
     let debugger_ports = DebuggerPorts::from_env();
     let local_hostname = trace_only || !config.feature.hostname;
     let process_info = EXECUTABLE_ARGS
@@ -370,7 +374,7 @@ fn layer_start(mut config: LayerConfig) {
         let new_connection = ProxyConnection::new(
             address,
             NewSessionRequest::New(process_info),
-            PROXY_CONNECTION_TIMEOUT,
+            proxy_connection_timeout,
         )
         .unwrap_or_else(|_| panic!("failed to initialize proxy connection at {address}"));
         PROXY_CONNECTION
@@ -626,7 +630,10 @@ pub(crate) unsafe extern "C" fn fork_detour() -> pid_t {
             let new_connection = ProxyConnection::new(
                 parent_connection.proxy_addr(),
                 NewSessionRequest::Forked(parent_connection.layer_id()),
-                PROXY_CONNECTION_TIMEOUT,
+                PROXY_CONNECTION_TIMEOUT
+                    .get()
+                    .copied()
+                    .expect("PROXY_CONNECTION_TIMEOUT should be set by now!"),
             )
             .expect("failed to establish proxy connection for child");
             PROXY_CONNECTION