From 243104855067542941ceefe2c4e9e8443da21dbf Mon Sep 17 00:00:00 2001 From: Ivan Kozlovic Date: Fri, 1 May 2020 15:22:52 -0600 Subject: [PATCH 1/2] [FIXED] Added jitter in the reconnect logic -Add ReconnectJitter option setter allowing user to set jitter for non and TLS connections (possibly different values) -Default for non-TLS will be 100ms and TLS 1sec -Add a CustomReconnectDelay handler that is passed the number of full url lists attempts. The user callback will then return a duration that is going to be used for the wait. It is the responsibility of the user to send a delay with some jitter if desired. Resolves #563 Signed-off-by: Ivan Kozlovic --- nats.go | 164 ++++++++++++++++++++++++++++++----------- nats_test.go | 126 ++++++++++++++++++++++++++++++- test/auth_test.go | 2 + test/basic_test.go | 1 + test/cluster_test.go | 7 ++ test/conn_test.go | 7 +- test/reconnect_test.go | 13 ++++ test/sub_test.go | 2 + 8 files changed, 278 insertions(+), 44 deletions(-) diff --git a/nats.go b/nats.go index 687479d60..b004d23c6 100644 --- a/nats.go +++ b/nats.go @@ -45,19 +45,21 @@ import ( // Default Constants const ( - Version = "1.9.2" - DefaultURL = "nats://127.0.0.1:4222" - DefaultPort = 4222 - DefaultMaxReconnect = 60 - DefaultReconnectWait = 2 * time.Second - DefaultTimeout = 2 * time.Second - DefaultPingInterval = 2 * time.Minute - DefaultMaxPingOut = 2 - DefaultMaxChanLen = 8192 // 8k - DefaultReconnectBufSize = 8 * 1024 * 1024 // 8MB - RequestChanLen = 8 - DefaultDrainTimeout = 30 * time.Second - LangString = "go" + Version = "1.9.3" + DefaultURL = "nats://127.0.0.1:4222" + DefaultPort = 4222 + DefaultMaxReconnect = 60 + DefaultReconnectWait = 2 * time.Second + DefaultReconnectJitter = 100 * time.Millisecond + DefaultReconnectJitterTLS = time.Second + DefaultTimeout = 2 * time.Second + DefaultPingInterval = 2 * time.Minute + DefaultMaxPingOut = 2 + DefaultMaxChanLen = 8192 // 8k + DefaultReconnectBufSize = 8 * 1024 * 1024 // 8MB + RequestChanLen = 8 + DefaultDrainTimeout = 30 * time.Second + LangString = "go" ) const ( @@ -127,15 +129,17 @@ func init() { // GetDefaultOptions returns default configuration options for the client. func GetDefaultOptions() Options { return Options{ - AllowReconnect: true, - MaxReconnect: DefaultMaxReconnect, - ReconnectWait: DefaultReconnectWait, - Timeout: DefaultTimeout, - PingInterval: DefaultPingInterval, - MaxPingsOut: DefaultMaxPingOut, - SubChanLen: DefaultMaxChanLen, - ReconnectBufSize: DefaultReconnectBufSize, - DrainTimeout: DefaultDrainTimeout, + AllowReconnect: true, + MaxReconnect: DefaultMaxReconnect, + ReconnectWait: DefaultReconnectWait, + ReconnectJitter: DefaultReconnectJitter, + ReconnectJitterTLS: DefaultReconnectJitterTLS, + Timeout: DefaultTimeout, + PingInterval: DefaultPingInterval, + MaxPingsOut: DefaultMaxPingOut, + SubChanLen: DefaultMaxChanLen, + ReconnectBufSize: DefaultReconnectBufSize, + DrainTimeout: DefaultDrainTimeout, } } @@ -182,6 +186,12 @@ type SignatureHandler func([]byte) ([]byte, error) // AuthTokenHandler is used to generate a new token. type AuthTokenHandler func() string +// ReconnectDelayHandler is used to get from the user the desired +// delay the library should pause before attempting to reconnect +// again. Note that this is invoked after the library tried the +// whole list of URLs and failed to reconnect. +type ReconnectDelayHandler func(attempts int) time.Duration + // asyncCB is used to preserve order for async callbacks. type asyncCB struct { f func() @@ -258,6 +268,24 @@ type Options struct { // to a server that we were already connected to previously. ReconnectWait time.Duration + // CustomReconnectDelayCB is invoked after the library tried every + // URL in the server list and failed to reconnect. It passes to the + // user the current number of attempts. This function returns the + // amount of time the library will sleep before attempting to reconnect + // again. It is strongly recommended that this value contains some + // jitter to prevent all connections to attempt reconnecting at the same time. + CustomReconnectDelayCB ReconnectDelayHandler + + // ReconnectJitter sets the upper bound for a random delay added to + // ReconnectWait during a reconnect when no TLS is used. + // Note that any jitter is capped with ReconnectJitterMax. + ReconnectJitter time.Duration + + // ReconnectJitterTLS sets the upper bound for a random delay added to + // ReconnectWait during a reconnect when TLS is used. + // Note that any jitter is capped with ReconnectJitterMax. + ReconnectJitterTLS time.Duration + // Timeout sets the timeout for a Dial operation on a connection. Timeout time.Duration @@ -411,6 +439,7 @@ type Conn struct { ptmr *time.Timer pout int ar bool // abort reconnect + rqch chan struct{} // New style response handler respSub string // The wildcard subject @@ -672,6 +701,24 @@ func MaxReconnects(max int) Option { } } +// ReconnectJitter is an Option to set the upper bound of a random delay added ReconnectWait. +func ReconnectJitter(jitter, jitterForTLS time.Duration) Option { + return func(o *Options) error { + o.ReconnectJitter = jitter + o.ReconnectJitterTLS = jitterForTLS + return nil + } +} + +// CustomReconnectDelay is an Option to set the CustomReconnectDelayCB option. +// See CustomReconnectDelayCB Option for more details. +func CustomReconnectDelay(cb ReconnectDelayHandler) Option { + return func(o *Options) error { + o.CustomReconnectDelayCB = cb + return nil + } +} + // PingInterval is an Option to set the period for client ping commands. func PingInterval(t time.Duration) Option { return func(o *Options) error { @@ -1396,6 +1443,7 @@ func (nc *Conn) setup() { nc.pongs = make([]chan struct{}, 0, 8) nc.fch = make(chan struct{}, flushChanSize) + nc.rqch = make(chan struct{}) // Setup scratch outbound buffer for PUB pub := nc.scratch[:len(_PUB_P_)] @@ -1818,33 +1866,63 @@ func (nc *Conn) doReconnect(err error) { // This is used to wait on go routines exit if we start them in the loop // but an error occurs after that. waitForGoRoutines := false + var rt *time.Timer + // Channel used to kick routine out of sleep when conn is closed. + rqch := nc.rqch + // Counter that is increased when the whole list of servers has been tried. + var wlf int + + var jitter time.Duration + var rw time.Duration + // If a custom reconnect delay handler is set, this takes precedence. + crd := nc.Opts.CustomReconnectDelayCB + if crd == nil { + rw = nc.Opts.ReconnectWait + // TODO: since we sleep only after the whole list has been tried, we can't + // rely on individual *srv to know if it is a TLS or non-TLS url. + // We have to pick which type of jitter to use, for now, we use these hints: + jitter = nc.Opts.ReconnectJitter + if nc.Opts.Secure || nc.Opts.TLSConfig != nil { + jitter = nc.Opts.ReconnectJitterTLS + } + } - for len(nc.srvPool) > 0 { + for i := 0; len(nc.srvPool) > 0; { cur, err := nc.selectNextServer() if err != nil { nc.err = err break } - sleepTime := int64(0) - - // Sleep appropriate amount of time before the - // connection attempt if connecting to same server - // we just got disconnected from.. - if time.Since(cur.lastAttempt) < nc.Opts.ReconnectWait { - sleepTime = int64(nc.Opts.ReconnectWait - time.Since(cur.lastAttempt)) - } - - // On Windows, createConn() will take more than a second when no - // server is running at that address. So it could be that the - // time elapsed between reconnect attempts is always > than - // the set option. Release the lock to give a chance to a parallel - // nc.Close() to break the loop. + doSleep := i+1 >= len(nc.srvPool) nc.mu.Unlock() - if sleepTime <= 0 { + + if !doSleep { + i++ + // Release the lock to give a chance to a concurrent nc.Close() to break the loop. runtime.Gosched() } else { - time.Sleep(time.Duration(sleepTime)) + i = 0 + var st time.Duration + if crd != nil { + wlf++ + st = crd(wlf) + } else { + st = rw + if jitter > 0 { + st += time.Duration(rand.Int63n(int64(jitter))) + } + } + if rt == nil { + rt = time.NewTimer(st) + } else { + rt.Reset(st) + } + select { + case <-rqch: + rt.Stop() + case <-rt.C: + } } // If the readLoop, etc.. go routines were started, wait for them to complete. if waitForGoRoutines { @@ -3655,9 +3733,13 @@ func (nc *Conn) close(status Status, doCBs bool, err error) { // Kick the Go routines so they fall out. nc.kickFlusher() - nc.mu.Unlock() - nc.mu.Lock() + // If the reconnect timer is waiting between a reconnect attempt, + // this will kick it out. + if nc.rqch != nil { + close(nc.rqch) + nc.rqch = nil + } // Clear any queued pongs, e.g. pending flush calls. nc.clearPendingFlushCalls() diff --git a/nats_test.go b/nats_test.go index bb1c653b5..4cea945e1 100644 --- a/nats_test.go +++ b/nats_test.go @@ -1502,6 +1502,7 @@ func TestExpiredUserCredentials(t *testing.T) { url := fmt.Sprintf("nats://127.0.0.1:%d", addr.Port) nc, err := Connect(url, ReconnectWait(25*time.Millisecond), + ReconnectJitter(0, 0), MaxReconnects(-1), ErrorHandler(func(_ *Conn, _ *Subscription, e error) { select { @@ -1583,6 +1584,7 @@ func TestExpiredUserCredentialsRenewal(t *testing.T) { nc, err := Connect(url, UserCredentials(chainedFile), ReconnectWait(25*time.Millisecond), + ReconnectJitter(0, 0), MaxReconnects(2), ReconnectHandler(func(nc *Conn) { rch <- true @@ -1817,7 +1819,7 @@ func TestNKeyOptionFromSeed(t *testing.T) { // Read connect and ping commands sent from the client br := bufio.NewReaderSize(conn, 10*1024) - line, _, _ := br.ReadLine() + line, _, err := br.ReadLine() if err != nil { errCh <- fmt.Errorf("expected CONNECT and PING from client, got: %s", err) return @@ -2075,6 +2077,7 @@ func TestAuthErrorOnReconnect(t *testing.T) { urls := fmt.Sprintf("nats://%s:%d, nats://%s:%d", o1.Host, o1.Port, o2.Host, o2.Port) nc, err := Connect(urls, ReconnectWait(25*time.Millisecond), + ReconnectJitter(0, 0), MaxReconnects(-1), DontRandomize(), DisconnectErrHandler(func(_ *Conn, e error) { @@ -2269,7 +2272,7 @@ func TestGetRTT(t *testing.T) { s := RunServerOnPort(-1) defer s.Shutdown() - nc, err := Connect(s.ClientURL(), ReconnectWait(10*time.Millisecond)) + nc, err := Connect(s.ClientURL(), ReconnectWait(10*time.Millisecond), ReconnectJitter(0, 0)) if err != nil { t.Fatalf("Expected to connect to server, got %v", err) } @@ -2389,3 +2392,122 @@ func TestNoPanicOnSrvPoolSizeChanging(t *testing.T) { } wg.Wait() } + +func TestReconnectWaitJitter(t *testing.T) { + s := RunServerOnPort(TEST_PORT) + defer s.Shutdown() + + rch := make(chan time.Time, 1) + nc, err := Connect(s.ClientURL(), + ReconnectWait(100*time.Millisecond), + ReconnectJitter(500*time.Millisecond, 0), + ReconnectHandler(func(_ *Conn) { + rch <- time.Now() + }), + ) + if err != nil { + t.Fatalf("Error during connect: %v", err) + } + defer nc.Close() + + s.Shutdown() + start := time.Now() + // Wait a bit so that the library tries a first time without waiting. + time.Sleep(50 * time.Millisecond) + s = RunServerOnPort(TEST_PORT) + defer s.Shutdown() + select { + case end := <-rch: + dur := end.Sub(start) + // We should wait at least the reconnect wait + random up to 500ms. + // Account for a bit of variation since we rely on the reconnect + // handler which is not invoked in place. + if dur < 90*time.Millisecond || dur > 800*time.Millisecond { + t.Fatalf("Wrong wait: %v", dur) + } + case <-time.After(5 * time.Second): + t.Fatalf("Should have reconnected") + } + nc.Close() + + // Use a long reconnect wait + nc, err = Connect(s.ClientURL(), ReconnectWait(10*time.Minute)) + if err != nil { + t.Fatalf("Error during connect: %v", err) + } + defer nc.Close() + + // Cause a disconnect + s.Shutdown() + // Wait a bit for the reconnect loop to go into wait mode. + time.Sleep(50 * time.Millisecond) + s = RunServerOnPort(TEST_PORT) + defer s.Shutdown() + // Now close and expect the reconnect go routine to return.. + nc.Close() + // Wait a bit to give a chance for the go routine to exit. + time.Sleep(50 * time.Millisecond) + buf := make([]byte, 100000) + n := runtime.Stack(buf, true) + if strings.Contains(string(buf[:n]), "doReconnect") { + t.Fatalf("doReconnect go routine still running:\n%s", buf[:n]) + } +} + +func TestCustomReconnectDelay(t *testing.T) { + s := RunServerOnPort(TEST_PORT) + defer s.Shutdown() + + expectedAttempt := 1 + errCh := make(chan error, 1) + cCh := make(chan bool, 1) + nc, err := Connect(s.ClientURL(), + CustomReconnectDelay(func(n int) time.Duration { + var err error + var delay time.Duration + if n != expectedAttempt { + err = fmt.Errorf("Expected attempt to be %v, got %v", expectedAttempt, n) + } else { + expectedAttempt++ + if n <= 4 { + delay = 100 * time.Millisecond + } + } + if err != nil { + select { + case errCh <- err: + default: + } + } + return delay + }), + MaxReconnects(4), + ClosedHandler(func(_ *Conn) { + cCh <- true + }), + ) + if err != nil { + t.Fatalf("Error during connect: %v", err) + } + defer nc.Close() + + // Cause disconnect + s.Shutdown() + + // We should be trying to reconnect 4 times + start := time.Now() + + // Wait on error or completion of test. + select { + case e := <-errCh: + if e != nil { + t.Fatal(e.Error()) + } + case <-cCh: + case <-time.After(2 * time.Second): + t.Fatalf("No CB invoked") + } + if dur := time.Since(start); dur >= 500*time.Millisecond { + t.Fatalf("Waited too long on each reconnect: %v", dur) + } +} diff --git a/test/auth_test.go b/test/auth_test.go index 4c2f4deac..8371a7549 100644 --- a/test/auth_test.go +++ b/test/auth_test.go @@ -117,6 +117,7 @@ func TestAuthFailAllowReconnect(t *testing.T) { copts.NoRandomize = true copts.MaxReconnect = 10 copts.ReconnectWait = 100 * time.Millisecond + nats.ReconnectJitter(0, 0)(&copts) copts.ReconnectedCB = func(_ *nats.Conn) { reconnectch <- true @@ -174,6 +175,7 @@ func TestTokenHandlerReconnect(t *testing.T) { copts.NoRandomize = true copts.MaxReconnect = 10 copts.ReconnectWait = 100 * time.Millisecond + nats.ReconnectJitter(0, 0)(&copts) copts.TokenHandler = func() string { return secret diff --git a/test/basic_test.go b/test/basic_test.go index 4667aea87..4e287729a 100644 --- a/test/basic_test.go +++ b/test/basic_test.go @@ -822,6 +822,7 @@ func TestOptions(t *testing.T) { nats.Name("myName"), nats.MaxReconnects(2), nats.ReconnectWait(50*time.Millisecond), + nats.ReconnectJitter(0, 0), nats.PingInterval(20*time.Millisecond)) if err != nil { t.Fatalf("Failed to connect: %v", err) diff --git a/test/cluster_test.go b/test/cluster_test.go index af129306a..32be512dd 100644 --- a/test/cluster_test.go +++ b/test/cluster_test.go @@ -271,6 +271,7 @@ func TestHotSpotReconnect(t *testing.T) { opts := []nats.Option{ nats.ReconnectWait(50 * time.Millisecond), + nats.ReconnectJitter(0, 0), nats.ReconnectHandler(func(_ *nats.Conn) { wg.Done() }), } @@ -389,6 +390,7 @@ func TestProperFalloutAfterMaxAttempts(t *testing.T) { } opts.NoRandomize = true opts.ReconnectWait = (25 * time.Millisecond) + nats.ReconnectJitter(0, 0)(&opts) dch := make(chan bool) opts.DisconnectedErrCB = func(_ *nats.Conn, _ error) { @@ -456,6 +458,7 @@ func TestProperFalloutAfterMaxAttemptsWithAuthMismatch(t *testing.T) { opts.MaxReconnect = 5 } opts.ReconnectWait = (25 * time.Millisecond) + nats.ReconnectJitter(0, 0)(&opts) dch := make(chan bool) opts.DisconnectedErrCB = func(_ *nats.Conn, _ error) { @@ -519,11 +522,13 @@ func TestTimeoutOnNoServers(t *testing.T) { opts.Servers = testServers[:2] opts.MaxReconnect = 2 opts.ReconnectWait = (100 * time.Millisecond) + nats.ReconnectJitter(0, 0)(&opts) } else { opts.Servers = testServers // 1 second total time wait opts.MaxReconnect = 10 opts.ReconnectWait = (100 * time.Millisecond) + nats.ReconnectJitter(0, 0)(&opts) } opts.NoRandomize = true @@ -584,6 +589,7 @@ func TestPingReconnect(t *testing.T) { opts.Servers = testServers opts.NoRandomize = true opts.ReconnectWait = 200 * time.Millisecond + nats.ReconnectJitter(0, 0)(&opts) opts.PingInterval = 50 * time.Millisecond opts.MaxPingsOut = -1 @@ -813,6 +819,7 @@ func TestServerPoolUpdatedWhenRouteGoesAway(t *testing.T) { nc, err = nats.Connect(s1Url, nats.MaxReconnects(10), nats.ReconnectWait(15*time.Millisecond), + nats.ReconnectJitter(0, 0), nats.SetCustomDialer(d), nats.ReconnectHandler(connHandler), nats.ClosedHandler(connHandler)) diff --git a/test/conn_test.go b/test/conn_test.go index 1643af88b..d03a7c30f 100644 --- a/test/conn_test.go +++ b/test/conn_test.go @@ -733,6 +733,7 @@ func TestCallbacksOrder(t *testing.T) { nats.ClosedHandler(cch), nats.ErrorHandler(ech), nats.ReconnectWait(50*time.Millisecond), + nats.ReconnectJitter(0, 0), nats.DontRandomize()) if err != nil { @@ -1158,6 +1159,7 @@ func TestErrStaleConnection(t *testing.T) { opts.ReconnectedCB = func(_ *nats.Conn) { rch <- true } opts.ClosedCB = func(_ *nats.Conn) { cch <- true } opts.ReconnectWait = 20 * time.Millisecond + nats.ReconnectJitter(0, 0)(&opts) opts.MaxReconnect = 100 opts.Servers = []string{natsURL} nc, err := opts.Connect() @@ -1247,6 +1249,7 @@ func TestServerErrorClosesConnection(t *testing.T) { opts.ReconnectedCB = func(_ *nats.Conn) { atomic.AddInt64(&reconnected, 1) } opts.ClosedCB = func(_ *nats.Conn) { cch <- true } opts.ReconnectWait = 20 * time.Millisecond + nats.ReconnectJitter(0, 0)(&opts) opts.MaxReconnect = 100 opts.Servers = []string{natsURL} nc, err := opts.Connect() @@ -1321,7 +1324,8 @@ func TestNoRaceOnLastError(t *testing.T) { nats.DisconnectHandler(dch), nats.ClosedHandler(cch), nats.MaxReconnects(-1), - nats.ReconnectWait(5*time.Millisecond)) + nats.ReconnectWait(5*time.Millisecond), + nats.ReconnectJitter(0, 0)) if err != nil { t.Fatalf("Unable to connect: %v\n", err) } @@ -1985,6 +1989,7 @@ func TestReceiveInfoWithEmptyConnectURLs(t *testing.T) { rch := make(chan bool) nc, err := nats.Connect("nats://127.0.0.1:4222", nats.ReconnectWait(50*time.Millisecond), + nats.ReconnectJitter(0, 0), nats.ReconnectHandler(func(_ *nats.Conn) { rch <- true })) diff --git a/test/reconnect_test.go b/test/reconnect_test.go index 28f8149eb..579c8bc1e 100644 --- a/test/reconnect_test.go +++ b/test/reconnect_test.go @@ -39,6 +39,16 @@ func TestReconnectTotalTime(t *testing.T) { } } +func TestDefaultReconnectJitter(t *testing.T) { + opts := nats.GetDefaultOptions() + if opts.ReconnectJitter != nats.DefaultReconnectJitter { + t.Fatalf("Expected default jitter for non TLS to be %v, got %v", nats.DefaultReconnectJitter, opts.ReconnectJitter) + } + if opts.ReconnectJitterTLS != nats.DefaultReconnectJitterTLS { + t.Fatalf("Expected default jitter for TLS to be %v, got %v", nats.DefaultReconnectJitterTLS, opts.ReconnectJitterTLS) + } +} + func TestReconnectDisallowedFlags(t *testing.T) { ts := startReconnectServer(t) defer ts.Shutdown() @@ -73,6 +83,7 @@ func TestReconnectAllowedFlags(t *testing.T) { opts.AllowReconnect = true opts.MaxReconnect = 2 opts.ReconnectWait = 1 * time.Second + nats.ReconnectJitter(0, 0)(&opts) opts.ClosedCB = func(_ *nats.Conn) { ch <- true @@ -435,6 +446,7 @@ func TestIsReconnectingAndStatus(t *testing.T) { opts.AllowReconnect = true opts.MaxReconnect = 10000 opts.ReconnectWait = 100 * time.Millisecond + nats.ReconnectJitter(0, 0)(&opts) opts.DisconnectedErrCB = func(_ *nats.Conn, _ error) { disconnectedch <- true @@ -504,6 +516,7 @@ func TestFullFlushChanDuringReconnect(t *testing.T) { opts.AllowReconnect = true opts.MaxReconnect = 10000 opts.ReconnectWait = 100 * time.Millisecond + nats.ReconnectJitter(0, 0)(&opts) opts.ReconnectedCB = func(_ *nats.Conn) { reconnectch <- true diff --git a/test/sub_test.go b/test/sub_test.go index 7187fe726..08fc19a94 100644 --- a/test/sub_test.go +++ b/test/sub_test.go @@ -144,6 +144,7 @@ func TestAutoUnsubAndReconnect(t *testing.T) { nc, err := nats.Connect(nats.DefaultURL, nats.ReconnectWait(50*time.Millisecond), + nats.ReconnectJitter(0, 0), nats.ReconnectHandler(func(_ *nats.Conn) { rch <- true })) if err != nil { t.Fatalf("Unable to connect: %v", err) @@ -201,6 +202,7 @@ func TestAutoUnsubWithParallelNextMsgCalls(t *testing.T) { nc, err := nats.Connect(nats.DefaultURL, nats.ReconnectWait(50*time.Millisecond), + nats.ReconnectJitter(0, 0), nats.ReconnectHandler(func(_ *nats.Conn) { rch <- true })) if err != nil { t.Fatalf("Unable to connect: %v", err) From b2be4bbb43e54388f4fdbabd1f23617390bfff96 Mon Sep 17 00:00:00 2001 From: Ivan Kozlovic Date: Tue, 12 May 2020 13:53:45 -0600 Subject: [PATCH 2/2] Removed lastAttempt since it is no longer used Signed-off-by: Ivan Kozlovic --- nats.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/nats.go b/nats.go index b004d23c6..036381f9c 100644 --- a/nats.go +++ b/nats.go @@ -517,13 +517,12 @@ type Statistics struct { // Tracks individual backend servers. type srv struct { - url *url.URL - didConnect bool - reconnects int - lastAttempt time.Time - lastErr error - isImplicit bool - tlsName string + url *url.URL + didConnect bool + reconnects int + lastErr error + isImplicit bool + tlsName string } type serverInfo struct { @@ -1299,8 +1298,6 @@ func (nc *Conn) createConn() (err error) { } if _, cur := nc.currentServer(); cur == nil { return ErrNoServers - } else { - cur.lastAttempt = time.Now() } // We will auto-expand host names if they resolve to multiple IPs