diff --git a/lease/lessor.go b/lease/lessor.go index 6584a6fb488b..e43c43dd6d10 100644 --- a/lease/lessor.go +++ b/lease/lessor.go @@ -18,6 +18,7 @@ import ( "encoding/binary" "errors" "math" + "math/rand" "sort" "sync" "sync/atomic" @@ -147,6 +148,9 @@ type lessor struct { stopC chan struct{} // doneC is a channel whose closure indicates that the lessor is stopped. doneC chan struct{} + + // 'true' when lessor just started promoting after recovery + promoting bool } func NewLessor(b backend.Backend, minLeaseTTL int64) Lessor { @@ -160,9 +164,10 @@ func newLessor(b backend.Backend, minLeaseTTL int64) *lessor { b: b, minLeaseTTL: minLeaseTTL, // expiredC is a small buffered chan to avoid unnecessary blocking. - expiredC: make(chan []*Lease, 16), - stopC: make(chan struct{}), - doneC: make(chan struct{}), + expiredC: make(chan []*Lease, 16), + stopC: make(chan struct{}), + doneC: make(chan struct{}), + promoting: true, } l.initAndRecover() @@ -323,11 +328,29 @@ func (le *lessor) Promote(extend time.Duration) { defer le.mu.Unlock() le.demotec = make(chan struct{}) + promoting := le.promoting // refresh the expiries of all leases. for _, l := range le.leaseMap { - l.refresh(extend) + ext := extend + if promoting { + // randomize expiry with 士10%, otherwise leases of same TTL + // will expire all at the same time, + ext += computeRandomDelta(l.ttl) + } + l.refresh(ext) + } + le.promoting = false +} + +func computeRandomDelta(seconds int64) time.Duration { + var delta int64 + if seconds > 10 { + delta = int64(float64(seconds) * 0.1 * rand.Float64()) + } else { + delta = rand.Int63n(10) } + return time.Duration(delta) * time.Second } func (le *lessor) Demote() { diff --git a/lease/lessor_test.go b/lease/lessor_test.go index bfada89932e4..4f8fa758f699 100644 --- a/lease/lessor_test.go +++ b/lease/lessor_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/coreos/etcd/mvcc/backend" + "github.com/coreos/etcd/pkg/monotime" ) const ( @@ -210,6 +211,52 @@ func TestLessorRenew(t *testing.T) { } } +// TestLessorRenewRandomize ensures Lessor renews with randomized expiry. +func TestLessorRenewRandomize(t *testing.T) { + dir, be := NewTestBackend(t) + defer os.RemoveAll(dir) + + le := newLessor(be, minLeaseTTL) + for i := LeaseID(1); i <= 10; i++ { + if _, err := le.Grant(i, 3600); err != nil { + t.Fatal(err) + } + } + + // simulate stop and recovery + le.Stop() + be.Close() + bcfg := backend.DefaultBackendConfig() + bcfg.Path = filepath.Join(dir, "be") + be = backend.New(bcfg) + defer be.Close() + le = newLessor(be, minLeaseTTL) + + now := monotime.Now() + + // first extend after recovery should randomize expiries + le.Promote(0) + + for _, l := range le.leaseMap { + leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9)) + pc := (float64(leftSeconds-3600) / float64(3600)) * 100 + if pc > 10.0 || pc < -10.0 || pc == 0 { // should be within 士10% + t.Fatalf("expected randomized expiry, got %d seconds (ttl: 3600)", leftSeconds) + } + } + + // second extend should not be randomized + le.Promote(0) + + for _, l := range le.leaseMap { + leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9)) + pc := (float64(leftSeconds-3600) / float64(3600)) * 100 + if pc > .5 || pc < -.5 { // should be close to original ttl 3600 + t.Fatalf("expected 3600-sec left, got %d", leftSeconds) + } + } +} + func TestLessorDetach(t *testing.T) { dir, be := NewTestBackend(t) defer os.RemoveAll(dir)