diff --git a/lease/lessor.go b/lease/lessor.go index 6584a6fb4..df8596ee3 100644 --- a/lease/lessor.go +++ b/lease/lessor.go @@ -18,6 +18,7 @@ import ( "encoding/binary" "errors" "math" + "math/rand" "sort" "sync" "sync/atomic" @@ -326,10 +327,22 @@ func (le *lessor) Promote(extend time.Duration) { // refresh the expiries of all leases. for _, l := range le.leaseMap { - l.refresh(extend) + // randomize expiry with 士10%, otherwise leases of same TTL + // will expire all at the same time, + l.refresh(extend + computeRandomDelta(l.ttl)) } } +func computeRandomDelta(seconds int64) time.Duration { + var delta int64 + if seconds > 10 { + delta = int64(float64(seconds) * 0.1 * rand.Float64()) + } else { + delta = rand.Int63n(10) + } + return time.Duration(delta) * time.Second +} + func (le *lessor) Demote() { le.mu.Lock() defer le.mu.Unlock() diff --git a/lease/lessor_test.go b/lease/lessor_test.go index bfada8993..93ea91c88 100644 --- a/lease/lessor_test.go +++ b/lease/lessor_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/coreos/etcd/mvcc/backend" + "github.com/coreos/etcd/pkg/monotime" ) const ( @@ -210,6 +211,41 @@ func TestLessorRenew(t *testing.T) { } } +// TestLessorRenewRandomize ensures Lessor renews with randomized expiry. +func TestLessorRenewRandomize(t *testing.T) { + dir, be := NewTestBackend(t) + defer os.RemoveAll(dir) + + le := newLessor(be, minLeaseTTL) + for i := LeaseID(1); i <= 10; i++ { + if _, err := le.Grant(i, 3600); err != nil { + t.Fatal(err) + } + } + + // simulate stop and recovery + le.Stop() + be.Close() + bcfg := backend.DefaultBackendConfig() + bcfg.Path = filepath.Join(dir, "be") + be = backend.New(bcfg) + defer be.Close() + le = newLessor(be, minLeaseTTL) + + now := monotime.Now() + + // extend after recovery should randomize expiries + le.Promote(0) + + for _, l := range le.leaseMap { + leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9)) + pc := (float64(leftSeconds-3600) / float64(3600)) * 100 + if pc > 10.0 || pc < -10.0 || pc == 0 { // should be within 士10% + t.Fatalf("expected randomized expiry, got %d seconds (ttl: 3600)", leftSeconds) + } + } +} + func TestLessorDetach(t *testing.T) { dir, be := NewTestBackend(t) defer os.RemoveAll(dir)