lease: randomize expiry on initial refresh call
Randomize the very first expiry on lease recovery to prevent recovered leases from expiring all at the same time. Address https://github.com/coreos/etcd/issues/8096. Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>release-3.3
parent
037e33e833
commit
5bba05703c
|
@ -18,6 +18,7 @@ import (
|
|||
"encoding/binary"
|
||||
"errors"
|
||||
"math"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
@ -326,10 +327,22 @@ func (le *lessor) Promote(extend time.Duration) {
|
|||
|
||||
// refresh the expiries of all leases.
|
||||
for _, l := range le.leaseMap {
|
||||
l.refresh(extend)
|
||||
// randomize expiry with 士10%, otherwise leases of same TTL
|
||||
// will expire all at the same time,
|
||||
l.refresh(extend + computeRandomDelta(l.ttl))
|
||||
}
|
||||
}
|
||||
|
||||
func computeRandomDelta(seconds int64) time.Duration {
|
||||
var delta int64
|
||||
if seconds > 10 {
|
||||
delta = int64(float64(seconds) * 0.1 * rand.Float64())
|
||||
} else {
|
||||
delta = rand.Int63n(10)
|
||||
}
|
||||
return time.Duration(delta) * time.Second
|
||||
}
|
||||
|
||||
func (le *lessor) Demote() {
|
||||
le.mu.Lock()
|
||||
defer le.mu.Unlock()
|
||||
|
|
|
@ -26,6 +26,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/coreos/etcd/mvcc/backend"
|
||||
"github.com/coreos/etcd/pkg/monotime"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -210,6 +211,41 @@ func TestLessorRenew(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestLessorRenewRandomize ensures Lessor renews with randomized expiry.
|
||||
func TestLessorRenewRandomize(t *testing.T) {
|
||||
dir, be := NewTestBackend(t)
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
le := newLessor(be, minLeaseTTL)
|
||||
for i := LeaseID(1); i <= 10; i++ {
|
||||
if _, err := le.Grant(i, 3600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// simulate stop and recovery
|
||||
le.Stop()
|
||||
be.Close()
|
||||
bcfg := backend.DefaultBackendConfig()
|
||||
bcfg.Path = filepath.Join(dir, "be")
|
||||
be = backend.New(bcfg)
|
||||
defer be.Close()
|
||||
le = newLessor(be, minLeaseTTL)
|
||||
|
||||
now := monotime.Now()
|
||||
|
||||
// extend after recovery should randomize expiries
|
||||
le.Promote(0)
|
||||
|
||||
for _, l := range le.leaseMap {
|
||||
leftSeconds := uint64(float64(l.expiry-now) * float64(1e-9))
|
||||
pc := (float64(leftSeconds-3600) / float64(3600)) * 100
|
||||
if pc > 10.0 || pc < -10.0 || pc == 0 { // should be within 士10%
|
||||
t.Fatalf("expected randomized expiry, got %d seconds (ttl: 3600)", leftSeconds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLessorDetach(t *testing.T) {
|
||||
dir, be := NewTestBackend(t)
|
||||
defer os.RemoveAll(dir)
|
||||
|
|
Loading…
Reference in New Issue