466 lines
11 KiB
Go
466 lines
11 KiB
Go
// Copyright 2015 The etcd Authors
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package lease
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sort"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/coreos/etcd/mvcc/backend"
|
|
)
|
|
|
|
const (
|
|
minLeaseTTL = int64(5)
|
|
minLeaseTTLDuration = time.Duration(minLeaseTTL) * time.Second
|
|
)
|
|
|
|
// TestLessorGrant ensures Lessor can grant wanted lease.
|
|
// The granted lease should have a unique ID with a term
|
|
// that is greater than minLeaseTTL.
|
|
func TestLessorGrant(t *testing.T) {
|
|
dir, be := NewTestBackend(t)
|
|
defer os.RemoveAll(dir)
|
|
defer be.Close()
|
|
|
|
le := newLessor(be, minLeaseTTL)
|
|
defer le.Stop()
|
|
le.Promote(0)
|
|
|
|
l, err := le.Grant(1, 1)
|
|
if err != nil {
|
|
t.Fatalf("could not grant lease 1 (%v)", err)
|
|
}
|
|
gl := le.Lookup(l.ID)
|
|
|
|
if !reflect.DeepEqual(gl, l) {
|
|
t.Errorf("lease = %v, want %v", gl, l)
|
|
}
|
|
if l.Remaining() < minLeaseTTLDuration-time.Second {
|
|
t.Errorf("term = %v, want at least %v", l.Remaining(), minLeaseTTLDuration-time.Second)
|
|
}
|
|
|
|
_, err = le.Grant(1, 1)
|
|
if err == nil {
|
|
t.Errorf("allocated the same lease")
|
|
}
|
|
|
|
var nl *Lease
|
|
nl, err = le.Grant(2, 1)
|
|
if err != nil {
|
|
t.Errorf("could not grant lease 2 (%v)", err)
|
|
}
|
|
if nl.ID == l.ID {
|
|
t.Errorf("new lease.id = %x, want != %x", nl.ID, l.ID)
|
|
}
|
|
|
|
be.BatchTx().Lock()
|
|
_, vs := be.BatchTx().UnsafeRange(leaseBucketName, int64ToBytes(int64(l.ID)), nil, 0)
|
|
if len(vs) != 1 {
|
|
t.Errorf("len(vs) = %d, want 1", len(vs))
|
|
}
|
|
be.BatchTx().Unlock()
|
|
}
|
|
|
|
// TestLeaseConcurrentKeys ensures Lease.Keys method calls are guarded
|
|
// from concurrent map writes on 'itemSet'.
|
|
func TestLeaseConcurrentKeys(t *testing.T) {
|
|
dir, be := NewTestBackend(t)
|
|
defer os.RemoveAll(dir)
|
|
defer be.Close()
|
|
|
|
le := newLessor(be, minLeaseTTL)
|
|
defer le.Stop()
|
|
le.SetRangeDeleter(func() TxnDelete { return newFakeDeleter(be) })
|
|
|
|
// grant a lease with long term (100 seconds) to
|
|
// avoid early termination during the test.
|
|
l, err := le.Grant(1, 100)
|
|
if err != nil {
|
|
t.Fatalf("could not grant lease for 100s ttl (%v)", err)
|
|
}
|
|
|
|
itemn := 10
|
|
items := make([]LeaseItem, itemn)
|
|
for i := 0; i < itemn; i++ {
|
|
items[i] = LeaseItem{Key: fmt.Sprintf("foo%d", i)}
|
|
}
|
|
if err = le.Attach(l.ID, items); err != nil {
|
|
t.Fatalf("failed to attach items to the lease: %v", err)
|
|
}
|
|
|
|
donec := make(chan struct{})
|
|
go func() {
|
|
le.Detach(l.ID, items)
|
|
close(donec)
|
|
}()
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(itemn)
|
|
for i := 0; i < itemn; i++ {
|
|
go func() {
|
|
defer wg.Done()
|
|
l.Keys()
|
|
}()
|
|
}
|
|
|
|
<-donec
|
|
wg.Wait()
|
|
}
|
|
|
|
// TestLessorRevoke ensures Lessor can revoke a lease.
|
|
// The items in the revoked lease should be removed from
|
|
// the backend.
|
|
// The revoked lease cannot be got from Lessor again.
|
|
func TestLessorRevoke(t *testing.T) {
|
|
dir, be := NewTestBackend(t)
|
|
defer os.RemoveAll(dir)
|
|
defer be.Close()
|
|
|
|
le := newLessor(be, minLeaseTTL)
|
|
defer le.Stop()
|
|
var fd *fakeDeleter
|
|
le.SetRangeDeleter(func() TxnDelete {
|
|
fd = newFakeDeleter(be)
|
|
return fd
|
|
})
|
|
|
|
// grant a lease with long term (100 seconds) to
|
|
// avoid early termination during the test.
|
|
l, err := le.Grant(1, 100)
|
|
if err != nil {
|
|
t.Fatalf("could not grant lease for 100s ttl (%v)", err)
|
|
}
|
|
|
|
items := []LeaseItem{
|
|
{"foo"},
|
|
{"bar"},
|
|
}
|
|
|
|
if err = le.Attach(l.ID, items); err != nil {
|
|
t.Fatalf("failed to attach items to the lease: %v", err)
|
|
}
|
|
|
|
if err = le.Revoke(l.ID); err != nil {
|
|
t.Fatal("failed to revoke lease:", err)
|
|
}
|
|
|
|
if le.Lookup(l.ID) != nil {
|
|
t.Errorf("got revoked lease %x", l.ID)
|
|
}
|
|
|
|
wdeleted := []string{"bar_", "foo_"}
|
|
sort.Sort(sort.StringSlice(fd.deleted))
|
|
if !reflect.DeepEqual(fd.deleted, wdeleted) {
|
|
t.Errorf("deleted= %v, want %v", fd.deleted, wdeleted)
|
|
}
|
|
|
|
be.BatchTx().Lock()
|
|
_, vs := be.BatchTx().UnsafeRange(leaseBucketName, int64ToBytes(int64(l.ID)), nil, 0)
|
|
if len(vs) != 0 {
|
|
t.Errorf("len(vs) = %d, want 0", len(vs))
|
|
}
|
|
be.BatchTx().Unlock()
|
|
}
|
|
|
|
// TestLessorRenew ensures Lessor can renew an existing lease.
|
|
func TestLessorRenew(t *testing.T) {
|
|
dir, be := NewTestBackend(t)
|
|
defer be.Close()
|
|
defer os.RemoveAll(dir)
|
|
|
|
le := newLessor(be, minLeaseTTL)
|
|
defer le.Stop()
|
|
le.Promote(0)
|
|
|
|
l, err := le.Grant(1, minLeaseTTL)
|
|
if err != nil {
|
|
t.Fatalf("failed to grant lease (%v)", err)
|
|
}
|
|
|
|
// manually change the ttl field
|
|
le.mu.Lock()
|
|
l.ttl = 10
|
|
le.mu.Unlock()
|
|
ttl, err := le.Renew(l.ID)
|
|
if err != nil {
|
|
t.Fatalf("failed to renew lease (%v)", err)
|
|
}
|
|
if ttl != l.ttl {
|
|
t.Errorf("ttl = %d, want %d", ttl, l.ttl)
|
|
}
|
|
|
|
l = le.Lookup(l.ID)
|
|
if l.Remaining() < 9*time.Second {
|
|
t.Errorf("failed to renew the lease")
|
|
}
|
|
}
|
|
|
|
// TestLessorRenewExtendPileup ensures Lessor extends leases on promotion if too many
|
|
// expire at the same time.
|
|
func TestLessorRenewExtendPileup(t *testing.T) {
|
|
oldRevokeRate := leaseRevokeRate
|
|
defer func() { leaseRevokeRate = oldRevokeRate }()
|
|
leaseRevokeRate = 10
|
|
|
|
dir, be := NewTestBackend(t)
|
|
defer os.RemoveAll(dir)
|
|
|
|
le := newLessor(be, minLeaseTTL)
|
|
ttl := int64(10)
|
|
for i := 1; i <= leaseRevokeRate*10; i++ {
|
|
if _, err := le.Grant(LeaseID(2*i), ttl); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// ttls that overlap spillover for ttl=10
|
|
if _, err := le.Grant(LeaseID(2*i+1), ttl+1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// simulate stop and recovery
|
|
le.Stop()
|
|
be.Close()
|
|
bcfg := backend.DefaultBackendConfig()
|
|
bcfg.Path = filepath.Join(dir, "be")
|
|
be = backend.New(bcfg)
|
|
defer be.Close()
|
|
le = newLessor(be, minLeaseTTL)
|
|
defer le.Stop()
|
|
|
|
// extend after recovery should extend expiration on lease pile-up
|
|
le.Promote(0)
|
|
|
|
windowCounts := make(map[int64]int)
|
|
for _, l := range le.leaseMap {
|
|
// round up slightly for baseline ttl
|
|
s := int64(l.Remaining().Seconds() + 0.1)
|
|
windowCounts[s]++
|
|
}
|
|
|
|
for i := ttl; i < ttl+20; i++ {
|
|
c := windowCounts[i]
|
|
if c > leaseRevokeRate {
|
|
t.Errorf("expected at most %d expiring at %ds, got %d", leaseRevokeRate, i, c)
|
|
}
|
|
if c < leaseRevokeRate/2 {
|
|
t.Errorf("expected at least %d expiring at %ds, got %d", leaseRevokeRate/2, i, c)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLessorDetach(t *testing.T) {
|
|
dir, be := NewTestBackend(t)
|
|
defer os.RemoveAll(dir)
|
|
defer be.Close()
|
|
|
|
le := newLessor(be, minLeaseTTL)
|
|
defer le.Stop()
|
|
le.SetRangeDeleter(func() TxnDelete { return newFakeDeleter(be) })
|
|
|
|
// grant a lease with long term (100 seconds) to
|
|
// avoid early termination during the test.
|
|
l, err := le.Grant(1, 100)
|
|
if err != nil {
|
|
t.Fatalf("could not grant lease for 100s ttl (%v)", err)
|
|
}
|
|
|
|
items := []LeaseItem{
|
|
{"foo"},
|
|
{"bar"},
|
|
}
|
|
|
|
if err := le.Attach(l.ID, items); err != nil {
|
|
t.Fatalf("failed to attach items to the lease: %v", err)
|
|
}
|
|
|
|
if err := le.Detach(l.ID, items[0:1]); err != nil {
|
|
t.Fatalf("failed to de-attach items to the lease: %v", err)
|
|
}
|
|
|
|
l = le.Lookup(l.ID)
|
|
if len(l.itemSet) != 1 {
|
|
t.Fatalf("len(l.itemSet) = %d, failed to de-attach items", len(l.itemSet))
|
|
}
|
|
if _, ok := l.itemSet[LeaseItem{"bar"}]; !ok {
|
|
t.Fatalf("de-attached wrong item, want %q exists", "bar")
|
|
}
|
|
}
|
|
|
|
// TestLessorRecover ensures Lessor recovers leases from
|
|
// persist backend.
|
|
func TestLessorRecover(t *testing.T) {
|
|
dir, be := NewTestBackend(t)
|
|
defer os.RemoveAll(dir)
|
|
defer be.Close()
|
|
|
|
le := newLessor(be, minLeaseTTL)
|
|
defer le.Stop()
|
|
l1, err1 := le.Grant(1, 10)
|
|
l2, err2 := le.Grant(2, 20)
|
|
if err1 != nil || err2 != nil {
|
|
t.Fatalf("could not grant initial leases (%v, %v)", err1, err2)
|
|
}
|
|
|
|
// Create a new lessor with the same backend
|
|
nle := newLessor(be, minLeaseTTL)
|
|
defer nle.Stop()
|
|
nl1 := nle.Lookup(l1.ID)
|
|
if nl1 == nil || nl1.ttl != l1.ttl {
|
|
t.Errorf("nl1 = %v, want nl1.ttl= %d", nl1.ttl, l1.ttl)
|
|
}
|
|
|
|
nl2 := nle.Lookup(l2.ID)
|
|
if nl2 == nil || nl2.ttl != l2.ttl {
|
|
t.Errorf("nl2 = %v, want nl2.ttl= %d", nl2.ttl, l2.ttl)
|
|
}
|
|
}
|
|
|
|
func TestLessorExpire(t *testing.T) {
|
|
dir, be := NewTestBackend(t)
|
|
defer os.RemoveAll(dir)
|
|
defer be.Close()
|
|
|
|
testMinTTL := int64(1)
|
|
|
|
le := newLessor(be, testMinTTL)
|
|
defer le.Stop()
|
|
|
|
le.Promote(1 * time.Second)
|
|
l, err := le.Grant(1, testMinTTL)
|
|
if err != nil {
|
|
t.Fatalf("failed to create lease: %v", err)
|
|
}
|
|
|
|
select {
|
|
case el := <-le.ExpiredLeasesC():
|
|
if el[0].ID != l.ID {
|
|
t.Fatalf("expired id = %x, want %x", el[0].ID, l.ID)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("failed to receive expired lease")
|
|
}
|
|
|
|
donec := make(chan struct{})
|
|
go func() {
|
|
// expired lease cannot be renewed
|
|
if _, err := le.Renew(l.ID); err != ErrLeaseNotFound {
|
|
t.Fatalf("unexpected renew")
|
|
}
|
|
donec <- struct{}{}
|
|
}()
|
|
|
|
select {
|
|
case <-donec:
|
|
t.Fatalf("renew finished before lease revocation")
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
|
|
// expired lease can be revoked
|
|
if err := le.Revoke(l.ID); err != nil {
|
|
t.Fatalf("failed to revoke expired lease: %v", err)
|
|
}
|
|
|
|
select {
|
|
case <-donec:
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("renew has not returned after lease revocation")
|
|
}
|
|
}
|
|
|
|
func TestLessorExpireAndDemote(t *testing.T) {
|
|
dir, be := NewTestBackend(t)
|
|
defer os.RemoveAll(dir)
|
|
defer be.Close()
|
|
|
|
testMinTTL := int64(1)
|
|
|
|
le := newLessor(be, testMinTTL)
|
|
defer le.Stop()
|
|
|
|
le.Promote(1 * time.Second)
|
|
l, err := le.Grant(1, testMinTTL)
|
|
if err != nil {
|
|
t.Fatalf("failed to create lease: %v", err)
|
|
}
|
|
|
|
select {
|
|
case el := <-le.ExpiredLeasesC():
|
|
if el[0].ID != l.ID {
|
|
t.Fatalf("expired id = %x, want %x", el[0].ID, l.ID)
|
|
}
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("failed to receive expired lease")
|
|
}
|
|
|
|
donec := make(chan struct{})
|
|
go func() {
|
|
// expired lease cannot be renewed
|
|
if _, err := le.Renew(l.ID); err != ErrNotPrimary {
|
|
t.Fatalf("unexpected renew: %v", err)
|
|
}
|
|
donec <- struct{}{}
|
|
}()
|
|
|
|
select {
|
|
case <-donec:
|
|
t.Fatalf("renew finished before demotion")
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
|
|
// demote will cause the renew request to fail with ErrNotPrimary
|
|
le.Demote()
|
|
|
|
select {
|
|
case <-donec:
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatalf("renew has not returned after lessor demotion")
|
|
}
|
|
}
|
|
|
|
type fakeDeleter struct {
|
|
deleted []string
|
|
tx backend.BatchTx
|
|
}
|
|
|
|
func newFakeDeleter(be backend.Backend) *fakeDeleter {
|
|
fd := &fakeDeleter{nil, be.BatchTx()}
|
|
fd.tx.Lock()
|
|
return fd
|
|
}
|
|
|
|
func (fd *fakeDeleter) End() { fd.tx.Unlock() }
|
|
|
|
func (fd *fakeDeleter) DeleteRange(key, end []byte) (int64, int64) {
|
|
fd.deleted = append(fd.deleted, string(key)+"_"+string(end))
|
|
return 0, 0
|
|
}
|
|
|
|
func NewTestBackend(t *testing.T) (string, backend.Backend) {
|
|
tmpPath, err := ioutil.TempDir("", "lease")
|
|
if err != nil {
|
|
t.Fatalf("failed to create tmpdir (%v)", err)
|
|
}
|
|
bcfg := backend.DefaultBackendConfig()
|
|
bcfg.Path = filepath.Join(tmpPath, "be")
|
|
return tmpPath, backend.New(bcfg)
|
|
}
|