etcdserver: fix sync tests
This is to fix possible testing failures caused by sync tests. Changes: 1. Get rid of time sleep operations, which introduces uncertainty. 2. Use fake Store.release-2.0
parent
ddfcb67ce3
commit
8ba801ec06
|
@ -4,7 +4,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"reflect"
|
"reflect"
|
||||||
"runtime"
|
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
@ -368,16 +367,8 @@ func TestDoProposalStopped(t *testing.T) {
|
||||||
|
|
||||||
// TestSync tests sync 1. is nonblocking 2. sends out SYNC request.
|
// TestSync tests sync 1. is nonblocking 2. sends out SYNC request.
|
||||||
func TestSync(t *testing.T) {
|
func TestSync(t *testing.T) {
|
||||||
n := raft.StartNode(0xBAD0, []int64{0xBAD0}, 10, 1)
|
n := &nodeProposeDataRecorder{}
|
||||||
n.Campaign(context.TODO())
|
|
||||||
select {
|
|
||||||
case <-n.Ready():
|
|
||||||
case <-time.After(time.Millisecond):
|
|
||||||
t.Fatalf("expect to receive ready within 1ms, but fail")
|
|
||||||
}
|
|
||||||
|
|
||||||
srv := &EtcdServer{
|
srv := &EtcdServer{
|
||||||
// TODO: use fake node for better testability
|
|
||||||
Node: n,
|
Node: n,
|
||||||
}
|
}
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
@ -388,46 +379,29 @@ func TestSync(t *testing.T) {
|
||||||
t.Errorf("CallSyncTime = %v, want < %v", d, time.Millisecond)
|
t.Errorf("CallSyncTime = %v, want < %v", d, time.Millisecond)
|
||||||
}
|
}
|
||||||
|
|
||||||
// give time for goroutine in sync to run
|
testutil.ForceGosched()
|
||||||
// TODO: use fake clock
|
data := n.data()
|
||||||
var ready raft.Ready
|
if len(data) != 1 {
|
||||||
select {
|
t.Fatalf("len(proposeData) = %d, want 1", len(data))
|
||||||
case ready = <-n.Ready():
|
|
||||||
case <-time.After(time.Millisecond):
|
|
||||||
t.Fatalf("expect to receive ready within 1ms, but fail")
|
|
||||||
}
|
}
|
||||||
|
var r pb.Request
|
||||||
if len(ready.CommittedEntries) != 1 {
|
if err := r.Unmarshal(data[0]); err != nil {
|
||||||
t.Fatalf("len(CommittedEntries) = %d, want 1", len(ready.CommittedEntries))
|
t.Fatalf("unmarshal request error: %v", err)
|
||||||
}
|
}
|
||||||
e := ready.CommittedEntries[0]
|
if r.Method != "SYNC" {
|
||||||
var req pb.Request
|
t.Errorf("method = %s, want SYNC", r.Method)
|
||||||
if err := req.Unmarshal(e.Data); err != nil {
|
|
||||||
t.Fatalf("unmarshal error: %v", err)
|
|
||||||
}
|
|
||||||
if req.Method != "SYNC" {
|
|
||||||
t.Errorf("method = %s, want SYNC", req.Method)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSyncFail tests the case that sync 1. is non-blocking 2. fails to
|
// TestSyncTimeout tests the case that sync 1. is non-blocking 2. cancel request
|
||||||
// propose SYNC request because there is no leader
|
// after timeout
|
||||||
func TestSyncFail(t *testing.T) {
|
func TestSyncTimeout(t *testing.T) {
|
||||||
// The node is run without Tick and Campaign, so it has no leader forever.
|
n := &nodeProposalBlockerRecorder{}
|
||||||
n := raft.StartNode(0xBAD0, []int64{0xBAD0}, 10, 1)
|
|
||||||
select {
|
|
||||||
case <-n.Ready():
|
|
||||||
case <-time.After(time.Millisecond):
|
|
||||||
t.Fatalf("expect to receive ready within 1ms, but fail")
|
|
||||||
}
|
|
||||||
|
|
||||||
srv := &EtcdServer{
|
srv := &EtcdServer{
|
||||||
// TODO: use fake node for better testability
|
|
||||||
Node: n,
|
Node: n,
|
||||||
}
|
}
|
||||||
routineN := runtime.NumGoroutine()
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
srv.sync(time.Millisecond)
|
srv.sync(0)
|
||||||
|
|
||||||
// check that sync is non-blocking
|
// check that sync is non-blocking
|
||||||
if d := time.Since(start); d > time.Millisecond {
|
if d := time.Since(start); d > time.Millisecond {
|
||||||
|
@ -436,36 +410,31 @@ func TestSyncFail(t *testing.T) {
|
||||||
|
|
||||||
// give time for goroutine in sync to cancel
|
// give time for goroutine in sync to cancel
|
||||||
// TODO: use fake clock
|
// TODO: use fake clock
|
||||||
time.Sleep(2 * time.Millisecond)
|
testutil.ForceGosched()
|
||||||
if g := runtime.NumGoroutine(); g != routineN {
|
w := []string{"Propose blocked"}
|
||||||
t.Errorf("NumGoroutine = %d, want %d", g, routineN)
|
if g := n.Action(); !reflect.DeepEqual(g, w) {
|
||||||
}
|
t.Errorf("action = %v, want %v", g, w)
|
||||||
select {
|
|
||||||
case g := <-n.Ready():
|
|
||||||
t.Errorf("ready = %+v, want no", g)
|
|
||||||
default:
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: TestNoSyncWhenNoLeader
|
||||||
|
|
||||||
func TestSyncTriggerDeleteExpriedKeys(t *testing.T) {
|
func TestSyncTriggerDeleteExpriedKeys(t *testing.T) {
|
||||||
n := raft.StartNode(0xBAD0, []int64{0xBAD0}, 10, 1)
|
n := raft.StartNode(0xBAD0, []int64{0xBAD0}, 10, 1)
|
||||||
n.Campaign(context.TODO())
|
n.Campaign(context.TODO())
|
||||||
st := &storeRecorder{}
|
st := &storeRecorder{}
|
||||||
syncInterval := 5 * time.Millisecond
|
|
||||||
syncTicker := time.NewTicker(syncInterval)
|
|
||||||
defer syncTicker.Stop()
|
|
||||||
srv := &EtcdServer{
|
srv := &EtcdServer{
|
||||||
// TODO: use fake node for better testability
|
// TODO: use fake node for better testability
|
||||||
Node: n,
|
Node: n,
|
||||||
Store: st,
|
Store: st,
|
||||||
Send: func(_ []raftpb.Message) {},
|
Send: func(_ []raftpb.Message) {},
|
||||||
Storage: &storageRecorder{},
|
Storage: &storageRecorder{},
|
||||||
SyncTicker: syncTicker.C,
|
SyncTicker: time.After(0),
|
||||||
}
|
}
|
||||||
srv.Start()
|
srv.Start()
|
||||||
// give time for sync request to be proposed and performed
|
// give time for sync request to be proposed and performed
|
||||||
// TODO: use fake clock
|
// TODO: use fake clock
|
||||||
time.Sleep(syncInterval + time.Millisecond)
|
testutil.ForceGosched()
|
||||||
srv.Stop()
|
srv.Stop()
|
||||||
|
|
||||||
action := st.Action()
|
action := st.Action()
|
||||||
|
@ -621,6 +590,19 @@ func TestGetBool(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGenID(t *testing.T) {
|
||||||
|
// Sanity check that the GenID function has been seeded appropriately
|
||||||
|
// (math/rand is seeded with 1 by default)
|
||||||
|
r := rand.NewSource(int64(1))
|
||||||
|
var n int64
|
||||||
|
for n == 0 {
|
||||||
|
n = r.Int63()
|
||||||
|
}
|
||||||
|
if n == GenID() {
|
||||||
|
t.Fatalf("GenID's rand seeded with 1!")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type recorder struct {
|
type recorder struct {
|
||||||
sync.Mutex
|
sync.Mutex
|
||||||
action []string
|
action []string
|
||||||
|
@ -764,15 +746,62 @@ func (n *readyNode) Compact(d []byte) {}
|
||||||
func (n *readyNode) AddNode(id int64) {}
|
func (n *readyNode) AddNode(id int64) {}
|
||||||
func (n *readyNode) RemoveNode(id int64) {}
|
func (n *readyNode) RemoveNode(id int64) {}
|
||||||
|
|
||||||
func TestGenID(t *testing.T) {
|
type nodeRecorder struct {
|
||||||
// Sanity check that the GenID function has been seeded appropriately
|
recorder
|
||||||
// (math/rand is seeded with 1 by default)
|
}
|
||||||
r := rand.NewSource(int64(1))
|
|
||||||
var n int64
|
func (n *nodeRecorder) Tick() {
|
||||||
for n == 0 {
|
n.record("Tick")
|
||||||
n = r.Int63()
|
}
|
||||||
}
|
func (n *nodeRecorder) Campaign(ctx context.Context) error {
|
||||||
if n == GenID() {
|
n.record("Campaign")
|
||||||
t.Fatalf("GenID's rand seeded with 1!")
|
return nil
|
||||||
}
|
}
|
||||||
|
func (n *nodeRecorder) Propose(ctx context.Context, data []byte) error {
|
||||||
|
n.record("Propose")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (n *nodeRecorder) Step(ctx context.Context, msg raftpb.Message) error {
|
||||||
|
n.record("Step")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (n *nodeRecorder) Ready() <-chan raft.Ready {
|
||||||
|
n.record("Ready")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (n *nodeRecorder) Stop() {
|
||||||
|
n.record("Stop")
|
||||||
|
}
|
||||||
|
func (n *nodeRecorder) Compact(d []byte) {
|
||||||
|
n.record("Compact")
|
||||||
|
}
|
||||||
|
|
||||||
|
type nodeProposeDataRecorder struct {
|
||||||
|
nodeRecorder
|
||||||
|
sync.Mutex
|
||||||
|
d [][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *nodeProposeDataRecorder) data() [][]byte {
|
||||||
|
n.Lock()
|
||||||
|
d := n.d
|
||||||
|
n.Unlock()
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
func (n *nodeProposeDataRecorder) Propose(ctx context.Context, data []byte) error {
|
||||||
|
n.nodeRecorder.Propose(ctx, data)
|
||||||
|
n.Lock()
|
||||||
|
n.d = append(n.d, data)
|
||||||
|
n.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type nodeProposalBlockerRecorder struct {
|
||||||
|
nodeRecorder
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *nodeProposalBlockerRecorder) Propose(ctx context.Context, data []byte) error {
|
||||||
|
<-ctx.Done()
|
||||||
|
n.record("Propose blocked")
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue