2014-05-06 10:28:14 +04:00
|
|
|
package raft
|
|
|
|
|
|
|
|
import (
|
2014-05-24 20:53:01 +04:00
|
|
|
"bytes"
|
2014-05-06 10:28:14 +04:00
|
|
|
"fmt"
|
|
|
|
"reflect"
|
|
|
|
"testing"
|
|
|
|
)
|
|
|
|
|
|
|
|
func TestLeaderElection(t *testing.T) {
|
|
|
|
tests := []struct {
|
2014-05-16 21:11:21 +04:00
|
|
|
*network
|
2014-05-06 10:28:14 +04:00
|
|
|
state stateType
|
|
|
|
}{
|
|
|
|
{newNetwork(nil, nil, nil), stateLeader},
|
|
|
|
{newNetwork(nil, nil, nopStepper), stateLeader},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper), stateCandidate},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil), stateCandidate},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil, nil), stateLeader},
|
2014-05-19 03:11:45 +04:00
|
|
|
|
|
|
|
// three nodes are have logs further along than 0
|
|
|
|
{
|
|
|
|
newNetwork(
|
|
|
|
nil,
|
2014-05-27 23:24:49 +04:00
|
|
|
&nsm{stateMachine{log: &log{ents: []Entry{{}, {Term: 1}}}}, nil},
|
|
|
|
&nsm{stateMachine{log: &log{ents: []Entry{{}, {Term: 2}}}}, nil},
|
|
|
|
&nsm{stateMachine{log: &log{ents: []Entry{{}, {Term: 1}, {Term: 3}}}}, nil},
|
2014-05-19 03:11:45 +04:00
|
|
|
nil,
|
|
|
|
),
|
|
|
|
stateFollower,
|
|
|
|
},
|
2014-05-19 05:05:03 +04:00
|
|
|
|
|
|
|
// logs converge
|
|
|
|
{
|
|
|
|
newNetwork(
|
2014-05-27 23:24:49 +04:00
|
|
|
&nsm{stateMachine{log: &log{ents: []Entry{{}, {Term: 1}}}}, nil},
|
2014-05-19 05:05:03 +04:00
|
|
|
nil,
|
2014-05-27 23:24:49 +04:00
|
|
|
&nsm{stateMachine{log: &log{ents: []Entry{{}, {Term: 2}}}}, nil},
|
|
|
|
&nsm{stateMachine{log: &log{ents: []Entry{{}, {Term: 1}}}}, nil},
|
2014-05-19 05:05:03 +04:00
|
|
|
nil,
|
|
|
|
),
|
|
|
|
stateLeader,
|
|
|
|
},
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
2014-05-24 00:30:04 +04:00
|
|
|
sm := tt.network.ss[0].(*nsm)
|
2014-05-06 10:28:14 +04:00
|
|
|
if sm.state != tt.state {
|
|
|
|
t.Errorf("#%d: state = %s, want %s", i, sm.state, tt.state)
|
|
|
|
}
|
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-24 20:53:01 +04:00
|
|
|
func TestLogReplication(t *testing.T) {
|
|
|
|
tests := []struct {
|
|
|
|
*network
|
2014-05-28 22:08:32 +04:00
|
|
|
msgs []Message
|
|
|
|
wcommitted int
|
2014-05-24 20:53:01 +04:00
|
|
|
}{
|
|
|
|
{
|
|
|
|
newNetwork(nil, nil, nil),
|
|
|
|
[]Message{
|
|
|
|
Message{To: 0, Type: msgProp, Data: []byte("somedata")},
|
|
|
|
},
|
|
|
|
1,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
newNetwork(nil, nil, nil),
|
|
|
|
[]Message{
|
|
|
|
Message{To: 0, Type: msgProp, Data: []byte("somedata")},
|
|
|
|
Message{To: 1, Type: msgHup},
|
|
|
|
Message{To: 1, Type: msgProp, Data: []byte("somedata")},
|
|
|
|
},
|
|
|
|
2,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
|
|
|
t.Logf("#%d: m = %+v", i, m)
|
|
|
|
})
|
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
|
|
|
|
|
|
|
for _, m := range tt.msgs {
|
|
|
|
tt.Step(m)
|
|
|
|
}
|
|
|
|
|
|
|
|
for j, ism := range tt.ss {
|
|
|
|
sm := ism.(*nsm)
|
|
|
|
|
2014-05-28 22:08:32 +04:00
|
|
|
if sm.log.committed != tt.wcommitted {
|
|
|
|
t.Errorf("#%d.%d: committed = %d, want %d", i, j, sm.log.committed, tt.wcommitted)
|
2014-05-24 20:53:01 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
ents := sm.nextEnts()
|
|
|
|
props := make([]Message, 0)
|
|
|
|
for _, m := range tt.msgs {
|
|
|
|
if m.Type == msgProp {
|
|
|
|
props = append(props, m)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for k, m := range props {
|
|
|
|
if !bytes.Equal(ents[k].Data, m.Data) {
|
|
|
|
t.Errorf("#%d.%d: data = %d, want %d", i, j, ents[k].Data, m.Data)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
func TestDualingCandidates(t *testing.T) {
|
2014-05-27 23:24:49 +04:00
|
|
|
a := &nsm{stateMachine{log: defaultLog()}, nil}
|
|
|
|
c := &nsm{stateMachine{log: defaultLog()}, nil}
|
2014-05-19 00:59:10 +04:00
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
tt := newNetwork(a, nil, c)
|
2014-05-19 00:59:10 +04:00
|
|
|
|
|
|
|
heal := false
|
|
|
|
next := stepperFunc(func(m Message) {
|
|
|
|
if heal {
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(m)
|
2014-05-19 00:59:10 +04:00
|
|
|
}
|
|
|
|
})
|
|
|
|
a.next = next
|
|
|
|
c.next = next
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
|
|
|
t.Logf("m = %+v", m)
|
|
|
|
})
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
|
|
|
tt.Step(Message{To: 2, Type: msgHup})
|
2014-05-16 21:11:21 +04:00
|
|
|
|
|
|
|
t.Log("healing")
|
2014-05-19 00:59:10 +04:00
|
|
|
heal = true
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 2, Type: msgHup})
|
2014-05-16 21:53:56 +04:00
|
|
|
|
|
|
|
tests := []struct {
|
2014-05-24 00:30:04 +04:00
|
|
|
sm *nsm
|
2014-05-16 21:53:56 +04:00
|
|
|
state stateType
|
2014-05-18 01:09:52 +04:00
|
|
|
term int
|
2014-05-16 21:53:56 +04:00
|
|
|
}{
|
|
|
|
{a, stateFollower, 2},
|
|
|
|
{c, stateLeader, 2},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
if g := tt.sm.state; g != tt.state {
|
|
|
|
t.Errorf("#%d: state = %s, want %s", i, g, tt.state)
|
|
|
|
}
|
|
|
|
if g := tt.sm.term; g != tt.term {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, tt.term)
|
|
|
|
}
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-27 23:24:49 +04:00
|
|
|
if g := diffLogs(defaultLog().ents, tt.logs()); g != nil {
|
2014-05-19 04:51:45 +04:00
|
|
|
for _, diff := range g {
|
|
|
|
t.Errorf("bag log:\n%s", diff)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestCandidateConcede(t *testing.T) {
|
2014-05-27 23:24:49 +04:00
|
|
|
a := &nsm{stateMachine{log: defaultLog()}, nil}
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
tt := newNetwork(a, nil, nil)
|
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
|
|
|
t.Logf("m = %+v", m)
|
|
|
|
})
|
|
|
|
|
|
|
|
a.next = nopStepper
|
|
|
|
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
|
|
|
tt.Step(Message{To: 2, Type: msgHup})
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
// heal the partition
|
|
|
|
a.next = tt
|
|
|
|
|
|
|
|
data := []byte("force follower")
|
|
|
|
// send a proposal to 2 to flush out a msgApp to 0
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 2, Type: msgProp, Data: data})
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
if g := a.state; g != stateFollower {
|
|
|
|
t.Errorf("state = %s, want %s", g, stateFollower)
|
|
|
|
}
|
|
|
|
if g := a.term; g != 1 {
|
|
|
|
t.Errorf("term = %d, want %d", g, 1)
|
|
|
|
}
|
|
|
|
wantLog := []Entry{{}, {Term: 1, Data: data}}
|
|
|
|
if g := diffLogs(wantLog, tt.logs()); g != nil {
|
2014-05-16 21:53:56 +04:00
|
|
|
for _, diff := range g {
|
|
|
|
t.Errorf("bag log:\n%s", diff)
|
|
|
|
}
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-16 21:53:56 +04:00
|
|
|
}
|
|
|
|
|
2014-05-28 22:59:01 +04:00
|
|
|
func TestSingleNodeCandidate(t *testing.T) {
|
|
|
|
tt := newNetwork(nil)
|
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
|
|
|
|
|
|
|
sm := tt.ss[0].(*nsm)
|
|
|
|
if sm.state != stateLeader {
|
|
|
|
t.Errorf("state = %d, want %d", sm.state, stateLeader)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-16 21:53:56 +04:00
|
|
|
func TestOldMessages(t *testing.T) {
|
|
|
|
tt := newNetwork(nil, nil, nil)
|
|
|
|
// make 0 leader @ term 3
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
2014-05-24 00:30:04 +04:00
|
|
|
tt.Step(Message{To: 1, Type: msgHup})
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
2014-05-16 21:53:56 +04:00
|
|
|
// pretend we're an old leader trying to make progress
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgApp, Term: 1, Entries: []Entry{{Term: 1}}})
|
2014-05-27 23:24:49 +04:00
|
|
|
if g := diffLogs(defaultLog().ents, tt.logs()); g != nil {
|
2014-05-16 21:11:21 +04:00
|
|
|
for _, diff := range g {
|
|
|
|
t.Errorf("bag log:\n%s", diff)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-18 04:18:58 +04:00
|
|
|
// TestOldMessagesReply - optimization - reply with new term.
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
func TestProposal(t *testing.T) {
|
|
|
|
tests := []struct {
|
2014-05-16 21:11:21 +04:00
|
|
|
*network
|
2014-05-18 01:09:52 +04:00
|
|
|
success bool
|
2014-05-06 10:28:14 +04:00
|
|
|
}{
|
2014-05-18 01:09:52 +04:00
|
|
|
{newNetwork(nil, nil, nil), true},
|
|
|
|
{newNetwork(nil, nil, nopStepper), true},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper), false},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil), false},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil, nil), true},
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-05-18 01:09:52 +04:00
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
|
|
|
t.Logf("#%d: m = %+v", i, m)
|
|
|
|
})
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
step := stepperFunc(func(m Message) {
|
|
|
|
defer func() {
|
2014-05-18 01:09:52 +04:00
|
|
|
// only recover is we expect it to panic so
|
|
|
|
// panics we don't expect go up.
|
|
|
|
if !tt.success {
|
2014-05-06 10:28:14 +04:00
|
|
|
e := recover()
|
|
|
|
if e != nil {
|
|
|
|
t.Logf("#%d: err: %s", i, e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(m)
|
2014-05-06 10:28:14 +04:00
|
|
|
})
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
data := []byte("somedata")
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
// promote 0 the leader
|
|
|
|
step(Message{To: 0, Type: msgHup})
|
|
|
|
step(Message{To: 0, Type: msgProp, Data: data})
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
var wantLog []Entry
|
|
|
|
if tt.success {
|
|
|
|
wantLog = []Entry{{}, {Term: 1, Data: data}}
|
|
|
|
} else {
|
2014-05-27 23:24:49 +04:00
|
|
|
wantLog = defaultLog().ents
|
2014-05-18 01:09:52 +04:00
|
|
|
}
|
|
|
|
if g := diffLogs(wantLog, tt.logs()); g != nil {
|
2014-05-06 10:28:14 +04:00
|
|
|
for _, diff := range g {
|
2014-05-18 01:09:52 +04:00
|
|
|
t.Errorf("#%d: diff:%s", i, diff)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
2014-05-24 00:30:04 +04:00
|
|
|
sm := tt.network.ss[0].(*nsm)
|
2014-05-06 10:28:14 +04:00
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestProposalByProxy(t *testing.T) {
|
2014-05-16 11:55:17 +04:00
|
|
|
data := []byte("somedata")
|
2014-05-18 01:09:52 +04:00
|
|
|
tests := []*network{
|
|
|
|
newNetwork(nil, nil, nil),
|
|
|
|
newNetwork(nil, nil, nopStepper),
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-05-16 21:11:21 +04:00
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
2014-05-06 10:28:14 +04:00
|
|
|
t.Logf("#%d: m = %+v", i, m)
|
|
|
|
})
|
|
|
|
|
|
|
|
// promote 0 the leader
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
// propose via follower
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 1, Type: msgProp, Data: []byte("somedata")})
|
2014-05-06 10:28:14 +04:00
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
wantLog := []Entry{{}, {Term: 1, Data: data}}
|
|
|
|
if g := diffLogs(wantLog, tt.logs()); g != nil {
|
2014-05-06 10:28:14 +04:00
|
|
|
for _, diff := range g {
|
2014-05-18 01:09:52 +04:00
|
|
|
t.Errorf("#%d: bad entry: %s", i, diff)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
2014-05-24 00:30:04 +04:00
|
|
|
sm := tt.ss[0].(*nsm)
|
2014-05-06 10:28:14 +04:00
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-24 20:53:01 +04:00
|
|
|
func TestCommit(t *testing.T) {
|
2014-05-22 03:02:15 +04:00
|
|
|
tests := []struct {
|
|
|
|
matches []int
|
|
|
|
logs []Entry
|
|
|
|
smTerm int
|
|
|
|
w int
|
|
|
|
}{
|
|
|
|
// odd
|
|
|
|
{[]int{2, 1, 1}, []Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 1}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
{[]int{2, 1, 2}, []Entry{{}, {Term: 1}, {Term: 2}}, 2, 2},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 2}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
|
|
|
|
// even
|
|
|
|
{[]int{2, 1, 1, 1}, []Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 1, 1}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
{[]int{2, 1, 1, 2}, []Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 1, 2}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
{[]int{2, 1, 2, 2}, []Entry{{}, {Term: 1}, {Term: 2}}, 2, 2},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 2, 2}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-05-29 03:34:40 +04:00
|
|
|
ins := make([]index, len(tt.matches))
|
2014-05-22 03:02:15 +04:00
|
|
|
for j := 0; j < len(ins); j++ {
|
2014-05-29 03:34:40 +04:00
|
|
|
ins[j] = index{tt.matches[j], tt.matches[j] + 1}
|
2014-05-22 03:02:15 +04:00
|
|
|
}
|
2014-05-27 23:24:49 +04:00
|
|
|
sm := &stateMachine{log: &log{ents: tt.logs}, ins: ins, k: len(ins), term: tt.smTerm}
|
2014-05-24 20:53:01 +04:00
|
|
|
sm.maybeCommit()
|
2014-05-28 22:08:32 +04:00
|
|
|
if g := sm.log.committed; g != tt.w {
|
|
|
|
t.Errorf("#%d: committed = %d, want %d", i, g, tt.w)
|
2014-05-22 03:02:15 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-18 11:17:46 +04:00
|
|
|
func TestVote(t *testing.T) {
|
|
|
|
tests := []struct {
|
2014-05-25 08:43:12 +04:00
|
|
|
state stateType
|
2014-05-18 11:17:46 +04:00
|
|
|
i, term int
|
2014-05-25 08:08:06 +04:00
|
|
|
voteFor int
|
2014-05-18 11:17:46 +04:00
|
|
|
w int
|
|
|
|
}{
|
2014-05-25 08:43:12 +04:00
|
|
|
{stateFollower, 0, 0, none, -1},
|
|
|
|
{stateFollower, 0, 1, none, -1},
|
|
|
|
{stateFollower, 0, 2, none, -1},
|
|
|
|
{stateFollower, 0, 3, none, 2},
|
|
|
|
|
|
|
|
{stateFollower, 1, 0, none, -1},
|
|
|
|
{stateFollower, 1, 1, none, -1},
|
|
|
|
{stateFollower, 1, 2, none, -1},
|
|
|
|
{stateFollower, 1, 3, none, 2},
|
|
|
|
|
|
|
|
{stateFollower, 2, 0, none, -1},
|
|
|
|
{stateFollower, 2, 1, none, -1},
|
|
|
|
{stateFollower, 2, 2, none, 2},
|
|
|
|
{stateFollower, 2, 3, none, 2},
|
|
|
|
|
|
|
|
{stateFollower, 3, 0, none, -1},
|
|
|
|
{stateFollower, 3, 1, none, -1},
|
|
|
|
{stateFollower, 3, 2, none, 2},
|
|
|
|
{stateFollower, 3, 3, none, 2},
|
|
|
|
|
|
|
|
{stateFollower, 3, 2, 1, 2},
|
|
|
|
{stateFollower, 3, 2, 0, -1},
|
|
|
|
|
|
|
|
{stateLeader, 3, 3, 0, -1},
|
|
|
|
{stateCandidate, 3, 3, 0, -1},
|
2014-05-18 11:17:46 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
called := false
|
2014-05-28 09:23:14 +04:00
|
|
|
sm := &nsm{
|
|
|
|
stateMachine{
|
|
|
|
state: tt.state,
|
|
|
|
vote: tt.voteFor,
|
|
|
|
log: &log{ents: []Entry{{}, {Term: 2}, {Term: 2}}},
|
|
|
|
},
|
|
|
|
nil,
|
|
|
|
}
|
2014-05-25 08:08:06 +04:00
|
|
|
|
2014-05-18 11:17:46 +04:00
|
|
|
sm.next = stepperFunc(func(m Message) {
|
|
|
|
called = true
|
|
|
|
if m.Index != tt.w {
|
|
|
|
t.Errorf("#%d, m.Index = %d, want %d", i, m.Index, tt.w)
|
|
|
|
}
|
|
|
|
})
|
2014-05-25 08:43:12 +04:00
|
|
|
sm.Step(Message{Type: msgVote, From: 1, Index: tt.i, LogTerm: tt.term})
|
2014-05-18 11:17:46 +04:00
|
|
|
if !called {
|
|
|
|
t.Fatal("#%d: not called", i)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-24 00:30:04 +04:00
|
|
|
func TestAllServerStepdown(t *testing.T) {
|
|
|
|
tests := []stateType{stateFollower, stateCandidate, stateLeader}
|
|
|
|
|
|
|
|
want := struct {
|
|
|
|
state stateType
|
|
|
|
term int
|
|
|
|
index int
|
|
|
|
}{stateFollower, 3, 1}
|
|
|
|
|
|
|
|
tmsgTypes := [...]messageType{msgVote, msgApp}
|
|
|
|
tterm := 3
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
sm := newStateMachine(3, 0)
|
|
|
|
switch tt {
|
|
|
|
case stateFollower:
|
|
|
|
sm.becomeFollower(1, 0)
|
|
|
|
case stateCandidate:
|
|
|
|
sm.becomeCandidate()
|
|
|
|
case stateLeader:
|
|
|
|
sm.becomeCandidate()
|
|
|
|
sm.becomeLeader()
|
|
|
|
}
|
|
|
|
|
|
|
|
for j, msgType := range tmsgTypes {
|
|
|
|
sm.Step(Message{Type: msgType, Term: tterm, LogTerm: tterm})
|
|
|
|
|
|
|
|
if sm.state != want.state {
|
|
|
|
t.Errorf("#%d.%d state = %v , want %v", i, j, sm.state, want.state)
|
|
|
|
}
|
|
|
|
if sm.term != want.term {
|
|
|
|
t.Errorf("#%d.%d term = %v , want %v", i, j, sm.term, want.term)
|
|
|
|
}
|
2014-05-27 23:24:49 +04:00
|
|
|
if len(sm.log.ents) != want.index {
|
|
|
|
t.Errorf("#%d.%d index = %v , want %v", i, j, len(sm.log.ents), want.index)
|
2014-05-24 00:30:04 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
func TestLogDiff(t *testing.T) {
|
|
|
|
a := []Entry{{}, {Term: 1}, {Term: 2}}
|
|
|
|
b := []Entry{{}, {Term: 1}, {Term: 2}}
|
|
|
|
c := []Entry{{}, {Term: 2}}
|
|
|
|
d := []Entry(nil)
|
|
|
|
|
|
|
|
w := []diff{
|
|
|
|
diff{1, []*Entry{{Term: 1}, {Term: 1}, {Term: 2}, nilLogEntry}},
|
|
|
|
diff{2, []*Entry{{Term: 2}, {Term: 2}, noEntry, nilLogEntry}},
|
|
|
|
}
|
|
|
|
|
|
|
|
if g := diffLogs(a, [][]Entry{b, c, d}); !reflect.DeepEqual(w, g) {
|
|
|
|
t.Errorf("g = %s", g)
|
|
|
|
t.Errorf("want %s", w)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
type network struct {
|
2014-05-19 10:34:55 +04:00
|
|
|
tee Interface
|
|
|
|
ss []Interface
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
// newNetwork initializes a network from nodes. A nil node will be replaced
|
|
|
|
// with a new *stateMachine. A *stateMachine will get its k, addr, and next
|
|
|
|
// fields set.
|
2014-05-19 10:34:55 +04:00
|
|
|
func newNetwork(nodes ...Interface) *network {
|
2014-05-16 21:11:21 +04:00
|
|
|
nt := &network{ss: nodes}
|
2014-05-06 10:28:14 +04:00
|
|
|
for i, n := range nodes {
|
|
|
|
switch v := n.(type) {
|
|
|
|
case nil:
|
2014-05-24 00:30:04 +04:00
|
|
|
nt.ss[i] = &nsm{*newStateMachine(len(nodes), i), nt}
|
|
|
|
case *nsm:
|
2014-05-06 10:28:14 +04:00
|
|
|
v.k = len(nodes)
|
|
|
|
v.addr = i
|
2014-05-19 03:11:45 +04:00
|
|
|
if v.next == nil {
|
|
|
|
v.next = nt
|
|
|
|
}
|
2014-05-19 00:59:10 +04:00
|
|
|
default:
|
|
|
|
nt.ss[i] = v
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return nt
|
|
|
|
}
|
|
|
|
|
2014-05-19 10:34:55 +04:00
|
|
|
func (nt network) Step(m Message) {
|
2014-05-16 21:11:21 +04:00
|
|
|
if nt.tee != nil {
|
2014-05-19 10:34:55 +04:00
|
|
|
nt.tee.Step(m)
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-19 10:34:55 +04:00
|
|
|
nt.ss[m.To].Step(m)
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
|
|
|
|
2014-05-16 11:55:17 +04:00
|
|
|
// logs returns all logs in nt prepended with want. If a node is not a
|
|
|
|
// *stateMachine, its log will be nil.
|
2014-05-18 01:09:52 +04:00
|
|
|
func (nt network) logs() [][]Entry {
|
|
|
|
ls := make([][]Entry, len(nt.ss))
|
2014-05-16 21:11:21 +04:00
|
|
|
for i, node := range nt.ss {
|
2014-05-24 00:30:04 +04:00
|
|
|
if sm, ok := node.(*nsm); ok {
|
2014-05-27 23:24:49 +04:00
|
|
|
ls[i] = sm.log.ents
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return ls
|
|
|
|
}
|
|
|
|
|
|
|
|
type diff struct {
|
|
|
|
i int
|
|
|
|
ents []*Entry // pointers so they can be nil for N/A
|
|
|
|
}
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
var noEntry = &Entry{}
|
|
|
|
var nilLogEntry = &Entry{}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
func (d diff) String() string {
|
|
|
|
s := fmt.Sprintf("[%d] ", d.i)
|
|
|
|
for i, e := range d.ents {
|
|
|
|
switch e {
|
2014-05-18 01:09:52 +04:00
|
|
|
case nilLogEntry:
|
|
|
|
s += fmt.Sprintf("o")
|
|
|
|
case noEntry:
|
|
|
|
s += fmt.Sprintf("-")
|
2014-05-06 10:28:14 +04:00
|
|
|
case nil:
|
|
|
|
s += fmt.Sprintf("<nil>")
|
|
|
|
default:
|
|
|
|
s += fmt.Sprintf("<%d:%q>", e.Term, string(e.Data))
|
|
|
|
}
|
|
|
|
if i != len(d.ents)-1 {
|
|
|
|
s += "\t\t"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return s
|
|
|
|
}
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
func diffLogs(base []Entry, logs [][]Entry) []diff {
|
2014-05-06 10:28:14 +04:00
|
|
|
var (
|
|
|
|
d []diff
|
|
|
|
max int
|
|
|
|
)
|
2014-05-18 01:09:52 +04:00
|
|
|
logs = append([][]Entry{base}, logs...)
|
2014-05-06 10:28:14 +04:00
|
|
|
for _, log := range logs {
|
|
|
|
if l := len(log); l > max {
|
|
|
|
max = l
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ediff := func(i int) (result []*Entry) {
|
|
|
|
e := make([]*Entry, len(logs))
|
|
|
|
found := false
|
|
|
|
for j, log := range logs {
|
|
|
|
if log == nil {
|
2014-05-18 01:09:52 +04:00
|
|
|
e[j] = nilLogEntry
|
2014-05-06 10:28:14 +04:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
if len(log) <= i {
|
2014-05-18 01:09:52 +04:00
|
|
|
e[j] = noEntry
|
2014-05-06 10:28:14 +04:00
|
|
|
found = true
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
e[j] = &log[i]
|
|
|
|
if j > 0 {
|
|
|
|
switch prev := e[j-1]; {
|
2014-05-18 01:09:52 +04:00
|
|
|
case prev == nilLogEntry:
|
|
|
|
case prev == noEntry:
|
2014-05-06 10:28:14 +04:00
|
|
|
case !reflect.DeepEqual(prev, e[j]):
|
|
|
|
found = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if found {
|
|
|
|
return e
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
for i := 0; i < max; i++ {
|
|
|
|
if e := ediff(i); e != nil {
|
|
|
|
d = append(d, diff{i, e})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return d
|
|
|
|
}
|
|
|
|
|
|
|
|
type stepperFunc func(Message)
|
|
|
|
|
2014-05-19 10:34:55 +04:00
|
|
|
func (f stepperFunc) Step(m Message) { f(m) }
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
var nopStepper = stepperFunc(func(Message) {})
|
2014-05-24 00:30:04 +04:00
|
|
|
|
|
|
|
type nsm struct {
|
|
|
|
stateMachine
|
|
|
|
next Interface
|
|
|
|
}
|
|
|
|
|
|
|
|
func (n *nsm) Step(m Message) {
|
|
|
|
(&n.stateMachine).Step(m)
|
|
|
|
ms := n.Msgs()
|
|
|
|
for _, m := range ms {
|
|
|
|
n.next.Step(m)
|
|
|
|
}
|
|
|
|
}
|
2014-05-27 23:24:49 +04:00
|
|
|
|
|
|
|
func defaultLog() *log {
|
|
|
|
return &log{ents: []Entry{{}}}
|
|
|
|
}
|