2014-05-06 10:28:14 +04:00
|
|
|
package raft
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"reflect"
|
|
|
|
"testing"
|
|
|
|
)
|
|
|
|
|
2014-05-16 11:55:17 +04:00
|
|
|
var defaultLog = []Entry{{}}
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
func TestLeaderElection(t *testing.T) {
|
|
|
|
tests := []struct {
|
2014-05-16 21:11:21 +04:00
|
|
|
*network
|
2014-05-06 10:28:14 +04:00
|
|
|
state stateType
|
|
|
|
}{
|
|
|
|
{newNetwork(nil, nil, nil), stateLeader},
|
|
|
|
{newNetwork(nil, nil, nopStepper), stateLeader},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper), stateCandidate},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil), stateCandidate},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil, nil), stateLeader},
|
2014-05-19 03:11:45 +04:00
|
|
|
|
|
|
|
// three nodes are have logs further along than 0
|
|
|
|
{
|
|
|
|
newNetwork(
|
|
|
|
nil,
|
|
|
|
&stateMachine{log: []Entry{{}, {Term: 1}}},
|
|
|
|
&stateMachine{log: []Entry{{}, {Term: 2}}},
|
2014-05-19 05:05:03 +04:00
|
|
|
&stateMachine{log: []Entry{{}, {Term: 1}, {Term: 3}}},
|
2014-05-19 03:11:45 +04:00
|
|
|
nil,
|
|
|
|
),
|
|
|
|
stateFollower,
|
|
|
|
},
|
2014-05-19 05:05:03 +04:00
|
|
|
|
|
|
|
// logs converge
|
|
|
|
{
|
|
|
|
newNetwork(
|
|
|
|
&stateMachine{log: []Entry{{}, {Term: 1}}},
|
|
|
|
nil,
|
|
|
|
&stateMachine{log: []Entry{{}, {Term: 2}}},
|
|
|
|
&stateMachine{log: []Entry{{}, {Term: 1}}},
|
|
|
|
nil,
|
|
|
|
),
|
|
|
|
stateLeader,
|
|
|
|
},
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
2014-05-16 21:11:21 +04:00
|
|
|
sm := tt.network.ss[0].(*stateMachine)
|
2014-05-06 10:28:14 +04:00
|
|
|
if sm.state != tt.state {
|
|
|
|
t.Errorf("#%d: state = %s, want %s", i, sm.state, tt.state)
|
|
|
|
}
|
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
func TestDualingCandidates(t *testing.T) {
|
2014-05-19 00:59:10 +04:00
|
|
|
a := &stateMachine{log: defaultLog}
|
|
|
|
c := &stateMachine{log: defaultLog}
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
tt := newNetwork(a, nil, c)
|
2014-05-19 00:59:10 +04:00
|
|
|
|
|
|
|
heal := false
|
|
|
|
next := stepperFunc(func(m Message) {
|
|
|
|
if heal {
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(m)
|
2014-05-19 00:59:10 +04:00
|
|
|
}
|
|
|
|
})
|
|
|
|
a.next = next
|
|
|
|
c.next = next
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
|
|
|
t.Logf("m = %+v", m)
|
|
|
|
})
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
|
|
|
tt.Step(Message{To: 2, Type: msgHup})
|
2014-05-16 21:11:21 +04:00
|
|
|
|
|
|
|
t.Log("healing")
|
2014-05-19 00:59:10 +04:00
|
|
|
heal = true
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 2, Type: msgHup})
|
2014-05-16 21:53:56 +04:00
|
|
|
|
|
|
|
tests := []struct {
|
2014-05-18 01:09:52 +04:00
|
|
|
sm *stateMachine
|
2014-05-16 21:53:56 +04:00
|
|
|
state stateType
|
2014-05-18 01:09:52 +04:00
|
|
|
term int
|
2014-05-16 21:53:56 +04:00
|
|
|
}{
|
|
|
|
{a, stateFollower, 2},
|
|
|
|
{c, stateLeader, 2},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
if g := tt.sm.state; g != tt.state {
|
|
|
|
t.Errorf("#%d: state = %s, want %s", i, g, tt.state)
|
|
|
|
}
|
|
|
|
if g := tt.sm.term; g != tt.term {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, tt.term)
|
|
|
|
}
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-18 01:09:52 +04:00
|
|
|
if g := diffLogs(defaultLog, tt.logs()); g != nil {
|
2014-05-19 04:51:45 +04:00
|
|
|
for _, diff := range g {
|
|
|
|
t.Errorf("bag log:\n%s", diff)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestCandidateConcede(t *testing.T) {
|
|
|
|
a := &stateMachine{log: defaultLog}
|
|
|
|
|
|
|
|
tt := newNetwork(a, nil, nil)
|
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
|
|
|
t.Logf("m = %+v", m)
|
|
|
|
})
|
|
|
|
|
|
|
|
a.next = nopStepper
|
|
|
|
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
|
|
|
tt.Step(Message{To: 2, Type: msgHup})
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
// heal the partition
|
|
|
|
a.next = tt
|
|
|
|
|
|
|
|
data := []byte("force follower")
|
|
|
|
// send a proposal to 2 to flush out a msgApp to 0
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 2, Type: msgProp, Data: data})
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
if g := a.state; g != stateFollower {
|
|
|
|
t.Errorf("state = %s, want %s", g, stateFollower)
|
|
|
|
}
|
|
|
|
if g := a.term; g != 1 {
|
|
|
|
t.Errorf("term = %d, want %d", g, 1)
|
|
|
|
}
|
|
|
|
wantLog := []Entry{{}, {Term: 1, Data: data}}
|
|
|
|
if g := diffLogs(wantLog, tt.logs()); g != nil {
|
2014-05-16 21:53:56 +04:00
|
|
|
for _, diff := range g {
|
|
|
|
t.Errorf("bag log:\n%s", diff)
|
|
|
|
}
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-16 21:53:56 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestOldMessages(t *testing.T) {
|
|
|
|
tt := newNetwork(nil, nil, nil)
|
|
|
|
// make 0 leader @ term 3
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
2014-05-16 21:53:56 +04:00
|
|
|
// pretend we're an old leader trying to make progress
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgApp, Term: 1, Entries: []Entry{{Term: 1}}})
|
2014-05-18 01:09:52 +04:00
|
|
|
if g := diffLogs(defaultLog, tt.logs()); g != nil {
|
2014-05-16 21:11:21 +04:00
|
|
|
for _, diff := range g {
|
|
|
|
t.Errorf("bag log:\n%s", diff)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-18 04:18:58 +04:00
|
|
|
// TestOldMessagesReply - optimization - reply with new term.
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
func TestProposal(t *testing.T) {
|
|
|
|
tests := []struct {
|
2014-05-16 21:11:21 +04:00
|
|
|
*network
|
2014-05-18 01:09:52 +04:00
|
|
|
success bool
|
2014-05-06 10:28:14 +04:00
|
|
|
}{
|
2014-05-18 01:09:52 +04:00
|
|
|
{newNetwork(nil, nil, nil), true},
|
|
|
|
{newNetwork(nil, nil, nopStepper), true},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper), false},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil), false},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil, nil), true},
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-05-18 01:09:52 +04:00
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
|
|
|
t.Logf("#%d: m = %+v", i, m)
|
|
|
|
})
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
step := stepperFunc(func(m Message) {
|
|
|
|
defer func() {
|
2014-05-18 01:09:52 +04:00
|
|
|
// only recover is we expect it to panic so
|
|
|
|
// panics we don't expect go up.
|
|
|
|
if !tt.success {
|
2014-05-06 10:28:14 +04:00
|
|
|
e := recover()
|
|
|
|
if e != nil {
|
|
|
|
t.Logf("#%d: err: %s", i, e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(m)
|
2014-05-06 10:28:14 +04:00
|
|
|
})
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
data := []byte("somedata")
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
// promote 0 the leader
|
|
|
|
step(Message{To: 0, Type: msgHup})
|
|
|
|
step(Message{To: 0, Type: msgProp, Data: data})
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
var wantLog []Entry
|
|
|
|
if tt.success {
|
|
|
|
wantLog = []Entry{{}, {Term: 1, Data: data}}
|
|
|
|
} else {
|
|
|
|
wantLog = defaultLog
|
|
|
|
}
|
|
|
|
if g := diffLogs(wantLog, tt.logs()); g != nil {
|
2014-05-06 10:28:14 +04:00
|
|
|
for _, diff := range g {
|
2014-05-18 01:09:52 +04:00
|
|
|
t.Errorf("#%d: diff:%s", i, diff)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
2014-05-16 21:11:21 +04:00
|
|
|
sm := tt.network.ss[0].(*stateMachine)
|
2014-05-06 10:28:14 +04:00
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestProposalByProxy(t *testing.T) {
|
2014-05-16 11:55:17 +04:00
|
|
|
data := []byte("somedata")
|
2014-05-18 01:09:52 +04:00
|
|
|
tests := []*network{
|
|
|
|
newNetwork(nil, nil, nil),
|
|
|
|
newNetwork(nil, nil, nopStepper),
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-05-16 21:11:21 +04:00
|
|
|
tt.tee = stepperFunc(func(m Message) {
|
2014-05-06 10:28:14 +04:00
|
|
|
t.Logf("#%d: m = %+v", i, m)
|
|
|
|
})
|
|
|
|
|
|
|
|
// promote 0 the leader
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 0, Type: msgHup})
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
// propose via follower
|
2014-05-19 10:34:55 +04:00
|
|
|
tt.Step(Message{To: 1, Type: msgProp, Data: []byte("somedata")})
|
2014-05-06 10:28:14 +04:00
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
wantLog := []Entry{{}, {Term: 1, Data: data}}
|
|
|
|
if g := diffLogs(wantLog, tt.logs()); g != nil {
|
2014-05-06 10:28:14 +04:00
|
|
|
for _, diff := range g {
|
2014-05-18 01:09:52 +04:00
|
|
|
t.Errorf("#%d: bad entry: %s", i, diff)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
2014-05-18 01:09:52 +04:00
|
|
|
sm := tt.ss[0].(*stateMachine)
|
2014-05-06 10:28:14 +04:00
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-18 11:17:46 +04:00
|
|
|
func TestVote(t *testing.T) {
|
|
|
|
tests := []struct {
|
|
|
|
i, term int
|
|
|
|
w int
|
|
|
|
}{
|
|
|
|
{0, 0, -1},
|
|
|
|
{0, 1, -1},
|
|
|
|
{0, 2, -1},
|
|
|
|
{0, 3, 2},
|
|
|
|
|
|
|
|
{1, 0, -1},
|
|
|
|
{1, 1, -1},
|
|
|
|
{1, 2, -1},
|
|
|
|
{1, 3, 2},
|
|
|
|
|
|
|
|
{2, 0, -1},
|
|
|
|
{2, 1, -1},
|
|
|
|
{2, 2, 2},
|
|
|
|
{2, 3, 2},
|
|
|
|
|
|
|
|
{3, 0, -1},
|
|
|
|
{3, 1, -1},
|
|
|
|
{3, 2, 2},
|
|
|
|
{3, 3, 2},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
called := false
|
|
|
|
sm := &stateMachine{log: []Entry{{}, {Term: 2}, {Term: 2}}}
|
|
|
|
sm.next = stepperFunc(func(m Message) {
|
|
|
|
called = true
|
|
|
|
if m.Index != tt.w {
|
|
|
|
t.Errorf("#%d, m.Index = %d, want %d", i, m.Index, tt.w)
|
|
|
|
}
|
|
|
|
})
|
2014-05-19 10:34:55 +04:00
|
|
|
sm.Step(Message{Type: msgVote, Index: tt.i, LogTerm: tt.term})
|
2014-05-18 11:17:46 +04:00
|
|
|
if !called {
|
|
|
|
t.Fatal("#%d: not called", i)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
func TestLogDiff(t *testing.T) {
|
|
|
|
a := []Entry{{}, {Term: 1}, {Term: 2}}
|
|
|
|
b := []Entry{{}, {Term: 1}, {Term: 2}}
|
|
|
|
c := []Entry{{}, {Term: 2}}
|
|
|
|
d := []Entry(nil)
|
|
|
|
|
|
|
|
w := []diff{
|
|
|
|
diff{1, []*Entry{{Term: 1}, {Term: 1}, {Term: 2}, nilLogEntry}},
|
|
|
|
diff{2, []*Entry{{Term: 2}, {Term: 2}, noEntry, nilLogEntry}},
|
|
|
|
}
|
|
|
|
|
|
|
|
if g := diffLogs(a, [][]Entry{b, c, d}); !reflect.DeepEqual(w, g) {
|
|
|
|
t.Errorf("g = %s", g)
|
|
|
|
t.Errorf("want %s", w)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
type network struct {
|
2014-05-19 10:34:55 +04:00
|
|
|
tee Interface
|
|
|
|
ss []Interface
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
// newNetwork initializes a network from nodes. A nil node will be replaced
|
|
|
|
// with a new *stateMachine. A *stateMachine will get its k, addr, and next
|
|
|
|
// fields set.
|
2014-05-19 10:34:55 +04:00
|
|
|
func newNetwork(nodes ...Interface) *network {
|
2014-05-16 21:11:21 +04:00
|
|
|
nt := &network{ss: nodes}
|
2014-05-06 10:28:14 +04:00
|
|
|
for i, n := range nodes {
|
|
|
|
switch v := n.(type) {
|
|
|
|
case nil:
|
2014-05-16 21:11:21 +04:00
|
|
|
nt.ss[i] = newStateMachine(len(nodes), i, nt)
|
2014-05-06 10:28:14 +04:00
|
|
|
case *stateMachine:
|
|
|
|
v.k = len(nodes)
|
|
|
|
v.addr = i
|
2014-05-19 03:11:45 +04:00
|
|
|
if v.next == nil {
|
|
|
|
v.next = nt
|
|
|
|
}
|
2014-05-19 00:59:10 +04:00
|
|
|
default:
|
|
|
|
nt.ss[i] = v
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return nt
|
|
|
|
}
|
|
|
|
|
2014-05-19 10:34:55 +04:00
|
|
|
func (nt network) Step(m Message) {
|
2014-05-16 21:11:21 +04:00
|
|
|
if nt.tee != nil {
|
2014-05-19 10:34:55 +04:00
|
|
|
nt.tee.Step(m)
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-19 10:34:55 +04:00
|
|
|
nt.ss[m.To].Step(m)
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
|
|
|
|
2014-05-16 11:55:17 +04:00
|
|
|
// logs returns all logs in nt prepended with want. If a node is not a
|
|
|
|
// *stateMachine, its log will be nil.
|
2014-05-18 01:09:52 +04:00
|
|
|
func (nt network) logs() [][]Entry {
|
|
|
|
ls := make([][]Entry, len(nt.ss))
|
2014-05-16 21:11:21 +04:00
|
|
|
for i, node := range nt.ss {
|
2014-05-06 10:28:14 +04:00
|
|
|
if sm, ok := node.(*stateMachine); ok {
|
|
|
|
ls[i] = sm.log
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ls
|
|
|
|
}
|
|
|
|
|
|
|
|
type diff struct {
|
|
|
|
i int
|
|
|
|
ents []*Entry // pointers so they can be nil for N/A
|
|
|
|
}
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
var noEntry = &Entry{}
|
|
|
|
var nilLogEntry = &Entry{}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
func (d diff) String() string {
|
|
|
|
s := fmt.Sprintf("[%d] ", d.i)
|
|
|
|
for i, e := range d.ents {
|
|
|
|
switch e {
|
2014-05-18 01:09:52 +04:00
|
|
|
case nilLogEntry:
|
|
|
|
s += fmt.Sprintf("o")
|
|
|
|
case noEntry:
|
|
|
|
s += fmt.Sprintf("-")
|
2014-05-06 10:28:14 +04:00
|
|
|
case nil:
|
|
|
|
s += fmt.Sprintf("<nil>")
|
|
|
|
default:
|
|
|
|
s += fmt.Sprintf("<%d:%q>", e.Term, string(e.Data))
|
|
|
|
}
|
|
|
|
if i != len(d.ents)-1 {
|
|
|
|
s += "\t\t"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return s
|
|
|
|
}
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
func diffLogs(base []Entry, logs [][]Entry) []diff {
|
2014-05-06 10:28:14 +04:00
|
|
|
var (
|
|
|
|
d []diff
|
|
|
|
max int
|
|
|
|
)
|
2014-05-18 01:09:52 +04:00
|
|
|
logs = append([][]Entry{base}, logs...)
|
2014-05-06 10:28:14 +04:00
|
|
|
for _, log := range logs {
|
|
|
|
if l := len(log); l > max {
|
|
|
|
max = l
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ediff := func(i int) (result []*Entry) {
|
|
|
|
e := make([]*Entry, len(logs))
|
|
|
|
found := false
|
|
|
|
for j, log := range logs {
|
|
|
|
if log == nil {
|
2014-05-18 01:09:52 +04:00
|
|
|
e[j] = nilLogEntry
|
2014-05-06 10:28:14 +04:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
if len(log) <= i {
|
2014-05-18 01:09:52 +04:00
|
|
|
e[j] = noEntry
|
2014-05-06 10:28:14 +04:00
|
|
|
found = true
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
e[j] = &log[i]
|
|
|
|
if j > 0 {
|
|
|
|
switch prev := e[j-1]; {
|
2014-05-18 01:09:52 +04:00
|
|
|
case prev == nilLogEntry:
|
|
|
|
case prev == noEntry:
|
2014-05-06 10:28:14 +04:00
|
|
|
case !reflect.DeepEqual(prev, e[j]):
|
|
|
|
found = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if found {
|
|
|
|
return e
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
for i := 0; i < max; i++ {
|
|
|
|
if e := ediff(i); e != nil {
|
|
|
|
d = append(d, diff{i, e})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return d
|
|
|
|
}
|
|
|
|
|
|
|
|
type stepperFunc func(Message)
|
|
|
|
|
2014-05-19 10:34:55 +04:00
|
|
|
func (f stepperFunc) Step(m Message) { f(m) }
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
var nopStepper = stepperFunc(func(Message) {})
|