2014-05-06 10:28:14 +04:00
|
|
|
package raft
|
|
|
|
|
|
|
|
import (
|
2014-05-24 20:53:01 +04:00
|
|
|
"bytes"
|
2014-06-06 02:02:12 +04:00
|
|
|
"math/rand"
|
2014-05-06 10:28:14 +04:00
|
|
|
"testing"
|
|
|
|
)
|
|
|
|
|
|
|
|
func TestLeaderElection(t *testing.T) {
|
|
|
|
tests := []struct {
|
2014-05-16 21:11:21 +04:00
|
|
|
*network
|
2014-05-06 10:28:14 +04:00
|
|
|
state stateType
|
|
|
|
}{
|
|
|
|
{newNetwork(nil, nil, nil), stateLeader},
|
|
|
|
{newNetwork(nil, nil, nopStepper), stateLeader},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper), stateCandidate},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil), stateCandidate},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil, nil), stateLeader},
|
2014-05-19 03:11:45 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
// three logs further along than 0
|
|
|
|
{newNetwork(nil, ents(1), ents(2), ents(1, 3), nil), stateFollower},
|
2014-05-19 05:05:03 +04:00
|
|
|
|
|
|
|
// logs converge
|
2014-06-06 02:02:12 +04:00
|
|
|
{newNetwork(ents(1), nil, ents(2), ents(1), nil), stateLeader},
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
|
|
|
sm := tt.network.peers[0].(*stateMachine)
|
2014-05-06 10:28:14 +04:00
|
|
|
if sm.state != tt.state {
|
|
|
|
t.Errorf("#%d: state = %s, want %s", i, sm.state, tt.state)
|
|
|
|
}
|
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-24 20:53:01 +04:00
|
|
|
func TestLogReplication(t *testing.T) {
|
|
|
|
tests := []struct {
|
|
|
|
*network
|
2014-05-28 22:08:32 +04:00
|
|
|
msgs []Message
|
|
|
|
wcommitted int
|
2014-05-24 20:53:01 +04:00
|
|
|
}{
|
|
|
|
{
|
|
|
|
newNetwork(nil, nil, nil),
|
|
|
|
[]Message{
|
2014-06-06 02:02:12 +04:00
|
|
|
{To: 0, Type: msgProp, Data: []byte("somedata")},
|
2014-05-24 20:53:01 +04:00
|
|
|
},
|
|
|
|
1,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
newNetwork(nil, nil, nil),
|
|
|
|
[]Message{
|
2014-06-06 02:02:12 +04:00
|
|
|
{To: 0, Type: msgProp, Data: []byte("somedata")},
|
|
|
|
{To: 1, Type: msgHup},
|
|
|
|
{To: 1, Type: msgProp, Data: []byte("somedata")},
|
2014-05-24 20:53:01 +04:00
|
|
|
},
|
|
|
|
2,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
2014-05-24 20:53:01 +04:00
|
|
|
|
|
|
|
for _, m := range tt.msgs {
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(m)
|
2014-05-24 20:53:01 +04:00
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
for j, x := range tt.network.peers {
|
|
|
|
sm := x.(*stateMachine)
|
2014-05-24 20:53:01 +04:00
|
|
|
|
2014-05-28 22:08:32 +04:00
|
|
|
if sm.log.committed != tt.wcommitted {
|
|
|
|
t.Errorf("#%d.%d: committed = %d, want %d", i, j, sm.log.committed, tt.wcommitted)
|
2014-05-24 20:53:01 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
ents := sm.nextEnts()
|
|
|
|
props := make([]Message, 0)
|
|
|
|
for _, m := range tt.msgs {
|
|
|
|
if m.Type == msgProp {
|
|
|
|
props = append(props, m)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for k, m := range props {
|
|
|
|
if !bytes.Equal(ents[k].Data, m.Data) {
|
|
|
|
t.Errorf("#%d.%d: data = %d, want %d", i, j, ents[k].Data, m.Data)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-29 02:30:01 +04:00
|
|
|
func TestSingleNodeCommit(t *testing.T) {
|
|
|
|
tt := newNetwork(nil)
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
|
|
|
tt.send(Message{To: 0, Type: msgProp, Data: []byte("some data")})
|
|
|
|
tt.send(Message{To: 0, Type: msgProp, Data: []byte("some data")})
|
2014-05-29 02:30:01 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
sm := tt.peers[0].(*stateMachine)
|
2014-05-29 02:30:01 +04:00
|
|
|
if sm.log.committed != 2 {
|
|
|
|
t.Errorf("committed = %d, want %d", sm.log.committed, 2)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-06-06 21:26:44 +04:00
|
|
|
func TestCannotCommitWithoutNewTermEntry(t *testing.T) {
|
|
|
|
tt := newNetwork(nil, nil, nil, nil, nil)
|
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
|
|
|
|
|
|
|
// 0 cannot reach 2,3,4
|
2014-06-06 22:01:55 +04:00
|
|
|
tt.cut(0, 2)
|
|
|
|
tt.cut(0, 3)
|
|
|
|
tt.cut(0, 4)
|
2014-06-06 21:26:44 +04:00
|
|
|
|
|
|
|
tt.send(Message{To: 0, Type: msgProp, Data: []byte("some data")})
|
|
|
|
tt.send(Message{To: 0, Type: msgProp, Data: []byte("some data")})
|
|
|
|
|
|
|
|
sm := tt.peers[0].(*stateMachine)
|
|
|
|
if sm.log.committed != 0 {
|
|
|
|
t.Errorf("committed = %d, want %d", sm.log.committed, 0)
|
|
|
|
}
|
|
|
|
|
|
|
|
// network recovery
|
|
|
|
tt.recover()
|
|
|
|
|
|
|
|
// elect 1 as the new leader with term 2
|
|
|
|
tt.send(Message{To: 1, Type: msgHup})
|
|
|
|
// send out a heartbeat
|
|
|
|
tt.send(Message{To: 1, Type: msgBeat})
|
|
|
|
|
|
|
|
// no log entries from previous term should be committed
|
|
|
|
sm = tt.peers[1].(*stateMachine)
|
|
|
|
if sm.log.committed != 0 {
|
|
|
|
t.Errorf("committed = %d, want %d", sm.log.committed, 0)
|
|
|
|
}
|
|
|
|
|
|
|
|
// after append a entry from the current term, all entries
|
|
|
|
// should be committed
|
|
|
|
tt.send(Message{To: 1, Type: msgProp, Data: []byte("some data")})
|
|
|
|
if sm.log.committed != 3 {
|
|
|
|
t.Errorf("committed = %d, want %d", sm.log.committed, 3)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
func TestDuelingCandidates(t *testing.T) {
|
|
|
|
a := newStateMachine(0, 0) // k, addr are set later
|
|
|
|
c := newStateMachine(0, 0)
|
2014-05-19 00:59:10 +04:00
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
tt := newNetwork(a, nil, c)
|
2014-06-06 22:01:55 +04:00
|
|
|
tt.cut(0, 2)
|
2014-05-19 00:59:10 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
|
|
|
tt.send(Message{To: 2, Type: msgHup})
|
2014-05-16 21:11:21 +04:00
|
|
|
|
2014-06-06 22:01:55 +04:00
|
|
|
tt.recover()
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 2, Type: msgHup})
|
2014-05-16 21:53:56 +04:00
|
|
|
|
|
|
|
tests := []struct {
|
2014-06-06 02:02:12 +04:00
|
|
|
sm *stateMachine
|
2014-05-16 21:53:56 +04:00
|
|
|
state stateType
|
2014-05-18 01:09:52 +04:00
|
|
|
term int
|
2014-05-16 21:53:56 +04:00
|
|
|
}{
|
|
|
|
{a, stateFollower, 2},
|
|
|
|
{c, stateLeader, 2},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
if g := tt.sm.state; g != tt.state {
|
|
|
|
t.Errorf("#%d: state = %s, want %s", i, g, tt.state)
|
|
|
|
}
|
|
|
|
if g := tt.sm.term; g != tt.term {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, tt.term)
|
|
|
|
}
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
|
|
|
|
base := ltoa(newLog())
|
|
|
|
for i, p := range tt.peers {
|
|
|
|
if sm, ok := p.(*stateMachine); ok {
|
|
|
|
l := ltoa(sm.log)
|
|
|
|
if g := diffu(base, l); g != "" {
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-19 04:51:45 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestCandidateConcede(t *testing.T) {
|
2014-06-06 02:02:12 +04:00
|
|
|
tt := newNetwork(nil, nil, nil)
|
|
|
|
tt.isolate(0)
|
2014-05-19 04:51:45 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
|
|
|
tt.send(Message{To: 2, Type: msgHup})
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
// heal the partition
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.recover()
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
data := []byte("force follower")
|
|
|
|
// send a proposal to 2 to flush out a msgApp to 0
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 2, Type: msgProp, Data: data})
|
2014-05-19 04:51:45 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
a := tt.peers[0].(*stateMachine)
|
2014-05-19 04:51:45 +04:00
|
|
|
if g := a.state; g != stateFollower {
|
|
|
|
t.Errorf("state = %s, want %s", g, stateFollower)
|
|
|
|
}
|
|
|
|
if g := a.term; g != 1 {
|
|
|
|
t.Errorf("term = %d, want %d", g, 1)
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
wantLog := ltoa(&log{ents: []Entry{{}, {Term: 1, Data: data}}, committed: 1})
|
|
|
|
for i, p := range tt.peers {
|
|
|
|
if sm, ok := p.(*stateMachine); ok {
|
|
|
|
l := ltoa(sm.log)
|
|
|
|
if g := diffu(wantLog, l); g != "" {
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-16 21:53:56 +04:00
|
|
|
}
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-16 21:53:56 +04:00
|
|
|
}
|
|
|
|
|
2014-05-28 22:59:01 +04:00
|
|
|
func TestSingleNodeCandidate(t *testing.T) {
|
|
|
|
tt := newNetwork(nil)
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
2014-05-28 22:59:01 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
sm := tt.peers[0].(*stateMachine)
|
2014-05-28 22:59:01 +04:00
|
|
|
if sm.state != stateLeader {
|
|
|
|
t.Errorf("state = %d, want %d", sm.state, stateLeader)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-16 21:53:56 +04:00
|
|
|
func TestOldMessages(t *testing.T) {
|
|
|
|
tt := newNetwork(nil, nil, nil)
|
|
|
|
// make 0 leader @ term 3
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
|
|
|
tt.send(Message{To: 1, Type: msgHup})
|
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
2014-05-16 21:53:56 +04:00
|
|
|
// pretend we're an old leader trying to make progress
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgApp, Term: 1, Entries: []Entry{{Term: 1}}})
|
|
|
|
|
|
|
|
base := ltoa(newLog())
|
|
|
|
for i, p := range tt.peers {
|
|
|
|
if sm, ok := p.(*stateMachine); ok {
|
|
|
|
l := ltoa(sm.log)
|
|
|
|
if g := diffu(base, l); g != "" {
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-18 04:18:58 +04:00
|
|
|
// TestOldMessagesReply - optimization - reply with new term.
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
func TestProposal(t *testing.T) {
|
|
|
|
tests := []struct {
|
2014-05-16 21:11:21 +04:00
|
|
|
*network
|
2014-05-18 01:09:52 +04:00
|
|
|
success bool
|
2014-05-06 10:28:14 +04:00
|
|
|
}{
|
2014-05-18 01:09:52 +04:00
|
|
|
{newNetwork(nil, nil, nil), true},
|
|
|
|
{newNetwork(nil, nil, nopStepper), true},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper), false},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil), false},
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil, nil), true},
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-06-06 02:02:12 +04:00
|
|
|
send := func(m Message) {
|
2014-05-06 10:28:14 +04:00
|
|
|
defer func() {
|
2014-05-18 01:09:52 +04:00
|
|
|
// only recover is we expect it to panic so
|
|
|
|
// panics we don't expect go up.
|
|
|
|
if !tt.success {
|
2014-05-06 10:28:14 +04:00
|
|
|
e := recover()
|
|
|
|
if e != nil {
|
|
|
|
t.Logf("#%d: err: %s", i, e)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(m)
|
|
|
|
}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
data := []byte("somedata")
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
// promote 0 the leader
|
2014-06-06 02:02:12 +04:00
|
|
|
send(Message{To: 0, Type: msgHup})
|
|
|
|
send(Message{To: 0, Type: msgProp, Data: data})
|
2014-05-06 10:28:14 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
wantLog := newLog()
|
2014-05-18 01:09:52 +04:00
|
|
|
if tt.success {
|
2014-06-06 02:02:12 +04:00
|
|
|
wantLog = &log{ents: []Entry{{}, {Term: 1, Data: data}}, committed: 1}
|
2014-05-18 01:09:52 +04:00
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
base := ltoa(wantLog)
|
|
|
|
for i, p := range tt.peers {
|
|
|
|
if sm, ok := p.(*stateMachine); ok {
|
|
|
|
l := ltoa(sm.log)
|
|
|
|
if g := diffu(base, l); g != "" {
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
sm := tt.network.peers[0].(*stateMachine)
|
2014-05-06 10:28:14 +04:00
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestProposalByProxy(t *testing.T) {
|
2014-05-16 11:55:17 +04:00
|
|
|
data := []byte("somedata")
|
2014-05-18 01:09:52 +04:00
|
|
|
tests := []*network{
|
|
|
|
newNetwork(nil, nil, nil),
|
|
|
|
newNetwork(nil, nil, nopStepper),
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
// promote 0 the leader
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 0, Type: msgHup})
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
// propose via follower
|
2014-06-06 02:02:12 +04:00
|
|
|
tt.send(Message{To: 1, Type: msgProp, Data: []byte("somedata")})
|
|
|
|
|
|
|
|
wantLog := &log{ents: []Entry{{}, {Term: 1, Data: data}}, committed: 1}
|
|
|
|
base := ltoa(wantLog)
|
|
|
|
for i, p := range tt.peers {
|
|
|
|
if sm, ok := p.(*stateMachine); ok {
|
|
|
|
l := ltoa(sm.log)
|
|
|
|
if g := diffu(base, l); g != "" {
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
sm := tt.peers[0].(*stateMachine)
|
2014-05-06 10:28:14 +04:00
|
|
|
if g := sm.term; g != 1 {
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-24 20:53:01 +04:00
|
|
|
func TestCommit(t *testing.T) {
|
2014-05-22 03:02:15 +04:00
|
|
|
tests := []struct {
|
|
|
|
matches []int
|
|
|
|
logs []Entry
|
|
|
|
smTerm int
|
|
|
|
w int
|
|
|
|
}{
|
2014-06-06 22:21:26 +04:00
|
|
|
// single
|
|
|
|
{[]int{1}, []Entry{{}, {Term: 1}}, 1, 1},
|
|
|
|
{[]int{1}, []Entry{{}, {Term: 1}}, 2, 0},
|
|
|
|
{[]int{2}, []Entry{{}, {Term: 1}, {Term: 2}}, 2, 2},
|
|
|
|
{[]int{1}, []Entry{{}, {Term: 2}}, 2, 1},
|
|
|
|
|
2014-05-22 03:02:15 +04:00
|
|
|
// odd
|
|
|
|
{[]int{2, 1, 1}, []Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 1}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
{[]int{2, 1, 2}, []Entry{{}, {Term: 1}, {Term: 2}}, 2, 2},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 2}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
|
|
|
|
// even
|
|
|
|
{[]int{2, 1, 1, 1}, []Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 1, 1}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
{[]int{2, 1, 1, 2}, []Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 1, 2}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
{[]int{2, 1, 2, 2}, []Entry{{}, {Term: 1}, {Term: 2}}, 2, 2},
|
2014-05-24 20:53:01 +04:00
|
|
|
{[]int{2, 1, 2, 2}, []Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-05-29 03:34:40 +04:00
|
|
|
ins := make([]index, len(tt.matches))
|
2014-05-22 03:02:15 +04:00
|
|
|
for j := 0; j < len(ins); j++ {
|
2014-05-29 03:34:40 +04:00
|
|
|
ins[j] = index{tt.matches[j], tt.matches[j] + 1}
|
2014-05-22 03:02:15 +04:00
|
|
|
}
|
2014-05-27 23:24:49 +04:00
|
|
|
sm := &stateMachine{log: &log{ents: tt.logs}, ins: ins, k: len(ins), term: tt.smTerm}
|
2014-05-24 20:53:01 +04:00
|
|
|
sm.maybeCommit()
|
2014-05-28 22:08:32 +04:00
|
|
|
if g := sm.log.committed; g != tt.w {
|
|
|
|
t.Errorf("#%d: committed = %d, want %d", i, g, tt.w)
|
2014-05-22 03:02:15 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-18 11:17:46 +04:00
|
|
|
func TestVote(t *testing.T) {
|
|
|
|
tests := []struct {
|
2014-05-25 08:43:12 +04:00
|
|
|
state stateType
|
2014-05-18 11:17:46 +04:00
|
|
|
i, term int
|
2014-05-25 08:08:06 +04:00
|
|
|
voteFor int
|
2014-05-18 11:17:46 +04:00
|
|
|
w int
|
|
|
|
}{
|
2014-05-25 08:43:12 +04:00
|
|
|
{stateFollower, 0, 0, none, -1},
|
|
|
|
{stateFollower, 0, 1, none, -1},
|
|
|
|
{stateFollower, 0, 2, none, -1},
|
|
|
|
{stateFollower, 0, 3, none, 2},
|
|
|
|
|
|
|
|
{stateFollower, 1, 0, none, -1},
|
|
|
|
{stateFollower, 1, 1, none, -1},
|
|
|
|
{stateFollower, 1, 2, none, -1},
|
|
|
|
{stateFollower, 1, 3, none, 2},
|
|
|
|
|
|
|
|
{stateFollower, 2, 0, none, -1},
|
|
|
|
{stateFollower, 2, 1, none, -1},
|
|
|
|
{stateFollower, 2, 2, none, 2},
|
|
|
|
{stateFollower, 2, 3, none, 2},
|
|
|
|
|
|
|
|
{stateFollower, 3, 0, none, -1},
|
|
|
|
{stateFollower, 3, 1, none, -1},
|
|
|
|
{stateFollower, 3, 2, none, 2},
|
|
|
|
{stateFollower, 3, 3, none, 2},
|
|
|
|
|
|
|
|
{stateFollower, 3, 2, 1, 2},
|
|
|
|
{stateFollower, 3, 2, 0, -1},
|
|
|
|
|
|
|
|
{stateLeader, 3, 3, 0, -1},
|
|
|
|
{stateCandidate, 3, 3, 0, -1},
|
2014-05-18 11:17:46 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
called := false
|
2014-06-06 02:02:12 +04:00
|
|
|
sm := &stateMachine{
|
|
|
|
state: tt.state,
|
|
|
|
vote: tt.voteFor,
|
|
|
|
log: &log{ents: []Entry{{}, {Term: 2}, {Term: 2}}},
|
2014-05-28 09:23:14 +04:00
|
|
|
}
|
2014-05-25 08:08:06 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
sm.Step(Message{Type: msgVote, From: 1, Index: tt.i, LogTerm: tt.term})
|
|
|
|
|
|
|
|
for _, m := range sm.Msgs() {
|
2014-05-18 11:17:46 +04:00
|
|
|
called = true
|
|
|
|
if m.Index != tt.w {
|
|
|
|
t.Errorf("#%d, m.Index = %d, want %d", i, m.Index, tt.w)
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
}
|
2014-05-18 11:17:46 +04:00
|
|
|
if !called {
|
|
|
|
t.Fatal("#%d: not called", i)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-06-07 03:27:29 +04:00
|
|
|
func TestStateTransition(t *testing.T) {
|
|
|
|
tests := []struct {
|
|
|
|
from stateType
|
|
|
|
to stateType
|
|
|
|
wallow bool
|
|
|
|
wterm int
|
|
|
|
wlead int
|
|
|
|
}{
|
|
|
|
{stateFollower, stateFollower, true, 1, none},
|
|
|
|
{stateFollower, stateCandidate, true, 1, none},
|
|
|
|
{stateFollower, stateLeader, false, -1, none},
|
|
|
|
|
|
|
|
{stateCandidate, stateFollower, true, 0, none},
|
|
|
|
{stateCandidate, stateCandidate, true, 1, none},
|
|
|
|
{stateCandidate, stateLeader, true, 0, 0},
|
|
|
|
|
|
|
|
{stateLeader, stateFollower, true, 1, none},
|
|
|
|
{stateLeader, stateCandidate, false, 1, none},
|
|
|
|
{stateLeader, stateLeader, true, 0, 0},
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
func() {
|
|
|
|
defer func() {
|
|
|
|
if r := recover(); r != nil {
|
|
|
|
if tt.wallow == true {
|
|
|
|
t.Errorf("%d: allow = %v, want %v", i, false, true)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
sm := newStateMachine(1, 0)
|
|
|
|
sm.state = tt.from
|
|
|
|
|
|
|
|
switch tt.to {
|
|
|
|
case stateFollower:
|
|
|
|
sm.becomeFollower(tt.wterm, tt.wlead)
|
|
|
|
case stateCandidate:
|
|
|
|
sm.becomeCandidate()
|
|
|
|
case stateLeader:
|
|
|
|
sm.becomeLeader()
|
|
|
|
}
|
|
|
|
|
|
|
|
if sm.term != tt.wterm {
|
|
|
|
t.Errorf("%d: term = %d, want %d", i, sm.term, tt.wterm)
|
|
|
|
}
|
|
|
|
if sm.lead != tt.wlead {
|
|
|
|
t.Errorf("%d: lead = %d, want %d", i, sm.lead, tt.wlead)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-24 00:30:04 +04:00
|
|
|
func TestAllServerStepdown(t *testing.T) {
|
|
|
|
tests := []stateType{stateFollower, stateCandidate, stateLeader}
|
|
|
|
|
|
|
|
want := struct {
|
|
|
|
state stateType
|
|
|
|
term int
|
|
|
|
index int
|
|
|
|
}{stateFollower, 3, 1}
|
|
|
|
|
|
|
|
tmsgTypes := [...]messageType{msgVote, msgApp}
|
|
|
|
tterm := 3
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
sm := newStateMachine(3, 0)
|
|
|
|
switch tt {
|
|
|
|
case stateFollower:
|
|
|
|
sm.becomeFollower(1, 0)
|
|
|
|
case stateCandidate:
|
|
|
|
sm.becomeCandidate()
|
|
|
|
case stateLeader:
|
|
|
|
sm.becomeCandidate()
|
|
|
|
sm.becomeLeader()
|
|
|
|
}
|
|
|
|
|
|
|
|
for j, msgType := range tmsgTypes {
|
|
|
|
sm.Step(Message{Type: msgType, Term: tterm, LogTerm: tterm})
|
|
|
|
|
|
|
|
if sm.state != want.state {
|
|
|
|
t.Errorf("#%d.%d state = %v , want %v", i, j, sm.state, want.state)
|
|
|
|
}
|
|
|
|
if sm.term != want.term {
|
|
|
|
t.Errorf("#%d.%d term = %v , want %v", i, j, sm.term, want.term)
|
|
|
|
}
|
2014-05-27 23:24:49 +04:00
|
|
|
if len(sm.log.ents) != want.index {
|
|
|
|
t.Errorf("#%d.%d index = %v , want %v", i, j, len(sm.log.ents), want.index)
|
2014-05-24 00:30:04 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-25 10:03:13 +04:00
|
|
|
func TestLeaderAppResp(t *testing.T) {
|
|
|
|
tests := []struct {
|
|
|
|
index int
|
|
|
|
wmsgNum int
|
|
|
|
windex int
|
|
|
|
wcommitted int
|
|
|
|
}{
|
|
|
|
{-1, 1, 1, 0}, // bad resp; leader does not commit; reply with log entries
|
|
|
|
{2, 2, 2, 2}, // good resp; leader commits; broadcast with commit index
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
// sm term is 1 after it becomes the leader.
|
|
|
|
// thus the last log term must be 1 to be committed.
|
|
|
|
sm := &stateMachine{addr: 0, k: 3, log: &log{ents: []Entry{{}, {Term: 0}, {Term: 1}}}}
|
|
|
|
sm.becomeCandidate()
|
|
|
|
sm.becomeLeader()
|
|
|
|
sm.Step(Message{From: 1, Type: msgAppResp, Index: tt.index, Term: sm.term})
|
|
|
|
msgs := sm.Msgs()
|
|
|
|
|
|
|
|
if len(msgs) != tt.wmsgNum {
|
|
|
|
t.Errorf("#%d msgNum = %d, want %d", i, len(msgs), tt.wmsgNum)
|
|
|
|
}
|
|
|
|
for j, msg := range msgs {
|
|
|
|
if msg.Index != tt.windex {
|
|
|
|
t.Errorf("#%d.%d index = %d, want %d", i, j, msg.Index, tt.windex)
|
|
|
|
}
|
|
|
|
if msg.Commit != tt.wcommitted {
|
|
|
|
t.Errorf("#%d.%d commit = %d, want %d", i, j, msg.Commit, tt.wcommitted)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
func ents(terms ...int) *stateMachine {
|
|
|
|
ents := []Entry{{}}
|
|
|
|
for _, term := range terms {
|
|
|
|
ents = append(ents, Entry{Term: term})
|
2014-05-18 01:09:52 +04:00
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
sm := &stateMachine{log: &log{ents: ents}}
|
|
|
|
sm.reset()
|
|
|
|
return sm
|
2014-05-18 01:09:52 +04:00
|
|
|
}
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
type network struct {
|
2014-06-06 02:02:12 +04:00
|
|
|
peers []Interface
|
|
|
|
dropm map[connem]float64
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
// newNetwork initializes a network from peers. A nil node will be replaced
|
|
|
|
// with a new *stateMachine. A *stateMachine will get its k, addr.
|
|
|
|
func newNetwork(peers ...Interface) *network {
|
|
|
|
for addr, p := range peers {
|
|
|
|
switch v := p.(type) {
|
2014-05-06 10:28:14 +04:00
|
|
|
case nil:
|
2014-06-06 02:02:12 +04:00
|
|
|
sm := newStateMachine(len(peers), addr)
|
|
|
|
peers[addr] = sm
|
|
|
|
case *stateMachine:
|
|
|
|
v.k = len(peers)
|
|
|
|
v.addr = addr
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
return &network{peers: peers, dropm: make(map[connem]float64)}
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
func (nw *network) send(msgs ...Message) {
|
|
|
|
for len(msgs) > 0 {
|
|
|
|
m := msgs[0]
|
|
|
|
p := nw.peers[m.To]
|
|
|
|
p.Step(m)
|
|
|
|
msgs = append(msgs[1:], nw.filter(p.Msgs())...)
|
2014-05-16 21:11:21 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
func (nw *network) drop(from, to int, perc float64) {
|
|
|
|
nw.dropm[connem{from, to}] = perc
|
|
|
|
}
|
|
|
|
|
2014-06-06 22:01:55 +04:00
|
|
|
func (nw *network) cut(one, other int) {
|
|
|
|
nw.drop(one, other, 1)
|
|
|
|
nw.drop(other, one, 1)
|
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
func (nw *network) isolate(addr int) {
|
|
|
|
for i := 0; i < len(nw.peers); i++ {
|
|
|
|
if i != addr {
|
|
|
|
nw.drop(addr, i, 1.0)
|
|
|
|
nw.drop(i, addr, 1.0)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
func (nw *network) recover() {
|
|
|
|
nw.dropm = make(map[connem]float64)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
func (nw *network) filter(msgs []Message) []Message {
|
|
|
|
mm := make([]Message, 0)
|
|
|
|
for _, m := range msgs {
|
|
|
|
switch m.Type {
|
|
|
|
case msgHup:
|
|
|
|
// hups never go over the network, so don't drop them but panic
|
|
|
|
panic("unexpected msgHup")
|
2014-05-06 10:28:14 +04:00
|
|
|
default:
|
2014-06-06 02:02:12 +04:00
|
|
|
perc := nw.dropm[connem{m.From, m.To}]
|
|
|
|
if n := rand.Float64(); n < perc {
|
2014-05-06 10:28:14 +04:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
mm = append(mm, m)
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
return mm
|
2014-05-06 10:28:14 +04:00
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
type connem struct {
|
|
|
|
from, to int
|
2014-05-24 00:30:04 +04:00
|
|
|
}
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
type blackHole struct{}
|
2014-05-27 23:24:49 +04:00
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
func (blackHole) Step(Message) {}
|
|
|
|
func (blackHole) Msgs() []Message { return nil }
|
|
|
|
|
|
|
|
var nopStepper = &blackHole{}
|