2014-10-18 02:41:22 +04:00
|
|
|
|
/*
|
|
|
|
|
Copyright 2014 CoreOS, Inc.
|
|
|
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
|
limitations under the License.
|
|
|
|
|
*/
|
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
|
package raft
|
|
|
|
|
|
|
|
|
|
import (
|
2014-05-24 20:53:01 +04:00
|
|
|
|
"bytes"
|
2014-09-12 10:18:51 +04:00
|
|
|
|
"fmt"
|
2014-10-07 16:34:15 +04:00
|
|
|
|
"math"
|
2014-06-06 02:02:12 +04:00
|
|
|
|
"math/rand"
|
2014-07-01 23:10:43 +04:00
|
|
|
|
"reflect"
|
2014-05-06 10:28:14 +04:00
|
|
|
|
"testing"
|
2014-08-28 05:53:18 +04:00
|
|
|
|
|
|
|
|
|
pb "github.com/coreos/etcd/raft/raftpb"
|
2014-05-06 10:28:14 +04:00
|
|
|
|
)
|
|
|
|
|
|
2014-08-24 04:56:06 +04:00
|
|
|
|
// nextEnts returns the appliable entries and updates the applied index
|
2014-11-13 00:23:42 +03:00
|
|
|
|
func nextEnts(r *raft, s *MemoryStorage) (ents []pb.Entry) {
|
2014-11-03 23:04:24 +03:00
|
|
|
|
// Transfer all unstable entries to "stable" storage.
|
2014-11-13 00:23:42 +03:00
|
|
|
|
s.Append(r.raftLog.unstableEntries())
|
2014-11-03 23:04:24 +03:00
|
|
|
|
r.raftLog.stableTo(r.raftLog.lastIndex())
|
|
|
|
|
|
2014-08-24 04:56:06 +04:00
|
|
|
|
ents = r.raftLog.nextEnts()
|
2014-11-07 01:13:35 +03:00
|
|
|
|
r.raftLog.appliedTo(r.raftLog.committed)
|
2014-08-24 04:56:06 +04:00
|
|
|
|
return ents
|
|
|
|
|
}
|
|
|
|
|
|
2014-08-23 00:24:33 +04:00
|
|
|
|
type Interface interface {
|
2014-08-28 05:53:18 +04:00
|
|
|
|
Step(m pb.Message) error
|
2014-10-30 00:04:45 +03:00
|
|
|
|
readMessages() []pb.Message
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (r *raft) readMessages() []pb.Message {
|
|
|
|
|
msgs := r.msgs
|
|
|
|
|
r.msgs = make([]pb.Message, 0)
|
|
|
|
|
|
|
|
|
|
return msgs
|
2014-08-23 00:24:33 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-11-21 04:58:23 +03:00
|
|
|
|
func TestProgressUpdate(t *testing.T) {
|
|
|
|
|
prevM, prevN := uint64(3), uint64(5)
|
|
|
|
|
tests := []struct {
|
|
|
|
|
update uint64
|
|
|
|
|
|
|
|
|
|
wm uint64
|
|
|
|
|
wn uint64
|
|
|
|
|
}{
|
|
|
|
|
{prevM - 1, prevM, prevN}, // do not decrease match, next
|
|
|
|
|
{prevM, prevM, prevN}, // do not decrease next
|
|
|
|
|
{prevM + 1, prevM + 1, prevN}, // increase match, do not decrease next
|
|
|
|
|
{prevM + 2, prevM + 2, prevN + 1}, // increase match, next
|
|
|
|
|
}
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
|
p := &progress{
|
|
|
|
|
match: prevM,
|
|
|
|
|
next: prevN,
|
|
|
|
|
}
|
|
|
|
|
p.update(tt.update)
|
|
|
|
|
if p.match != tt.wm {
|
|
|
|
|
t.Errorf("#%d: match=%d, want %d", i, p.match, tt.wm)
|
|
|
|
|
}
|
|
|
|
|
if p.next != tt.wn {
|
|
|
|
|
t.Errorf("#%d: next=%d, want %d", i, p.next, tt.wn)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-30 08:57:25 +03:00
|
|
|
|
func TestProgressMaybeDecr(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
|
|
|
|
m uint64
|
|
|
|
|
n uint64
|
|
|
|
|
to uint64
|
|
|
|
|
|
|
|
|
|
w bool
|
|
|
|
|
wn uint64
|
|
|
|
|
}{
|
|
|
|
|
{
|
|
|
|
|
// match != 0 is always false
|
|
|
|
|
1, 0, 0, false, 0,
|
|
|
|
|
},
|
|
|
|
|
{
|
2014-11-18 22:42:08 +03:00
|
|
|
|
// match != 0 and to is greater than match
|
|
|
|
|
// directly decrease to match+1
|
|
|
|
|
5, 10, 5, false, 10,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
// match != 0 and to is greater than match
|
|
|
|
|
// directly decrease to match+1
|
|
|
|
|
5, 10, 4, false, 10,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
// match != 0 and to is not greater than match
|
|
|
|
|
5, 10, 9, true, 6,
|
2014-10-30 08:57:25 +03:00
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
// next-1 != to is always false
|
|
|
|
|
0, 0, 0, false, 0,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
// next-1 != to is always false
|
|
|
|
|
0, 10, 5, false, 10,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
// next>1 = decremented by 1
|
|
|
|
|
0, 10, 9, true, 9,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
// next>1 = decremented by 1
|
|
|
|
|
0, 2, 1, true, 1,
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
// next<=1 = reset to 1
|
|
|
|
|
0, 1, 0, true, 1,
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
|
p := &progress{
|
|
|
|
|
match: tt.m,
|
|
|
|
|
next: tt.n,
|
|
|
|
|
}
|
|
|
|
|
if g := p.maybeDecrTo(tt.to); g != tt.w {
|
|
|
|
|
t.Errorf("#%d: maybeDecrTo=%t, want %t", i, g, tt.w)
|
|
|
|
|
}
|
|
|
|
|
if gm := p.match; gm != tt.m {
|
|
|
|
|
t.Errorf("#%d: match=%d, want %d", i, gm, tt.m)
|
|
|
|
|
}
|
|
|
|
|
if gn := p.next; gn != tt.wn {
|
|
|
|
|
t.Errorf("#%d: next=%d, want %d", i, gn, tt.wn)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
|
func TestLeaderElection(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
2014-05-16 21:11:21 +04:00
|
|
|
|
*network
|
2014-09-16 04:35:02 +04:00
|
|
|
|
state StateType
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}{
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{newNetwork(nil, nil, nil), StateLeader},
|
|
|
|
|
{newNetwork(nil, nil, nopStepper), StateLeader},
|
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper), StateCandidate},
|
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil), StateCandidate},
|
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil, nil), StateLeader},
|
2014-05-19 03:11:45 +04:00
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
|
// three logs further along than 0
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{newNetwork(nil, ents(1), ents(2), ents(1, 3), nil), StateFollower},
|
2014-05-19 05:05:03 +04:00
|
|
|
|
|
|
|
|
|
// logs converge
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{newNetwork(ents(1), nil, ents(2), ents(1), nil), StateLeader},
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm := tt.network.peers[1].(*raft)
|
2014-05-06 10:28:14 +04:00
|
|
|
|
if sm.state != tt.state {
|
|
|
|
|
t.Errorf("#%d: state = %s, want %s", i, sm.state, tt.state)
|
|
|
|
|
}
|
2014-08-23 01:19:30 +04:00
|
|
|
|
if g := sm.Term; g != 1 {
|
2014-05-06 10:28:14 +04:00
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-24 20:53:01 +04:00
|
|
|
|
func TestLogReplication(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
|
|
|
|
*network
|
2014-08-28 05:53:18 +04:00
|
|
|
|
msgs []pb.Message
|
2014-10-08 14:29:53 +04:00
|
|
|
|
wcommitted uint64
|
2014-05-24 20:53:01 +04:00
|
|
|
|
}{
|
|
|
|
|
{
|
|
|
|
|
newNetwork(nil, nil, nil),
|
2014-08-28 05:53:18 +04:00
|
|
|
|
[]pb.Message{
|
2014-10-12 11:34:22 +04:00
|
|
|
|
{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("somedata")}}},
|
2014-05-24 20:53:01 +04:00
|
|
|
|
},
|
2014-06-26 22:55:56 +04:00
|
|
|
|
2,
|
2014-05-24 20:53:01 +04:00
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
newNetwork(nil, nil, nil),
|
2014-08-28 05:53:18 +04:00
|
|
|
|
[]pb.Message{
|
2014-10-12 11:34:22 +04:00
|
|
|
|
{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("somedata")}}},
|
|
|
|
|
{From: 1, To: 2, Type: pb.MsgHup},
|
|
|
|
|
{From: 1, To: 2, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("somedata")}}},
|
2014-05-24 20:53:01 +04:00
|
|
|
|
},
|
2014-06-26 22:55:56 +04:00
|
|
|
|
4,
|
2014-05-24 20:53:01 +04:00
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
2014-05-24 20:53:01 +04:00
|
|
|
|
|
|
|
|
|
for _, m := range tt.msgs {
|
2014-06-06 02:02:12 +04:00
|
|
|
|
tt.send(m)
|
2014-05-24 20:53:01 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
|
for j, x := range tt.network.peers {
|
2014-08-23 00:24:33 +04:00
|
|
|
|
sm := x.(*raft)
|
2014-05-24 20:53:01 +04:00
|
|
|
|
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.committed != tt.wcommitted {
|
|
|
|
|
t.Errorf("#%d.%d: committed = %d, want %d", i, j, sm.raftLog.committed, tt.wcommitted)
|
2014-05-24 20:53:01 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-10-08 02:22:35 +04:00
|
|
|
|
ents := []pb.Entry{}
|
2014-11-13 00:23:42 +03:00
|
|
|
|
for _, e := range nextEnts(sm, tt.network.storage[j]) {
|
2014-06-26 22:55:56 +04:00
|
|
|
|
if e.Data != nil {
|
|
|
|
|
ents = append(ents, e)
|
|
|
|
|
}
|
|
|
|
|
}
|
2014-10-08 02:22:35 +04:00
|
|
|
|
props := []pb.Message{}
|
2014-05-24 20:53:01 +04:00
|
|
|
|
for _, m := range tt.msgs {
|
2014-10-12 11:34:22 +04:00
|
|
|
|
if m.Type == pb.MsgProp {
|
2014-05-24 20:53:01 +04:00
|
|
|
|
props = append(props, m)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
for k, m := range props {
|
2014-06-06 00:00:18 +04:00
|
|
|
|
if !bytes.Equal(ents[k].Data, m.Entries[0].Data) {
|
|
|
|
|
t.Errorf("#%d.%d: data = %d, want %d", i, j, ents[k].Data, m.Entries[0].Data)
|
2014-05-24 20:53:01 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-29 02:30:01 +04:00
|
|
|
|
func TestSingleNodeCommit(t *testing.T) {
|
|
|
|
|
tt := newNetwork(nil)
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("some data")}}})
|
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("some data")}}})
|
2014-05-29 02:30:01 +04:00
|
|
|
|
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm := tt.peers[1].(*raft)
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.committed != 3 {
|
|
|
|
|
t.Errorf("committed = %d, want %d", sm.raftLog.committed, 3)
|
2014-05-29 02:30:01 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-06-26 22:55:56 +04:00
|
|
|
|
// TestCannotCommitWithoutNewTermEntry tests the entries cannot be committed
|
|
|
|
|
// when leader changes, no new proposal comes in and ChangeTerm proposal is
|
|
|
|
|
// filtered.
|
2014-06-06 21:26:44 +04:00
|
|
|
|
func TestCannotCommitWithoutNewTermEntry(t *testing.T) {
|
|
|
|
|
tt := newNetwork(nil, nil, nil, nil, nil)
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
2014-06-06 21:26:44 +04:00
|
|
|
|
|
|
|
|
|
// 0 cannot reach 2,3,4
|
2014-09-12 06:23:05 +04:00
|
|
|
|
tt.cut(1, 3)
|
|
|
|
|
tt.cut(1, 4)
|
|
|
|
|
tt.cut(1, 5)
|
2014-06-06 21:26:44 +04:00
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("some data")}}})
|
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("some data")}}})
|
2014-06-06 21:26:44 +04:00
|
|
|
|
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm := tt.peers[1].(*raft)
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.committed != 1 {
|
|
|
|
|
t.Errorf("committed = %d, want %d", sm.raftLog.committed, 1)
|
2014-06-06 21:26:44 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// network recovery
|
|
|
|
|
tt.recover()
|
2014-06-26 22:55:56 +04:00
|
|
|
|
// avoid committing ChangeTerm proposal
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.ignore(pb.MsgApp)
|
2014-06-06 21:26:44 +04:00
|
|
|
|
|
|
|
|
|
// elect 1 as the new leader with term 2
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup})
|
2014-06-06 21:26:44 +04:00
|
|
|
|
|
|
|
|
|
// no log entries from previous term should be committed
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm = tt.peers[2].(*raft)
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.committed != 1 {
|
|
|
|
|
t.Errorf("committed = %d, want %d", sm.raftLog.committed, 1)
|
2014-06-06 21:26:44 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-06-26 22:55:56 +04:00
|
|
|
|
tt.recover()
|
|
|
|
|
|
|
|
|
|
// still be able to append a entry
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 2, To: 2, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("some data")}}})
|
2014-06-26 22:55:56 +04:00
|
|
|
|
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.committed != 5 {
|
|
|
|
|
t.Errorf("committed = %d, want %d", sm.raftLog.committed, 5)
|
2014-06-26 22:55:56 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestCommitWithoutNewTermEntry tests the entries could be committed
|
|
|
|
|
// when leader changes, no new proposal comes in.
|
|
|
|
|
func TestCommitWithoutNewTermEntry(t *testing.T) {
|
|
|
|
|
tt := newNetwork(nil, nil, nil, nil, nil)
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
2014-06-26 22:55:56 +04:00
|
|
|
|
|
|
|
|
|
// 0 cannot reach 2,3,4
|
2014-09-12 06:23:05 +04:00
|
|
|
|
tt.cut(1, 3)
|
|
|
|
|
tt.cut(1, 4)
|
|
|
|
|
tt.cut(1, 5)
|
2014-06-26 22:55:56 +04:00
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("some data")}}})
|
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("some data")}}})
|
2014-06-26 22:55:56 +04:00
|
|
|
|
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm := tt.peers[1].(*raft)
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.committed != 1 {
|
|
|
|
|
t.Errorf("committed = %d, want %d", sm.raftLog.committed, 1)
|
2014-06-26 22:55:56 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// network recovery
|
|
|
|
|
tt.recover()
|
|
|
|
|
|
|
|
|
|
// elect 1 as the new leader with term 2
|
|
|
|
|
// after append a ChangeTerm entry from the current term, all entries
|
|
|
|
|
// should be committed
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup})
|
2014-06-26 22:55:56 +04:00
|
|
|
|
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.committed != 4 {
|
|
|
|
|
t.Errorf("committed = %d, want %d", sm.raftLog.committed, 4)
|
2014-06-06 21:26:44 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
|
func TestDuelingCandidates(t *testing.T) {
|
2014-11-13 00:23:42 +03:00
|
|
|
|
a := newRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage())
|
|
|
|
|
b := newRaft(2, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage())
|
|
|
|
|
c := newRaft(3, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage())
|
2014-05-19 00:59:10 +04:00
|
|
|
|
|
2014-06-26 22:55:56 +04:00
|
|
|
|
nt := newNetwork(a, b, c)
|
2014-09-12 06:23:05 +04:00
|
|
|
|
nt.cut(1, 3)
|
2014-05-19 00:59:10 +04:00
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
|
|
|
|
nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup})
|
2014-05-16 21:11:21 +04:00
|
|
|
|
|
2014-06-26 22:55:56 +04:00
|
|
|
|
nt.recover()
|
2014-10-12 11:34:22 +04:00
|
|
|
|
nt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup})
|
2014-05-16 21:53:56 +04:00
|
|
|
|
|
2014-11-03 23:04:24 +03:00
|
|
|
|
wlog := &raftLog{
|
|
|
|
|
storage: &MemoryStorage{ents: []pb.Entry{{}, pb.Entry{Data: nil, Term: 1, Index: 1}}},
|
|
|
|
|
committed: 1,
|
|
|
|
|
unstable: 2,
|
|
|
|
|
}
|
2014-05-16 21:53:56 +04:00
|
|
|
|
tests := []struct {
|
2014-08-23 00:24:33 +04:00
|
|
|
|
sm *raft
|
2014-09-16 04:35:02 +04:00
|
|
|
|
state StateType
|
2014-10-08 14:29:53 +04:00
|
|
|
|
term uint64
|
2014-07-24 03:15:25 +04:00
|
|
|
|
raftLog *raftLog
|
2014-05-16 21:53:56 +04:00
|
|
|
|
}{
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{a, StateFollower, 2, wlog},
|
|
|
|
|
{b, StateFollower, 2, wlog},
|
2014-11-13 00:23:42 +03:00
|
|
|
|
{c, StateFollower, 2, newLog(NewMemoryStorage())},
|
2014-05-16 21:53:56 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
|
if g := tt.sm.state; g != tt.state {
|
|
|
|
|
t.Errorf("#%d: state = %s, want %s", i, g, tt.state)
|
|
|
|
|
}
|
2014-08-23 01:19:30 +04:00
|
|
|
|
if g := tt.sm.Term; g != tt.term {
|
2014-05-16 21:53:56 +04:00
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, tt.term)
|
|
|
|
|
}
|
2014-07-24 03:15:25 +04:00
|
|
|
|
base := ltoa(tt.raftLog)
|
2014-10-08 14:29:53 +04:00
|
|
|
|
if sm, ok := nt.peers[1+uint64(i)].(*raft); ok {
|
2014-07-24 03:15:25 +04:00
|
|
|
|
l := ltoa(sm.raftLog)
|
2014-06-06 02:02:12 +04:00
|
|
|
|
if g := diffu(base, l); g != "" {
|
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-19 04:51:45 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestCandidateConcede(t *testing.T) {
|
2014-06-06 02:02:12 +04:00
|
|
|
|
tt := newNetwork(nil, nil, nil)
|
2014-09-12 06:23:05 +04:00
|
|
|
|
tt.isolate(1)
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
|
|
|
|
tt.send(pb.Message{From: 3, To: 3, Type: pb.MsgHup})
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
|
|
// heal the partition
|
2014-06-06 02:02:12 +04:00
|
|
|
|
tt.recover()
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
|
|
|
|
data := []byte("force follower")
|
2014-10-30 09:21:38 +03:00
|
|
|
|
// send a proposal to 2 to flush out a MsgApp to 0
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 3, To: 3, Type: pb.MsgProp, Entries: []pb.Entry{{Data: data}}})
|
2014-05-19 04:51:45 +04:00
|
|
|
|
|
2014-09-12 06:23:05 +04:00
|
|
|
|
a := tt.peers[1].(*raft)
|
2014-09-16 04:35:02 +04:00
|
|
|
|
if g := a.state; g != StateFollower {
|
|
|
|
|
t.Errorf("state = %s, want %s", g, StateFollower)
|
2014-05-19 04:51:45 +04:00
|
|
|
|
}
|
2014-08-23 01:19:30 +04:00
|
|
|
|
if g := a.Term; g != 1 {
|
2014-05-19 04:51:45 +04:00
|
|
|
|
t.Errorf("term = %d, want %d", g, 1)
|
|
|
|
|
}
|
2014-11-03 23:04:24 +03:00
|
|
|
|
wantLog := ltoa(&raftLog{
|
|
|
|
|
storage: &MemoryStorage{
|
|
|
|
|
ents: []pb.Entry{{}, {Data: nil, Term: 1, Index: 1}, {Term: 1, Index: 2, Data: data}},
|
|
|
|
|
},
|
|
|
|
|
unstable: 3,
|
|
|
|
|
committed: 2,
|
|
|
|
|
})
|
2014-06-06 02:02:12 +04:00
|
|
|
|
for i, p := range tt.peers {
|
2014-08-23 00:24:33 +04:00
|
|
|
|
if sm, ok := p.(*raft); ok {
|
2014-07-24 03:15:25 +04:00
|
|
|
|
l := ltoa(sm.raftLog)
|
2014-06-06 02:02:12 +04:00
|
|
|
|
if g := diffu(wantLog, l); g != "" {
|
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-16 21:53:56 +04:00
|
|
|
|
}
|
2014-05-16 21:11:21 +04:00
|
|
|
|
}
|
2014-05-16 21:53:56 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-05-28 22:59:01 +04:00
|
|
|
|
func TestSingleNodeCandidate(t *testing.T) {
|
|
|
|
|
tt := newNetwork(nil)
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
2014-05-28 22:59:01 +04:00
|
|
|
|
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm := tt.peers[1].(*raft)
|
2014-09-16 04:35:02 +04:00
|
|
|
|
if sm.state != StateLeader {
|
|
|
|
|
t.Errorf("state = %d, want %d", sm.state, StateLeader)
|
2014-05-28 22:59:01 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-16 21:53:56 +04:00
|
|
|
|
func TestOldMessages(t *testing.T) {
|
|
|
|
|
tt := newNetwork(nil, nil, nil)
|
|
|
|
|
// make 0 leader @ term 3
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
|
|
|
|
tt.send(pb.Message{From: 2, To: 2, Type: pb.MsgHup})
|
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
2014-05-16 21:53:56 +04:00
|
|
|
|
// pretend we're an old leader trying to make progress
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgApp, Term: 1, Entries: []pb.Entry{{Term: 1}}})
|
2014-06-06 02:02:12 +04:00
|
|
|
|
|
2014-07-24 03:15:25 +04:00
|
|
|
|
l := &raftLog{
|
2014-11-03 23:04:24 +03:00
|
|
|
|
storage: &MemoryStorage{
|
|
|
|
|
ents: []pb.Entry{
|
|
|
|
|
{}, {Data: nil, Term: 1, Index: 1},
|
|
|
|
|
{Data: nil, Term: 2, Index: 2}, {Data: nil, Term: 3, Index: 3},
|
|
|
|
|
},
|
2014-06-26 22:55:56 +04:00
|
|
|
|
},
|
2014-11-03 23:04:24 +03:00
|
|
|
|
unstable: 4,
|
2014-06-26 22:55:56 +04:00
|
|
|
|
committed: 3,
|
|
|
|
|
}
|
|
|
|
|
base := ltoa(l)
|
2014-06-06 02:02:12 +04:00
|
|
|
|
for i, p := range tt.peers {
|
2014-08-23 00:24:33 +04:00
|
|
|
|
if sm, ok := p.(*raft); ok {
|
2014-07-24 03:15:25 +04:00
|
|
|
|
l := ltoa(sm.raftLog)
|
2014-06-06 02:02:12 +04:00
|
|
|
|
if g := diffu(base, l); g != "" {
|
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-16 21:11:21 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-18 04:18:58 +04:00
|
|
|
|
// TestOldMessagesReply - optimization - reply with new term.
|
|
|
|
|
|
2014-05-06 10:28:14 +04:00
|
|
|
|
func TestProposal(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
2014-05-16 21:11:21 +04:00
|
|
|
|
*network
|
2014-05-18 01:09:52 +04:00
|
|
|
|
success bool
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}{
|
2014-05-18 01:09:52 +04:00
|
|
|
|
{newNetwork(nil, nil, nil), true},
|
|
|
|
|
{newNetwork(nil, nil, nopStepper), true},
|
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper), false},
|
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil), false},
|
|
|
|
|
{newNetwork(nil, nopStepper, nopStepper, nil, nil), true},
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-08-28 05:53:18 +04:00
|
|
|
|
send := func(m pb.Message) {
|
2014-05-06 10:28:14 +04:00
|
|
|
|
defer func() {
|
2014-05-18 01:09:52 +04:00
|
|
|
|
// only recover is we expect it to panic so
|
|
|
|
|
// panics we don't expect go up.
|
|
|
|
|
if !tt.success {
|
2014-05-06 10:28:14 +04:00
|
|
|
|
e := recover()
|
|
|
|
|
if e != nil {
|
|
|
|
|
t.Logf("#%d: err: %s", i, e)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}()
|
2014-06-06 02:02:12 +04:00
|
|
|
|
tt.send(m)
|
|
|
|
|
}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
2014-05-18 01:09:52 +04:00
|
|
|
|
data := []byte("somedata")
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
|
|
// promote 0 the leader
|
2014-10-12 11:34:22 +04:00
|
|
|
|
send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
|
|
|
|
send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: data}}})
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
2014-11-13 00:23:42 +03:00
|
|
|
|
wantLog := newLog(NewMemoryStorage())
|
2014-05-18 01:09:52 +04:00
|
|
|
|
if tt.success {
|
2014-11-03 23:04:24 +03:00
|
|
|
|
wantLog = &raftLog{
|
|
|
|
|
storage: &MemoryStorage{
|
|
|
|
|
ents: []pb.Entry{{}, {Data: nil, Term: 1, Index: 1}, {Term: 1, Index: 2, Data: data}},
|
|
|
|
|
},
|
|
|
|
|
unstable: 3,
|
|
|
|
|
committed: 2}
|
2014-05-18 01:09:52 +04:00
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
|
base := ltoa(wantLog)
|
|
|
|
|
for i, p := range tt.peers {
|
2014-08-23 00:24:33 +04:00
|
|
|
|
if sm, ok := p.(*raft); ok {
|
2014-07-24 03:15:25 +04:00
|
|
|
|
l := ltoa(sm.raftLog)
|
2014-06-06 02:02:12 +04:00
|
|
|
|
if g := diffu(base, l); g != "" {
|
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm := tt.network.peers[1].(*raft)
|
2014-08-23 01:19:30 +04:00
|
|
|
|
if g := sm.Term; g != 1 {
|
2014-05-06 10:28:14 +04:00
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestProposalByProxy(t *testing.T) {
|
2014-05-16 11:55:17 +04:00
|
|
|
|
data := []byte("somedata")
|
2014-05-18 01:09:52 +04:00
|
|
|
|
tests := []*network{
|
|
|
|
|
newNetwork(nil, nil, nil),
|
|
|
|
|
newNetwork(nil, nil, nopStepper),
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
|
// promote 0 the leader
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
|
|
|
|
// propose via follower
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tt.send(pb.Message{From: 2, To: 2, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("somedata")}}})
|
2014-06-06 02:02:12 +04:00
|
|
|
|
|
2014-11-03 23:04:24 +03:00
|
|
|
|
wantLog := &raftLog{
|
|
|
|
|
storage: &MemoryStorage{
|
|
|
|
|
ents: []pb.Entry{{}, {Data: nil, Term: 1, Index: 1}, {Term: 1, Data: data, Index: 2}},
|
|
|
|
|
},
|
|
|
|
|
unstable: 3,
|
|
|
|
|
committed: 2}
|
2014-06-06 02:02:12 +04:00
|
|
|
|
base := ltoa(wantLog)
|
|
|
|
|
for i, p := range tt.peers {
|
2014-08-23 00:24:33 +04:00
|
|
|
|
if sm, ok := p.(*raft); ok {
|
2014-07-24 03:15:25 +04:00
|
|
|
|
l := ltoa(sm.raftLog)
|
2014-06-06 02:02:12 +04:00
|
|
|
|
if g := diffu(base, l); g != "" {
|
|
|
|
|
t.Errorf("#%d: diff:\n%s", i, g)
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
t.Logf("#%d: empty log", i)
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm := tt.peers[1].(*raft)
|
2014-08-23 01:19:30 +04:00
|
|
|
|
if g := sm.Term; g != 1 {
|
2014-05-06 10:28:14 +04:00
|
|
|
|
t.Errorf("#%d: term = %d, want %d", i, g, 1)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-24 20:53:01 +04:00
|
|
|
|
func TestCommit(t *testing.T) {
|
2014-05-22 03:02:15 +04:00
|
|
|
|
tests := []struct {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
matches []uint64
|
2014-08-28 05:53:18 +04:00
|
|
|
|
logs []pb.Entry
|
2014-10-08 14:29:53 +04:00
|
|
|
|
smTerm uint64
|
|
|
|
|
w uint64
|
2014-05-22 03:02:15 +04:00
|
|
|
|
}{
|
2014-06-06 22:21:26 +04:00
|
|
|
|
// single
|
2014-10-08 14:29:53 +04:00
|
|
|
|
{[]uint64{1}, []pb.Entry{{}, {Term: 1}}, 1, 1},
|
|
|
|
|
{[]uint64{1}, []pb.Entry{{}, {Term: 1}}, 2, 0},
|
|
|
|
|
{[]uint64{2}, []pb.Entry{{}, {Term: 1}, {Term: 2}}, 2, 2},
|
|
|
|
|
{[]uint64{1}, []pb.Entry{{}, {Term: 2}}, 2, 1},
|
2014-06-06 22:21:26 +04:00
|
|
|
|
|
2014-05-22 03:02:15 +04:00
|
|
|
|
// odd
|
2014-10-08 14:29:53 +04:00
|
|
|
|
{[]uint64{2, 1, 1}, []pb.Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
|
|
|
|
{[]uint64{2, 1, 1}, []pb.Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
|
|
|
|
{[]uint64{2, 1, 2}, []pb.Entry{{}, {Term: 1}, {Term: 2}}, 2, 2},
|
|
|
|
|
{[]uint64{2, 1, 2}, []pb.Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
|
|
|
|
|
|
// even
|
2014-10-08 14:29:53 +04:00
|
|
|
|
{[]uint64{2, 1, 1, 1}, []pb.Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
|
|
|
|
{[]uint64{2, 1, 1, 1}, []pb.Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
|
|
|
|
{[]uint64{2, 1, 1, 2}, []pb.Entry{{}, {Term: 1}, {Term: 2}}, 1, 1},
|
|
|
|
|
{[]uint64{2, 1, 1, 2}, []pb.Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
|
|
|
|
{[]uint64{2, 1, 2, 2}, []pb.Entry{{}, {Term: 1}, {Term: 2}}, 2, 2},
|
|
|
|
|
{[]uint64{2, 1, 2, 2}, []pb.Entry{{}, {Term: 1}, {Term: 1}}, 2, 0},
|
2014-05-22 03:02:15 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
prs := make(map[uint64]*progress)
|
2014-06-10 03:45:42 +04:00
|
|
|
|
for j := 0; j < len(tt.matches); j++ {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
prs[uint64(j)] = &progress{tt.matches[j], tt.matches[j] + 1}
|
2014-05-22 03:02:15 +04:00
|
|
|
|
}
|
2014-11-03 23:04:24 +03:00
|
|
|
|
sm := &raft{
|
|
|
|
|
raftLog: &raftLog{storage: &MemoryStorage{ents: tt.logs}, unstable: uint64(len(tt.logs))},
|
|
|
|
|
prs: prs,
|
|
|
|
|
HardState: pb.HardState{Term: tt.smTerm},
|
|
|
|
|
}
|
2014-05-24 20:53:01 +04:00
|
|
|
|
sm.maybeCommit()
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if g := sm.raftLog.committed; g != tt.w {
|
2014-05-28 22:08:32 +04:00
|
|
|
|
t.Errorf("#%d: committed = %d, want %d", i, g, tt.w)
|
2014-05-22 03:02:15 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-07 16:34:15 +04:00
|
|
|
|
func TestIsElectionTimeout(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
|
|
|
|
elapse int
|
2014-10-08 03:41:17 +04:00
|
|
|
|
wprobability float64
|
2014-10-07 16:34:15 +04:00
|
|
|
|
round bool
|
|
|
|
|
}{
|
|
|
|
|
{5, 0, false},
|
|
|
|
|
{13, 0.3, true},
|
|
|
|
|
{15, 0.5, true},
|
|
|
|
|
{18, 0.8, true},
|
|
|
|
|
{20, 1, false},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-11-13 00:23:42 +03:00
|
|
|
|
sm := newRaft(1, []uint64{1}, 10, 1, NewMemoryStorage())
|
2014-10-07 16:34:15 +04:00
|
|
|
|
sm.elapsed = tt.elapse
|
|
|
|
|
c := 0
|
|
|
|
|
for j := 0; j < 10000; j++ {
|
|
|
|
|
if sm.isElectionTimeout() {
|
|
|
|
|
c++
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
got := float64(c) / 10000.0
|
|
|
|
|
if tt.round {
|
|
|
|
|
got = math.Floor(got*10+0.5) / 10.0
|
|
|
|
|
}
|
2014-10-08 03:41:17 +04:00
|
|
|
|
if got != tt.wprobability {
|
|
|
|
|
t.Errorf("#%d: possibility = %v, want %v", i, got, tt.wprobability)
|
2014-10-07 16:34:15 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-09-04 21:58:22 +04:00
|
|
|
|
// ensure that the Step function ignores the message from old term and does not pass it to the
|
|
|
|
|
// acutal stepX function.
|
|
|
|
|
func TestStepIgnoreOldTermMsg(t *testing.T) {
|
|
|
|
|
called := false
|
|
|
|
|
fakeStep := func(r *raft, m pb.Message) {
|
|
|
|
|
called = true
|
|
|
|
|
}
|
2014-11-13 00:23:42 +03:00
|
|
|
|
sm := newRaft(1, []uint64{1}, 10, 1, NewMemoryStorage())
|
2014-09-04 21:58:22 +04:00
|
|
|
|
sm.step = fakeStep
|
|
|
|
|
sm.Term = 2
|
2014-10-12 11:34:22 +04:00
|
|
|
|
sm.Step(pb.Message{Type: pb.MsgApp, Term: sm.Term - 1})
|
2014-09-04 21:58:22 +04:00
|
|
|
|
if called == true {
|
|
|
|
|
t.Errorf("stepFunc called = %v , want %v", called, false)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-07-14 21:58:41 +04:00
|
|
|
|
// TestHandleMsgApp ensures:
|
|
|
|
|
// 1. Reply false if log doesn’t contain an entry at prevLogIndex whose term matches prevLogTerm.
|
|
|
|
|
// 2. If an existing entry conflicts with a new one (same index but different terms),
|
|
|
|
|
// delete the existing entry and all that follow it; append any new entries not already in the log.
|
|
|
|
|
// 3. If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry).
|
|
|
|
|
func TestHandleMsgApp(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
2014-08-28 05:53:18 +04:00
|
|
|
|
m pb.Message
|
2014-10-08 14:29:53 +04:00
|
|
|
|
wIndex uint64
|
|
|
|
|
wCommit uint64
|
2014-10-01 23:59:30 +04:00
|
|
|
|
wReject bool
|
2014-07-14 21:58:41 +04:00
|
|
|
|
}{
|
|
|
|
|
// Ensure 1
|
2014-10-12 11:34:22 +04:00
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 3, Index: 2, Commit: 3}, 2, 0, true}, // previous log mismatch
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 3, Index: 3, Commit: 3}, 2, 0, true}, // previous log non-exist
|
2014-07-14 21:58:41 +04:00
|
|
|
|
|
|
|
|
|
// Ensure 2
|
2014-10-12 11:34:22 +04:00
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 1, Index: 1, Commit: 1}, 2, 1, false},
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 0, Index: 0, Commit: 1, Entries: []pb.Entry{{Term: 2}}}, 1, 1, false},
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 2, Index: 2, Commit: 3, Entries: []pb.Entry{{Term: 2}, {Term: 2}}}, 4, 3, false},
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 2, Index: 2, Commit: 4, Entries: []pb.Entry{{Term: 2}}}, 3, 3, false},
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 1, Index: 1, Commit: 4, Entries: []pb.Entry{{Term: 2}}}, 2, 2, false},
|
2014-07-14 21:58:41 +04:00
|
|
|
|
|
|
|
|
|
// Ensure 3
|
2014-10-12 11:34:22 +04:00
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 1, LogTerm: 1, Index: 1, Commit: 3}, 2, 1, false}, // match entry 1, commit upto last new entry 1
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 1, LogTerm: 1, Index: 1, Commit: 3, Entries: []pb.Entry{{Term: 2}}}, 2, 2, false}, // match entry 1, commit upto last new entry 2
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 2, Index: 2, Commit: 3}, 2, 2, false}, // match entry 2, commit upto last new entry 2
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, LogTerm: 2, Index: 2, Commit: 4}, 2, 2, false}, // commit upto log.last()
|
2014-07-14 21:58:41 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-08-23 00:24:33 +04:00
|
|
|
|
sm := &raft{
|
2014-09-16 04:35:02 +04:00
|
|
|
|
state: StateFollower,
|
|
|
|
|
HardState: pb.HardState{Term: 2},
|
2014-11-03 23:04:24 +03:00
|
|
|
|
raftLog: &raftLog{
|
|
|
|
|
committed: 0,
|
|
|
|
|
storage: &MemoryStorage{ents: []pb.Entry{{}, {Term: 1}, {Term: 2}}},
|
|
|
|
|
unstable: 3,
|
|
|
|
|
},
|
2014-07-14 21:58:41 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
sm.handleAppendEntries(tt.m)
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.lastIndex() != tt.wIndex {
|
|
|
|
|
t.Errorf("#%d: lastIndex = %d, want %d", i, sm.raftLog.lastIndex(), tt.wIndex)
|
2014-07-14 21:58:41 +04:00
|
|
|
|
}
|
2014-07-24 03:15:25 +04:00
|
|
|
|
if sm.raftLog.committed != tt.wCommit {
|
|
|
|
|
t.Errorf("#%d: committed = %d, want %d", i, sm.raftLog.committed, tt.wCommit)
|
2014-07-14 21:58:41 +04:00
|
|
|
|
}
|
2014-10-30 00:04:45 +03:00
|
|
|
|
m := sm.readMessages()
|
2014-07-14 21:58:41 +04:00
|
|
|
|
if len(m) != 1 {
|
2014-09-29 06:38:37 +04:00
|
|
|
|
t.Fatalf("#%d: msg = nil, want 1", i)
|
2014-07-14 21:58:41 +04:00
|
|
|
|
}
|
2014-10-01 23:59:30 +04:00
|
|
|
|
if m[0].Reject != tt.wReject {
|
|
|
|
|
t.Errorf("#%d: reject = %v, want %v", i, m[0].Reject, tt.wReject)
|
2014-07-14 21:58:41 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-11-18 03:58:49 +03:00
|
|
|
|
// TestHandleHeartbeat ensures that the follower commits to the commit in the message.
|
|
|
|
|
func TestHandleHeartbeat(t *testing.T) {
|
|
|
|
|
commit := uint64(2)
|
|
|
|
|
tests := []struct {
|
|
|
|
|
m pb.Message
|
|
|
|
|
wCommit uint64
|
|
|
|
|
}{
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, Commit: commit + 1}, commit + 1},
|
|
|
|
|
{pb.Message{Type: pb.MsgApp, Term: 2, Commit: commit - 1}, commit}, // do not decrease commit
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-11-20 01:05:16 +03:00
|
|
|
|
storage := NewMemoryStorage()
|
2014-11-25 04:13:47 +03:00
|
|
|
|
storage.Append([]pb.Entry{{Index: 1, Term: 1}, {Index: 2, Term: 2}, {Index: 3, Term: 3}})
|
2014-11-18 03:58:49 +03:00
|
|
|
|
sm := &raft{
|
|
|
|
|
state: StateFollower,
|
|
|
|
|
HardState: pb.HardState{Term: 2},
|
2014-11-20 01:05:16 +03:00
|
|
|
|
raftLog: newLog(storage),
|
2014-11-18 03:58:49 +03:00
|
|
|
|
}
|
|
|
|
|
sm.raftLog.commitTo(commit)
|
|
|
|
|
sm.handleHeartbeat(tt.m)
|
|
|
|
|
if sm.raftLog.committed != tt.wCommit {
|
|
|
|
|
t.Errorf("#%d: committed = %d, want %d", i, sm.raftLog.committed, tt.wCommit)
|
|
|
|
|
}
|
|
|
|
|
m := sm.readMessages()
|
|
|
|
|
if len(m) != 0 {
|
|
|
|
|
t.Fatalf("#%d: msg = nil, want 0", i)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-06-11 19:58:26 +04:00
|
|
|
|
func TestRecvMsgVote(t *testing.T) {
|
2014-05-18 11:17:46 +04:00
|
|
|
|
tests := []struct {
|
2014-09-16 04:35:02 +04:00
|
|
|
|
state StateType
|
2014-10-08 14:29:53 +04:00
|
|
|
|
i, term uint64
|
|
|
|
|
voteFor uint64
|
2014-10-01 23:59:30 +04:00
|
|
|
|
wreject bool
|
2014-05-18 11:17:46 +04:00
|
|
|
|
}{
|
2014-09-29 06:34:15 +04:00
|
|
|
|
{StateFollower, 0, 0, None, true},
|
|
|
|
|
{StateFollower, 0, 1, None, true},
|
|
|
|
|
{StateFollower, 0, 2, None, true},
|
|
|
|
|
{StateFollower, 0, 3, None, false},
|
|
|
|
|
|
|
|
|
|
{StateFollower, 1, 0, None, true},
|
|
|
|
|
{StateFollower, 1, 1, None, true},
|
|
|
|
|
{StateFollower, 1, 2, None, true},
|
|
|
|
|
{StateFollower, 1, 3, None, false},
|
|
|
|
|
|
|
|
|
|
{StateFollower, 2, 0, None, true},
|
|
|
|
|
{StateFollower, 2, 1, None, true},
|
|
|
|
|
{StateFollower, 2, 2, None, false},
|
|
|
|
|
{StateFollower, 2, 3, None, false},
|
|
|
|
|
|
|
|
|
|
{StateFollower, 3, 0, None, true},
|
|
|
|
|
{StateFollower, 3, 1, None, true},
|
|
|
|
|
{StateFollower, 3, 2, None, false},
|
|
|
|
|
{StateFollower, 3, 3, None, false},
|
|
|
|
|
|
|
|
|
|
{StateFollower, 3, 2, 2, false},
|
|
|
|
|
{StateFollower, 3, 2, 1, true},
|
|
|
|
|
|
|
|
|
|
{StateLeader, 3, 3, 1, true},
|
|
|
|
|
{StateCandidate, 3, 3, 1, true},
|
2014-05-18 11:17:46 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-11-13 00:23:42 +03:00
|
|
|
|
sm := newRaft(1, []uint64{1}, 10, 1, NewMemoryStorage())
|
2014-09-03 21:16:33 +04:00
|
|
|
|
sm.state = tt.state
|
|
|
|
|
switch tt.state {
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateFollower:
|
2014-09-03 21:16:33 +04:00
|
|
|
|
sm.step = stepFollower
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateCandidate:
|
2014-09-03 21:16:33 +04:00
|
|
|
|
sm.step = stepCandidate
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateLeader:
|
2014-09-03 21:16:33 +04:00
|
|
|
|
sm.step = stepLeader
|
2014-05-28 09:23:14 +04:00
|
|
|
|
}
|
2014-09-16 04:35:02 +04:00
|
|
|
|
sm.HardState = pb.HardState{Vote: tt.voteFor}
|
2014-11-03 23:04:24 +03:00
|
|
|
|
sm.raftLog = &raftLog{
|
2014-11-25 04:13:47 +03:00
|
|
|
|
storage: &MemoryStorage{ents: []pb.Entry{{}, {Index: 1, Term: 2}, {Index: 2, Term: 2}}},
|
2014-11-03 23:04:24 +03:00
|
|
|
|
unstable: 3,
|
|
|
|
|
}
|
2014-05-25 08:08:06 +04:00
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
sm.Step(pb.Message{Type: pb.MsgVote, From: 2, Index: tt.i, LogTerm: tt.term})
|
2014-06-06 02:02:12 +04:00
|
|
|
|
|
2014-10-30 00:04:45 +03:00
|
|
|
|
msgs := sm.readMessages()
|
2014-06-17 22:19:56 +04:00
|
|
|
|
if g := len(msgs); g != 1 {
|
2014-09-29 06:34:15 +04:00
|
|
|
|
t.Fatalf("#%d: len(msgs) = %d, want 1", i, g)
|
2014-06-11 19:58:26 +04:00
|
|
|
|
continue
|
2014-06-06 02:02:12 +04:00
|
|
|
|
}
|
2014-10-01 23:59:30 +04:00
|
|
|
|
if g := msgs[0].Reject; g != tt.wreject {
|
2014-10-08 23:07:11 +04:00
|
|
|
|
t.Errorf("#%d, m.Reject = %v, want %v", i, g, tt.wreject)
|
2014-05-18 11:17:46 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-06-07 03:27:29 +04:00
|
|
|
|
func TestStateTransition(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
2014-09-16 04:35:02 +04:00
|
|
|
|
from StateType
|
|
|
|
|
to StateType
|
2014-06-07 03:27:29 +04:00
|
|
|
|
wallow bool
|
2014-10-08 14:29:53 +04:00
|
|
|
|
wterm uint64
|
|
|
|
|
wlead uint64
|
2014-06-07 03:27:29 +04:00
|
|
|
|
}{
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{StateFollower, StateFollower, true, 1, None},
|
|
|
|
|
{StateFollower, StateCandidate, true, 1, None},
|
2014-10-08 14:29:53 +04:00
|
|
|
|
{StateFollower, StateLeader, false, 0, None},
|
2014-06-07 03:27:29 +04:00
|
|
|
|
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{StateCandidate, StateFollower, true, 0, None},
|
|
|
|
|
{StateCandidate, StateCandidate, true, 1, None},
|
|
|
|
|
{StateCandidate, StateLeader, true, 0, 1},
|
2014-06-07 03:27:29 +04:00
|
|
|
|
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{StateLeader, StateFollower, true, 1, None},
|
|
|
|
|
{StateLeader, StateCandidate, false, 1, None},
|
|
|
|
|
{StateLeader, StateLeader, true, 0, 1},
|
2014-06-07 03:27:29 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
|
func() {
|
|
|
|
|
defer func() {
|
|
|
|
|
if r := recover(); r != nil {
|
|
|
|
|
if tt.wallow == true {
|
|
|
|
|
t.Errorf("%d: allow = %v, want %v", i, false, true)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
|
2014-11-13 00:23:42 +03:00
|
|
|
|
sm := newRaft(1, []uint64{1}, 10, 1, NewMemoryStorage())
|
2014-06-07 03:27:29 +04:00
|
|
|
|
sm.state = tt.from
|
|
|
|
|
|
|
|
|
|
switch tt.to {
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateFollower:
|
2014-06-07 03:27:29 +04:00
|
|
|
|
sm.becomeFollower(tt.wterm, tt.wlead)
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateCandidate:
|
2014-06-07 03:27:29 +04:00
|
|
|
|
sm.becomeCandidate()
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateLeader:
|
2014-06-07 03:27:29 +04:00
|
|
|
|
sm.becomeLeader()
|
|
|
|
|
}
|
|
|
|
|
|
2014-08-23 01:19:30 +04:00
|
|
|
|
if sm.Term != tt.wterm {
|
|
|
|
|
t.Errorf("%d: term = %d, want %d", i, sm.Term, tt.wterm)
|
2014-06-07 03:27:29 +04:00
|
|
|
|
}
|
2014-08-24 04:40:25 +04:00
|
|
|
|
if sm.lead != tt.wlead {
|
2014-06-07 03:27:29 +04:00
|
|
|
|
t.Errorf("%d: lead = %d, want %d", i, sm.lead, tt.wlead)
|
|
|
|
|
}
|
|
|
|
|
}()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-24 00:30:04 +04:00
|
|
|
|
func TestAllServerStepdown(t *testing.T) {
|
2014-06-26 22:55:56 +04:00
|
|
|
|
tests := []struct {
|
2014-09-16 04:35:02 +04:00
|
|
|
|
state StateType
|
2014-06-26 22:55:56 +04:00
|
|
|
|
|
2014-09-16 04:35:02 +04:00
|
|
|
|
wstate StateType
|
2014-10-08 14:29:53 +04:00
|
|
|
|
wterm uint64
|
|
|
|
|
windex uint64
|
2014-06-26 22:55:56 +04:00
|
|
|
|
}{
|
2014-11-18 00:37:46 +03:00
|
|
|
|
{StateFollower, StateFollower, 3, 0},
|
|
|
|
|
{StateCandidate, StateFollower, 3, 0},
|
|
|
|
|
{StateLeader, StateFollower, 3, 1},
|
2014-06-26 22:55:56 +04:00
|
|
|
|
}
|
2014-05-24 00:30:04 +04:00
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
tmsgTypes := [...]pb.MessageType{pb.MsgVote, pb.MsgApp}
|
2014-10-08 14:29:53 +04:00
|
|
|
|
tterm := uint64(3)
|
2014-05-24 00:30:04 +04:00
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-11-13 00:23:42 +03:00
|
|
|
|
sm := newRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage())
|
2014-06-26 22:55:56 +04:00
|
|
|
|
switch tt.state {
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateFollower:
|
2014-09-15 09:44:59 +04:00
|
|
|
|
sm.becomeFollower(1, None)
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateCandidate:
|
2014-05-24 00:30:04 +04:00
|
|
|
|
sm.becomeCandidate()
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateLeader:
|
2014-05-24 00:30:04 +04:00
|
|
|
|
sm.becomeCandidate()
|
|
|
|
|
sm.becomeLeader()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for j, msgType := range tmsgTypes {
|
2014-09-12 06:23:05 +04:00
|
|
|
|
sm.Step(pb.Message{From: 2, Type: msgType, Term: tterm, LogTerm: tterm})
|
2014-05-24 00:30:04 +04:00
|
|
|
|
|
2014-06-26 22:55:56 +04:00
|
|
|
|
if sm.state != tt.wstate {
|
|
|
|
|
t.Errorf("#%d.%d state = %v , want %v", i, j, sm.state, tt.wstate)
|
2014-05-24 00:30:04 +04:00
|
|
|
|
}
|
2014-08-23 01:19:30 +04:00
|
|
|
|
if sm.Term != tt.wterm {
|
|
|
|
|
t.Errorf("#%d.%d term = %v , want %v", i, j, sm.Term, tt.wterm)
|
2014-05-24 00:30:04 +04:00
|
|
|
|
}
|
2014-11-18 00:37:46 +03:00
|
|
|
|
if uint64(sm.raftLog.lastIndex()) != tt.windex {
|
|
|
|
|
t.Errorf("#%d.%d index = %v , want %v", i, j, sm.raftLog.lastIndex(), tt.windex)
|
|
|
|
|
}
|
2014-11-03 23:04:24 +03:00
|
|
|
|
if uint64(len(sm.raftLog.allEntries())) != tt.windex {
|
2014-11-18 00:37:46 +03:00
|
|
|
|
t.Errorf("#%d.%d len(ents) = %v , want %v", i, j, len(sm.raftLog.allEntries()), tt.windex)
|
2014-05-24 00:30:04 +04:00
|
|
|
|
}
|
2014-10-08 14:29:53 +04:00
|
|
|
|
wlead := uint64(2)
|
2014-10-12 11:34:22 +04:00
|
|
|
|
if msgType == pb.MsgVote {
|
2014-09-15 09:44:59 +04:00
|
|
|
|
wlead = None
|
2014-07-15 09:39:44 +04:00
|
|
|
|
}
|
2014-08-24 04:40:25 +04:00
|
|
|
|
if sm.lead != wlead {
|
2014-09-15 09:44:59 +04:00
|
|
|
|
t.Errorf("#%d, sm.lead = %d, want %d", i, sm.lead, None)
|
2014-07-15 09:39:44 +04:00
|
|
|
|
}
|
2014-05-24 00:30:04 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-25 10:03:13 +04:00
|
|
|
|
func TestLeaderAppResp(t *testing.T) {
|
2014-11-18 22:42:08 +03:00
|
|
|
|
// initial progress: match = 0; next = 3
|
2014-05-25 10:03:13 +04:00
|
|
|
|
tests := []struct {
|
2014-10-24 22:50:21 +04:00
|
|
|
|
index uint64
|
|
|
|
|
reject bool
|
|
|
|
|
// progress
|
|
|
|
|
wmatch uint64
|
|
|
|
|
wnext uint64
|
|
|
|
|
// message
|
2014-05-25 10:03:13 +04:00
|
|
|
|
wmsgNum int
|
2014-10-08 14:29:53 +04:00
|
|
|
|
windex uint64
|
|
|
|
|
wcommitted uint64
|
2014-05-25 10:03:13 +04:00
|
|
|
|
}{
|
2014-10-24 22:50:21 +04:00
|
|
|
|
{3, true, 0, 3, 0, 0, 0}, // stale resp; no replies
|
|
|
|
|
{2, true, 0, 2, 1, 1, 0}, // denied resp; leader does not commit; decrese next and send probing msg
|
2014-11-18 22:42:08 +03:00
|
|
|
|
{2, false, 2, 4, 2, 2, 2}, // accept resp; leader commits; broadcast with commit index
|
2014-10-24 22:50:21 +04:00
|
|
|
|
{0, false, 0, 3, 0, 0, 0}, // ignore heartbeat replies
|
2014-05-25 10:03:13 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
|
// sm term is 1 after it becomes the leader.
|
|
|
|
|
// thus the last log term must be 1 to be committed.
|
2014-11-13 00:23:42 +03:00
|
|
|
|
sm := newRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage())
|
2014-11-03 23:04:24 +03:00
|
|
|
|
sm.raftLog = &raftLog{
|
2014-11-25 04:13:47 +03:00
|
|
|
|
storage: &MemoryStorage{ents: []pb.Entry{{}, {Index: 1, Term: 0}, {Index: 2, Term: 1}}},
|
2014-11-03 23:04:24 +03:00
|
|
|
|
unstable: 3,
|
|
|
|
|
}
|
2014-05-25 10:03:13 +04:00
|
|
|
|
sm.becomeCandidate()
|
|
|
|
|
sm.becomeLeader()
|
2014-10-30 00:04:45 +03:00
|
|
|
|
sm.readMessages()
|
2014-10-12 11:34:22 +04:00
|
|
|
|
sm.Step(pb.Message{From: 2, Type: pb.MsgAppResp, Index: tt.index, Term: sm.Term, Reject: tt.reject})
|
2014-10-24 22:50:21 +04:00
|
|
|
|
|
|
|
|
|
p := sm.prs[2]
|
|
|
|
|
if p.match != tt.wmatch {
|
|
|
|
|
t.Errorf("#%d match = %d, want %d", i, p.match, tt.wmatch)
|
|
|
|
|
}
|
|
|
|
|
if p.next != tt.wnext {
|
|
|
|
|
t.Errorf("#%d next = %d, want %d", i, p.next, tt.wnext)
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-30 00:04:45 +03:00
|
|
|
|
msgs := sm.readMessages()
|
2014-05-25 10:03:13 +04:00
|
|
|
|
|
|
|
|
|
if len(msgs) != tt.wmsgNum {
|
|
|
|
|
t.Errorf("#%d msgNum = %d, want %d", i, len(msgs), tt.wmsgNum)
|
|
|
|
|
}
|
|
|
|
|
for j, msg := range msgs {
|
|
|
|
|
if msg.Index != tt.windex {
|
|
|
|
|
t.Errorf("#%d.%d index = %d, want %d", i, j, msg.Index, tt.windex)
|
|
|
|
|
}
|
|
|
|
|
if msg.Commit != tt.wcommitted {
|
|
|
|
|
t.Errorf("#%d.%d commit = %d, want %d", i, j, msg.Commit, tt.wcommitted)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-09-15 20:58:22 +04:00
|
|
|
|
// When the leader receives a heartbeat tick, it should
|
2014-10-30 09:21:38 +03:00
|
|
|
|
// send a MsgApp with m.Index = 0, m.LogTerm=0 and empty entries.
|
2014-09-15 20:58:22 +04:00
|
|
|
|
func TestBcastBeat(t *testing.T) {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
offset := uint64(1000)
|
2014-09-15 20:58:22 +04:00
|
|
|
|
// make a state machine with log.offset = 1000
|
|
|
|
|
s := pb.Snapshot{
|
2014-11-20 00:17:50 +03:00
|
|
|
|
Metadata: pb.SnapshotMetadata{
|
|
|
|
|
Index: offset,
|
|
|
|
|
Term: 1,
|
|
|
|
|
ConfState: pb.ConfState{Nodes: []uint64{1, 2, 3}},
|
|
|
|
|
},
|
2014-09-15 20:58:22 +04:00
|
|
|
|
}
|
2014-11-20 00:17:50 +03:00
|
|
|
|
storage := NewMemoryStorage()
|
|
|
|
|
storage.ApplySnapshot(s)
|
2014-11-22 01:22:20 +03:00
|
|
|
|
sm := newRaft(1, nil, 10, 1, storage)
|
2014-09-15 20:58:22 +04:00
|
|
|
|
sm.Term = 1
|
|
|
|
|
|
|
|
|
|
sm.becomeCandidate()
|
|
|
|
|
sm.becomeLeader()
|
|
|
|
|
for i := 0; i < 10; i++ {
|
2014-11-25 04:13:47 +03:00
|
|
|
|
sm.appendEntry(pb.Entry{Index: uint64(i) + 1})
|
2014-09-15 20:58:22 +04:00
|
|
|
|
}
|
2014-11-18 02:44:57 +03:00
|
|
|
|
// slow follower
|
|
|
|
|
sm.prs[2].match, sm.prs[2].next = 5, 6
|
|
|
|
|
// normal follower
|
|
|
|
|
sm.prs[3].match, sm.prs[3].next = sm.raftLog.lastIndex(), sm.raftLog.lastIndex()+1
|
2014-09-15 20:58:22 +04:00
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
sm.Step(pb.Message{Type: pb.MsgBeat})
|
2014-10-30 00:04:45 +03:00
|
|
|
|
msgs := sm.readMessages()
|
2014-09-29 21:44:20 +04:00
|
|
|
|
if len(msgs) != 2 {
|
2014-10-30 09:21:38 +03:00
|
|
|
|
t.Fatalf("len(msgs) = %v, want 2", len(msgs))
|
2014-09-15 20:58:22 +04:00
|
|
|
|
}
|
2014-11-18 02:44:57 +03:00
|
|
|
|
wantCommitMap := map[uint64]uint64{
|
|
|
|
|
2: min(sm.raftLog.committed, sm.prs[2].match),
|
|
|
|
|
3: min(sm.raftLog.committed, sm.prs[3].match),
|
|
|
|
|
}
|
2014-09-29 21:44:20 +04:00
|
|
|
|
for i, m := range msgs {
|
2014-10-12 11:34:22 +04:00
|
|
|
|
if m.Type != pb.MsgApp {
|
|
|
|
|
t.Fatalf("#%d: type = %v, want = %v", i, m.Type, pb.MsgApp)
|
2014-09-15 20:58:22 +04:00
|
|
|
|
}
|
2014-09-29 21:44:20 +04:00
|
|
|
|
if m.Index != 0 {
|
|
|
|
|
t.Fatalf("#%d: prevIndex = %d, want %d", i, m.Index, 0)
|
2014-09-15 20:58:22 +04:00
|
|
|
|
}
|
2014-09-29 21:44:20 +04:00
|
|
|
|
if m.LogTerm != 0 {
|
|
|
|
|
t.Fatalf("#%d: prevTerm = %d, want %d", i, m.LogTerm, 0)
|
2014-09-15 20:58:22 +04:00
|
|
|
|
}
|
2014-11-18 02:44:57 +03:00
|
|
|
|
if wantCommitMap[m.To] == 0 {
|
2014-09-29 21:44:20 +04:00
|
|
|
|
t.Fatalf("#%d: unexpected to %d", i, m.To)
|
|
|
|
|
} else {
|
2014-11-18 02:44:57 +03:00
|
|
|
|
if m.Commit != wantCommitMap[m.To] {
|
|
|
|
|
t.Fatalf("#%d: commit = %d, want %d", i, m.Commit, wantCommitMap[m.To])
|
|
|
|
|
}
|
|
|
|
|
delete(wantCommitMap, m.To)
|
2014-09-15 20:58:22 +04:00
|
|
|
|
}
|
|
|
|
|
if len(m.Entries) != 0 {
|
|
|
|
|
t.Fatalf("#%d: len(entries) = %d, want 0", i, len(m.Entries))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-30 09:21:38 +03:00
|
|
|
|
// tests the output of the statemachine when receiving MsgBeat
|
2014-06-07 00:40:24 +04:00
|
|
|
|
func TestRecvMsgBeat(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
2014-09-16 04:35:02 +04:00
|
|
|
|
state StateType
|
2014-06-07 00:40:24 +04:00
|
|
|
|
wMsg int
|
|
|
|
|
}{
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{StateLeader, 2},
|
2014-10-30 09:21:38 +03:00
|
|
|
|
// candidate and follower should ignore MsgBeat
|
2014-09-16 04:35:02 +04:00
|
|
|
|
{StateCandidate, 0},
|
|
|
|
|
{StateFollower, 0},
|
2014-06-07 00:40:24 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-11-13 00:23:42 +03:00
|
|
|
|
sm := newRaft(1, []uint64{1, 2, 3}, 10, 1, NewMemoryStorage())
|
2014-11-25 04:13:47 +03:00
|
|
|
|
sm.raftLog = &raftLog{storage: &MemoryStorage{ents: []pb.Entry{{}, {Index: 1, Term: 0}, {Index: 2, Term: 1}}}}
|
2014-08-23 01:19:30 +04:00
|
|
|
|
sm.Term = 1
|
2014-06-07 00:40:24 +04:00
|
|
|
|
sm.state = tt.state
|
2014-09-03 21:16:33 +04:00
|
|
|
|
switch tt.state {
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateFollower:
|
2014-09-03 21:16:33 +04:00
|
|
|
|
sm.step = stepFollower
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateCandidate:
|
2014-09-03 21:16:33 +04:00
|
|
|
|
sm.step = stepCandidate
|
2014-09-16 04:35:02 +04:00
|
|
|
|
case StateLeader:
|
2014-09-03 21:16:33 +04:00
|
|
|
|
sm.step = stepLeader
|
|
|
|
|
}
|
2014-10-12 11:34:22 +04:00
|
|
|
|
sm.Step(pb.Message{From: 1, To: 1, Type: pb.MsgBeat})
|
2014-06-07 00:40:24 +04:00
|
|
|
|
|
2014-10-30 00:04:45 +03:00
|
|
|
|
msgs := sm.readMessages()
|
2014-06-07 00:40:24 +04:00
|
|
|
|
if len(msgs) != tt.wMsg {
|
|
|
|
|
t.Errorf("%d: len(msgs) = %d, want %d", i, len(msgs), tt.wMsg)
|
|
|
|
|
}
|
|
|
|
|
for _, m := range msgs {
|
2014-10-12 11:34:22 +04:00
|
|
|
|
if m.Type != pb.MsgApp {
|
|
|
|
|
t.Errorf("%d: msg.type = %v, want %v", i, m.Type, pb.MsgApp)
|
2014-06-07 00:40:24 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-11-18 22:42:08 +03:00
|
|
|
|
func TestLeaderIncreaseNext(t *testing.T) {
|
|
|
|
|
previousEnts := []pb.Entry{{Term: 1, Index: 1}, {Term: 1, Index: 2}, {Term: 1, Index: 3}}
|
|
|
|
|
tests := []struct {
|
|
|
|
|
// progress
|
|
|
|
|
match uint64
|
|
|
|
|
next uint64
|
|
|
|
|
|
|
|
|
|
wnext uint64
|
|
|
|
|
}{
|
|
|
|
|
// match is not zero, optimistically increase next
|
|
|
|
|
// previous entries + noop entry + propose + 1
|
|
|
|
|
{1, 2, uint64(len(previousEnts) + 1 + 1 + 1)},
|
|
|
|
|
// match is zero, not optimistically increase next
|
|
|
|
|
{0, 2, 2},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
2014-11-20 01:05:16 +03:00
|
|
|
|
sm := newRaft(1, []uint64{1, 2}, 10, 1, NewMemoryStorage())
|
2014-11-18 22:42:08 +03:00
|
|
|
|
sm.raftLog.append(0, previousEnts...)
|
|
|
|
|
sm.becomeCandidate()
|
|
|
|
|
sm.becomeLeader()
|
|
|
|
|
sm.prs[2].match, sm.prs[2].next = tt.match, tt.next
|
|
|
|
|
sm.Step(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Data: []byte("somedata")}}})
|
|
|
|
|
|
|
|
|
|
p := sm.prs[2]
|
|
|
|
|
if p.next != tt.wnext {
|
|
|
|
|
t.Errorf("#%d next = %d, want %d", i, p.next, tt.wnext)
|
|
|
|
|
}
|
|
|
|
|
}
|
2014-06-07 00:40:24 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-07-01 23:10:43 +04:00
|
|
|
|
func TestRestore(t *testing.T) {
|
2014-08-28 05:53:18 +04:00
|
|
|
|
s := pb.Snapshot{
|
2014-11-20 00:17:50 +03:00
|
|
|
|
Metadata: pb.SnapshotMetadata{
|
|
|
|
|
Index: 11, // magic number
|
|
|
|
|
Term: 11, // magic number
|
|
|
|
|
ConfState: pb.ConfState{Nodes: []uint64{1, 2, 3}},
|
|
|
|
|
},
|
2014-07-01 23:10:43 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-11-20 00:17:50 +03:00
|
|
|
|
storage := NewMemoryStorage()
|
|
|
|
|
sm := newRaft(1, []uint64{1, 2}, 10, 1, storage)
|
2014-08-01 00:03:23 +04:00
|
|
|
|
if ok := sm.restore(s); !ok {
|
|
|
|
|
t.Fatal("restore fail, want succeed")
|
|
|
|
|
}
|
2014-07-01 23:10:43 +04:00
|
|
|
|
|
2014-11-20 00:17:50 +03:00
|
|
|
|
if sm.raftLog.lastIndex() != s.Metadata.Index {
|
|
|
|
|
t.Errorf("log.lastIndex = %d, want %d", sm.raftLog.lastIndex(), s.Metadata.Index)
|
2014-08-01 02:18:44 +04:00
|
|
|
|
}
|
2014-11-20 00:17:50 +03:00
|
|
|
|
if sm.raftLog.term(s.Metadata.Index) != s.Metadata.Term {
|
|
|
|
|
t.Errorf("log.lastTerm = %d, want %d", sm.raftLog.term(s.Metadata.Index), s.Metadata.Term)
|
2014-08-01 02:18:44 +04:00
|
|
|
|
}
|
2014-10-08 11:33:55 +04:00
|
|
|
|
sg := sm.nodes()
|
2014-11-20 00:17:50 +03:00
|
|
|
|
if !reflect.DeepEqual(sg, s.Metadata.ConfState.Nodes) {
|
|
|
|
|
t.Errorf("sm.Nodes = %+v, want %+v", sg, s.Metadata.ConfState.Nodes)
|
2014-07-01 23:10:43 +04:00
|
|
|
|
}
|
2014-08-01 00:03:23 +04:00
|
|
|
|
|
|
|
|
|
if ok := sm.restore(s); ok {
|
|
|
|
|
t.Fatal("restore succeed, want fail")
|
|
|
|
|
}
|
2014-07-01 23:10:43 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-07-02 23:49:58 +04:00
|
|
|
|
func TestProvideSnap(t *testing.T) {
|
|
|
|
|
// restore the statemachin from a snapshot
|
|
|
|
|
// so it has a compacted log and a snapshot
|
2014-11-20 00:17:50 +03:00
|
|
|
|
s := pb.Snapshot{
|
|
|
|
|
Metadata: pb.SnapshotMetadata{
|
|
|
|
|
Index: 11, // magic number
|
|
|
|
|
Term: 11, // magic number
|
|
|
|
|
ConfState: pb.ConfState{Nodes: []uint64{1, 2}},
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
storage := NewMemoryStorage()
|
|
|
|
|
sm := newRaft(1, []uint64{1}, 10, 1, storage)
|
2014-07-02 23:49:58 +04:00
|
|
|
|
sm.restore(s)
|
|
|
|
|
|
|
|
|
|
sm.becomeCandidate()
|
|
|
|
|
sm.becomeLeader()
|
|
|
|
|
|
|
|
|
|
// force set the next of node 1, so that
|
|
|
|
|
// node 1 needs a snapshot
|
2014-11-03 23:04:24 +03:00
|
|
|
|
sm.prs[2].next = sm.raftLog.firstIndex()
|
2014-07-02 23:49:58 +04:00
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
sm.Step(pb.Message{From: 2, To: 1, Type: pb.MsgAppResp, Index: sm.prs[2].next - 1, Reject: true})
|
2014-10-30 00:04:45 +03:00
|
|
|
|
msgs := sm.readMessages()
|
2014-07-02 23:49:58 +04:00
|
|
|
|
if len(msgs) != 1 {
|
2014-09-29 06:28:12 +04:00
|
|
|
|
t.Fatalf("len(msgs) = %d, want 1", len(msgs))
|
2014-07-02 23:49:58 +04:00
|
|
|
|
}
|
2014-09-29 21:44:20 +04:00
|
|
|
|
m := msgs[0]
|
2014-10-12 11:34:22 +04:00
|
|
|
|
if m.Type != pb.MsgSnap {
|
|
|
|
|
t.Errorf("m.Type = %v, want %v", m.Type, pb.MsgSnap)
|
2014-07-02 23:49:58 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestRestoreFromSnapMsg(t *testing.T) {
|
2014-08-28 05:53:18 +04:00
|
|
|
|
s := pb.Snapshot{
|
2014-11-20 00:17:50 +03:00
|
|
|
|
Metadata: pb.SnapshotMetadata{
|
|
|
|
|
Index: 11, // magic number
|
|
|
|
|
Term: 11, // magic number
|
|
|
|
|
ConfState: pb.ConfState{Nodes: []uint64{1, 2}},
|
|
|
|
|
},
|
2014-07-02 23:49:58 +04:00
|
|
|
|
}
|
2014-10-12 11:34:22 +04:00
|
|
|
|
m := pb.Message{Type: pb.MsgSnap, From: 1, Term: 2, Snapshot: s}
|
2014-07-02 23:49:58 +04:00
|
|
|
|
|
2014-11-13 00:23:42 +03:00
|
|
|
|
sm := newRaft(2, []uint64{1, 2}, 10, 1, NewMemoryStorage())
|
2014-07-02 23:49:58 +04:00
|
|
|
|
sm.Step(m)
|
|
|
|
|
|
2014-11-20 00:17:50 +03:00
|
|
|
|
// TODO(bdarnell): what should this test?
|
2014-07-02 23:49:58 +04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestSlowNodeRestore(t *testing.T) {
|
|
|
|
|
nt := newNetwork(nil, nil, nil)
|
2014-10-12 11:34:22 +04:00
|
|
|
|
nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgHup})
|
2014-07-02 23:49:58 +04:00
|
|
|
|
|
2014-09-12 06:23:05 +04:00
|
|
|
|
nt.isolate(3)
|
2014-10-17 02:32:57 +04:00
|
|
|
|
for j := 0; j <= 100; j++ {
|
2014-10-12 11:34:22 +04:00
|
|
|
|
nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{}}})
|
2014-07-02 23:49:58 +04:00
|
|
|
|
}
|
2014-09-12 06:23:05 +04:00
|
|
|
|
lead := nt.peers[1].(*raft)
|
2014-11-13 00:23:42 +03:00
|
|
|
|
nextEnts(lead, nt.storage[1])
|
2014-11-22 00:37:55 +03:00
|
|
|
|
nt.storage[1].Compact(lead.raftLog.applied, &pb.ConfState{Nodes: lead.nodes()}, nil)
|
2014-07-02 23:49:58 +04:00
|
|
|
|
|
|
|
|
|
nt.recover()
|
2014-09-29 21:44:20 +04:00
|
|
|
|
// trigger a snapshot
|
2014-10-12 11:34:22 +04:00
|
|
|
|
nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{}}})
|
2014-09-12 06:23:05 +04:00
|
|
|
|
follower := nt.peers[3].(*raft)
|
2014-07-02 23:49:58 +04:00
|
|
|
|
|
2014-09-29 21:44:20 +04:00
|
|
|
|
// trigger a commit
|
2014-10-12 11:34:22 +04:00
|
|
|
|
nt.send(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{}}})
|
2014-09-29 21:44:20 +04:00
|
|
|
|
if follower.raftLog.committed != lead.raftLog.committed {
|
|
|
|
|
t.Errorf("follower.comitted = %d, want %d", follower.raftLog.committed, lead.raftLog.committed)
|
2014-07-02 23:49:58 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-09-23 23:02:44 +04:00
|
|
|
|
// TestStepConfig tests that when raft step msgProp in EntryConfChange type,
|
2014-09-19 23:35:56 +04:00
|
|
|
|
// it appends the entry to log and sets pendingConf to be true.
|
|
|
|
|
func TestStepConfig(t *testing.T) {
|
|
|
|
|
// a raft that cannot make progress
|
2014-11-13 00:23:42 +03:00
|
|
|
|
r := newRaft(1, []uint64{1, 2}, 10, 1, NewMemoryStorage())
|
2014-09-19 23:35:56 +04:00
|
|
|
|
r.becomeCandidate()
|
|
|
|
|
r.becomeLeader()
|
|
|
|
|
index := r.raftLog.lastIndex()
|
2014-10-12 11:34:22 +04:00
|
|
|
|
r.Step(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Type: pb.EntryConfChange}}})
|
2014-09-19 23:35:56 +04:00
|
|
|
|
if g := r.raftLog.lastIndex(); g != index+1 {
|
|
|
|
|
t.Errorf("index = %d, want %d", g, index+1)
|
|
|
|
|
}
|
|
|
|
|
if r.pendingConf != true {
|
|
|
|
|
t.Errorf("pendingConf = %v, want true", r.pendingConf)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestStepIgnoreConfig tests that if raft step the second msgProp in
|
2014-09-23 23:02:44 +04:00
|
|
|
|
// EntryConfChange type when the first one is uncommitted, the node will deny
|
2014-09-19 23:35:56 +04:00
|
|
|
|
// the proposal and keep its original state.
|
|
|
|
|
func TestStepIgnoreConfig(t *testing.T) {
|
|
|
|
|
// a raft that cannot make progress
|
2014-11-13 00:23:42 +03:00
|
|
|
|
r := newRaft(1, []uint64{1, 2}, 10, 1, NewMemoryStorage())
|
2014-09-19 23:35:56 +04:00
|
|
|
|
r.becomeCandidate()
|
|
|
|
|
r.becomeLeader()
|
2014-10-12 11:34:22 +04:00
|
|
|
|
r.Step(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Type: pb.EntryConfChange}}})
|
2014-09-19 23:35:56 +04:00
|
|
|
|
index := r.raftLog.lastIndex()
|
|
|
|
|
pendingConf := r.pendingConf
|
2014-10-12 11:34:22 +04:00
|
|
|
|
r.Step(pb.Message{From: 1, To: 1, Type: pb.MsgProp, Entries: []pb.Entry{{Type: pb.EntryConfChange}}})
|
2014-09-19 23:35:56 +04:00
|
|
|
|
if g := r.raftLog.lastIndex(); g != index {
|
|
|
|
|
t.Errorf("index = %d, want %d", g, index)
|
|
|
|
|
}
|
|
|
|
|
if r.pendingConf != pendingConf {
|
|
|
|
|
t.Errorf("pendingConf = %v, want %v", r.pendingConf, pendingConf)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestRecoverPendingConfig tests that new leader recovers its pendingConf flag
|
|
|
|
|
// based on uncommitted entries.
|
|
|
|
|
func TestRecoverPendingConfig(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
2014-09-21 09:48:17 +04:00
|
|
|
|
entType pb.EntryType
|
2014-09-19 23:35:56 +04:00
|
|
|
|
wpending bool
|
|
|
|
|
}{
|
2014-09-21 09:48:17 +04:00
|
|
|
|
{pb.EntryNormal, false},
|
2014-09-23 23:02:44 +04:00
|
|
|
|
{pb.EntryConfChange, true},
|
2014-09-19 23:35:56 +04:00
|
|
|
|
}
|
|
|
|
|
for i, tt := range tests {
|
2014-11-13 00:23:42 +03:00
|
|
|
|
r := newRaft(1, []uint64{1, 2}, 10, 1, NewMemoryStorage())
|
2014-09-19 23:35:56 +04:00
|
|
|
|
r.appendEntry(pb.Entry{Type: tt.entType})
|
|
|
|
|
r.becomeCandidate()
|
|
|
|
|
r.becomeLeader()
|
|
|
|
|
if r.pendingConf != tt.wpending {
|
|
|
|
|
t.Errorf("#%d: pendingConf = %v, want %v", i, r.pendingConf, tt.wpending)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestRecoverDoublePendingConfig tests that new leader will panic if
|
|
|
|
|
// there exist two uncommitted config entries.
|
|
|
|
|
func TestRecoverDoublePendingConfig(t *testing.T) {
|
|
|
|
|
func() {
|
|
|
|
|
defer func() {
|
|
|
|
|
if err := recover(); err == nil {
|
|
|
|
|
t.Errorf("expect panic, but nothing happens")
|
|
|
|
|
}
|
|
|
|
|
}()
|
2014-11-13 00:23:42 +03:00
|
|
|
|
r := newRaft(1, []uint64{1, 2}, 10, 1, NewMemoryStorage())
|
2014-09-23 23:02:44 +04:00
|
|
|
|
r.appendEntry(pb.Entry{Type: pb.EntryConfChange})
|
|
|
|
|
r.appendEntry(pb.Entry{Type: pb.EntryConfChange})
|
2014-09-19 23:35:56 +04:00
|
|
|
|
r.becomeCandidate()
|
|
|
|
|
r.becomeLeader()
|
|
|
|
|
}()
|
|
|
|
|
}
|
|
|
|
|
|
2014-09-30 06:54:20 +04:00
|
|
|
|
// TestAddNode tests that addNode could update pendingConf and nodes correctly.
|
2014-09-19 23:35:56 +04:00
|
|
|
|
func TestAddNode(t *testing.T) {
|
2014-11-13 00:23:42 +03:00
|
|
|
|
r := newRaft(1, []uint64{1}, 10, 1, NewMemoryStorage())
|
2014-09-19 23:35:56 +04:00
|
|
|
|
r.pendingConf = true
|
|
|
|
|
r.addNode(2)
|
|
|
|
|
if r.pendingConf != false {
|
|
|
|
|
t.Errorf("pendingConf = %v, want false", r.pendingConf)
|
|
|
|
|
}
|
|
|
|
|
nodes := r.nodes()
|
2014-10-08 14:29:53 +04:00
|
|
|
|
wnodes := []uint64{1, 2}
|
2014-09-19 23:35:56 +04:00
|
|
|
|
if !reflect.DeepEqual(nodes, wnodes) {
|
|
|
|
|
t.Errorf("nodes = %v, want %v", nodes, wnodes)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-09-30 06:54:20 +04:00
|
|
|
|
// TestRemoveNode tests that removeNode could update pendingConf, nodes and
|
|
|
|
|
// and removed list correctly.
|
2014-09-19 23:35:56 +04:00
|
|
|
|
func TestRemoveNode(t *testing.T) {
|
2014-11-13 00:23:42 +03:00
|
|
|
|
r := newRaft(1, []uint64{1, 2}, 10, 1, NewMemoryStorage())
|
2014-09-19 23:35:56 +04:00
|
|
|
|
r.pendingConf = true
|
|
|
|
|
r.removeNode(2)
|
|
|
|
|
if r.pendingConf != false {
|
|
|
|
|
t.Errorf("pendingConf = %v, want false", r.pendingConf)
|
|
|
|
|
}
|
2014-10-08 14:29:53 +04:00
|
|
|
|
w := []uint64{1}
|
2014-09-19 23:35:56 +04:00
|
|
|
|
if g := r.nodes(); !reflect.DeepEqual(g, w) {
|
|
|
|
|
t.Errorf("nodes = %v, want %v", g, w)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-09-24 21:19:41 +04:00
|
|
|
|
func TestPromotable(t *testing.T) {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
id := uint64(1)
|
2014-09-24 10:09:21 +04:00
|
|
|
|
tests := []struct {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
peers []uint64
|
2014-09-24 21:19:41 +04:00
|
|
|
|
wp bool
|
2014-09-24 10:09:21 +04:00
|
|
|
|
}{
|
2014-10-08 14:29:53 +04:00
|
|
|
|
{[]uint64{1}, true},
|
|
|
|
|
{[]uint64{1, 2, 3}, true},
|
|
|
|
|
{[]uint64{}, false},
|
|
|
|
|
{[]uint64{2, 3}, false},
|
2014-09-24 10:09:21 +04:00
|
|
|
|
}
|
|
|
|
|
for i, tt := range tests {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
r := &raft{id: id, prs: make(map[uint64]*progress)}
|
2014-09-24 21:19:41 +04:00
|
|
|
|
for _, id := range tt.peers {
|
|
|
|
|
r.prs[id] = &progress{}
|
2014-09-24 10:09:21 +04:00
|
|
|
|
}
|
2014-09-24 21:19:41 +04:00
|
|
|
|
if g := r.promotable(); g != tt.wp {
|
|
|
|
|
t.Errorf("#%d: promotable = %v, want %v", i, g, tt.wp)
|
2014-09-24 10:09:21 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-11-12 08:11:09 +03:00
|
|
|
|
func TestRaftNodes(t *testing.T) {
|
|
|
|
|
tests := []struct {
|
|
|
|
|
ids []uint64
|
|
|
|
|
wids []uint64
|
|
|
|
|
}{
|
|
|
|
|
{
|
|
|
|
|
[]uint64{1, 2, 3},
|
|
|
|
|
[]uint64{1, 2, 3},
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
[]uint64{3, 2, 1},
|
|
|
|
|
[]uint64{1, 2, 3},
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
for i, tt := range tests {
|
2014-11-13 22:21:09 +03:00
|
|
|
|
r := newRaft(1, tt.ids, 10, 1, NewMemoryStorage())
|
2014-11-12 08:11:09 +03:00
|
|
|
|
if !reflect.DeepEqual(r.nodes(), tt.wids) {
|
|
|
|
|
t.Errorf("#%d: nodes = %+v, want %+v", i, r.nodes(), tt.wids)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-08 14:29:53 +04:00
|
|
|
|
func ents(terms ...uint64) *raft {
|
2014-08-28 05:53:18 +04:00
|
|
|
|
ents := []pb.Entry{{}}
|
2014-11-25 04:13:47 +03:00
|
|
|
|
for i, term := range terms {
|
|
|
|
|
ents = append(ents, pb.Entry{Index: uint64(i), Term: term})
|
2014-05-18 01:09:52 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-11-03 23:04:24 +03:00
|
|
|
|
sm := &raft{
|
|
|
|
|
raftLog: &raftLog{
|
|
|
|
|
storage: &MemoryStorage{ents: ents},
|
|
|
|
|
unstable: uint64(len(ents)),
|
|
|
|
|
},
|
|
|
|
|
}
|
2014-06-16 05:52:10 +04:00
|
|
|
|
sm.reset(0)
|
2014-06-06 02:02:12 +04:00
|
|
|
|
return sm
|
2014-05-18 01:09:52 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-05-16 21:11:21 +04:00
|
|
|
|
type network struct {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
peers map[uint64]Interface
|
2014-11-13 00:23:42 +03:00
|
|
|
|
storage map[uint64]*MemoryStorage
|
2014-06-26 22:55:56 +04:00
|
|
|
|
dropm map[connem]float64
|
2014-10-12 11:34:22 +04:00
|
|
|
|
ignorem map[pb.MessageType]bool
|
2014-05-16 21:11:21 +04:00
|
|
|
|
}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
|
2014-06-18 04:35:19 +04:00
|
|
|
|
// newNetwork initializes a network from peers.
|
|
|
|
|
// A nil node will be replaced with a new *stateMachine.
|
|
|
|
|
// A *stateMachine will get its k, id.
|
2014-10-29 01:46:56 +03:00
|
|
|
|
// When using stateMachine, the address list is always [1, n].
|
2014-06-06 02:02:12 +04:00
|
|
|
|
func newNetwork(peers ...Interface) *network {
|
2014-06-18 04:35:19 +04:00
|
|
|
|
size := len(peers)
|
2014-10-29 01:46:56 +03:00
|
|
|
|
peerAddrs := idsBySize(size)
|
2014-06-10 03:45:42 +04:00
|
|
|
|
|
2014-10-08 14:29:53 +04:00
|
|
|
|
npeers := make(map[uint64]Interface, size)
|
2014-11-13 00:23:42 +03:00
|
|
|
|
nstorage := make(map[uint64]*MemoryStorage, size)
|
2014-06-18 04:35:19 +04:00
|
|
|
|
|
2014-09-10 03:33:09 +04:00
|
|
|
|
for i, p := range peers {
|
|
|
|
|
id := peerAddrs[i]
|
2014-06-06 02:02:12 +04:00
|
|
|
|
switch v := p.(type) {
|
2014-05-06 10:28:14 +04:00
|
|
|
|
case nil:
|
2014-11-13 00:23:42 +03:00
|
|
|
|
nstorage[id] = NewMemoryStorage()
|
|
|
|
|
sm := newRaft(id, peerAddrs, 10, 1, nstorage[id])
|
2014-09-10 03:33:09 +04:00
|
|
|
|
npeers[id] = sm
|
2014-08-23 00:24:33 +04:00
|
|
|
|
case *raft:
|
2014-09-10 03:33:09 +04:00
|
|
|
|
v.id = id
|
2014-10-08 14:29:53 +04:00
|
|
|
|
v.prs = make(map[uint64]*progress)
|
2014-06-18 04:35:19 +04:00
|
|
|
|
for i := 0; i < size; i++ {
|
2014-09-10 03:33:09 +04:00
|
|
|
|
v.prs[peerAddrs[i]] = &progress{}
|
2014-06-10 03:45:42 +04:00
|
|
|
|
}
|
2014-06-16 05:52:10 +04:00
|
|
|
|
v.reset(0)
|
2014-09-10 03:33:09 +04:00
|
|
|
|
npeers[id] = v
|
|
|
|
|
case *blackHole:
|
|
|
|
|
npeers[id] = v
|
2014-06-18 04:35:19 +04:00
|
|
|
|
default:
|
2014-09-12 10:18:51 +04:00
|
|
|
|
panic(fmt.Sprintf("unexpected state machine type: %T", p))
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
2014-06-26 22:55:56 +04:00
|
|
|
|
return &network{
|
|
|
|
|
peers: npeers,
|
2014-11-13 00:23:42 +03:00
|
|
|
|
storage: nstorage,
|
2014-06-26 22:55:56 +04:00
|
|
|
|
dropm: make(map[connem]float64),
|
2014-10-12 11:34:22 +04:00
|
|
|
|
ignorem: make(map[pb.MessageType]bool),
|
2014-06-26 22:55:56 +04:00
|
|
|
|
}
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-08-28 05:53:18 +04:00
|
|
|
|
func (nw *network) send(msgs ...pb.Message) {
|
2014-06-06 02:02:12 +04:00
|
|
|
|
for len(msgs) > 0 {
|
|
|
|
|
m := msgs[0]
|
|
|
|
|
p := nw.peers[m.To]
|
|
|
|
|
p.Step(m)
|
2014-10-30 00:04:45 +03:00
|
|
|
|
msgs = append(msgs[1:], nw.filter(p.readMessages())...)
|
2014-05-16 21:11:21 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-08 14:29:53 +04:00
|
|
|
|
func (nw *network) drop(from, to uint64, perc float64) {
|
2014-06-06 02:02:12 +04:00
|
|
|
|
nw.dropm[connem{from, to}] = perc
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-08 14:29:53 +04:00
|
|
|
|
func (nw *network) cut(one, other uint64) {
|
2014-06-06 22:01:55 +04:00
|
|
|
|
nw.drop(one, other, 1)
|
|
|
|
|
nw.drop(other, one, 1)
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-08 14:29:53 +04:00
|
|
|
|
func (nw *network) isolate(id uint64) {
|
2014-06-06 02:02:12 +04:00
|
|
|
|
for i := 0; i < len(nw.peers); i++ {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
nid := uint64(i) + 1
|
2014-07-09 22:53:27 +04:00
|
|
|
|
if nid != id {
|
|
|
|
|
nw.drop(id, nid, 1.0)
|
|
|
|
|
nw.drop(nid, id, 1.0)
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-12 11:34:22 +04:00
|
|
|
|
func (nw *network) ignore(t pb.MessageType) {
|
2014-06-26 22:55:56 +04:00
|
|
|
|
nw.ignorem[t] = true
|
|
|
|
|
}
|
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
|
func (nw *network) recover() {
|
|
|
|
|
nw.dropm = make(map[connem]float64)
|
2014-10-12 11:34:22 +04:00
|
|
|
|
nw.ignorem = make(map[pb.MessageType]bool)
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-08-28 05:53:18 +04:00
|
|
|
|
func (nw *network) filter(msgs []pb.Message) []pb.Message {
|
2014-10-08 02:22:35 +04:00
|
|
|
|
mm := []pb.Message{}
|
2014-06-06 02:02:12 +04:00
|
|
|
|
for _, m := range msgs {
|
2014-06-26 22:55:56 +04:00
|
|
|
|
if nw.ignorem[m.Type] {
|
|
|
|
|
continue
|
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
|
switch m.Type {
|
2014-10-12 11:34:22 +04:00
|
|
|
|
case pb.MsgHup:
|
2014-06-06 02:02:12 +04:00
|
|
|
|
// hups never go over the network, so don't drop them but panic
|
|
|
|
|
panic("unexpected msgHup")
|
2014-05-06 10:28:14 +04:00
|
|
|
|
default:
|
2014-06-06 02:02:12 +04:00
|
|
|
|
perc := nw.dropm[connem{m.From, m.To}]
|
|
|
|
|
if n := rand.Float64(); n < perc {
|
2014-05-06 10:28:14 +04:00
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
|
mm = append(mm, m)
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
2014-06-06 02:02:12 +04:00
|
|
|
|
return mm
|
2014-05-06 10:28:14 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
|
type connem struct {
|
2014-10-08 14:29:53 +04:00
|
|
|
|
from, to uint64
|
2014-05-24 00:30:04 +04:00
|
|
|
|
}
|
|
|
|
|
|
2014-06-06 02:02:12 +04:00
|
|
|
|
type blackHole struct{}
|
2014-05-27 23:24:49 +04:00
|
|
|
|
|
2014-08-28 05:53:18 +04:00
|
|
|
|
func (blackHole) Step(pb.Message) error { return nil }
|
2014-10-30 00:04:45 +03:00
|
|
|
|
func (blackHole) readMessages() []pb.Message { return nil }
|
2014-06-06 02:02:12 +04:00
|
|
|
|
|
|
|
|
|
var nopStepper = &blackHole{}
|
2014-10-29 01:46:56 +03:00
|
|
|
|
|
|
|
|
|
func idsBySize(size int) []uint64 {
|
|
|
|
|
ids := make([]uint64, size)
|
|
|
|
|
for i := 0; i < size; i++ {
|
|
|
|
|
ids[i] = 1 + uint64(i)
|
|
|
|
|
}
|
|
|
|
|
return ids
|
|
|
|
|
}
|