2016-05-13 06:49:15 +03:00
// Copyright 2015 The etcd Authors
2015-01-25 06:19:16 +03:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2014-10-18 02:41:22 +04:00
2014-05-06 10:28:14 +04:00
package raft
import (
2014-05-24 20:53:01 +04:00
"bytes"
2014-09-12 10:18:51 +04:00
"fmt"
2014-10-07 16:34:15 +04:00
"math"
2014-06-06 02:02:12 +04:00
"math/rand"
2014-07-01 23:10:43 +04:00
"reflect"
2014-05-06 10:28:14 +04:00
"testing"
2014-08-28 05:53:18 +04:00
pb "github.com/coreos/etcd/raft/raftpb"
2014-05-06 10:28:14 +04:00
)
2014-08-24 04:56:06 +04:00
// nextEnts returns the appliable entries and updates the applied index
2014-11-13 00:23:42 +03:00
func nextEnts ( r * raft , s * MemoryStorage ) ( ents [ ] pb . Entry ) {
2014-11-03 23:04:24 +03:00
// Transfer all unstable entries to "stable" storage.
2014-11-13 00:23:42 +03:00
s . Append ( r . raftLog . unstableEntries ( ) )
2014-11-29 01:13:07 +03:00
r . raftLog . stableTo ( r . raftLog . lastIndex ( ) , r . raftLog . lastTerm ( ) )
2014-11-03 23:04:24 +03:00
2014-08-24 04:56:06 +04:00
ents = r . raftLog . nextEnts ( )
2014-11-07 01:13:35 +03:00
r . raftLog . appliedTo ( r . raftLog . committed )
2014-08-24 04:56:06 +04:00
return ents
}
2016-02-01 08:42:39 +03:00
type stateMachine interface {
2014-08-28 05:53:18 +04:00
Step ( m pb . Message ) error
2014-10-30 00:04:45 +03:00
readMessages ( ) [ ] pb . Message
}
func ( r * raft ) readMessages ( ) [ ] pb . Message {
msgs := r . msgs
r . msgs = make ( [ ] pb . Message , 0 )
return msgs
2014-08-23 00:24:33 +04:00
}
2015-03-20 04:06:51 +03:00
func TestProgressBecomeProbe ( t * testing . T ) {
2015-03-16 10:46:16 +03:00
match := uint64 ( 1 )
tests := [ ] struct {
p * Progress
wnext uint64
} {
{
2015-03-20 04:06:51 +03:00
& Progress { State : ProgressStateReplicate , Match : match , Next : 5 , ins : newInflights ( 256 ) } ,
2015-03-16 10:46:16 +03:00
2 ,
} ,
{
// snapshot finish
2015-03-20 04:06:51 +03:00
& Progress { State : ProgressStateSnapshot , Match : match , Next : 5 , PendingSnapshot : 10 , ins : newInflights ( 256 ) } ,
2015-03-16 10:46:16 +03:00
11 ,
} ,
{
// snapshot failure
2015-03-20 04:06:51 +03:00
& Progress { State : ProgressStateSnapshot , Match : match , Next : 5 , PendingSnapshot : 0 , ins : newInflights ( 256 ) } ,
2015-03-16 10:46:16 +03:00
2 ,
} ,
}
for i , tt := range tests {
tt . p . becomeProbe ( )
if tt . p . State != ProgressStateProbe {
t . Errorf ( "#%d: state = %s, want %s" , i , tt . p . State , ProgressStateProbe )
}
if tt . p . Match != match {
t . Errorf ( "#%d: match = %d, want %d" , i , tt . p . Match , match )
}
if tt . p . Next != tt . wnext {
t . Errorf ( "#%d: next = %d, want %d" , i , tt . p . Next , tt . wnext )
}
}
}
2015-03-20 04:06:51 +03:00
func TestProgressBecomeReplicate ( t * testing . T ) {
p := & Progress { State : ProgressStateProbe , Match : 1 , Next : 5 , ins : newInflights ( 256 ) }
2015-03-16 10:46:16 +03:00
p . becomeReplicate ( )
if p . State != ProgressStateReplicate {
t . Errorf ( "state = %s, want %s" , p . State , ProgressStateReplicate )
}
if p . Match != 1 {
t . Errorf ( "match = %d, want 1" , p . Match )
}
if w := p . Match + 1 ; p . Next != w {
t . Errorf ( "next = %d, want %d" , p . Next , w )
}
}
2015-03-20 04:06:51 +03:00
func TestProgressBecomeSnapshot ( t * testing . T ) {
p := & Progress { State : ProgressStateProbe , Match : 1 , Next : 5 , ins : newInflights ( 256 ) }
2015-03-16 10:46:16 +03:00
p . becomeSnapshot ( 10 )
if p . State != ProgressStateSnapshot {
t . Errorf ( "state = %s, want %s" , p . State , ProgressStateSnapshot )
}
if p . Match != 1 {
t . Errorf ( "match = %d, want 1" , p . Match )
}
if p . PendingSnapshot != 10 {
t . Errorf ( "pendingSnapshot = %d, want 10" , p . PendingSnapshot )
}
}
2014-11-21 04:58:23 +03:00
func TestProgressUpdate ( t * testing . T ) {
prevM , prevN := uint64 ( 3 ) , uint64 ( 5 )
tests := [ ] struct {
update uint64
2015-03-16 10:46:16 +03:00
wm uint64
wn uint64
wok bool
2014-11-21 04:58:23 +03:00
} {
2015-03-16 10:46:16 +03:00
{ prevM - 1 , prevM , prevN , false } , // do not decrease match, next
{ prevM , prevM , prevN , false } , // do not decrease next
{ prevM + 1 , prevM + 1 , prevN , true } , // increase match, do not decrease next
{ prevM + 2 , prevM + 2 , prevN + 1 , true } , // increase match, next
2014-11-21 04:58:23 +03:00
}
for i , tt := range tests {
2015-01-20 21:26:22 +03:00
p := & Progress {
Match : prevM ,
Next : prevN ,
2014-11-21 04:58:23 +03:00
}
2015-03-16 10:46:16 +03:00
ok := p . maybeUpdate ( tt . update )
if ok != tt . wok {
t . Errorf ( "#%d: ok= %v, want %v" , i , ok , tt . wok )
}
2015-01-20 21:26:22 +03:00
if p . Match != tt . wm {
t . Errorf ( "#%d: match= %d, want %d" , i , p . Match , tt . wm )
2014-11-21 04:58:23 +03:00
}
2015-01-20 21:26:22 +03:00
if p . Next != tt . wn {
t . Errorf ( "#%d: next= %d, want %d" , i , p . Next , tt . wn )
2014-11-21 04:58:23 +03:00
}
}
}
2014-10-30 08:57:25 +03:00
func TestProgressMaybeDecr ( t * testing . T ) {
tests := [ ] struct {
2015-03-16 10:46:16 +03:00
state ProgressStateType
2014-12-31 06:42:59 +03:00
m uint64
n uint64
rejected uint64
last uint64
2014-10-30 08:57:25 +03:00
w bool
wn uint64
} {
{
2015-03-16 10:46:16 +03:00
// state replicate and rejected is not greater than match
ProgressStateReplicate , 5 , 10 , 5 , 5 , false , 10 ,
2014-10-30 08:57:25 +03:00
} ,
{
2015-03-16 10:46:16 +03:00
// state replicate and rejected is not greater than match
ProgressStateReplicate , 5 , 10 , 4 , 4 , false , 10 ,
2014-11-18 22:42:08 +03:00
} ,
{
2015-03-16 10:46:16 +03:00
// state replicate and rejected is greater than match
2014-11-18 22:42:08 +03:00
// directly decrease to match+1
2015-03-16 10:46:16 +03:00
ProgressStateReplicate , 5 , 10 , 9 , 9 , true , 6 ,
2014-10-30 08:57:25 +03:00
} ,
{
2014-12-31 06:42:59 +03:00
// next-1 != rejected is always false
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 0 , 0 , 0 , false , 0 ,
2014-10-30 08:57:25 +03:00
} ,
{
2014-12-31 06:42:59 +03:00
// next-1 != rejected is always false
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 10 , 5 , 5 , false , 10 ,
2014-10-30 08:57:25 +03:00
} ,
{
// next>1 = decremented by 1
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 10 , 9 , 9 , true , 9 ,
2014-10-30 08:57:25 +03:00
} ,
{
// next>1 = decremented by 1
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 2 , 1 , 1 , true , 1 ,
2014-10-30 08:57:25 +03:00
} ,
{
// next<=1 = reset to 1
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 1 , 0 , 0 , true , 1 ,
2014-12-31 06:42:59 +03:00
} ,
{
// decrease to min(rejected, last+1)
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 10 , 9 , 2 , true , 3 ,
2014-12-31 06:42:59 +03:00
} ,
{
// rejected < 1, reset to 1
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 10 , 9 , 0 , true , 1 ,
2014-10-30 08:57:25 +03:00
} ,
}
for i , tt := range tests {
2015-01-20 21:26:22 +03:00
p := & Progress {
2015-03-16 10:46:16 +03:00
State : tt . state ,
2015-01-20 21:26:22 +03:00
Match : tt . m ,
Next : tt . n ,
2014-10-30 08:57:25 +03:00
}
2014-12-31 06:42:59 +03:00
if g := p . maybeDecrTo ( tt . rejected , tt . last ) ; g != tt . w {
2014-12-18 23:02:15 +03:00
t . Errorf ( "#%d: maybeDecrTo= %t, want %t" , i , g , tt . w )
2014-10-30 08:57:25 +03:00
}
2015-01-20 21:26:22 +03:00
if gm := p . Match ; gm != tt . m {
2014-12-18 23:02:15 +03:00
t . Errorf ( "#%d: match= %d, want %d" , i , gm , tt . m )
2014-10-30 08:57:25 +03:00
}
2015-01-20 21:26:22 +03:00
if gn := p . Next ; gn != tt . wn {
2014-12-18 23:02:15 +03:00
t . Errorf ( "#%d: next= %d, want %d" , i , gn , tt . wn )
2014-10-30 08:57:25 +03:00
}
}
}
2015-03-16 10:46:16 +03:00
func TestProgressIsPaused ( t * testing . T ) {
2014-12-18 23:02:15 +03:00
tests := [ ] struct {
2015-03-16 10:46:16 +03:00
state ProgressStateType
paused bool
2014-12-18 23:02:15 +03:00
w bool
} {
2015-03-16 10:46:16 +03:00
{ ProgressStateProbe , false , false } ,
{ ProgressStateProbe , true , true } ,
{ ProgressStateReplicate , false , false } ,
{ ProgressStateReplicate , true , false } ,
{ ProgressStateSnapshot , false , true } ,
{ ProgressStateSnapshot , true , true } ,
2014-12-18 23:02:15 +03:00
}
for i , tt := range tests {
2015-01-20 21:26:22 +03:00
p := & Progress {
2015-03-16 10:46:16 +03:00
State : tt . state ,
Paused : tt . paused ,
2015-03-20 04:06:51 +03:00
ins : newInflights ( 256 ) ,
2014-12-18 23:02:15 +03:00
}
2015-03-16 10:46:16 +03:00
if g := p . isPaused ( ) ; g != tt . w {
2016-01-31 05:15:56 +03:00
t . Errorf ( "#%d: paused= %t, want %t" , i , g , tt . w )
2014-12-18 23:02:15 +03:00
}
}
}
2015-03-16 10:46:16 +03:00
// TestProgressResume ensures that progress.maybeUpdate and progress.maybeDecrTo
// will reset progress.paused.
func TestProgressResume ( t * testing . T ) {
2015-01-20 21:26:22 +03:00
p := & Progress {
2015-03-16 10:46:16 +03:00
Next : 2 ,
Paused : true ,
2014-12-18 23:02:15 +03:00
}
2014-12-31 06:42:59 +03:00
p . maybeDecrTo ( 1 , 1 )
2016-04-03 04:27:54 +03:00
if p . Paused {
2015-03-16 10:46:16 +03:00
t . Errorf ( "paused= %v, want false" , p . Paused )
2014-12-18 23:02:15 +03:00
}
2015-03-16 10:46:16 +03:00
p . Paused = true
p . maybeUpdate ( 2 )
2016-04-03 04:27:54 +03:00
if p . Paused {
2015-03-16 10:46:16 +03:00
t . Errorf ( "paused= %v, want false" , p . Paused )
2014-12-18 23:02:15 +03:00
}
}
2015-03-16 10:46:16 +03:00
// TestProgressResumeByHeartbeat ensures raft.heartbeat reset progress.paused by heartbeat.
func TestProgressResumeByHeartbeat ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
2014-12-18 23:02:15 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
2015-03-16 10:46:16 +03:00
r . prs [ 2 ] . Paused = true
2014-12-18 23:02:15 +03:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
2016-04-03 04:27:54 +03:00
if r . prs [ 2 ] . Paused {
2015-03-16 10:46:16 +03:00
t . Errorf ( "paused = %v, want false" , r . prs [ 2 ] . Paused )
2014-12-18 23:02:15 +03:00
}
}
2015-03-16 10:46:16 +03:00
func TestProgressPaused ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
2014-12-18 23:02:15 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
ms := r . readMessages ( )
if len ( ms ) != 1 {
t . Errorf ( "len(ms) = %d, want 1" , len ( ms ) )
}
}
2014-05-06 10:28:14 +04:00
func TestLeaderElection ( t * testing . T ) {
tests := [ ] struct {
2014-05-16 21:11:21 +04:00
* network
2014-09-16 04:35:02 +04:00
state StateType
2014-05-06 10:28:14 +04:00
} {
2014-09-16 04:35:02 +04:00
{ newNetwork ( nil , nil , nil ) , StateLeader } ,
{ newNetwork ( nil , nil , nopStepper ) , StateLeader } ,
{ newNetwork ( nil , nopStepper , nopStepper ) , StateCandidate } ,
{ newNetwork ( nil , nopStepper , nopStepper , nil ) , StateCandidate } ,
{ newNetwork ( nil , nopStepper , nopStepper , nil , nil ) , StateLeader } ,
2014-05-19 03:11:45 +04:00
2014-06-06 02:02:12 +04:00
// three logs further along than 0
2014-09-16 04:35:02 +04:00
{ newNetwork ( nil , ents ( 1 ) , ents ( 2 ) , ents ( 1 , 3 ) , nil ) , StateFollower } ,
2014-05-19 05:05:03 +04:00
// logs converge
2014-09-16 04:35:02 +04:00
{ newNetwork ( ents ( 1 ) , nil , ents ( 2 ) , ents ( 1 ) , nil ) , StateLeader } ,
2014-05-06 10:28:14 +04:00
}
for i , tt := range tests {
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-09-12 06:23:05 +04:00
sm := tt . network . peers [ 1 ] . ( * raft )
2014-05-06 10:28:14 +04:00
if sm . state != tt . state {
t . Errorf ( "#%d: state = %s, want %s" , i , sm . state , tt . state )
}
2014-08-23 01:19:30 +04:00
if g := sm . Term ; g != 1 {
2014-05-06 10:28:14 +04:00
t . Errorf ( "#%d: term = %d, want %d" , i , g , 1 )
}
}
}
2014-05-24 20:53:01 +04:00
func TestLogReplication ( t * testing . T ) {
tests := [ ] struct {
* network
2014-08-28 05:53:18 +04:00
msgs [ ] pb . Message
2014-10-08 14:29:53 +04:00
wcommitted uint64
2014-05-24 20:53:01 +04:00
} {
{
newNetwork ( nil , nil , nil ) ,
2014-08-28 05:53:18 +04:00
[ ] pb . Message {
2014-10-12 11:34:22 +04:00
{ From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
2014-05-24 20:53:01 +04:00
} ,
2014-06-26 22:55:56 +04:00
2 ,
2014-05-24 20:53:01 +04:00
} ,
{
newNetwork ( nil , nil , nil ) ,
2014-08-28 05:53:18 +04:00
[ ] pb . Message {
2014-10-12 11:34:22 +04:00
{ From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
{ From : 1 , To : 2 , Type : pb . MsgHup } ,
{ From : 1 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
2014-05-24 20:53:01 +04:00
} ,
2014-06-26 22:55:56 +04:00
4 ,
2014-05-24 20:53:01 +04:00
} ,
}
for i , tt := range tests {
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-24 20:53:01 +04:00
for _ , m := range tt . msgs {
2014-06-06 02:02:12 +04:00
tt . send ( m )
2014-05-24 20:53:01 +04:00
}
2014-06-06 02:02:12 +04:00
for j , x := range tt . network . peers {
2014-08-23 00:24:33 +04:00
sm := x . ( * raft )
2014-05-24 20:53:01 +04:00
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != tt . wcommitted {
t . Errorf ( "#%d.%d: committed = %d, want %d" , i , j , sm . raftLog . committed , tt . wcommitted )
2014-05-24 20:53:01 +04:00
}
2014-10-08 02:22:35 +04:00
ents := [ ] pb . Entry { }
2014-11-13 00:23:42 +03:00
for _ , e := range nextEnts ( sm , tt . network . storage [ j ] ) {
2014-06-26 22:55:56 +04:00
if e . Data != nil {
ents = append ( ents , e )
}
}
2014-10-08 02:22:35 +04:00
props := [ ] pb . Message { }
2014-05-24 20:53:01 +04:00
for _ , m := range tt . msgs {
2014-10-12 11:34:22 +04:00
if m . Type == pb . MsgProp {
2014-05-24 20:53:01 +04:00
props = append ( props , m )
}
}
for k , m := range props {
2014-06-06 00:00:18 +04:00
if ! bytes . Equal ( ents [ k ] . Data , m . Entries [ 0 ] . Data ) {
t . Errorf ( "#%d.%d: data = %d, want %d" , i , j , ents [ k ] . Data , m . Entries [ 0 ] . Data )
2014-05-24 20:53:01 +04:00
}
}
}
}
}
2014-05-29 02:30:01 +04:00
func TestSingleNodeCommit ( t * testing . T ) {
tt := newNetwork ( nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-05-29 02:30:01 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 3 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 3 )
2014-05-29 02:30:01 +04:00
}
}
2014-06-26 22:55:56 +04:00
// TestCannotCommitWithoutNewTermEntry tests the entries cannot be committed
// when leader changes, no new proposal comes in and ChangeTerm proposal is
// filtered.
2014-06-06 21:26:44 +04:00
func TestCannotCommitWithoutNewTermEntry ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil , nil , nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-06-06 21:26:44 +04:00
// 0 cannot reach 2,3,4
2014-09-12 06:23:05 +04:00
tt . cut ( 1 , 3 )
tt . cut ( 1 , 4 )
tt . cut ( 1 , 5 )
2014-06-06 21:26:44 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-06-06 21:26:44 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-06 21:26:44 +04:00
}
// network recovery
tt . recover ( )
2014-06-26 22:55:56 +04:00
// avoid committing ChangeTerm proposal
2014-10-12 11:34:22 +04:00
tt . ignore ( pb . MsgApp )
2014-06-06 21:26:44 +04:00
2014-12-18 23:02:15 +03:00
// elect 2 as the new leader with term 2
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
2014-06-06 21:26:44 +04:00
// no log entries from previous term should be committed
2014-09-12 06:23:05 +04:00
sm = tt . peers [ 2 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-06 21:26:44 +04:00
}
2014-06-26 22:55:56 +04:00
tt . recover ( )
2014-12-18 23:02:15 +03:00
// send heartbeat; reset wait
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgBeat } )
// append an entry at current term
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-12-18 23:02:15 +03:00
// expect the committed to be advanced
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 5 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 5 )
2014-06-26 22:55:56 +04:00
}
}
// TestCommitWithoutNewTermEntry tests the entries could be committed
// when leader changes, no new proposal comes in.
func TestCommitWithoutNewTermEntry ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil , nil , nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-06-26 22:55:56 +04:00
// 0 cannot reach 2,3,4
2014-09-12 06:23:05 +04:00
tt . cut ( 1 , 3 )
tt . cut ( 1 , 4 )
tt . cut ( 1 , 5 )
2014-06-26 22:55:56 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-06-26 22:55:56 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-26 22:55:56 +04:00
}
// network recovery
tt . recover ( )
// elect 1 as the new leader with term 2
// after append a ChangeTerm entry from the current term, all entries
// should be committed
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
2014-06-26 22:55:56 +04:00
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 4 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 4 )
2014-06-06 21:26:44 +04:00
}
}
2014-06-06 02:02:12 +04:00
func TestDuelingCandidates ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-05-19 00:59:10 +04:00
2014-06-26 22:55:56 +04:00
nt := newNetwork ( a , b , c )
2014-09-12 06:23:05 +04:00
nt . cut ( 1 , 3 )
2014-05-19 00:59:10 +04:00
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-16 21:11:21 +04:00
2016-05-19 05:38:10 +03:00
// 1 becomes leader since it receives votes from 1 and 2
2016-05-18 09:20:02 +03:00
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "state = %s, want %s" , sm . state , StateLeader )
}
2016-05-19 05:38:10 +03:00
// 3 stays as candidate since it receives a vote from 3 and a rejection from 2
2016-05-18 09:20:02 +03:00
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , sm . state , StateCandidate )
}
2014-06-26 22:55:56 +04:00
nt . recover ( )
2016-05-19 05:38:10 +03:00
// candidate 3 now increases its term and tries to vote again
// we expect it to disrupt the leader 1 since it has a higher term
// 3 will be follower again since both 1 and 2 rejects its vote request since 3 does not have a long enough log
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-16 21:53:56 +04:00
2014-11-03 23:04:24 +03:00
wlog := & raftLog {
2015-08-22 04:52:16 +03:00
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } } } ,
2014-11-03 23:04:24 +03:00
committed : 1 ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 2 } ,
2014-11-03 23:04:24 +03:00
}
2014-05-16 21:53:56 +04:00
tests := [ ] struct {
2014-08-23 00:24:33 +04:00
sm * raft
2014-09-16 04:35:02 +04:00
state StateType
2014-10-08 14:29:53 +04:00
term uint64
2014-07-24 03:15:25 +04:00
raftLog * raftLog
2014-05-16 21:53:56 +04:00
} {
2014-09-16 04:35:02 +04:00
{ a , StateFollower , 2 , wlog } ,
{ b , StateFollower , 2 , wlog } ,
2015-08-10 18:04:38 +03:00
{ c , StateFollower , 2 , newLog ( NewMemoryStorage ( ) , raftLogger ) } ,
2014-05-16 21:53:56 +04:00
}
for i , tt := range tests {
if g := tt . sm . state ; g != tt . state {
t . Errorf ( "#%d: state = %s, want %s" , i , g , tt . state )
}
2014-08-23 01:19:30 +04:00
if g := tt . sm . Term ; g != tt . term {
2014-05-16 21:53:56 +04:00
t . Errorf ( "#%d: term = %d, want %d" , i , g , tt . term )
}
2014-07-24 03:15:25 +04:00
base := ltoa ( tt . raftLog )
2014-10-08 14:29:53 +04:00
if sm , ok := nt . peers [ 1 + uint64 ( i ) ] . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-19 04:51:45 +04:00
}
}
}
func TestCandidateConcede ( t * testing . T ) {
2014-06-06 02:02:12 +04:00
tt := newNetwork ( nil , nil , nil )
2014-09-12 06:23:05 +04:00
tt . isolate ( 1 )
2014-05-19 04:51:45 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-19 04:51:45 +04:00
// heal the partition
2014-06-06 02:02:12 +04:00
tt . recover ( )
2014-12-18 23:02:15 +03:00
// send heartbeat; reset wait
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgBeat } )
2014-05-19 04:51:45 +04:00
data := [ ] byte ( "force follower" )
2014-12-19 04:10:37 +03:00
// send a proposal to 3 to flush out a MsgApp to 1
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : data } } } )
2014-12-19 04:10:37 +03:00
// send heartbeat; flush out commit
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgBeat } )
2014-05-19 04:51:45 +04:00
2014-09-12 06:23:05 +04:00
a := tt . peers [ 1 ] . ( * raft )
2014-09-16 04:35:02 +04:00
if g := a . state ; g != StateFollower {
t . Errorf ( "state = %s, want %s" , g , StateFollower )
2014-05-19 04:51:45 +04:00
}
2014-08-23 01:19:30 +04:00
if g := a . Term ; g != 1 {
2014-05-19 04:51:45 +04:00
t . Errorf ( "term = %d, want %d" , g , 1 )
}
2014-11-03 23:04:24 +03:00
wantLog := ltoa ( & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Index : 2 , Data : data } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 ,
} )
2014-06-06 02:02:12 +04:00
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( wantLog , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-16 21:53:56 +04:00
}
2014-05-16 21:11:21 +04:00
}
2014-05-16 21:53:56 +04:00
}
2014-05-28 22:59:01 +04:00
func TestSingleNodeCandidate ( t * testing . T ) {
tt := newNetwork ( nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-28 22:59:01 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-09-16 04:35:02 +04:00
if sm . state != StateLeader {
t . Errorf ( "state = %d, want %d" , sm . state , StateLeader )
2014-05-28 22:59:01 +04:00
}
}
2014-05-16 21:53:56 +04:00
func TestOldMessages ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil )
// make 0 leader @ term 3
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-12-18 23:02:15 +03:00
// pretend we're an old leader trying to make progress; this entry is expected to be ignored.
tt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgApp , Term : 2 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } } } )
// commit a new entry
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
2014-06-06 02:02:12 +04:00
2015-02-18 00:15:48 +03:00
ilog := & raftLog {
2014-11-03 23:04:24 +03:00
storage : & MemoryStorage {
ents : [ ] pb . Entry {
{ } , { Data : nil , Term : 1 , Index : 1 } ,
{ Data : nil , Term : 2 , Index : 2 } , { Data : nil , Term : 3 , Index : 3 } ,
2014-12-18 23:02:15 +03:00
{ Data : [ ] byte ( "somedata" ) , Term : 3 , Index : 4 } ,
2014-11-03 23:04:24 +03:00
} ,
2014-06-26 22:55:56 +04:00
} ,
2014-12-18 23:02:15 +03:00
unstable : unstable { offset : 5 } ,
committed : 4 ,
2014-06-26 22:55:56 +04:00
}
2015-02-18 00:15:48 +03:00
base := ltoa ( ilog )
2014-06-06 02:02:12 +04:00
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-16 21:11:21 +04:00
}
}
}
2014-05-18 04:18:58 +04:00
// TestOldMessagesReply - optimization - reply with new term.
2014-05-06 10:28:14 +04:00
func TestProposal ( t * testing . T ) {
tests := [ ] struct {
2014-05-16 21:11:21 +04:00
* network
2014-05-18 01:09:52 +04:00
success bool
2014-05-06 10:28:14 +04:00
} {
2014-05-18 01:09:52 +04:00
{ newNetwork ( nil , nil , nil ) , true } ,
{ newNetwork ( nil , nil , nopStepper ) , true } ,
{ newNetwork ( nil , nopStepper , nopStepper ) , false } ,
{ newNetwork ( nil , nopStepper , nopStepper , nil ) , false } ,
{ newNetwork ( nil , nopStepper , nopStepper , nil , nil ) , true } ,
2014-05-06 10:28:14 +04:00
}
2015-02-18 00:15:48 +03:00
for j , tt := range tests {
2014-08-28 05:53:18 +04:00
send := func ( m pb . Message ) {
2014-05-06 10:28:14 +04:00
defer func ( ) {
2014-05-18 01:09:52 +04:00
// only recover is we expect it to panic so
// panics we don't expect go up.
if ! tt . success {
2014-05-06 10:28:14 +04:00
e := recover ( )
if e != nil {
2015-02-18 00:15:48 +03:00
t . Logf ( "#%d: err: %s" , j , e )
2014-05-06 10:28:14 +04:00
}
}
} ( )
2014-06-06 02:02:12 +04:00
tt . send ( m )
}
2014-05-06 10:28:14 +04:00
2014-05-18 01:09:52 +04:00
data := [ ] byte ( "somedata" )
2014-05-06 10:28:14 +04:00
// promote 0 the leader
2014-10-12 11:34:22 +04:00
send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : data } } } )
2014-05-06 10:28:14 +04:00
2015-08-10 18:04:38 +03:00
wantLog := newLog ( NewMemoryStorage ( ) , raftLogger )
2014-05-18 01:09:52 +04:00
if tt . success {
2014-11-03 23:04:24 +03:00
wantLog = & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Index : 2 , Data : data } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 }
2014-05-18 01:09:52 +04:00
}
2014-06-06 02:02:12 +04:00
base := ltoa ( wantLog )
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-06 10:28:14 +04:00
}
}
2014-09-12 06:23:05 +04:00
sm := tt . network . peers [ 1 ] . ( * raft )
2014-08-23 01:19:30 +04:00
if g := sm . Term ; g != 1 {
2015-02-18 00:15:48 +03:00
t . Errorf ( "#%d: term = %d, want %d" , j , g , 1 )
2014-05-06 10:28:14 +04:00
}
}
}
func TestProposalByProxy ( t * testing . T ) {
2014-05-16 11:55:17 +04:00
data := [ ] byte ( "somedata" )
2014-05-18 01:09:52 +04:00
tests := [ ] * network {
newNetwork ( nil , nil , nil ) ,
newNetwork ( nil , nil , nopStepper ) ,
2014-05-06 10:28:14 +04:00
}
2015-02-18 00:15:48 +03:00
for j , tt := range tests {
2014-05-06 10:28:14 +04:00
// promote 0 the leader
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-06 10:28:14 +04:00
// propose via follower
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
2014-06-06 02:02:12 +04:00
2014-11-03 23:04:24 +03:00
wantLog := & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Data : data , Index : 2 } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 }
2014-06-06 02:02:12 +04:00
base := ltoa ( wantLog )
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-06 10:28:14 +04:00
}
}
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-08-23 01:19:30 +04:00
if g := sm . Term ; g != 1 {
2015-02-18 00:15:48 +03:00
t . Errorf ( "#%d: term = %d, want %d" , j , g , 1 )
2014-05-06 10:28:14 +04:00
}
}
}
2014-05-24 20:53:01 +04:00
func TestCommit ( t * testing . T ) {
2014-05-22 03:02:15 +04:00
tests := [ ] struct {
2014-10-08 14:29:53 +04:00
matches [ ] uint64
2014-08-28 05:53:18 +04:00
logs [ ] pb . Entry
2014-10-08 14:29:53 +04:00
smTerm uint64
w uint64
2014-05-22 03:02:15 +04:00
} {
2014-06-06 22:21:26 +04:00
// single
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } } , 1 , 1 } ,
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 2 } } , 2 , 1 } ,
2014-06-06 22:21:26 +04:00
2014-05-22 03:02:15 +04:00
// odd
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 2 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 2 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
2014-05-22 03:02:15 +04:00
// even
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 2 , 1 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 2 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 2 , 1 , 2 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
2014-05-22 03:02:15 +04:00
}
for i , tt := range tests {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
storage . Append ( tt . logs )
storage . hardState = pb . HardState { Term : tt . smTerm }
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 5 , 1 , storage )
2014-06-10 03:45:42 +04:00
for j := 0 ; j < len ( tt . matches ) ; j ++ {
2014-12-15 22:25:35 +03:00
sm . setProgress ( uint64 ( j ) + 1 , tt . matches [ j ] , tt . matches [ j ] + 1 )
2014-11-03 23:04:24 +03:00
}
2014-05-24 20:53:01 +04:00
sm . maybeCommit ( )
2014-07-24 03:15:25 +04:00
if g := sm . raftLog . committed ; g != tt . w {
2014-05-28 22:08:32 +04:00
t . Errorf ( "#%d: committed = %d, want %d" , i , g , tt . w )
2014-05-22 03:02:15 +04:00
}
}
}
2016-04-06 15:41:46 +03:00
func TestPastElectionTimeout ( t * testing . T ) {
2014-10-07 16:34:15 +04:00
tests := [ ] struct {
elapse int
2014-10-08 03:41:17 +04:00
wprobability float64
2014-10-07 16:34:15 +04:00
round bool
} {
{ 5 , 0 , false } ,
2016-04-01 20:09:09 +03:00
{ 10 , 0.1 , true } ,
{ 13 , 0.4 , true } ,
{ 15 , 0.6 , true } ,
{ 18 , 0.9 , true } ,
2014-10-07 16:34:15 +04:00
{ 20 , 1 , false } ,
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2015-11-24 08:59:25 +03:00
sm . electionElapsed = tt . elapse
2014-10-07 16:34:15 +04:00
c := 0
for j := 0 ; j < 10000 ; j ++ {
2016-04-01 20:09:09 +03:00
sm . resetRandomizedElectionTimeout ( )
if sm . pastElectionTimeout ( ) {
2014-10-07 16:34:15 +04:00
c ++
}
}
got := float64 ( c ) / 10000.0
if tt . round {
got = math . Floor ( got * 10 + 0.5 ) / 10.0
}
2014-10-08 03:41:17 +04:00
if got != tt . wprobability {
2016-04-06 15:41:46 +03:00
t . Errorf ( "#%d: probability = %v, want %v" , i , got , tt . wprobability )
2014-10-07 16:34:15 +04:00
}
}
}
2014-09-04 21:58:22 +04:00
// ensure that the Step function ignores the message from old term and does not pass it to the
2015-09-11 01:06:56 +03:00
// actual stepX function.
2014-09-04 21:58:22 +04:00
func TestStepIgnoreOldTermMsg ( t * testing . T ) {
called := false
fakeStep := func ( r * raft , m pb . Message ) {
called = true
}
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-04 21:58:22 +04:00
sm . step = fakeStep
sm . Term = 2
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { Type : pb . MsgApp , Term : sm . Term - 1 } )
2016-04-03 04:27:54 +03:00
if called {
2014-09-04 21:58:22 +04:00
t . Errorf ( "stepFunc called = %v , want %v" , called , false )
}
}
2014-07-14 21:58:41 +04:00
// TestHandleMsgApp ensures:
// 1. Reply false if log doesn’ t contain an entry at prevLogIndex whose term matches prevLogTerm.
// 2. If an existing entry conflicts with a new one (same index but different terms),
// delete the existing entry and all that follow it; append any new entries not already in the log.
// 3. If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry).
func TestHandleMsgApp ( t * testing . T ) {
tests := [ ] struct {
2014-08-28 05:53:18 +04:00
m pb . Message
2014-10-08 14:29:53 +04:00
wIndex uint64
wCommit uint64
2014-10-01 23:59:30 +04:00
wReject bool
2014-07-14 21:58:41 +04:00
} {
// Ensure 1
2014-10-12 11:34:22 +04:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 3 , Index : 2 , Commit : 3 } , 2 , 0 , true } , // previous log mismatch
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 3 , Index : 3 , Commit : 3 } , 2 , 0 , true } , // previous log non-exist
2014-07-14 21:58:41 +04:00
// Ensure 2
2014-10-12 11:34:22 +04:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 1 , Index : 1 , Commit : 1 } , 2 , 1 , false } ,
2014-12-02 19:50:15 +03:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 0 , Index : 0 , Commit : 1 , Entries : [ ] pb . Entry { { Index : 1 , Term : 2 } } } , 1 , 1 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 3 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } , { Index : 4 , Term : 2 } } } , 4 , 3 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 4 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } } } , 3 , 3 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 1 , Index : 1 , Commit : 4 , Entries : [ ] pb . Entry { { Index : 2 , Term : 2 } } } , 2 , 2 , false } ,
2014-07-14 21:58:41 +04:00
// Ensure 3
2014-12-05 01:12:55 +03:00
{ pb . Message { Type : pb . MsgApp , Term : 1 , LogTerm : 1 , Index : 1 , Commit : 3 } , 2 , 1 , false } , // match entry 1, commit up to last new entry 1
{ pb . Message { Type : pb . MsgApp , Term : 1 , LogTerm : 1 , Index : 1 , Commit : 3 , Entries : [ ] pb . Entry { { Index : 2 , Term : 2 } } } , 2 , 2 , false } , // match entry 1, commit up to last new entry 2
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 3 } , 2 , 2 , false } , // match entry 2, commit up to last new entry 2
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 4 } , 2 , 2 , false } , // commit up to log.last()
2014-07-14 21:58:41 +04:00
}
for i , tt := range tests {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
2014-12-15 22:25:35 +03:00
sm . becomeFollower ( 2 , None )
2014-07-14 21:58:41 +04:00
sm . handleAppendEntries ( tt . m )
2014-07-24 03:15:25 +04:00
if sm . raftLog . lastIndex ( ) != tt . wIndex {
t . Errorf ( "#%d: lastIndex = %d, want %d" , i , sm . raftLog . lastIndex ( ) , tt . wIndex )
2014-07-14 21:58:41 +04:00
}
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != tt . wCommit {
t . Errorf ( "#%d: committed = %d, want %d" , i , sm . raftLog . committed , tt . wCommit )
2014-07-14 21:58:41 +04:00
}
2014-10-30 00:04:45 +03:00
m := sm . readMessages ( )
2014-07-14 21:58:41 +04:00
if len ( m ) != 1 {
2014-09-29 06:38:37 +04:00
t . Fatalf ( "#%d: msg = nil, want 1" , i )
2014-07-14 21:58:41 +04:00
}
2014-10-01 23:59:30 +04:00
if m [ 0 ] . Reject != tt . wReject {
t . Errorf ( "#%d: reject = %v, want %v" , i , m [ 0 ] . Reject , tt . wReject )
2014-07-14 21:58:41 +04:00
}
}
}
2014-11-18 03:58:49 +03:00
// TestHandleHeartbeat ensures that the follower commits to the commit in the message.
func TestHandleHeartbeat ( t * testing . T ) {
commit := uint64 ( 2 )
tests := [ ] struct {
m pb . Message
wCommit uint64
} {
2016-05-20 09:04:08 +03:00
{ pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeat , Term : 2 , Commit : commit + 1 } , commit + 1 } ,
{ pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeat , Term : 2 , Commit : commit - 1 } , commit } , // do not decrease commit
2014-11-18 03:58:49 +03:00
}
for i , tt := range tests {
2014-11-20 01:05:16 +03:00
storage := NewMemoryStorage ( )
2014-11-25 04:13:47 +03:00
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } , { Index : 3 , Term : 3 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , storage )
2014-12-15 22:25:35 +03:00
sm . becomeFollower ( 2 , 2 )
2014-11-18 03:58:49 +03:00
sm . raftLog . commitTo ( commit )
sm . handleHeartbeat ( tt . m )
if sm . raftLog . committed != tt . wCommit {
t . Errorf ( "#%d: committed = %d, want %d" , i , sm . raftLog . committed , tt . wCommit )
}
m := sm . readMessages ( )
2015-01-14 22:42:30 +03:00
if len ( m ) != 1 {
t . Fatalf ( "#%d: msg = nil, want 1" , i )
}
if m [ 0 ] . Type != pb . MsgHeartbeatResp {
t . Errorf ( "#%d: type = %v, want MsgHeartbeatResp" , i , m [ 0 ] . Type )
2014-11-18 03:58:49 +03:00
}
}
}
2015-01-14 22:42:30 +03:00
// TestHandleHeartbeatResp ensures that we re-send log entries when we get a heartbeat response.
func TestHandleHeartbeatResp ( t * testing . T ) {
storage := NewMemoryStorage ( )
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } , { Index : 3 , Term : 3 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , storage )
2015-01-14 22:42:30 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
sm . raftLog . commitTo ( sm . raftLog . lastIndex ( ) )
// A heartbeat response from a node that is behind; re-send MsgApp
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs := sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
}
if msgs [ 0 ] . Type != pb . MsgApp {
t . Errorf ( "type = %v, want MsgApp" , msgs [ 0 ] . Type )
}
// A second heartbeat response with no AppResp does not re-send because we are in the wait state.
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs = sm . readMessages ( )
if len ( msgs ) != 0 {
t . Fatalf ( "len(msgs) = %d, want 0" , len ( msgs ) )
}
// Send a heartbeat to reset the wait state; next heartbeat will re-send MsgApp.
sm . bcastHeartbeat ( )
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs = sm . readMessages ( )
if len ( msgs ) != 2 {
t . Fatalf ( "len(msgs) = %d, want 2" , len ( msgs ) )
}
if msgs [ 0 ] . Type != pb . MsgHeartbeat {
t . Errorf ( "type = %v, want MsgHeartbeat" , msgs [ 0 ] . Type )
}
if msgs [ 1 ] . Type != pb . MsgApp {
t . Errorf ( "type = %v, want MsgApp" , msgs [ 1 ] . Type )
}
// Once we have an MsgAppResp, heartbeats no longer send MsgApp.
sm . Step ( pb . Message {
From : 2 ,
Type : pb . MsgAppResp ,
Index : msgs [ 1 ] . Index + uint64 ( len ( msgs [ 1 ] . Entries ) ) ,
} )
2015-01-28 01:40:14 +03:00
// Consume the message sent in response to MsgAppResp
sm . readMessages ( )
2015-01-14 22:42:30 +03:00
sm . bcastHeartbeat ( ) // reset wait state
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs = sm . readMessages ( )
if len ( msgs ) != 1 {
2015-01-28 01:40:14 +03:00
t . Fatalf ( "len(msgs) = %d, want 1: %+v" , len ( msgs ) , msgs )
2015-01-14 22:42:30 +03:00
}
if msgs [ 0 ] . Type != pb . MsgHeartbeat {
t . Errorf ( "type = %v, want MsgHeartbeat" , msgs [ 0 ] . Type )
}
}
2016-02-01 08:42:39 +03:00
// TestMsgAppRespWaitReset verifies the resume behavior of a leader
2015-01-28 01:40:14 +03:00
// MsgAppResp.
func TestMsgAppRespWaitReset ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
2015-01-28 01:40:14 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
// The new leader has just emitted a new Term 4 entry; consume those messages
// from the outgoing queue.
sm . bcastAppend ( )
sm . readMessages ( )
// Node 2 acks the first entry, making it committed.
sm . Step ( pb . Message {
From : 2 ,
Type : pb . MsgAppResp ,
Index : 1 ,
} )
2016-01-26 23:18:55 +03:00
if sm . raftLog . committed != 1 {
t . Fatalf ( "expected committed to be 1, got %d" , sm . raftLog . committed )
2015-01-28 01:40:14 +03:00
}
// Also consume the MsgApp messages that update Commit on the followers.
sm . readMessages ( )
// A new command is now proposed on node 1.
sm . Step ( pb . Message {
From : 1 ,
Type : pb . MsgProp ,
Entries : [ ] pb . Entry { { } } ,
} )
// The command is broadcast to all nodes not in the wait state.
// Node 2 left the wait state due to its MsgAppResp, but node 3 is still waiting.
msgs := sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "expected 1 message, got %d: %+v" , len ( msgs ) , msgs )
}
if msgs [ 0 ] . Type != pb . MsgApp || msgs [ 0 ] . To != 2 {
2015-12-16 07:18:21 +03:00
t . Errorf ( "expected MsgApp to node 2, got %v to %d" , msgs [ 0 ] . Type , msgs [ 0 ] . To )
2015-01-28 01:40:14 +03:00
}
if len ( msgs [ 0 ] . Entries ) != 1 || msgs [ 0 ] . Entries [ 0 ] . Index != 2 {
t . Errorf ( "expected to send entry 2, but got %v" , msgs [ 0 ] . Entries )
}
// Now Node 3 acks the first entry. This releases the wait and entry 2 is sent.
sm . Step ( pb . Message {
From : 3 ,
Type : pb . MsgAppResp ,
Index : 1 ,
} )
msgs = sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "expected 1 message, got %d: %+v" , len ( msgs ) , msgs )
}
if msgs [ 0 ] . Type != pb . MsgApp || msgs [ 0 ] . To != 3 {
2015-12-16 07:18:21 +03:00
t . Errorf ( "expected MsgApp to node 3, got %v to %d" , msgs [ 0 ] . Type , msgs [ 0 ] . To )
2015-01-28 01:40:14 +03:00
}
if len ( msgs [ 0 ] . Entries ) != 1 || msgs [ 0 ] . Entries [ 0 ] . Index != 2 {
t . Errorf ( "expected to send entry 2, but got %v" , msgs [ 0 ] . Entries )
}
}
2014-06-11 19:58:26 +04:00
func TestRecvMsgVote ( t * testing . T ) {
2014-05-18 11:17:46 +04:00
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
state StateType
2014-10-08 14:29:53 +04:00
i , term uint64
voteFor uint64
2014-10-01 23:59:30 +04:00
wreject bool
2014-05-18 11:17:46 +04:00
} {
2014-09-29 06:34:15 +04:00
{ StateFollower , 0 , 0 , None , true } ,
{ StateFollower , 0 , 1 , None , true } ,
{ StateFollower , 0 , 2 , None , true } ,
{ StateFollower , 0 , 3 , None , false } ,
{ StateFollower , 1 , 0 , None , true } ,
{ StateFollower , 1 , 1 , None , true } ,
{ StateFollower , 1 , 2 , None , true } ,
{ StateFollower , 1 , 3 , None , false } ,
{ StateFollower , 2 , 0 , None , true } ,
{ StateFollower , 2 , 1 , None , true } ,
{ StateFollower , 2 , 2 , None , false } ,
{ StateFollower , 2 , 3 , None , false } ,
{ StateFollower , 3 , 0 , None , true } ,
{ StateFollower , 3 , 1 , None , true } ,
{ StateFollower , 3 , 2 , None , false } ,
{ StateFollower , 3 , 3 , None , false } ,
{ StateFollower , 3 , 2 , 2 , false } ,
{ StateFollower , 3 , 2 , 1 , true } ,
{ StateLeader , 3 , 3 , 1 , true } ,
{ StateCandidate , 3 , 3 , 1 , true } ,
2014-05-18 11:17:46 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-03 21:16:33 +04:00
sm . state = tt . state
switch tt . state {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-09-03 21:16:33 +04:00
sm . step = stepFollower
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-09-03 21:16:33 +04:00
sm . step = stepCandidate
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-09-03 21:16:33 +04:00
sm . step = stepLeader
2014-05-28 09:23:14 +04:00
}
2016-01-26 23:18:55 +03:00
sm . Vote = tt . voteFor
2014-11-03 23:04:24 +03:00
sm . raftLog = & raftLog {
2014-11-25 04:13:47 +03:00
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 2 } , { Index : 2 , Term : 2 } } } ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
}
2014-05-25 08:08:06 +04:00
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { Type : pb . MsgVote , From : 2 , Index : tt . i , LogTerm : tt . term } )
2014-06-06 02:02:12 +04:00
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-06-17 22:19:56 +04:00
if g := len ( msgs ) ; g != 1 {
2014-09-29 06:34:15 +04:00
t . Fatalf ( "#%d: len(msgs) = %d, want 1" , i , g )
2014-06-11 19:58:26 +04:00
continue
2014-06-06 02:02:12 +04:00
}
2014-10-01 23:59:30 +04:00
if g := msgs [ 0 ] . Reject ; g != tt . wreject {
2014-10-08 23:07:11 +04:00
t . Errorf ( "#%d, m.Reject = %v, want %v" , i , g , tt . wreject )
2014-05-18 11:17:46 +04:00
}
}
}
2014-06-07 03:27:29 +04:00
func TestStateTransition ( t * testing . T ) {
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
from StateType
to StateType
2014-06-07 03:27:29 +04:00
wallow bool
2014-10-08 14:29:53 +04:00
wterm uint64
wlead uint64
2014-06-07 03:27:29 +04:00
} {
2014-09-16 04:35:02 +04:00
{ StateFollower , StateFollower , true , 1 , None } ,
{ StateFollower , StateCandidate , true , 1 , None } ,
2014-10-08 14:29:53 +04:00
{ StateFollower , StateLeader , false , 0 , None } ,
2014-06-07 03:27:29 +04:00
2014-09-16 04:35:02 +04:00
{ StateCandidate , StateFollower , true , 0 , None } ,
{ StateCandidate , StateCandidate , true , 1 , None } ,
{ StateCandidate , StateLeader , true , 0 , 1 } ,
2014-06-07 03:27:29 +04:00
2014-09-16 04:35:02 +04:00
{ StateLeader , StateFollower , true , 1 , None } ,
{ StateLeader , StateCandidate , false , 1 , None } ,
{ StateLeader , StateLeader , true , 0 , 1 } ,
2014-06-07 03:27:29 +04:00
}
for i , tt := range tests {
func ( ) {
defer func ( ) {
if r := recover ( ) ; r != nil {
2016-04-03 04:27:54 +03:00
if tt . wallow {
2014-06-07 03:27:29 +04:00
t . Errorf ( "%d: allow = %v, want %v" , i , false , true )
}
}
} ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-06-07 03:27:29 +04:00
sm . state = tt . from
switch tt . to {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-06-07 03:27:29 +04:00
sm . becomeFollower ( tt . wterm , tt . wlead )
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-06-07 03:27:29 +04:00
sm . becomeCandidate ( )
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-06-07 03:27:29 +04:00
sm . becomeLeader ( )
}
2014-08-23 01:19:30 +04:00
if sm . Term != tt . wterm {
t . Errorf ( "%d: term = %d, want %d" , i , sm . Term , tt . wterm )
2014-06-07 03:27:29 +04:00
}
2014-08-24 04:40:25 +04:00
if sm . lead != tt . wlead {
2014-06-07 03:27:29 +04:00
t . Errorf ( "%d: lead = %d, want %d" , i , sm . lead , tt . wlead )
}
} ( )
}
}
2014-05-24 00:30:04 +04:00
func TestAllServerStepdown ( t * testing . T ) {
2014-06-26 22:55:56 +04:00
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
state StateType
2014-06-26 22:55:56 +04:00
2014-09-16 04:35:02 +04:00
wstate StateType
2014-10-08 14:29:53 +04:00
wterm uint64
windex uint64
2014-06-26 22:55:56 +04:00
} {
2014-11-18 00:37:46 +03:00
{ StateFollower , StateFollower , 3 , 0 } ,
{ StateCandidate , StateFollower , 3 , 0 } ,
{ StateLeader , StateFollower , 3 , 1 } ,
2014-06-26 22:55:56 +04:00
}
2014-05-24 00:30:04 +04:00
2014-10-12 11:34:22 +04:00
tmsgTypes := [ ... ] pb . MessageType { pb . MsgVote , pb . MsgApp }
2014-10-08 14:29:53 +04:00
tterm := uint64 ( 3 )
2014-05-24 00:30:04 +04:00
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-06-26 22:55:56 +04:00
switch tt . state {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-09-15 09:44:59 +04:00
sm . becomeFollower ( 1 , None )
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-05-24 00:30:04 +04:00
sm . becomeCandidate ( )
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-05-24 00:30:04 +04:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
}
for j , msgType := range tmsgTypes {
2014-09-12 06:23:05 +04:00
sm . Step ( pb . Message { From : 2 , Type : msgType , Term : tterm , LogTerm : tterm } )
2014-05-24 00:30:04 +04:00
2014-06-26 22:55:56 +04:00
if sm . state != tt . wstate {
t . Errorf ( "#%d.%d state = %v , want %v" , i , j , sm . state , tt . wstate )
2014-05-24 00:30:04 +04:00
}
2014-08-23 01:19:30 +04:00
if sm . Term != tt . wterm {
t . Errorf ( "#%d.%d term = %v , want %v" , i , j , sm . Term , tt . wterm )
2014-05-24 00:30:04 +04:00
}
2014-11-18 00:37:46 +03:00
if uint64 ( sm . raftLog . lastIndex ( ) ) != tt . windex {
t . Errorf ( "#%d.%d index = %v , want %v" , i , j , sm . raftLog . lastIndex ( ) , tt . windex )
}
2014-11-03 23:04:24 +03:00
if uint64 ( len ( sm . raftLog . allEntries ( ) ) ) != tt . windex {
2014-11-18 00:37:46 +03:00
t . Errorf ( "#%d.%d len(ents) = %v , want %v" , i , j , len ( sm . raftLog . allEntries ( ) ) , tt . windex )
2014-05-24 00:30:04 +04:00
}
2014-10-08 14:29:53 +04:00
wlead := uint64 ( 2 )
2014-10-12 11:34:22 +04:00
if msgType == pb . MsgVote {
2014-09-15 09:44:59 +04:00
wlead = None
2014-07-15 09:39:44 +04:00
}
2014-08-24 04:40:25 +04:00
if sm . lead != wlead {
2014-09-15 09:44:59 +04:00
t . Errorf ( "#%d, sm.lead = %d, want %d" , i , sm . lead , None )
2014-07-15 09:39:44 +04:00
}
2014-05-24 00:30:04 +04:00
}
}
}
2015-11-24 08:59:25 +03:00
func TestLeaderStepdownWhenQuorumActive ( t * testing . T ) {
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
sm . checkQuorum = true
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < sm . electionTimeout + 1 ; i ++ {
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp , Term : sm . Term } )
sm . tick ( )
}
if sm . state != StateLeader {
t . Errorf ( "state = %v, want %v" , sm . state , StateLeader )
}
}
func TestLeaderStepdownWhenQuorumLost ( t * testing . T ) {
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
sm . checkQuorum = true
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < sm . electionTimeout + 1 ; i ++ {
sm . tick ( )
}
if sm . state != StateFollower {
t . Errorf ( "state = %v, want %v" , sm . state , StateFollower )
}
}
2016-05-27 12:23:18 +03:00
func TestLeaderSupersedingWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
// Prevent campaigning from b
b . randomizedElectionTimeout = b . electionTimeout + 1
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// Peer b rejected c's vote since its electionElapsed had not reached to electionTimeout
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
// Letting b's electionElapsed reach to electionTimeout
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if c . state != StateLeader {
t . Errorf ( "state = %s, want %s" , c . state , StateLeader )
}
}
func TestLeaderElectionWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
// Letting b's electionElapsed reach to timeout so that it can vote for a
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
for i := 0 ; i < a . electionTimeout ; i ++ {
a . tick ( )
}
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if a . state != StateFollower {
t . Errorf ( "state = %s, want %s" , a . state , StateFollower )
}
if c . state != StateLeader {
t . Errorf ( "state = %s, want %s" , c . state , StateLeader )
}
}
// TestFreeStuckCandidateWithCheckQuorum ensures that a candidate with a higher term
// can disrupt the leader even if the leader still "officially" holds the lease, The
// leader is expected to step down and adopt the candidate's term
func TestFreeStuckCandidateWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 1 )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
if c . Term != b . Term + 1 {
t . Errorf ( "term = %d, want %d" , c . Term , b . Term + 1 )
}
// Vote again for safety
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
if c . Term != b . Term + 2 {
t . Errorf ( "term = %d, want %d" , c . Term , b . Term + 2 )
}
nt . recover ( )
nt . send ( pb . Message { From : 1 , To : 3 , Type : pb . MsgHeartbeat , Term : a . Term } )
// Disrupt the leader so that the stuck peer is freed
if a . state != StateFollower {
t . Errorf ( "state = %s, want %s" , a . state , StateFollower )
}
if c . Term != a . Term {
t . Errorf ( "term = %d, want %d" , c . Term , a . Term )
}
}
func TestNonPromotableVoterWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
nt := newNetwork ( a , b )
// Need to remove 2 again to make it a non-promotable node since newNetwork overwritten some internal states
b . delProgress ( 2 )
if b . promotable ( ) {
t . Fatalf ( "promotable = %v, want false" , b . promotable ( ) )
}
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if b . lead != 1 {
t . Errorf ( "lead = %d, want 1" , b . lead )
}
}
2014-05-25 10:03:13 +04:00
func TestLeaderAppResp ( t * testing . T ) {
2014-11-18 22:42:08 +03:00
// initial progress: match = 0; next = 3
2014-05-25 10:03:13 +04:00
tests := [ ] struct {
2014-10-24 22:50:21 +04:00
index uint64
reject bool
// progress
wmatch uint64
wnext uint64
// message
2014-05-25 10:03:13 +04:00
wmsgNum int
2014-10-08 14:29:53 +04:00
windex uint64
wcommitted uint64
2014-05-25 10:03:13 +04:00
} {
2014-10-24 22:50:21 +04:00
{ 3 , true , 0 , 3 , 0 , 0 , 0 } , // stale resp; no replies
2014-12-05 01:12:55 +03:00
{ 2 , true , 0 , 2 , 1 , 1 , 0 } , // denied resp; leader does not commit; decrease next and send probing msg
2014-11-18 22:42:08 +03:00
{ 2 , false , 2 , 4 , 2 , 2 , 2 } , // accept resp; leader commits; broadcast with commit index
2014-10-24 22:50:21 +04:00
{ 0 , false , 0 , 3 , 0 , 0 , 0 } , // ignore heartbeat replies
2014-05-25 10:03:13 +04:00
}
for i , tt := range tests {
// sm term is 1 after it becomes the leader.
// thus the last log term must be 1 to be committed.
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-11-03 23:04:24 +03:00
sm . raftLog = & raftLog {
2014-11-25 04:13:47 +03:00
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 0 } , { Index : 2 , Term : 1 } } } ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
}
2014-05-25 10:03:13 +04:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
2014-10-30 00:04:45 +03:00
sm . readMessages ( )
2014-12-31 06:42:59 +03:00
sm . Step ( pb . Message { From : 2 , Type : pb . MsgAppResp , Index : tt . index , Term : sm . Term , Reject : tt . reject , RejectHint : tt . index } )
2014-10-24 22:50:21 +04:00
p := sm . prs [ 2 ]
2015-01-20 21:26:22 +03:00
if p . Match != tt . wmatch {
t . Errorf ( "#%d match = %d, want %d" , i , p . Match , tt . wmatch )
2014-10-24 22:50:21 +04:00
}
2015-01-20 21:26:22 +03:00
if p . Next != tt . wnext {
t . Errorf ( "#%d next = %d, want %d" , i , p . Next , tt . wnext )
2014-10-24 22:50:21 +04:00
}
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-05-25 10:03:13 +04:00
if len ( msgs ) != tt . wmsgNum {
t . Errorf ( "#%d msgNum = %d, want %d" , i , len ( msgs ) , tt . wmsgNum )
}
for j , msg := range msgs {
if msg . Index != tt . windex {
t . Errorf ( "#%d.%d index = %d, want %d" , i , j , msg . Index , tt . windex )
}
if msg . Commit != tt . wcommitted {
t . Errorf ( "#%d.%d commit = %d, want %d" , i , j , msg . Commit , tt . wcommitted )
}
}
}
}
2014-09-15 20:58:22 +04:00
// When the leader receives a heartbeat tick, it should
2014-10-30 09:21:38 +03:00
// send a MsgApp with m.Index = 0, m.LogTerm=0 and empty entries.
2014-09-15 20:58:22 +04:00
func TestBcastBeat ( t * testing . T ) {
2014-10-08 14:29:53 +04:00
offset := uint64 ( 1000 )
2014-09-15 20:58:22 +04:00
// make a state machine with log.offset = 1000
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : offset ,
Term : 1 ,
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 , 3 } } ,
} ,
2014-09-15 20:58:22 +04:00
}
2014-11-20 00:17:50 +03:00
storage := NewMemoryStorage ( )
storage . ApplySnapshot ( s )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , nil , 10 , 1 , storage )
2014-09-15 20:58:22 +04:00
sm . Term = 1
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < 10 ; i ++ {
2014-11-25 04:13:47 +03:00
sm . appendEntry ( pb . Entry { Index : uint64 ( i ) + 1 } )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
// slow follower
2015-01-20 21:26:22 +03:00
sm . prs [ 2 ] . Match , sm . prs [ 2 ] . Next = 5 , 6
2014-11-18 02:44:57 +03:00
// normal follower
2015-01-20 21:26:22 +03:00
sm . prs [ 3 ] . Match , sm . prs [ 3 ] . Next = sm . raftLog . lastIndex ( ) , sm . raftLog . lastIndex ( ) + 1
2014-09-15 20:58:22 +04:00
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { Type : pb . MsgBeat } )
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-09-29 21:44:20 +04:00
if len ( msgs ) != 2 {
2014-10-30 09:21:38 +03:00
t . Fatalf ( "len(msgs) = %v, want 2" , len ( msgs ) )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
wantCommitMap := map [ uint64 ] uint64 {
2015-01-20 21:26:22 +03:00
2 : min ( sm . raftLog . committed , sm . prs [ 2 ] . Match ) ,
3 : min ( sm . raftLog . committed , sm . prs [ 3 ] . Match ) ,
2014-11-18 02:44:57 +03:00
}
2014-09-29 21:44:20 +04:00
for i , m := range msgs {
2014-12-04 08:32:12 +03:00
if m . Type != pb . MsgHeartbeat {
t . Fatalf ( "#%d: type = %v, want = %v" , i , m . Type , pb . MsgHeartbeat )
2014-09-15 20:58:22 +04:00
}
2014-09-29 21:44:20 +04:00
if m . Index != 0 {
t . Fatalf ( "#%d: prevIndex = %d, want %d" , i , m . Index , 0 )
2014-09-15 20:58:22 +04:00
}
2014-09-29 21:44:20 +04:00
if m . LogTerm != 0 {
t . Fatalf ( "#%d: prevTerm = %d, want %d" , i , m . LogTerm , 0 )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
if wantCommitMap [ m . To ] == 0 {
2014-09-29 21:44:20 +04:00
t . Fatalf ( "#%d: unexpected to %d" , i , m . To )
} else {
2014-11-18 02:44:57 +03:00
if m . Commit != wantCommitMap [ m . To ] {
t . Fatalf ( "#%d: commit = %d, want %d" , i , m . Commit , wantCommitMap [ m . To ] )
}
delete ( wantCommitMap , m . To )
2014-09-15 20:58:22 +04:00
}
if len ( m . Entries ) != 0 {
t . Fatalf ( "#%d: len(entries) = %d, want 0" , i , len ( m . Entries ) )
}
}
}
2016-01-31 05:15:56 +03:00
// tests the output of the state machine when receiving MsgBeat
2014-06-07 00:40:24 +04:00
func TestRecvMsgBeat ( t * testing . T ) {
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
state StateType
2014-06-07 00:40:24 +04:00
wMsg int
} {
2014-09-16 04:35:02 +04:00
{ StateLeader , 2 } ,
2014-10-30 09:21:38 +03:00
// candidate and follower should ignore MsgBeat
2014-09-16 04:35:02 +04:00
{ StateCandidate , 0 } ,
{ StateFollower , 0 } ,
2014-06-07 00:40:24 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-11-25 04:13:47 +03:00
sm . raftLog = & raftLog { storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 0 } , { Index : 2 , Term : 1 } } } }
2014-08-23 01:19:30 +04:00
sm . Term = 1
2014-06-07 00:40:24 +04:00
sm . state = tt . state
2014-09-03 21:16:33 +04:00
switch tt . state {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-09-03 21:16:33 +04:00
sm . step = stepFollower
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-09-03 21:16:33 +04:00
sm . step = stepCandidate
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-09-03 21:16:33 +04:00
sm . step = stepLeader
}
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
2014-06-07 00:40:24 +04:00
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-06-07 00:40:24 +04:00
if len ( msgs ) != tt . wMsg {
t . Errorf ( "%d: len(msgs) = %d, want %d" , i , len ( msgs ) , tt . wMsg )
}
for _ , m := range msgs {
2014-12-04 08:32:12 +03:00
if m . Type != pb . MsgHeartbeat {
t . Errorf ( "%d: msg.type = %v, want %v" , i , m . Type , pb . MsgHeartbeat )
2014-06-07 00:40:24 +04:00
}
}
}
}
2014-11-18 22:42:08 +03:00
func TestLeaderIncreaseNext ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
tests := [ ] struct {
// progress
2015-03-16 10:46:16 +03:00
state ProgressStateType
2014-11-18 22:42:08 +03:00
next uint64
wnext uint64
} {
2015-03-16 10:46:16 +03:00
// state replicate, optimistically increase next
2014-11-18 22:42:08 +03:00
// previous entries + noop entry + propose + 1
2015-03-16 10:46:16 +03:00
{ ProgressStateReplicate , 2 , uint64 ( len ( previousEnts ) + 1 + 1 + 1 ) } ,
// state probe, not optimistically increase next
{ ProgressStateProbe , 2 , 2 } ,
2014-11-18 22:42:08 +03:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-12-02 19:50:15 +03:00
sm . raftLog . append ( previousEnts ... )
2014-11-18 22:42:08 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
2015-03-16 10:46:16 +03:00
sm . prs [ 2 ] . State = tt . state
sm . prs [ 2 ] . Next = tt . next
2014-11-18 22:42:08 +03:00
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
p := sm . prs [ 2 ]
2015-01-20 21:26:22 +03:00
if p . Next != tt . wnext {
t . Errorf ( "#%d next = %d, want %d" , i , p . Next , tt . wnext )
2014-11-18 22:42:08 +03:00
}
}
2014-06-07 00:40:24 +04:00
}
2015-03-16 10:46:16 +03:00
func TestSendAppendForProgressProbe ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-02-05 01:41:14 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
2015-03-16 10:46:16 +03:00
r . prs [ 2 ] . becomeProbe ( )
2015-02-05 01:41:14 +03:00
2015-03-16 10:46:16 +03:00
// each round is a heartbeat
2015-02-05 01:41:14 +03:00
for i := 0 ; i < 3 ; i ++ {
2015-03-16 10:46:16 +03:00
// we expect that raft will only send out one msgAPP per heartbeat timeout
r . appendEntry ( pb . Entry { Data : [ ] byte ( "somedata" ) } )
r . sendAppend ( 2 )
2015-02-05 01:41:14 +03:00
msg := r . readMessages ( )
if len ( msg ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msg ) , 1 )
}
2015-03-16 10:46:16 +03:00
if msg [ 0 ] . Index != 0 {
t . Errorf ( "index = %d, want %d" , msg [ 0 ] . Index , 0 )
2015-02-05 01:41:14 +03:00
}
2016-04-03 04:27:54 +03:00
if ! r . prs [ 2 ] . Paused {
2015-03-16 10:46:16 +03:00
t . Errorf ( "paused = %v, want true" , r . prs [ 2 ] . Paused )
2015-02-05 01:41:14 +03:00
}
2015-02-28 22:56:55 +03:00
for j := 0 ; j < 10 ; j ++ {
2015-03-16 10:46:16 +03:00
r . appendEntry ( pb . Entry { Data : [ ] byte ( "somedata" ) } )
r . sendAppend ( 2 )
2015-02-05 01:41:14 +03:00
if l := len ( r . readMessages ( ) ) ; l != 0 {
t . Errorf ( "len(msg) = %d, want %d" , l , 0 )
}
}
// do a heartbeat
2015-02-28 22:56:55 +03:00
for j := 0 ; j < r . heartbeatTimeout ; j ++ {
2015-02-05 01:41:14 +03:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
}
// consume the heartbeat
msg = r . readMessages ( )
if len ( msg ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msg ) , 1 )
}
if msg [ 0 ] . Type != pb . MsgHeartbeat {
2015-12-16 07:18:21 +03:00
t . Errorf ( "type = %v, want %v" , msg [ 0 ] . Type , pb . MsgHeartbeat )
2015-02-05 01:41:14 +03:00
}
}
2015-03-16 10:46:16 +03:00
}
func TestSendAppendForProgressReplicate ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
r . prs [ 2 ] . becomeReplicate ( )
2015-02-05 01:41:14 +03:00
for i := 0 ; i < 10 ; i ++ {
2015-03-16 10:46:16 +03:00
r . appendEntry ( pb . Entry { Data : [ ] byte ( "somedata" ) } )
r . sendAppend ( 2 )
2015-02-05 01:41:14 +03:00
msgs := r . readMessages ( )
if len ( msgs ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msgs ) , 1 )
}
}
}
2015-03-16 10:46:16 +03:00
func TestSendAppendForProgressSnapshot ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
r . prs [ 2 ] . becomeSnapshot ( 10 )
for i := 0 ; i < 10 ; i ++ {
r . appendEntry ( pb . Entry { Data : [ ] byte ( "somedata" ) } )
r . sendAppend ( 2 )
msgs := r . readMessages ( )
if len ( msgs ) != 0 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msgs ) , 0 )
}
}
}
func TestRecvMsgUnreachable ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
s := NewMemoryStorage ( )
s . Append ( previousEnts )
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , s )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
// set node 2 to state replicate
r . prs [ 2 ] . Match = 3
r . prs [ 2 ] . becomeReplicate ( )
r . prs [ 2 ] . optimisticUpdate ( 5 )
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgUnreachable } )
if r . prs [ 2 ] . State != ProgressStateProbe {
t . Errorf ( "state = %s, want %s" , r . prs [ 2 ] . State , ProgressStateProbe )
}
if wnext := r . prs [ 2 ] . Match + 1 ; r . prs [ 2 ] . Next != wnext {
t . Errorf ( "next = %d, want %d" , r . prs [ 2 ] . Next , wnext )
}
}
2014-07-01 23:10:43 +04:00
func TestRestore ( t * testing . T ) {
2014-08-28 05:53:18 +04:00
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 , 3 } } ,
} ,
2014-07-01 23:10:43 +04:00
}
2014-11-20 00:17:50 +03:00
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , storage )
2014-08-01 00:03:23 +04:00
if ok := sm . restore ( s ) ; ! ok {
t . Fatal ( "restore fail, want succeed" )
}
2014-07-01 23:10:43 +04:00
2014-11-20 00:17:50 +03:00
if sm . raftLog . lastIndex ( ) != s . Metadata . Index {
t . Errorf ( "log.lastIndex = %d, want %d" , sm . raftLog . lastIndex ( ) , s . Metadata . Index )
2014-08-01 02:18:44 +04:00
}
2015-06-13 08:24:40 +03:00
if mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) != s . Metadata . Term {
t . Errorf ( "log.lastTerm = %d, want %d" , mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) , s . Metadata . Term )
2014-08-01 02:18:44 +04:00
}
2014-10-08 11:33:55 +04:00
sg := sm . nodes ( )
2014-11-20 00:17:50 +03:00
if ! reflect . DeepEqual ( sg , s . Metadata . ConfState . Nodes ) {
t . Errorf ( "sm.Nodes = %+v, want %+v" , sg , s . Metadata . ConfState . Nodes )
2014-07-01 23:10:43 +04:00
}
2014-08-01 00:03:23 +04:00
if ok := sm . restore ( s ) ; ok {
t . Fatal ( "restore succeed, want fail" )
}
2014-07-01 23:10:43 +04:00
}
2014-12-03 08:34:14 +03:00
func TestRestoreIgnoreSnapshot ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
commit := uint64 ( 1 )
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , storage )
2014-12-03 08:34:14 +03:00
sm . raftLog . append ( previousEnts ... )
sm . raftLog . commitTo ( commit )
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : commit ,
Term : 1 ,
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } } ,
} ,
}
// ignore snapshot
if ok := sm . restore ( s ) ; ok {
t . Errorf ( "restore = %t, want %t" , ok , false )
}
if sm . raftLog . committed != commit {
t . Errorf ( "commit = %d, want %d" , sm . raftLog . committed , commit )
}
// ignore snapshot and fast forward commit
s . Metadata . Index = commit + 1
if ok := sm . restore ( s ) ; ok {
t . Errorf ( "restore = %t, want %t" , ok , false )
}
if sm . raftLog . committed != commit + 1 {
t . Errorf ( "commit = %d, want %d" , sm . raftLog . committed , commit + 1 )
}
}
2014-07-02 23:49:58 +04:00
func TestProvideSnap ( t * testing . T ) {
2015-12-10 23:17:30 +03:00
// restore the state machine from a snapshot so it has a compacted log and a snapshot
2014-11-20 00:17:50 +03:00
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } } ,
} ,
}
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
2014-07-02 23:49:58 +04:00
sm . restore ( s )
sm . becomeCandidate ( )
sm . becomeLeader ( )
2015-12-10 23:17:30 +03:00
// force set the next of node 2, so that node 2 needs a snapshot
2015-01-20 21:26:22 +03:00
sm . prs [ 2 ] . Next = sm . raftLog . firstIndex ( )
sm . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgAppResp , Index : sm . prs [ 2 ] . Next - 1 , Reject : true } )
2015-12-10 23:17:30 +03:00
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-07-02 23:49:58 +04:00
if len ( msgs ) != 1 {
2014-09-29 06:28:12 +04:00
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
2014-07-02 23:49:58 +04:00
}
2014-09-29 21:44:20 +04:00
m := msgs [ 0 ]
2014-10-12 11:34:22 +04:00
if m . Type != pb . MsgSnap {
t . Errorf ( "m.Type = %v, want %v" , m . Type , pb . MsgSnap )
2014-07-02 23:49:58 +04:00
}
}
2015-12-10 23:17:30 +03:00
func TestIgnoreProvidingSnap ( t * testing . T ) {
// restore the state machine from a snapshot so it has a compacted log and a snapshot
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } } ,
} ,
}
storage := NewMemoryStorage ( )
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
sm . restore ( s )
sm . becomeCandidate ( )
sm . becomeLeader ( )
// force set the next of node 2, so that node 2 needs a snapshot
// change node 2 to be inactive, expect node 1 ignore sending snapshot to 2
sm . prs [ 2 ] . Next = sm . raftLog . firstIndex ( ) - 1
sm . prs [ 2 ] . RecentActive = false
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
msgs := sm . readMessages ( )
if len ( msgs ) != 0 {
t . Errorf ( "len(msgs) = %d, want 0" , len ( msgs ) )
}
}
2014-07-02 23:49:58 +04:00
func TestRestoreFromSnapMsg ( t * testing . T ) {
2014-08-28 05:53:18 +04:00
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } } ,
} ,
2014-07-02 23:49:58 +04:00
}
2014-10-12 11:34:22 +04:00
m := pb . Message { Type : pb . MsgSnap , From : 1 , Term : 2 , Snapshot : s }
2014-07-02 23:49:58 +04:00
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 2 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-07-02 23:49:58 +04:00
sm . Step ( m )
2014-11-20 00:17:50 +03:00
// TODO(bdarnell): what should this test?
2014-07-02 23:49:58 +04:00
}
func TestSlowNodeRestore ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-07-02 23:49:58 +04:00
2014-09-12 06:23:05 +04:00
nt . isolate ( 3 )
2014-10-17 02:32:57 +04:00
for j := 0 ; j <= 100 ; j ++ {
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2014-07-02 23:49:58 +04:00
}
2014-09-12 06:23:05 +04:00
lead := nt . peers [ 1 ] . ( * raft )
2014-11-13 00:23:42 +03:00
nextEnts ( lead , nt . storage [ 1 ] )
2015-01-23 21:50:43 +03:00
nt . storage [ 1 ] . CreateSnapshot ( lead . raftLog . applied , & pb . ConfState { Nodes : lead . nodes ( ) } , nil )
nt . storage [ 1 ] . Compact ( lead . raftLog . applied )
2014-07-02 23:49:58 +04:00
nt . recover ( )
2015-12-10 23:17:30 +03:00
// send heartbeats so that the leader can learn everyone is active.
// node 3 will only be considered as active when node 1 receives a reply from it.
for {
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
if lead . prs [ 3 ] . RecentActive {
break
}
}
2014-09-29 21:44:20 +04:00
// trigger a snapshot
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2015-12-10 23:17:30 +03:00
2014-09-12 06:23:05 +04:00
follower := nt . peers [ 3 ] . ( * raft )
2014-07-02 23:49:58 +04:00
2014-09-29 21:44:20 +04:00
// trigger a commit
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2014-09-29 21:44:20 +04:00
if follower . raftLog . committed != lead . raftLog . committed {
2014-12-05 01:12:55 +03:00
t . Errorf ( "follower.committed = %d, want %d" , follower . raftLog . committed , lead . raftLog . committed )
2014-07-02 23:49:58 +04:00
}
}
2014-09-23 23:02:44 +04:00
// TestStepConfig tests that when raft step msgProp in EntryConfChange type,
2014-09-19 23:35:56 +04:00
// it appends the entry to log and sets pendingConf to be true.
func TestStepConfig ( t * testing . T ) {
// a raft that cannot make progress
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
index := r . raftLog . lastIndex ( )
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-09-19 23:35:56 +04:00
if g := r . raftLog . lastIndex ( ) ; g != index + 1 {
t . Errorf ( "index = %d, want %d" , g , index + 1 )
}
2016-04-03 04:27:54 +03:00
if ! r . pendingConf {
2014-09-19 23:35:56 +04:00
t . Errorf ( "pendingConf = %v, want true" , r . pendingConf )
}
}
// TestStepIgnoreConfig tests that if raft step the second msgProp in
2014-12-10 09:13:42 +03:00
// EntryConfChange type when the first one is uncommitted, the node will set
// the proposal to noop and keep its original state.
2014-09-19 23:35:56 +04:00
func TestStepIgnoreConfig ( t * testing . T ) {
// a raft that cannot make progress
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-09-19 23:35:56 +04:00
index := r . raftLog . lastIndex ( )
pendingConf := r . pendingConf
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-12-10 09:13:42 +03:00
wents := [ ] pb . Entry { { Type : pb . EntryNormal , Term : 1 , Index : 3 , Data : nil } }
2015-06-13 08:24:40 +03:00
ents , err := r . raftLog . entries ( index + 1 , noLimit )
if err != nil {
t . Fatalf ( "unexpected error %v" , err )
}
if ! reflect . DeepEqual ( ents , wents ) {
2014-12-10 09:13:42 +03:00
t . Errorf ( "ents = %+v, want %+v" , ents , wents )
2014-09-19 23:35:56 +04:00
}
if r . pendingConf != pendingConf {
t . Errorf ( "pendingConf = %v, want %v" , r . pendingConf , pendingConf )
}
}
// TestRecoverPendingConfig tests that new leader recovers its pendingConf flag
// based on uncommitted entries.
func TestRecoverPendingConfig ( t * testing . T ) {
tests := [ ] struct {
2014-09-21 09:48:17 +04:00
entType pb . EntryType
2014-09-19 23:35:56 +04:00
wpending bool
} {
2014-09-21 09:48:17 +04:00
{ pb . EntryNormal , false } ,
2014-09-23 23:02:44 +04:00
{ pb . EntryConfChange , true } ,
2014-09-19 23:35:56 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . appendEntry ( pb . Entry { Type : tt . entType } )
r . becomeCandidate ( )
r . becomeLeader ( )
if r . pendingConf != tt . wpending {
t . Errorf ( "#%d: pendingConf = %v, want %v" , i , r . pendingConf , tt . wpending )
}
}
}
// TestRecoverDoublePendingConfig tests that new leader will panic if
// there exist two uncommitted config entries.
func TestRecoverDoublePendingConfig ( t * testing . T ) {
func ( ) {
defer func ( ) {
if err := recover ( ) ; err == nil {
t . Errorf ( "expect panic, but nothing happens" )
}
} ( )
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-23 23:02:44 +04:00
r . appendEntry ( pb . Entry { Type : pb . EntryConfChange } )
r . appendEntry ( pb . Entry { Type : pb . EntryConfChange } )
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
} ( )
}
2014-09-30 06:54:20 +04:00
// TestAddNode tests that addNode could update pendingConf and nodes correctly.
2014-09-19 23:35:56 +04:00
func TestAddNode ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . pendingConf = true
r . addNode ( 2 )
2016-04-03 04:27:54 +03:00
if r . pendingConf {
2014-09-19 23:35:56 +04:00
t . Errorf ( "pendingConf = %v, want false" , r . pendingConf )
}
nodes := r . nodes ( )
2014-10-08 14:29:53 +04:00
wnodes := [ ] uint64 { 1 , 2 }
2014-09-19 23:35:56 +04:00
if ! reflect . DeepEqual ( nodes , wnodes ) {
t . Errorf ( "nodes = %v, want %v" , nodes , wnodes )
}
}
2014-09-30 06:54:20 +04:00
// TestRemoveNode tests that removeNode could update pendingConf, nodes and
// and removed list correctly.
2014-09-19 23:35:56 +04:00
func TestRemoveNode ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . pendingConf = true
r . removeNode ( 2 )
2016-04-03 04:27:54 +03:00
if r . pendingConf {
2014-09-19 23:35:56 +04:00
t . Errorf ( "pendingConf = %v, want false" , r . pendingConf )
}
2014-10-08 14:29:53 +04:00
w := [ ] uint64 { 1 }
2014-09-19 23:35:56 +04:00
if g := r . nodes ( ) ; ! reflect . DeepEqual ( g , w ) {
t . Errorf ( "nodes = %v, want %v" , g , w )
}
2016-05-16 19:58:57 +03:00
// remove all nodes from cluster
r . removeNode ( 1 )
w = [ ] uint64 { }
if g := r . nodes ( ) ; ! reflect . DeepEqual ( g , w ) {
t . Errorf ( "nodes = %v, want %v" , g , w )
}
2014-09-19 23:35:56 +04:00
}
2014-09-24 21:19:41 +04:00
func TestPromotable ( t * testing . T ) {
2014-10-08 14:29:53 +04:00
id := uint64 ( 1 )
2014-09-24 10:09:21 +04:00
tests := [ ] struct {
2014-10-08 14:29:53 +04:00
peers [ ] uint64
2014-09-24 21:19:41 +04:00
wp bool
2014-09-24 10:09:21 +04:00
} {
2014-10-08 14:29:53 +04:00
{ [ ] uint64 { 1 } , true } ,
{ [ ] uint64 { 1 , 2 , 3 } , true } ,
{ [ ] uint64 { } , false } ,
{ [ ] uint64 { 2 , 3 } , false } ,
2014-09-24 10:09:21 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( id , tt . peers , 5 , 1 , NewMemoryStorage ( ) )
2014-09-24 21:19:41 +04:00
if g := r . promotable ( ) ; g != tt . wp {
t . Errorf ( "#%d: promotable = %v, want %v" , i , g , tt . wp )
2014-09-24 10:09:21 +04:00
}
}
}
2014-11-12 08:11:09 +03:00
func TestRaftNodes ( t * testing . T ) {
tests := [ ] struct {
ids [ ] uint64
wids [ ] uint64
} {
{
[ ] uint64 { 1 , 2 , 3 } ,
[ ] uint64 { 1 , 2 , 3 } ,
} ,
{
[ ] uint64 { 3 , 2 , 1 } ,
[ ] uint64 { 1 , 2 , 3 } ,
} ,
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , tt . ids , 10 , 1 , NewMemoryStorage ( ) )
2014-11-12 08:11:09 +03:00
if ! reflect . DeepEqual ( r . nodes ( ) , tt . wids ) {
t . Errorf ( "#%d: nodes = %+v, want %+v" , i , r . nodes ( ) , tt . wids )
}
}
}
2015-11-17 03:32:12 +03:00
func TestCampaignWhileLeader ( t * testing . T ) {
r := newTestRaft ( 1 , [ ] uint64 { 1 } , 5 , 1 , NewMemoryStorage ( ) )
if r . state != StateFollower {
t . Errorf ( "expected new node to be follower but got %s" , r . state )
}
// We don't call campaign() directly because it comes after the check
// for our current state.
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if r . state != StateLeader {
t . Errorf ( "expected single-node election to become leader but got %s" , r . state )
}
term := r . Term
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if r . state != StateLeader {
t . Errorf ( "expected to remain leader but got %s" , r . state )
}
if r . Term != term {
t . Errorf ( "expected to remain in term %v but got %v" , term , r . Term )
}
}
2016-01-20 18:03:12 +03:00
// TestCommitAfterRemoveNode verifies that pending commands can become
// committed when a config change reduces the quorum requirements.
func TestCommitAfterRemoveNode ( t * testing . T ) {
// Create a cluster with two nodes.
s := NewMemoryStorage ( )
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , s )
r . becomeCandidate ( )
r . becomeLeader ( )
// Begin to remove the second node.
cc := pb . ConfChange {
Type : pb . ConfChangeRemoveNode ,
NodeID : 2 ,
}
ccData , err := cc . Marshal ( )
if err != nil {
t . Fatal ( err )
}
r . Step ( pb . Message {
Type : pb . MsgProp ,
Entries : [ ] pb . Entry {
{ Type : pb . EntryConfChange , Data : ccData } ,
} ,
} )
// Stabilize the log and make sure nothing is committed yet.
if ents := nextEnts ( r , s ) ; len ( ents ) > 0 {
t . Fatalf ( "unexpected committed entries: %v" , ents )
}
ccIndex := r . raftLog . lastIndex ( )
// While the config change is pending, make another proposal.
r . Step ( pb . Message {
Type : pb . MsgProp ,
Entries : [ ] pb . Entry {
{ Type : pb . EntryNormal , Data : [ ] byte ( "hello" ) } ,
} ,
} )
// Node 2 acknowledges the config change, committing it.
r . Step ( pb . Message {
Type : pb . MsgAppResp ,
From : 2 ,
Index : ccIndex ,
} )
ents := nextEnts ( r , s )
if len ( ents ) != 2 {
t . Fatalf ( "expected two committed entries, got %v" , ents )
}
if ents [ 0 ] . Type != pb . EntryNormal || ents [ 0 ] . Data != nil {
t . Fatalf ( "expected ents[0] to be empty, but got %v" , ents [ 0 ] )
}
if ents [ 1 ] . Type != pb . EntryConfChange {
t . Fatalf ( "expected ents[1] to be EntryConfChange, got %v" , ents [ 1 ] )
}
// Apply the config change. This reduces quorum requirements so the
// pending command can now commit.
r . removeNode ( 2 )
ents = nextEnts ( r , s )
if len ( ents ) != 1 || ents [ 0 ] . Type != pb . EntryNormal ||
string ( ents [ 0 ] . Data ) != "hello" {
t . Fatalf ( "expected one committed EntryNormal, got %v" , ents )
}
}
2016-04-11 09:16:56 +03:00
// TestLeaderTransferToUpToDateNode verifies transferring should succeed
2016-03-24 14:59:41 +03:00
// if the transferee has the most up-to-date log entires when transfer starts.
func TestLeaderTransferToUpToDateNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
if lead . lead != 1 {
t . Fatalf ( "after election leader is %x, want 1" , lead . lead )
}
// Transfer leadership to 2.
nt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
// After some log replication, transfer leadership back to 1.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . send ( pb . Message { From : 1 , To : 2 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferToSlowFollower ( t * testing . T ) {
defaultLogger . EnableDebug ( )
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . recover ( )
lead := nt . peers [ 1 ] . ( * raft )
if lead . prs [ 3 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x for node 3, want %x" , lead . prs [ 3 ] . Match , 1 )
}
// Transfer leadership to 3 when node 3 is lack of log.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 3 )
}
func TestLeaderTransferAfterSnapshot ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
lead := nt . peers [ 1 ] . ( * raft )
nextEnts ( lead , nt . storage [ 1 ] )
nt . storage [ 1 ] . CreateSnapshot ( lead . raftLog . applied , & pb . ConfState { Nodes : lead . nodes ( ) } , nil )
nt . storage [ 1 ] . Compact ( lead . raftLog . applied )
nt . recover ( )
if lead . prs [ 3 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x for node 3, want %x" , lead . prs [ 3 ] . Match , 1 )
}
// Transfer leadership to 3 when node 3 is lack of snapshot.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
// Send pb.MsgHeartbeatResp to leader to trigger a snapshot for node 3.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgHeartbeatResp } )
checkLeaderTransferState ( t , lead , StateFollower , 3 )
}
func TestLeaderTransferToSelf ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to self, there will be noop.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferToNonExistingNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to non-existing node, there will be noop.
nt . send ( pb . Message { From : 4 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferTimeout ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node, wait for timeout.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . electionTimeout - lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferIgnoreProposal ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node to let transfer pending, then send proposal.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
if lead . prs [ 1 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x, want %x" , lead . prs [ 1 ] . Match , 1 )
}
}
func TestLeaderTransferReceiveHigherTermVote ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node to let transfer pending.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup , Index : 1 , Term : 2 } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
}
func TestLeaderTransferRemoveNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . ignore ( pb . MsgTimeoutNow )
lead := nt . peers [ 1 ] . ( * raft )
2016-04-11 09:16:56 +03:00
// The leadTransferee is removed when leadship transferring.
2016-03-24 14:59:41 +03:00
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
lead . removeNode ( 3 )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
// TestLeaderTransferBack verifies leadership can transfer back to self when last transfer is pending.
func TestLeaderTransferBack ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
// Transfer leadership back to self.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
// TestLeaderTransferSecondTransferToAnotherNode verifies leader can transfer to another node
// when last transfer is pending.
func TestLeaderTransferSecondTransferToAnotherNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
// Transfer leadership to another node.
nt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
}
// TestLeaderTransferSecondTransferToSameNode verifies second transfer leader request
// to the same node should not extend the timeout while the first one is pending.
func TestLeaderTransferSecondTransferToSameNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
// Second transfer leadership request to the same node.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
for i := 0 ; i < lead . electionTimeout - lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func checkLeaderTransferState ( t * testing . T , r * raft , state StateType , lead uint64 ) {
if r . state != state || r . lead != lead {
2016-04-11 09:16:56 +03:00
t . Fatalf ( "after transferring, node has state %v lead %v, want state %v lead %v" , r . state , r . lead , state , lead )
2016-03-24 14:59:41 +03:00
}
if r . leadTransferee != None {
2016-04-11 09:16:56 +03:00
t . Fatalf ( "after transferring, node has leadTransferee %v, want leadTransferee %v" , r . leadTransferee , None )
2016-03-24 14:59:41 +03:00
}
}
2014-10-08 14:29:53 +04:00
func ents ( terms ... uint64 ) * raft {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
2014-11-25 04:13:47 +03:00
for i , term := range terms {
2014-12-15 22:25:35 +03:00
storage . Append ( [ ] pb . Entry { { Index : uint64 ( i + 1 ) , Term : term } } )
2014-11-03 23:04:24 +03:00
}
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { } , 5 , 1 , storage )
2014-06-16 05:52:10 +04:00
sm . reset ( 0 )
2014-06-06 02:02:12 +04:00
return sm
2014-05-18 01:09:52 +04:00
}
2014-05-16 21:11:21 +04:00
type network struct {
2016-02-01 08:42:39 +03:00
peers map [ uint64 ] stateMachine
2014-11-13 00:23:42 +03:00
storage map [ uint64 ] * MemoryStorage
2014-06-26 22:55:56 +04:00
dropm map [ connem ] float64
2014-10-12 11:34:22 +04:00
ignorem map [ pb . MessageType ] bool
2014-05-16 21:11:21 +04:00
}
2014-05-06 10:28:14 +04:00
2014-06-18 04:35:19 +04:00
// newNetwork initializes a network from peers.
// A nil node will be replaced with a new *stateMachine.
// A *stateMachine will get its k, id.
2014-10-29 01:46:56 +03:00
// When using stateMachine, the address list is always [1, n].
2016-02-01 08:42:39 +03:00
func newNetwork ( peers ... stateMachine ) * network {
2014-06-18 04:35:19 +04:00
size := len ( peers )
2014-10-29 01:46:56 +03:00
peerAddrs := idsBySize ( size )
2014-06-10 03:45:42 +04:00
2016-02-01 08:42:39 +03:00
npeers := make ( map [ uint64 ] stateMachine , size )
2014-11-13 00:23:42 +03:00
nstorage := make ( map [ uint64 ] * MemoryStorage , size )
2014-06-18 04:35:19 +04:00
2015-02-18 00:15:48 +03:00
for j , p := range peers {
id := peerAddrs [ j ]
2014-06-06 02:02:12 +04:00
switch v := p . ( type ) {
2014-05-06 10:28:14 +04:00
case nil :
2014-11-13 00:23:42 +03:00
nstorage [ id ] = NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( id , peerAddrs , 10 , 1 , nstorage [ id ] )
2014-09-10 03:33:09 +04:00
npeers [ id ] = sm
2014-08-23 00:24:33 +04:00
case * raft :
2014-09-10 03:33:09 +04:00
v . id = id
2015-01-20 21:26:22 +03:00
v . prs = make ( map [ uint64 ] * Progress )
2014-06-18 04:35:19 +04:00
for i := 0 ; i < size ; i ++ {
2015-01-20 21:26:22 +03:00
v . prs [ peerAddrs [ i ] ] = & Progress { }
2014-06-10 03:45:42 +04:00
}
2014-06-16 05:52:10 +04:00
v . reset ( 0 )
2014-09-10 03:33:09 +04:00
npeers [ id ] = v
case * blackHole :
npeers [ id ] = v
2014-06-18 04:35:19 +04:00
default :
2014-09-12 10:18:51 +04:00
panic ( fmt . Sprintf ( "unexpected state machine type: %T" , p ) )
2014-05-06 10:28:14 +04:00
}
}
2014-06-26 22:55:56 +04:00
return & network {
peers : npeers ,
2014-11-13 00:23:42 +03:00
storage : nstorage ,
2014-06-26 22:55:56 +04:00
dropm : make ( map [ connem ] float64 ) ,
2014-10-12 11:34:22 +04:00
ignorem : make ( map [ pb . MessageType ] bool ) ,
2014-06-26 22:55:56 +04:00
}
2014-05-06 10:28:14 +04:00
}
2014-08-28 05:53:18 +04:00
func ( nw * network ) send ( msgs ... pb . Message ) {
2014-06-06 02:02:12 +04:00
for len ( msgs ) > 0 {
m := msgs [ 0 ]
p := nw . peers [ m . To ]
p . Step ( m )
2014-10-30 00:04:45 +03:00
msgs = append ( msgs [ 1 : ] , nw . filter ( p . readMessages ( ) ) ... )
2014-05-16 21:11:21 +04:00
}
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) drop ( from , to uint64 , perc float64 ) {
2014-06-06 02:02:12 +04:00
nw . dropm [ connem { from , to } ] = perc
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) cut ( one , other uint64 ) {
2014-06-06 22:01:55 +04:00
nw . drop ( one , other , 1 )
nw . drop ( other , one , 1 )
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) isolate ( id uint64 ) {
2014-06-06 02:02:12 +04:00
for i := 0 ; i < len ( nw . peers ) ; i ++ {
2014-10-08 14:29:53 +04:00
nid := uint64 ( i ) + 1
2014-07-09 22:53:27 +04:00
if nid != id {
nw . drop ( id , nid , 1.0 )
nw . drop ( nid , id , 1.0 )
2014-05-06 10:28:14 +04:00
}
}
}
2014-10-12 11:34:22 +04:00
func ( nw * network ) ignore ( t pb . MessageType ) {
2014-06-26 22:55:56 +04:00
nw . ignorem [ t ] = true
}
2014-06-06 02:02:12 +04:00
func ( nw * network ) recover ( ) {
nw . dropm = make ( map [ connem ] float64 )
2014-10-12 11:34:22 +04:00
nw . ignorem = make ( map [ pb . MessageType ] bool )
2014-05-06 10:28:14 +04:00
}
2014-08-28 05:53:18 +04:00
func ( nw * network ) filter ( msgs [ ] pb . Message ) [ ] pb . Message {
2014-10-08 02:22:35 +04:00
mm := [ ] pb . Message { }
2014-06-06 02:02:12 +04:00
for _ , m := range msgs {
2014-06-26 22:55:56 +04:00
if nw . ignorem [ m . Type ] {
continue
}
2014-06-06 02:02:12 +04:00
switch m . Type {
2014-10-12 11:34:22 +04:00
case pb . MsgHup :
2014-06-06 02:02:12 +04:00
// hups never go over the network, so don't drop them but panic
panic ( "unexpected msgHup" )
2014-05-06 10:28:14 +04:00
default :
2014-06-06 02:02:12 +04:00
perc := nw . dropm [ connem { m . From , m . To } ]
if n := rand . Float64 ( ) ; n < perc {
2014-05-06 10:28:14 +04:00
continue
}
}
2014-06-06 02:02:12 +04:00
mm = append ( mm , m )
2014-05-06 10:28:14 +04:00
}
2014-06-06 02:02:12 +04:00
return mm
2014-05-06 10:28:14 +04:00
}
2014-06-06 02:02:12 +04:00
type connem struct {
2014-10-08 14:29:53 +04:00
from , to uint64
2014-05-24 00:30:04 +04:00
}
2014-06-06 02:02:12 +04:00
type blackHole struct { }
2014-05-27 23:24:49 +04:00
2014-08-28 05:53:18 +04:00
func ( blackHole ) Step ( pb . Message ) error { return nil }
2014-10-30 00:04:45 +03:00
func ( blackHole ) readMessages ( ) [ ] pb . Message { return nil }
2014-06-06 02:02:12 +04:00
var nopStepper = & blackHole { }
2014-10-29 01:46:56 +03:00
func idsBySize ( size int ) [ ] uint64 {
ids := make ( [ ] uint64 , size )
for i := 0 ; i < size ; i ++ {
ids [ i ] = 1 + uint64 ( i )
}
return ids
}
2015-03-22 04:15:58 +03:00
2015-03-24 22:36:42 +03:00
func newTestConfig ( id uint64 , peers [ ] uint64 , election , heartbeat int , storage Storage ) * Config {
return & Config {
2015-03-22 04:15:58 +03:00
ID : id ,
2015-03-24 21:10:07 +03:00
peers : peers ,
2015-03-22 04:15:58 +03:00
ElectionTick : election ,
HeartbeatTick : heartbeat ,
Storage : storage ,
MaxSizePerMsg : noLimit ,
MaxInflightMsgs : 256 ,
}
2015-03-24 22:36:42 +03:00
}
2015-03-22 04:15:58 +03:00
2015-03-24 22:36:42 +03:00
func newTestRaft ( id uint64 , peers [ ] uint64 , election , heartbeat int , storage Storage ) * raft {
return newRaft ( newTestConfig ( id , peers , election , heartbeat , storage ) )
2015-03-22 04:15:58 +03:00
}