2016-05-13 06:49:15 +03:00
// Copyright 2015 The etcd Authors
2015-01-25 06:19:16 +03:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2014-10-18 02:41:22 +04:00
2014-05-06 10:28:14 +04:00
package raft
import (
2014-05-24 20:53:01 +04:00
"bytes"
2014-09-12 10:18:51 +04:00
"fmt"
2014-10-07 16:34:15 +04:00
"math"
2014-06-06 02:02:12 +04:00
"math/rand"
2014-07-01 23:10:43 +04:00
"reflect"
2018-08-07 06:30:07 +03:00
"strings"
2014-05-06 10:28:14 +04:00
"testing"
2014-08-28 05:53:18 +04:00
2018-08-29 03:11:06 +03:00
pb "go.etcd.io/etcd/raft/raftpb"
2014-05-06 10:28:14 +04:00
)
2014-08-24 04:56:06 +04:00
// nextEnts returns the appliable entries and updates the applied index
2014-11-13 00:23:42 +03:00
func nextEnts ( r * raft , s * MemoryStorage ) ( ents [ ] pb . Entry ) {
2014-11-03 23:04:24 +03:00
// Transfer all unstable entries to "stable" storage.
2014-11-13 00:23:42 +03:00
s . Append ( r . raftLog . unstableEntries ( ) )
2014-11-29 01:13:07 +03:00
r . raftLog . stableTo ( r . raftLog . lastIndex ( ) , r . raftLog . lastTerm ( ) )
2014-11-03 23:04:24 +03:00
2014-08-24 04:56:06 +04:00
ents = r . raftLog . nextEnts ( )
2014-11-07 01:13:35 +03:00
r . raftLog . appliedTo ( r . raftLog . committed )
2014-08-24 04:56:06 +04:00
return ents
}
2016-02-01 08:42:39 +03:00
type stateMachine interface {
2014-08-28 05:53:18 +04:00
Step ( m pb . Message ) error
2014-10-30 00:04:45 +03:00
readMessages ( ) [ ] pb . Message
}
func ( r * raft ) readMessages ( ) [ ] pb . Message {
msgs := r . msgs
r . msgs = make ( [ ] pb . Message , 0 )
return msgs
2014-08-23 00:24:33 +04:00
}
2015-03-20 04:06:51 +03:00
func TestProgressBecomeProbe ( t * testing . T ) {
2015-03-16 10:46:16 +03:00
match := uint64 ( 1 )
tests := [ ] struct {
p * Progress
wnext uint64
} {
{
2015-03-20 04:06:51 +03:00
& Progress { State : ProgressStateReplicate , Match : match , Next : 5 , ins : newInflights ( 256 ) } ,
2015-03-16 10:46:16 +03:00
2 ,
} ,
{
// snapshot finish
2015-03-20 04:06:51 +03:00
& Progress { State : ProgressStateSnapshot , Match : match , Next : 5 , PendingSnapshot : 10 , ins : newInflights ( 256 ) } ,
2015-03-16 10:46:16 +03:00
11 ,
} ,
{
// snapshot failure
2015-03-20 04:06:51 +03:00
& Progress { State : ProgressStateSnapshot , Match : match , Next : 5 , PendingSnapshot : 0 , ins : newInflights ( 256 ) } ,
2015-03-16 10:46:16 +03:00
2 ,
} ,
}
for i , tt := range tests {
tt . p . becomeProbe ( )
if tt . p . State != ProgressStateProbe {
t . Errorf ( "#%d: state = %s, want %s" , i , tt . p . State , ProgressStateProbe )
}
if tt . p . Match != match {
t . Errorf ( "#%d: match = %d, want %d" , i , tt . p . Match , match )
}
if tt . p . Next != tt . wnext {
t . Errorf ( "#%d: next = %d, want %d" , i , tt . p . Next , tt . wnext )
}
}
}
2015-03-20 04:06:51 +03:00
func TestProgressBecomeReplicate ( t * testing . T ) {
p := & Progress { State : ProgressStateProbe , Match : 1 , Next : 5 , ins : newInflights ( 256 ) }
2015-03-16 10:46:16 +03:00
p . becomeReplicate ( )
if p . State != ProgressStateReplicate {
t . Errorf ( "state = %s, want %s" , p . State , ProgressStateReplicate )
}
if p . Match != 1 {
t . Errorf ( "match = %d, want 1" , p . Match )
}
if w := p . Match + 1 ; p . Next != w {
t . Errorf ( "next = %d, want %d" , p . Next , w )
}
}
2015-03-20 04:06:51 +03:00
func TestProgressBecomeSnapshot ( t * testing . T ) {
p := & Progress { State : ProgressStateProbe , Match : 1 , Next : 5 , ins : newInflights ( 256 ) }
2015-03-16 10:46:16 +03:00
p . becomeSnapshot ( 10 )
if p . State != ProgressStateSnapshot {
t . Errorf ( "state = %s, want %s" , p . State , ProgressStateSnapshot )
}
if p . Match != 1 {
t . Errorf ( "match = %d, want 1" , p . Match )
}
if p . PendingSnapshot != 10 {
t . Errorf ( "pendingSnapshot = %d, want 10" , p . PendingSnapshot )
}
}
2014-11-21 04:58:23 +03:00
func TestProgressUpdate ( t * testing . T ) {
prevM , prevN := uint64 ( 3 ) , uint64 ( 5 )
tests := [ ] struct {
update uint64
2015-03-16 10:46:16 +03:00
wm uint64
wn uint64
wok bool
2014-11-21 04:58:23 +03:00
} {
2015-03-16 10:46:16 +03:00
{ prevM - 1 , prevM , prevN , false } , // do not decrease match, next
{ prevM , prevM , prevN , false } , // do not decrease next
{ prevM + 1 , prevM + 1 , prevN , true } , // increase match, do not decrease next
{ prevM + 2 , prevM + 2 , prevN + 1 , true } , // increase match, next
2014-11-21 04:58:23 +03:00
}
for i , tt := range tests {
2015-01-20 21:26:22 +03:00
p := & Progress {
Match : prevM ,
Next : prevN ,
2014-11-21 04:58:23 +03:00
}
2015-03-16 10:46:16 +03:00
ok := p . maybeUpdate ( tt . update )
if ok != tt . wok {
t . Errorf ( "#%d: ok= %v, want %v" , i , ok , tt . wok )
}
2015-01-20 21:26:22 +03:00
if p . Match != tt . wm {
t . Errorf ( "#%d: match= %d, want %d" , i , p . Match , tt . wm )
2014-11-21 04:58:23 +03:00
}
2015-01-20 21:26:22 +03:00
if p . Next != tt . wn {
t . Errorf ( "#%d: next= %d, want %d" , i , p . Next , tt . wn )
2014-11-21 04:58:23 +03:00
}
}
}
2014-10-30 08:57:25 +03:00
func TestProgressMaybeDecr ( t * testing . T ) {
tests := [ ] struct {
2015-03-16 10:46:16 +03:00
state ProgressStateType
2014-12-31 06:42:59 +03:00
m uint64
n uint64
rejected uint64
last uint64
2014-10-30 08:57:25 +03:00
w bool
wn uint64
} {
{
2015-03-16 10:46:16 +03:00
// state replicate and rejected is not greater than match
ProgressStateReplicate , 5 , 10 , 5 , 5 , false , 10 ,
2014-10-30 08:57:25 +03:00
} ,
{
2015-03-16 10:46:16 +03:00
// state replicate and rejected is not greater than match
ProgressStateReplicate , 5 , 10 , 4 , 4 , false , 10 ,
2014-11-18 22:42:08 +03:00
} ,
{
2015-03-16 10:46:16 +03:00
// state replicate and rejected is greater than match
2014-11-18 22:42:08 +03:00
// directly decrease to match+1
2015-03-16 10:46:16 +03:00
ProgressStateReplicate , 5 , 10 , 9 , 9 , true , 6 ,
2014-10-30 08:57:25 +03:00
} ,
{
2014-12-31 06:42:59 +03:00
// next-1 != rejected is always false
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 0 , 0 , 0 , false , 0 ,
2014-10-30 08:57:25 +03:00
} ,
{
2014-12-31 06:42:59 +03:00
// next-1 != rejected is always false
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 10 , 5 , 5 , false , 10 ,
2014-10-30 08:57:25 +03:00
} ,
{
// next>1 = decremented by 1
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 10 , 9 , 9 , true , 9 ,
2014-10-30 08:57:25 +03:00
} ,
{
// next>1 = decremented by 1
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 2 , 1 , 1 , true , 1 ,
2014-10-30 08:57:25 +03:00
} ,
{
// next<=1 = reset to 1
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 1 , 0 , 0 , true , 1 ,
2014-12-31 06:42:59 +03:00
} ,
{
// decrease to min(rejected, last+1)
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 10 , 9 , 2 , true , 3 ,
2014-12-31 06:42:59 +03:00
} ,
{
// rejected < 1, reset to 1
2015-03-16 10:46:16 +03:00
ProgressStateProbe , 0 , 10 , 9 , 0 , true , 1 ,
2014-10-30 08:57:25 +03:00
} ,
}
for i , tt := range tests {
2015-01-20 21:26:22 +03:00
p := & Progress {
2015-03-16 10:46:16 +03:00
State : tt . state ,
2015-01-20 21:26:22 +03:00
Match : tt . m ,
Next : tt . n ,
2014-10-30 08:57:25 +03:00
}
2014-12-31 06:42:59 +03:00
if g := p . maybeDecrTo ( tt . rejected , tt . last ) ; g != tt . w {
2014-12-18 23:02:15 +03:00
t . Errorf ( "#%d: maybeDecrTo= %t, want %t" , i , g , tt . w )
2014-10-30 08:57:25 +03:00
}
2015-01-20 21:26:22 +03:00
if gm := p . Match ; gm != tt . m {
2014-12-18 23:02:15 +03:00
t . Errorf ( "#%d: match= %d, want %d" , i , gm , tt . m )
2014-10-30 08:57:25 +03:00
}
2015-01-20 21:26:22 +03:00
if gn := p . Next ; gn != tt . wn {
2014-12-18 23:02:15 +03:00
t . Errorf ( "#%d: next= %d, want %d" , i , gn , tt . wn )
2014-10-30 08:57:25 +03:00
}
}
}
2015-03-16 10:46:16 +03:00
func TestProgressIsPaused ( t * testing . T ) {
2014-12-18 23:02:15 +03:00
tests := [ ] struct {
2015-03-16 10:46:16 +03:00
state ProgressStateType
paused bool
2014-12-18 23:02:15 +03:00
w bool
} {
2015-03-16 10:46:16 +03:00
{ ProgressStateProbe , false , false } ,
{ ProgressStateProbe , true , true } ,
{ ProgressStateReplicate , false , false } ,
{ ProgressStateReplicate , true , false } ,
{ ProgressStateSnapshot , false , true } ,
{ ProgressStateSnapshot , true , true } ,
2014-12-18 23:02:15 +03:00
}
for i , tt := range tests {
2015-01-20 21:26:22 +03:00
p := & Progress {
2015-03-16 10:46:16 +03:00
State : tt . state ,
Paused : tt . paused ,
2015-03-20 04:06:51 +03:00
ins : newInflights ( 256 ) ,
2014-12-18 23:02:15 +03:00
}
2016-12-04 08:14:08 +03:00
if g := p . IsPaused ( ) ; g != tt . w {
2016-01-31 05:15:56 +03:00
t . Errorf ( "#%d: paused= %t, want %t" , i , g , tt . w )
2014-12-18 23:02:15 +03:00
}
}
}
2015-03-16 10:46:16 +03:00
// TestProgressResume ensures that progress.maybeUpdate and progress.maybeDecrTo
// will reset progress.paused.
func TestProgressResume ( t * testing . T ) {
2015-01-20 21:26:22 +03:00
p := & Progress {
2015-03-16 10:46:16 +03:00
Next : 2 ,
Paused : true ,
2014-12-18 23:02:15 +03:00
}
2014-12-31 06:42:59 +03:00
p . maybeDecrTo ( 1 , 1 )
2016-04-03 04:27:54 +03:00
if p . Paused {
2015-03-16 10:46:16 +03:00
t . Errorf ( "paused= %v, want false" , p . Paused )
2014-12-18 23:02:15 +03:00
}
2015-03-16 10:46:16 +03:00
p . Paused = true
p . maybeUpdate ( 2 )
2016-04-03 04:27:54 +03:00
if p . Paused {
2015-03-16 10:46:16 +03:00
t . Errorf ( "paused= %v, want false" , p . Paused )
2014-12-18 23:02:15 +03:00
}
}
2017-01-09 10:42:54 +03:00
// TestProgressResumeByHeartbeatResp ensures raft.heartbeat reset progress.paused by heartbeat response.
2016-12-19 23:24:21 +03:00
func TestProgressResumeByHeartbeatResp ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
2014-12-18 23:02:15 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
2015-03-16 10:46:16 +03:00
r . prs [ 2 ] . Paused = true
2014-12-18 23:02:15 +03:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
2016-12-19 23:24:21 +03:00
if ! r . prs [ 2 ] . Paused {
2017-01-09 10:42:54 +03:00
t . Errorf ( "paused = %v, want true" , r . prs [ 2 ] . Paused )
2016-12-19 23:24:21 +03:00
}
r . prs [ 2 ] . becomeReplicate ( )
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeatResp } )
2016-04-03 04:27:54 +03:00
if r . prs [ 2 ] . Paused {
2015-03-16 10:46:16 +03:00
t . Errorf ( "paused = %v, want false" , r . prs [ 2 ] . Paused )
2014-12-18 23:02:15 +03:00
}
}
2015-03-16 10:46:16 +03:00
func TestProgressPaused ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
2014-12-18 23:02:15 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
ms := r . readMessages ( )
if len ( ms ) != 1 {
t . Errorf ( "len(ms) = %d, want 1" , len ( ms ) )
}
}
2018-08-07 06:30:07 +03:00
func TestProgressFlowControl ( t * testing . T ) {
cfg := newTestConfig ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
cfg . MaxInflightMsgs = 3
cfg . MaxSizePerMsg = 2048
r := newRaft ( cfg )
r . becomeCandidate ( )
r . becomeLeader ( )
// Throw away all the messages relating to the initial election.
r . readMessages ( )
// While node 2 is in probe state, propose a bunch of entries.
r . prs [ 2 ] . becomeProbe ( )
blob := [ ] byte ( strings . Repeat ( "a" , 1000 ) )
for i := 0 ; i < 10 ; i ++ {
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : blob } } } )
}
ms := r . readMessages ( )
// First append has two entries: the empty entry to confirm the
// election, and the first proposal (only one proposal gets sent
// because we're in probe state).
if len ( ms ) != 1 || ms [ 0 ] . Type != pb . MsgApp {
t . Fatalf ( "expected 1 MsgApp, got %v" , ms )
}
if len ( ms [ 0 ] . Entries ) != 2 {
t . Fatalf ( "expected 2 entries, got %d" , len ( ms [ 0 ] . Entries ) )
}
if len ( ms [ 0 ] . Entries [ 0 ] . Data ) != 0 || len ( ms [ 0 ] . Entries [ 1 ] . Data ) != 1000 {
t . Fatalf ( "unexpected entry sizes: %v" , ms [ 0 ] . Entries )
}
// When this append is acked, we change to replicate state and can
// send multiple messages at once.
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgAppResp , Index : ms [ 0 ] . Entries [ 1 ] . Index } )
ms = r . readMessages ( )
if len ( ms ) != 3 {
t . Fatalf ( "expected 3 messages, got %d" , len ( ms ) )
}
for i , m := range ms {
if m . Type != pb . MsgApp {
t . Errorf ( "%d: expected MsgApp, got %s" , i , m . Type )
}
if len ( m . Entries ) != 2 {
t . Errorf ( "%d: expected 2 entries, got %d" , i , len ( m . Entries ) )
}
}
// Ack all three of those messages together and get the last two
// messages (containing three entries).
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgAppResp , Index : ms [ 2 ] . Entries [ 1 ] . Index } )
ms = r . readMessages ( )
if len ( ms ) != 2 {
t . Fatalf ( "expected 2 messages, got %d" , len ( ms ) )
}
for i , m := range ms {
if m . Type != pb . MsgApp {
t . Errorf ( "%d: expected MsgApp, got %s" , i , m . Type )
}
}
if len ( ms [ 0 ] . Entries ) != 2 {
t . Errorf ( "%d: expected 2 entries, got %d" , 0 , len ( ms [ 0 ] . Entries ) )
}
if len ( ms [ 1 ] . Entries ) != 1 {
t . Errorf ( "%d: expected 1 entry, got %d" , 1 , len ( ms [ 1 ] . Entries ) )
}
}
2014-05-06 10:28:14 +04:00
func TestLeaderElection ( t * testing . T ) {
2016-10-25 17:31:44 +03:00
testLeaderElection ( t , false )
}
func TestLeaderElectionPreVote ( t * testing . T ) {
testLeaderElection ( t , true )
}
func testLeaderElection ( t * testing . T , preVote bool ) {
var cfg func ( * Config )
2018-07-05 20:12:45 +03:00
candState := StateCandidate
2018-05-01 12:05:08 +03:00
candTerm := uint64 ( 1 )
2016-10-25 17:31:44 +03:00
if preVote {
cfg = preVoteConfig
2018-05-01 12:05:08 +03:00
// In pre-vote mode, an election that fails to complete
// leaves the node in pre-candidate state without advancing
// the term.
candState = StatePreCandidate
candTerm = 0
2016-10-25 17:31:44 +03:00
}
tests := [ ] struct {
* network
state StateType
expTerm uint64
} {
{ newNetworkWithConfig ( cfg , nil , nil , nil ) , StateLeader , 1 } ,
{ newNetworkWithConfig ( cfg , nil , nil , nopStepper ) , StateLeader , 1 } ,
2018-05-01 12:05:08 +03:00
{ newNetworkWithConfig ( cfg , nil , nopStepper , nopStepper ) , candState , candTerm } ,
{ newNetworkWithConfig ( cfg , nil , nopStepper , nopStepper , nil ) , candState , candTerm } ,
2016-10-25 17:31:44 +03:00
{ newNetworkWithConfig ( cfg , nil , nopStepper , nopStepper , nil , nil ) , StateLeader , 1 } ,
// three logs further along than 0, but in the same term so rejections
// are returned instead of the votes being ignored.
2016-12-03 12:29:15 +03:00
{ newNetworkWithConfig ( cfg ,
nil , entsWithConfig ( cfg , 1 ) , entsWithConfig ( cfg , 1 ) , entsWithConfig ( cfg , 1 , 1 ) , nil ) ,
StateFollower , 1 } ,
2016-10-25 17:31:44 +03:00
}
for i , tt := range tests {
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
sm := tt . network . peers [ 1 ] . ( * raft )
2018-05-01 12:05:08 +03:00
if sm . state != tt . state {
t . Errorf ( "#%d: state = %s, want %s" , i , sm . state , tt . state )
2016-10-25 17:31:44 +03:00
}
2018-05-01 12:05:08 +03:00
if g := sm . Term ; g != tt . expTerm {
t . Errorf ( "#%d: term = %d, want %d" , i , g , tt . expTerm )
2016-10-10 09:32:40 +03:00
}
2014-05-06 10:28:14 +04:00
}
2016-10-10 09:32:40 +03:00
}
2014-05-06 10:28:14 +04:00
2017-11-11 05:38:21 +03:00
// TestLearnerElectionTimeout verfies that the leader should not start election even
// when times out.
func TestLearnerElectionTimeout ( t * testing . T ) {
n1 := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
// n2 is learner. Learner should not start election even when times out.
setRandomizedElectionTimeout ( n2 , n2 . electionTimeout )
for i := 0 ; i < n2 . electionTimeout ; i ++ {
n2 . tick ( )
}
if n2 . state != StateFollower {
t . Errorf ( "peer 2 state: %s, want %s" , n2 . state , StateFollower )
}
}
2018-01-03 04:34:12 +03:00
// TestLearnerPromotion verifies that the learner should not election until
2017-11-11 05:38:21 +03:00
// it is promoted to a normal peer.
func TestLearnerPromotion ( t * testing . T ) {
n1 := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
nt := newNetwork ( n1 , n2 )
if n1 . state == StateLeader {
t . Error ( "peer 1 state is leader, want not" , n1 . state )
}
// n1 should become leader
setRandomizedElectionTimeout ( n1 , n1 . electionTimeout )
for i := 0 ; i < n1 . electionTimeout ; i ++ {
n1 . tick ( )
}
if n1 . state != StateLeader {
t . Errorf ( "peer 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Errorf ( "peer 2 state: %s, want %s" , n2 . state , StateFollower )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
n1 . addNode ( 2 )
n2 . addNode ( 2 )
if n2 . isLearner {
t . Error ( "peer 2 is learner, want not" )
}
// n2 start election, should become leader
setRandomizedElectionTimeout ( n2 , n2 . electionTimeout )
for i := 0 ; i < n2 . electionTimeout ; i ++ {
n2 . tick ( )
}
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgBeat } )
if n1 . state != StateFollower {
t . Errorf ( "peer 1 state: %s, want %s" , n1 . state , StateFollower )
}
if n2 . state != StateLeader {
t . Errorf ( "peer 2 state: %s, want %s" , n2 . state , StateLeader )
}
}
// TestLearnerCannotVote checks that a learner can't vote even it receives a valid Vote request.
func TestLearnerCannotVote ( t * testing . T ) {
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 . becomeFollower ( 1 , None )
n2 . Step ( pb . Message { From : 1 , To : 2 , Term : 2 , Type : pb . MsgVote , LogTerm : 11 , Index : 11 } )
if len ( n2 . msgs ) != 0 {
t . Errorf ( "expect learner not to vote, but received %v messages" , n2 . msgs )
}
}
2016-10-25 17:31:44 +03:00
func TestLeaderCycle ( t * testing . T ) {
testLeaderCycle ( t , false )
}
func TestLeaderCyclePreVote ( t * testing . T ) {
testLeaderCycle ( t , true )
}
// testLeaderCycle verifies that each node in a cluster can campaign
2016-10-10 09:32:40 +03:00
// and be elected in turn. This ensures that elections (including
// pre-vote) work when not starting from a clean slate (as they do in
// TestLeaderElection)
2016-10-25 17:31:44 +03:00
func testLeaderCycle ( t * testing . T , preVote bool ) {
var cfg func ( * Config )
if preVote {
cfg = preVoteConfig
}
n := newNetworkWithConfig ( cfg , nil , nil , nil )
for campaignerID := uint64 ( 1 ) ; campaignerID <= 3 ; campaignerID ++ {
n . send ( pb . Message { From : campaignerID , To : campaignerID , Type : pb . MsgHup } )
for _ , peer := range n . peers {
sm := peer . ( * raft )
if sm . id == campaignerID && sm . state != StateLeader {
t . Errorf ( "preVote=%v: campaigning node %d state = %v, want StateLeader" ,
preVote , sm . id , sm . state )
} else if sm . id != campaignerID && sm . state != StateFollower {
t . Errorf ( "preVote=%v: after campaign of node %d, " +
"node %d had state = %v, want StateFollower" ,
preVote , campaignerID , sm . id , sm . state )
2016-10-10 09:32:40 +03:00
}
2014-05-06 10:28:14 +04:00
}
}
}
2016-12-03 12:29:15 +03:00
// TestLeaderElectionOverwriteNewerLogs tests a scenario in which a
// newly-elected leader does *not* have the newest (i.e. highest term)
// log entries, and must overwrite higher-term log entries with
// lower-term ones.
func TestLeaderElectionOverwriteNewerLogs ( t * testing . T ) {
testLeaderElectionOverwriteNewerLogs ( t , false )
}
func TestLeaderElectionOverwriteNewerLogsPreVote ( t * testing . T ) {
testLeaderElectionOverwriteNewerLogs ( t , true )
}
func testLeaderElectionOverwriteNewerLogs ( t * testing . T , preVote bool ) {
var cfg func ( * Config )
if preVote {
cfg = preVoteConfig
}
// This network represents the results of the following sequence of
// events:
// - Node 1 won the election in term 1.
// - Node 1 replicated a log entry to node 2 but died before sending
// it to other nodes.
// - Node 3 won the second election in term 2.
// - Node 3 wrote an entry to its logs but died without sending it
// to any other nodes.
//
// At this point, nodes 1, 2, and 3 all have uncommitted entries in
// their logs and could win an election at term 3. The winner's log
// entry overwrites the losers'. (TestLeaderSyncFollowerLog tests
// the case where older log entries are overwritten, so this test
// focuses on the case where the newer entries are lost).
n := newNetworkWithConfig ( cfg ,
entsWithConfig ( cfg , 1 ) , // Node 1: Won first election
entsWithConfig ( cfg , 1 ) , // Node 2: Got logs from node 1
entsWithConfig ( cfg , 2 ) , // Node 3: Won second election
votedWithConfig ( cfg , 3 , 2 ) , // Node 4: Voted but didn't get logs
votedWithConfig ( cfg , 3 , 2 ) ) // Node 5: Voted but didn't get logs
// Node 1 campaigns. The election fails because a quorum of nodes
// know about the election that already happened at term 2. Node 1's
// term is pushed ahead to 2.
n . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
sm1 := n . peers [ 1 ] . ( * raft )
if sm1 . state != StateFollower {
t . Errorf ( "state = %s, want StateFollower" , sm1 . state )
}
if sm1 . Term != 2 {
t . Errorf ( "term = %d, want 2" , sm1 . Term )
}
// Node 1 campaigns again with a higher term. This time it succeeds.
n . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if sm1 . state != StateLeader {
t . Errorf ( "state = %s, want StateLeader" , sm1 . state )
}
if sm1 . Term != 3 {
t . Errorf ( "term = %d, want 3" , sm1 . Term )
}
// Now all nodes agree on a log entry with term 1 at index 1 (and
// term 3 at index 2).
for i := range n . peers {
sm := n . peers [ i ] . ( * raft )
entries := sm . raftLog . allEntries ( )
if len ( entries ) != 2 {
t . Fatalf ( "node %d: len(entries) == %d, want 2" , i , len ( entries ) )
}
if entries [ 0 ] . Term != 1 {
t . Errorf ( "node %d: term at index 1 == %d, want 1" , i , entries [ 0 ] . Term )
}
if entries [ 1 ] . Term != 3 {
t . Errorf ( "node %d: term at index 2 == %d, want 3" , i , entries [ 1 ] . Term )
}
}
}
2016-10-19 06:07:43 +03:00
func TestVoteFromAnyState ( t * testing . T ) {
2016-10-25 17:31:44 +03:00
testVoteFromAnyState ( t , pb . MsgVote )
}
2016-10-19 06:07:43 +03:00
2016-10-25 17:31:44 +03:00
func TestPreVoteFromAnyState ( t * testing . T ) {
testVoteFromAnyState ( t , pb . MsgPreVote )
}
func testVoteFromAnyState ( t * testing . T , vt pb . MessageType ) {
for st := StateType ( 0 ) ; st < numStates ; st ++ {
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
r . Term = 1
switch st {
case StateFollower :
r . becomeFollower ( r . Term , 3 )
case StatePreCandidate :
r . becomePreCandidate ( )
case StateCandidate :
r . becomeCandidate ( )
case StateLeader :
r . becomeCandidate ( )
r . becomeLeader ( )
}
// Note that setting our state above may have advanced r.Term
// past its initial value.
origTerm := r . Term
newTerm := r . Term + 1
msg := pb . Message {
From : 2 ,
To : 1 ,
Type : vt ,
Term : newTerm ,
LogTerm : newTerm ,
Index : 42 ,
}
if err := r . Step ( msg ) ; err != nil {
t . Errorf ( "%s,%s: Step failed: %s" , vt , st , err )
}
if len ( r . msgs ) != 1 {
t . Errorf ( "%s,%s: %d response messages, want 1: %+v" , vt , st , len ( r . msgs ) , r . msgs )
} else {
resp := r . msgs [ 0 ]
if resp . Type != voteRespMsgType ( vt ) {
t . Errorf ( "%s,%s: response message is %s, want %s" ,
vt , st , resp . Type , voteRespMsgType ( vt ) )
}
if resp . Reject {
t . Errorf ( "%s,%s: unexpected rejection" , vt , st )
2016-10-19 06:07:43 +03:00
}
2016-10-25 17:31:44 +03:00
}
2016-10-19 06:07:43 +03:00
2016-10-25 17:31:44 +03:00
// If this was a real vote, we reset our state and term.
if vt == pb . MsgVote {
if r . state != StateFollower {
2016-12-26 09:31:59 +03:00
t . Errorf ( "%s,%s: state %s, want %s" , vt , st , r . state , StateFollower )
2016-10-19 06:07:43 +03:00
}
2016-10-25 17:31:44 +03:00
if r . Term != newTerm {
t . Errorf ( "%s,%s: term %d, want %d" , vt , st , r . Term , newTerm )
2016-10-19 06:07:43 +03:00
}
2016-10-25 17:31:44 +03:00
if r . Vote != 2 {
t . Errorf ( "%s,%s: vote %d, want 2" , vt , st , r . Vote )
2016-10-19 06:07:43 +03:00
}
2016-10-25 17:31:44 +03:00
} else {
// In a prevote, nothing changes.
if r . state != st {
t . Errorf ( "%s,%s: state %s, want %s" , vt , st , r . state , st )
}
if r . Term != origTerm {
t . Errorf ( "%s,%s: term %d, want %d" , vt , st , r . Term , origTerm )
}
// if st == StateFollower or StatePreCandidate, r hasn't voted yet.
// In StateCandidate or StateLeader, it's voted for itself.
if r . Vote != None && r . Vote != 1 {
t . Errorf ( "%s,%s: vote %d, want %d or 1" , vt , st , r . Vote , None )
2016-10-19 06:07:43 +03:00
}
}
}
}
2014-05-24 20:53:01 +04:00
func TestLogReplication ( t * testing . T ) {
tests := [ ] struct {
* network
2014-08-28 05:53:18 +04:00
msgs [ ] pb . Message
2014-10-08 14:29:53 +04:00
wcommitted uint64
2014-05-24 20:53:01 +04:00
} {
{
newNetwork ( nil , nil , nil ) ,
2014-08-28 05:53:18 +04:00
[ ] pb . Message {
2014-10-12 11:34:22 +04:00
{ From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
2014-05-24 20:53:01 +04:00
} ,
2014-06-26 22:55:56 +04:00
2 ,
2014-05-24 20:53:01 +04:00
} ,
{
newNetwork ( nil , nil , nil ) ,
2014-08-28 05:53:18 +04:00
[ ] pb . Message {
2014-10-12 11:34:22 +04:00
{ From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
{ From : 1 , To : 2 , Type : pb . MsgHup } ,
{ From : 1 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
2014-05-24 20:53:01 +04:00
} ,
2014-06-26 22:55:56 +04:00
4 ,
2014-05-24 20:53:01 +04:00
} ,
}
for i , tt := range tests {
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-24 20:53:01 +04:00
for _ , m := range tt . msgs {
2014-06-06 02:02:12 +04:00
tt . send ( m )
2014-05-24 20:53:01 +04:00
}
2014-06-06 02:02:12 +04:00
for j , x := range tt . network . peers {
2014-08-23 00:24:33 +04:00
sm := x . ( * raft )
2014-05-24 20:53:01 +04:00
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != tt . wcommitted {
t . Errorf ( "#%d.%d: committed = %d, want %d" , i , j , sm . raftLog . committed , tt . wcommitted )
2014-05-24 20:53:01 +04:00
}
2014-10-08 02:22:35 +04:00
ents := [ ] pb . Entry { }
2014-11-13 00:23:42 +03:00
for _ , e := range nextEnts ( sm , tt . network . storage [ j ] ) {
2014-06-26 22:55:56 +04:00
if e . Data != nil {
ents = append ( ents , e )
}
}
2014-10-08 02:22:35 +04:00
props := [ ] pb . Message { }
2014-05-24 20:53:01 +04:00
for _ , m := range tt . msgs {
2014-10-12 11:34:22 +04:00
if m . Type == pb . MsgProp {
2014-05-24 20:53:01 +04:00
props = append ( props , m )
}
}
for k , m := range props {
2014-06-06 00:00:18 +04:00
if ! bytes . Equal ( ents [ k ] . Data , m . Entries [ 0 ] . Data ) {
t . Errorf ( "#%d.%d: data = %d, want %d" , i , j , ents [ k ] . Data , m . Entries [ 0 ] . Data )
2014-05-24 20:53:01 +04:00
}
}
}
}
}
2017-11-11 05:38:21 +03:00
// TestLearnerLogReplication tests that a learner can receive entries from the leader.
func TestLearnerLogReplication ( t * testing . T ) {
n1 := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
nt := newNetwork ( n1 , n2 )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
setRandomizedElectionTimeout ( n1 , n1 . electionTimeout )
for i := 0 ; i < n1 . electionTimeout ; i ++ {
n1 . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
// n1 is leader and n2 is learner
if n1 . state != StateLeader {
t . Errorf ( "peer 1 state: %s, want %s" , n1 . state , StateLeader )
}
if ! n2 . isLearner {
t . Error ( "peer 2 state: not learner, want yes" )
}
nextCommitted := n1 . raftLog . committed + 1
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
if n1 . raftLog . committed != nextCommitted {
t . Errorf ( "peer 1 wants committed to %d, but still %d" , nextCommitted , n1 . raftLog . committed )
}
if n1 . raftLog . committed != n2 . raftLog . committed {
t . Errorf ( "peer 2 wants committed to %d, but still %d" , n1 . raftLog . committed , n2 . raftLog . committed )
}
match := n1 . getProgress ( 2 ) . Match
if match != n2 . raftLog . committed {
2018-02-26 21:03:25 +03:00
t . Errorf ( "progress 2 of leader 1 wants match %d, but got %d" , n2 . raftLog . committed , match )
2017-11-11 05:38:21 +03:00
}
}
2014-05-29 02:30:01 +04:00
func TestSingleNodeCommit ( t * testing . T ) {
tt := newNetwork ( nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-05-29 02:30:01 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 3 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 3 )
2014-05-29 02:30:01 +04:00
}
}
2014-06-26 22:55:56 +04:00
// TestCannotCommitWithoutNewTermEntry tests the entries cannot be committed
// when leader changes, no new proposal comes in and ChangeTerm proposal is
// filtered.
2014-06-06 21:26:44 +04:00
func TestCannotCommitWithoutNewTermEntry ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil , nil , nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-06-06 21:26:44 +04:00
// 0 cannot reach 2,3,4
2014-09-12 06:23:05 +04:00
tt . cut ( 1 , 3 )
tt . cut ( 1 , 4 )
tt . cut ( 1 , 5 )
2014-06-06 21:26:44 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-06-06 21:26:44 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-06 21:26:44 +04:00
}
// network recovery
tt . recover ( )
2014-06-26 22:55:56 +04:00
// avoid committing ChangeTerm proposal
2014-10-12 11:34:22 +04:00
tt . ignore ( pb . MsgApp )
2014-06-06 21:26:44 +04:00
2014-12-18 23:02:15 +03:00
// elect 2 as the new leader with term 2
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
2014-06-06 21:26:44 +04:00
// no log entries from previous term should be committed
2014-09-12 06:23:05 +04:00
sm = tt . peers [ 2 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-06 21:26:44 +04:00
}
2014-06-26 22:55:56 +04:00
tt . recover ( )
2014-12-18 23:02:15 +03:00
// send heartbeat; reset wait
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgBeat } )
// append an entry at current term
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-12-18 23:02:15 +03:00
// expect the committed to be advanced
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 5 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 5 )
2014-06-26 22:55:56 +04:00
}
}
// TestCommitWithoutNewTermEntry tests the entries could be committed
// when leader changes, no new proposal comes in.
func TestCommitWithoutNewTermEntry ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil , nil , nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-06-26 22:55:56 +04:00
// 0 cannot reach 2,3,4
2014-09-12 06:23:05 +04:00
tt . cut ( 1 , 3 )
tt . cut ( 1 , 4 )
tt . cut ( 1 , 5 )
2014-06-26 22:55:56 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-06-26 22:55:56 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-26 22:55:56 +04:00
}
// network recovery
tt . recover ( )
// elect 1 as the new leader with term 2
// after append a ChangeTerm entry from the current term, all entries
// should be committed
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
2014-06-26 22:55:56 +04:00
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 4 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 4 )
2014-06-06 21:26:44 +04:00
}
}
2014-06-06 02:02:12 +04:00
func TestDuelingCandidates ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-05-19 00:59:10 +04:00
2014-06-26 22:55:56 +04:00
nt := newNetwork ( a , b , c )
2014-09-12 06:23:05 +04:00
nt . cut ( 1 , 3 )
2014-05-19 00:59:10 +04:00
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-16 21:11:21 +04:00
2016-05-19 05:38:10 +03:00
// 1 becomes leader since it receives votes from 1 and 2
2016-05-18 09:20:02 +03:00
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "state = %s, want %s" , sm . state , StateLeader )
}
2016-05-19 05:38:10 +03:00
// 3 stays as candidate since it receives a vote from 3 and a rejection from 2
2016-05-18 09:20:02 +03:00
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , sm . state , StateCandidate )
}
2014-06-26 22:55:56 +04:00
nt . recover ( )
2016-05-19 05:38:10 +03:00
// candidate 3 now increases its term and tries to vote again
// we expect it to disrupt the leader 1 since it has a higher term
// 3 will be follower again since both 1 and 2 rejects its vote request since 3 does not have a long enough log
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-16 21:53:56 +04:00
2014-11-03 23:04:24 +03:00
wlog := & raftLog {
2015-08-22 04:52:16 +03:00
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } } } ,
2014-11-03 23:04:24 +03:00
committed : 1 ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 2 } ,
2014-11-03 23:04:24 +03:00
}
2014-05-16 21:53:56 +04:00
tests := [ ] struct {
2014-08-23 00:24:33 +04:00
sm * raft
2014-09-16 04:35:02 +04:00
state StateType
2014-10-08 14:29:53 +04:00
term uint64
2014-07-24 03:15:25 +04:00
raftLog * raftLog
2014-05-16 21:53:56 +04:00
} {
2014-09-16 04:35:02 +04:00
{ a , StateFollower , 2 , wlog } ,
{ b , StateFollower , 2 , wlog } ,
2015-08-10 18:04:38 +03:00
{ c , StateFollower , 2 , newLog ( NewMemoryStorage ( ) , raftLogger ) } ,
2014-05-16 21:53:56 +04:00
}
for i , tt := range tests {
if g := tt . sm . state ; g != tt . state {
t . Errorf ( "#%d: state = %s, want %s" , i , g , tt . state )
}
2014-08-23 01:19:30 +04:00
if g := tt . sm . Term ; g != tt . term {
2014-05-16 21:53:56 +04:00
t . Errorf ( "#%d: term = %d, want %d" , i , g , tt . term )
}
2014-07-24 03:15:25 +04:00
base := ltoa ( tt . raftLog )
2014-10-08 14:29:53 +04:00
if sm , ok := nt . peers [ 1 + uint64 ( i ) ] . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-19 04:51:45 +04:00
}
}
}
2016-10-10 09:32:40 +03:00
func TestDuelingPreCandidates ( t * testing . T ) {
cfgA := newTestConfig ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
cfgB := newTestConfig ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
cfgC := newTestConfig ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
cfgA . PreVote = true
cfgB . PreVote = true
cfgC . PreVote = true
a := newRaft ( cfgA )
b := newRaft ( cfgB )
c := newRaft ( cfgC )
nt := newNetwork ( a , b , c )
nt . cut ( 1 , 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// 1 becomes leader since it receives votes from 1 and 2
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "state = %s, want %s" , sm . state , StateLeader )
}
// 3 campaigns then reverts to follower when its PreVote is rejected
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateFollower {
t . Errorf ( "state = %s, want %s" , sm . state , StateFollower )
}
nt . recover ( )
// Candidate 3 now increases its term and tries to vote again.
// With PreVote, it does not disrupt the leader.
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
wlog := & raftLog {
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } } } ,
committed : 1 ,
unstable : unstable { offset : 2 } ,
}
tests := [ ] struct {
sm * raft
state StateType
term uint64
raftLog * raftLog
} {
{ a , StateLeader , 1 , wlog } ,
{ b , StateFollower , 1 , wlog } ,
{ c , StateFollower , 1 , newLog ( NewMemoryStorage ( ) , raftLogger ) } ,
}
for i , tt := range tests {
if g := tt . sm . state ; g != tt . state {
t . Errorf ( "#%d: state = %s, want %s" , i , g , tt . state )
}
if g := tt . sm . Term ; g != tt . term {
t . Errorf ( "#%d: term = %d, want %d" , i , g , tt . term )
}
base := ltoa ( tt . raftLog )
if sm , ok := nt . peers [ 1 + uint64 ( i ) ] . ( * raft ) ; ok {
l := ltoa ( sm . raftLog )
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
}
}
}
2014-05-19 04:51:45 +04:00
func TestCandidateConcede ( t * testing . T ) {
2014-06-06 02:02:12 +04:00
tt := newNetwork ( nil , nil , nil )
2014-09-12 06:23:05 +04:00
tt . isolate ( 1 )
2014-05-19 04:51:45 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-19 04:51:45 +04:00
// heal the partition
2014-06-06 02:02:12 +04:00
tt . recover ( )
2014-12-18 23:02:15 +03:00
// send heartbeat; reset wait
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgBeat } )
2014-05-19 04:51:45 +04:00
data := [ ] byte ( "force follower" )
2014-12-19 04:10:37 +03:00
// send a proposal to 3 to flush out a MsgApp to 1
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : data } } } )
2014-12-19 04:10:37 +03:00
// send heartbeat; flush out commit
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgBeat } )
2014-05-19 04:51:45 +04:00
2014-09-12 06:23:05 +04:00
a := tt . peers [ 1 ] . ( * raft )
2014-09-16 04:35:02 +04:00
if g := a . state ; g != StateFollower {
t . Errorf ( "state = %s, want %s" , g , StateFollower )
2014-05-19 04:51:45 +04:00
}
2014-08-23 01:19:30 +04:00
if g := a . Term ; g != 1 {
2014-05-19 04:51:45 +04:00
t . Errorf ( "term = %d, want %d" , g , 1 )
}
2014-11-03 23:04:24 +03:00
wantLog := ltoa ( & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Index : 2 , Data : data } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 ,
} )
2014-06-06 02:02:12 +04:00
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( wantLog , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-16 21:53:56 +04:00
}
2014-05-16 21:11:21 +04:00
}
2014-05-16 21:53:56 +04:00
}
2014-05-28 22:59:01 +04:00
func TestSingleNodeCandidate ( t * testing . T ) {
tt := newNetwork ( nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-28 22:59:01 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-09-16 04:35:02 +04:00
if sm . state != StateLeader {
t . Errorf ( "state = %d, want %d" , sm . state , StateLeader )
2014-05-28 22:59:01 +04:00
}
}
2016-10-10 09:32:40 +03:00
func TestSingleNodePreCandidate ( t * testing . T ) {
tt := newNetworkWithConfig ( preVoteConfig , nil )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
sm := tt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "state = %d, want %d" , sm . state , StateLeader )
}
}
2014-05-16 21:53:56 +04:00
func TestOldMessages ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil )
// make 0 leader @ term 3
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-12-18 23:02:15 +03:00
// pretend we're an old leader trying to make progress; this entry is expected to be ignored.
tt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgApp , Term : 2 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } } } )
// commit a new entry
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
2014-06-06 02:02:12 +04:00
2015-02-18 00:15:48 +03:00
ilog := & raftLog {
2014-11-03 23:04:24 +03:00
storage : & MemoryStorage {
ents : [ ] pb . Entry {
{ } , { Data : nil , Term : 1 , Index : 1 } ,
{ Data : nil , Term : 2 , Index : 2 } , { Data : nil , Term : 3 , Index : 3 } ,
2014-12-18 23:02:15 +03:00
{ Data : [ ] byte ( "somedata" ) , Term : 3 , Index : 4 } ,
2014-11-03 23:04:24 +03:00
} ,
2014-06-26 22:55:56 +04:00
} ,
2014-12-18 23:02:15 +03:00
unstable : unstable { offset : 5 } ,
committed : 4 ,
2014-06-26 22:55:56 +04:00
}
2015-02-18 00:15:48 +03:00
base := ltoa ( ilog )
2014-06-06 02:02:12 +04:00
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-16 21:11:21 +04:00
}
}
}
2014-05-18 04:18:58 +04:00
// TestOldMessagesReply - optimization - reply with new term.
2014-05-06 10:28:14 +04:00
func TestProposal ( t * testing . T ) {
tests := [ ] struct {
2014-05-16 21:11:21 +04:00
* network
2014-05-18 01:09:52 +04:00
success bool
2014-05-06 10:28:14 +04:00
} {
2014-05-18 01:09:52 +04:00
{ newNetwork ( nil , nil , nil ) , true } ,
{ newNetwork ( nil , nil , nopStepper ) , true } ,
{ newNetwork ( nil , nopStepper , nopStepper ) , false } ,
{ newNetwork ( nil , nopStepper , nopStepper , nil ) , false } ,
{ newNetwork ( nil , nopStepper , nopStepper , nil , nil ) , true } ,
2014-05-06 10:28:14 +04:00
}
2015-02-18 00:15:48 +03:00
for j , tt := range tests {
2014-08-28 05:53:18 +04:00
send := func ( m pb . Message ) {
2014-05-06 10:28:14 +04:00
defer func ( ) {
2014-05-18 01:09:52 +04:00
// only recover is we expect it to panic so
// panics we don't expect go up.
if ! tt . success {
2014-05-06 10:28:14 +04:00
e := recover ( )
if e != nil {
2015-02-18 00:15:48 +03:00
t . Logf ( "#%d: err: %s" , j , e )
2014-05-06 10:28:14 +04:00
}
}
} ( )
2014-06-06 02:02:12 +04:00
tt . send ( m )
}
2014-05-06 10:28:14 +04:00
2014-05-18 01:09:52 +04:00
data := [ ] byte ( "somedata" )
2014-05-06 10:28:14 +04:00
// promote 0 the leader
2014-10-12 11:34:22 +04:00
send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : data } } } )
2014-05-06 10:28:14 +04:00
2015-08-10 18:04:38 +03:00
wantLog := newLog ( NewMemoryStorage ( ) , raftLogger )
2014-05-18 01:09:52 +04:00
if tt . success {
2014-11-03 23:04:24 +03:00
wantLog = & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Index : 2 , Data : data } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 }
2014-05-18 01:09:52 +04:00
}
2014-06-06 02:02:12 +04:00
base := ltoa ( wantLog )
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-06 10:28:14 +04:00
}
}
2014-09-12 06:23:05 +04:00
sm := tt . network . peers [ 1 ] . ( * raft )
2014-08-23 01:19:30 +04:00
if g := sm . Term ; g != 1 {
2015-02-18 00:15:48 +03:00
t . Errorf ( "#%d: term = %d, want %d" , j , g , 1 )
2014-05-06 10:28:14 +04:00
}
}
}
func TestProposalByProxy ( t * testing . T ) {
2014-05-16 11:55:17 +04:00
data := [ ] byte ( "somedata" )
2014-05-18 01:09:52 +04:00
tests := [ ] * network {
newNetwork ( nil , nil , nil ) ,
newNetwork ( nil , nil , nopStepper ) ,
2014-05-06 10:28:14 +04:00
}
2015-02-18 00:15:48 +03:00
for j , tt := range tests {
2014-05-06 10:28:14 +04:00
// promote 0 the leader
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-06 10:28:14 +04:00
// propose via follower
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
2014-06-06 02:02:12 +04:00
2014-11-03 23:04:24 +03:00
wantLog := & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Data : data , Index : 2 } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 }
2014-06-06 02:02:12 +04:00
base := ltoa ( wantLog )
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-06 10:28:14 +04:00
}
}
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-08-23 01:19:30 +04:00
if g := sm . Term ; g != 1 {
2015-02-18 00:15:48 +03:00
t . Errorf ( "#%d: term = %d, want %d" , j , g , 1 )
2014-05-06 10:28:14 +04:00
}
}
}
2014-05-24 20:53:01 +04:00
func TestCommit ( t * testing . T ) {
2014-05-22 03:02:15 +04:00
tests := [ ] struct {
2014-10-08 14:29:53 +04:00
matches [ ] uint64
2014-08-28 05:53:18 +04:00
logs [ ] pb . Entry
2014-10-08 14:29:53 +04:00
smTerm uint64
w uint64
2014-05-22 03:02:15 +04:00
} {
2014-06-06 22:21:26 +04:00
// single
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } } , 1 , 1 } ,
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 2 } } , 2 , 1 } ,
2014-06-06 22:21:26 +04:00
2014-05-22 03:02:15 +04:00
// odd
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 2 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 2 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
2014-05-22 03:02:15 +04:00
// even
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 2 , 1 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 2 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 2 , 1 , 2 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
2014-05-22 03:02:15 +04:00
}
for i , tt := range tests {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
storage . Append ( tt . logs )
storage . hardState = pb . HardState { Term : tt . smTerm }
2018-05-10 01:18:57 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 2 , storage )
2014-06-10 03:45:42 +04:00
for j := 0 ; j < len ( tt . matches ) ; j ++ {
2017-11-11 05:38:21 +03:00
sm . setProgress ( uint64 ( j ) + 1 , tt . matches [ j ] , tt . matches [ j ] + 1 , false )
2014-11-03 23:04:24 +03:00
}
2014-05-24 20:53:01 +04:00
sm . maybeCommit ( )
2014-07-24 03:15:25 +04:00
if g := sm . raftLog . committed ; g != tt . w {
2014-05-28 22:08:32 +04:00
t . Errorf ( "#%d: committed = %d, want %d" , i , g , tt . w )
2014-05-22 03:02:15 +04:00
}
}
}
2016-04-06 15:41:46 +03:00
func TestPastElectionTimeout ( t * testing . T ) {
2014-10-07 16:34:15 +04:00
tests := [ ] struct {
elapse int
2014-10-08 03:41:17 +04:00
wprobability float64
2014-10-07 16:34:15 +04:00
round bool
} {
{ 5 , 0 , false } ,
2016-04-01 20:09:09 +03:00
{ 10 , 0.1 , true } ,
{ 13 , 0.4 , true } ,
{ 15 , 0.6 , true } ,
{ 18 , 0.9 , true } ,
2014-10-07 16:34:15 +04:00
{ 20 , 1 , false } ,
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2015-11-24 08:59:25 +03:00
sm . electionElapsed = tt . elapse
2014-10-07 16:34:15 +04:00
c := 0
for j := 0 ; j < 10000 ; j ++ {
2016-04-01 20:09:09 +03:00
sm . resetRandomizedElectionTimeout ( )
if sm . pastElectionTimeout ( ) {
2014-10-07 16:34:15 +04:00
c ++
}
}
got := float64 ( c ) / 10000.0
if tt . round {
got = math . Floor ( got * 10 + 0.5 ) / 10.0
}
2014-10-08 03:41:17 +04:00
if got != tt . wprobability {
2016-04-06 15:41:46 +03:00
t . Errorf ( "#%d: probability = %v, want %v" , i , got , tt . wprobability )
2014-10-07 16:34:15 +04:00
}
}
}
2014-09-04 21:58:22 +04:00
// ensure that the Step function ignores the message from old term and does not pass it to the
2015-09-11 01:06:56 +03:00
// actual stepX function.
2014-09-04 21:58:22 +04:00
func TestStepIgnoreOldTermMsg ( t * testing . T ) {
called := false
2018-01-11 07:43:55 +03:00
fakeStep := func ( r * raft , m pb . Message ) error {
2014-09-04 21:58:22 +04:00
called = true
2018-01-11 07:43:55 +03:00
return nil
2014-09-04 21:58:22 +04:00
}
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-04 21:58:22 +04:00
sm . step = fakeStep
sm . Term = 2
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { Type : pb . MsgApp , Term : sm . Term - 1 } )
2016-04-03 04:27:54 +03:00
if called {
2014-09-04 21:58:22 +04:00
t . Errorf ( "stepFunc called = %v , want %v" , called , false )
}
}
2014-07-14 21:58:41 +04:00
// TestHandleMsgApp ensures:
// 1. Reply false if log doesn’ t contain an entry at prevLogIndex whose term matches prevLogTerm.
// 2. If an existing entry conflicts with a new one (same index but different terms),
// delete the existing entry and all that follow it; append any new entries not already in the log.
// 3. If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry).
func TestHandleMsgApp ( t * testing . T ) {
tests := [ ] struct {
2014-08-28 05:53:18 +04:00
m pb . Message
2014-10-08 14:29:53 +04:00
wIndex uint64
wCommit uint64
2014-10-01 23:59:30 +04:00
wReject bool
2014-07-14 21:58:41 +04:00
} {
// Ensure 1
2014-10-12 11:34:22 +04:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 3 , Index : 2 , Commit : 3 } , 2 , 0 , true } , // previous log mismatch
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 3 , Index : 3 , Commit : 3 } , 2 , 0 , true } , // previous log non-exist
2014-07-14 21:58:41 +04:00
// Ensure 2
2014-10-12 11:34:22 +04:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 1 , Index : 1 , Commit : 1 } , 2 , 1 , false } ,
2014-12-02 19:50:15 +03:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 0 , Index : 0 , Commit : 1 , Entries : [ ] pb . Entry { { Index : 1 , Term : 2 } } } , 1 , 1 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 3 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } , { Index : 4 , Term : 2 } } } , 4 , 3 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 4 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } } } , 3 , 3 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 1 , Index : 1 , Commit : 4 , Entries : [ ] pb . Entry { { Index : 2 , Term : 2 } } } , 2 , 2 , false } ,
2014-07-14 21:58:41 +04:00
// Ensure 3
2014-12-05 01:12:55 +03:00
{ pb . Message { Type : pb . MsgApp , Term : 1 , LogTerm : 1 , Index : 1 , Commit : 3 } , 2 , 1 , false } , // match entry 1, commit up to last new entry 1
{ pb . Message { Type : pb . MsgApp , Term : 1 , LogTerm : 1 , Index : 1 , Commit : 3 , Entries : [ ] pb . Entry { { Index : 2 , Term : 2 } } } , 2 , 2 , false } , // match entry 1, commit up to last new entry 2
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 3 } , 2 , 2 , false } , // match entry 2, commit up to last new entry 2
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 4 } , 2 , 2 , false } , // commit up to log.last()
2014-07-14 21:58:41 +04:00
}
for i , tt := range tests {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
2014-12-15 22:25:35 +03:00
sm . becomeFollower ( 2 , None )
2014-07-14 21:58:41 +04:00
sm . handleAppendEntries ( tt . m )
2014-07-24 03:15:25 +04:00
if sm . raftLog . lastIndex ( ) != tt . wIndex {
t . Errorf ( "#%d: lastIndex = %d, want %d" , i , sm . raftLog . lastIndex ( ) , tt . wIndex )
2014-07-14 21:58:41 +04:00
}
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != tt . wCommit {
t . Errorf ( "#%d: committed = %d, want %d" , i , sm . raftLog . committed , tt . wCommit )
2014-07-14 21:58:41 +04:00
}
2014-10-30 00:04:45 +03:00
m := sm . readMessages ( )
2014-07-14 21:58:41 +04:00
if len ( m ) != 1 {
2014-09-29 06:38:37 +04:00
t . Fatalf ( "#%d: msg = nil, want 1" , i )
2014-07-14 21:58:41 +04:00
}
2014-10-01 23:59:30 +04:00
if m [ 0 ] . Reject != tt . wReject {
t . Errorf ( "#%d: reject = %v, want %v" , i , m [ 0 ] . Reject , tt . wReject )
2014-07-14 21:58:41 +04:00
}
}
}
2014-11-18 03:58:49 +03:00
// TestHandleHeartbeat ensures that the follower commits to the commit in the message.
func TestHandleHeartbeat ( t * testing . T ) {
commit := uint64 ( 2 )
tests := [ ] struct {
m pb . Message
wCommit uint64
} {
2016-05-20 09:04:08 +03:00
{ pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeat , Term : 2 , Commit : commit + 1 } , commit + 1 } ,
{ pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeat , Term : 2 , Commit : commit - 1 } , commit } , // do not decrease commit
2014-11-18 03:58:49 +03:00
}
for i , tt := range tests {
2014-11-20 01:05:16 +03:00
storage := NewMemoryStorage ( )
2014-11-25 04:13:47 +03:00
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } , { Index : 3 , Term : 3 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , storage )
2014-12-15 22:25:35 +03:00
sm . becomeFollower ( 2 , 2 )
2014-11-18 03:58:49 +03:00
sm . raftLog . commitTo ( commit )
sm . handleHeartbeat ( tt . m )
if sm . raftLog . committed != tt . wCommit {
t . Errorf ( "#%d: committed = %d, want %d" , i , sm . raftLog . committed , tt . wCommit )
}
m := sm . readMessages ( )
2015-01-14 22:42:30 +03:00
if len ( m ) != 1 {
t . Fatalf ( "#%d: msg = nil, want 1" , i )
}
if m [ 0 ] . Type != pb . MsgHeartbeatResp {
t . Errorf ( "#%d: type = %v, want MsgHeartbeatResp" , i , m [ 0 ] . Type )
2014-11-18 03:58:49 +03:00
}
}
}
2015-01-14 22:42:30 +03:00
// TestHandleHeartbeatResp ensures that we re-send log entries when we get a heartbeat response.
func TestHandleHeartbeatResp ( t * testing . T ) {
storage := NewMemoryStorage ( )
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } , { Index : 3 , Term : 3 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , storage )
2015-01-14 22:42:30 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
sm . raftLog . commitTo ( sm . raftLog . lastIndex ( ) )
// A heartbeat response from a node that is behind; re-send MsgApp
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs := sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
}
if msgs [ 0 ] . Type != pb . MsgApp {
t . Errorf ( "type = %v, want MsgApp" , msgs [ 0 ] . Type )
}
2016-12-19 23:24:21 +03:00
// A second heartbeat response generates another MsgApp re-send
2015-01-14 22:42:30 +03:00
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs = sm . readMessages ( )
2016-12-19 23:24:21 +03:00
if len ( msgs ) != 1 {
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
2015-01-14 22:42:30 +03:00
}
2016-12-19 23:24:21 +03:00
if msgs [ 0 ] . Type != pb . MsgApp {
t . Errorf ( "type = %v, want MsgApp" , msgs [ 0 ] . Type )
2015-01-14 22:42:30 +03:00
}
// Once we have an MsgAppResp, heartbeats no longer send MsgApp.
sm . Step ( pb . Message {
From : 2 ,
Type : pb . MsgAppResp ,
2016-12-19 23:24:21 +03:00
Index : msgs [ 0 ] . Index + uint64 ( len ( msgs [ 0 ] . Entries ) ) ,
2015-01-14 22:42:30 +03:00
} )
2015-01-28 01:40:14 +03:00
// Consume the message sent in response to MsgAppResp
sm . readMessages ( )
2015-01-14 22:42:30 +03:00
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs = sm . readMessages ( )
2016-12-19 23:24:21 +03:00
if len ( msgs ) != 0 {
t . Fatalf ( "len(msgs) = %d, want 0: %+v" , len ( msgs ) , msgs )
2015-01-14 22:42:30 +03:00
}
}
2017-03-22 23:59:02 +03:00
// TestRaftFreesReadOnlyMem ensures raft will free read request from
// readOnly readIndexQueue and pendingReadIndex map.
2018-08-29 03:11:06 +03:00
// related issue: https://go.etcd.io/etcd/issues/7571
2017-03-22 23:59:02 +03:00
func TestRaftFreesReadOnlyMem ( t * testing . T ) {
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
sm . becomeCandidate ( )
sm . becomeLeader ( )
sm . raftLog . commitTo ( sm . raftLog . lastIndex ( ) )
ctx := [ ] byte ( "ctx" )
// leader starts linearizable read request.
// more info: raft dissertation 6.4, step 2.
sm . Step ( pb . Message { From : 2 , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : ctx } } } )
msgs := sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
}
if msgs [ 0 ] . Type != pb . MsgHeartbeat {
t . Fatalf ( "type = %v, want MsgHeartbeat" , msgs [ 0 ] . Type )
}
if ! bytes . Equal ( msgs [ 0 ] . Context , ctx ) {
t . Fatalf ( "Context = %v, want %v" , msgs [ 0 ] . Context , ctx )
}
if len ( sm . readOnly . readIndexQueue ) != 1 {
t . Fatalf ( "len(readIndexQueue) = %v, want 1" , len ( sm . readOnly . readIndexQueue ) )
}
if len ( sm . readOnly . pendingReadIndex ) != 1 {
t . Fatalf ( "len(pendingReadIndex) = %v, want 1" , len ( sm . readOnly . pendingReadIndex ) )
}
if _ , ok := sm . readOnly . pendingReadIndex [ string ( ctx ) ] ; ! ok {
t . Fatalf ( "can't find context %v in pendingReadIndex " , ctx )
}
// heartbeat responses from majority of followers (1 in this case)
// acknowledge the authority of the leader.
// more info: raft dissertation 6.4, step 3.
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp , Context : ctx } )
if len ( sm . readOnly . readIndexQueue ) != 0 {
t . Fatalf ( "len(readIndexQueue) = %v, want 0" , len ( sm . readOnly . readIndexQueue ) )
}
if len ( sm . readOnly . pendingReadIndex ) != 0 {
t . Fatalf ( "len(pendingReadIndex) = %v, want 0" , len ( sm . readOnly . pendingReadIndex ) )
}
if _ , ok := sm . readOnly . pendingReadIndex [ string ( ctx ) ] ; ok {
t . Fatalf ( "found context %v in pendingReadIndex, want none" , ctx )
}
}
2016-02-01 08:42:39 +03:00
// TestMsgAppRespWaitReset verifies the resume behavior of a leader
2015-01-28 01:40:14 +03:00
// MsgAppResp.
func TestMsgAppRespWaitReset ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
2015-01-28 01:40:14 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
// The new leader has just emitted a new Term 4 entry; consume those messages
// from the outgoing queue.
sm . bcastAppend ( )
sm . readMessages ( )
// Node 2 acks the first entry, making it committed.
sm . Step ( pb . Message {
From : 2 ,
Type : pb . MsgAppResp ,
Index : 1 ,
} )
2016-01-26 23:18:55 +03:00
if sm . raftLog . committed != 1 {
t . Fatalf ( "expected committed to be 1, got %d" , sm . raftLog . committed )
2015-01-28 01:40:14 +03:00
}
// Also consume the MsgApp messages that update Commit on the followers.
sm . readMessages ( )
// A new command is now proposed on node 1.
sm . Step ( pb . Message {
From : 1 ,
Type : pb . MsgProp ,
Entries : [ ] pb . Entry { { } } ,
} )
// The command is broadcast to all nodes not in the wait state.
// Node 2 left the wait state due to its MsgAppResp, but node 3 is still waiting.
msgs := sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "expected 1 message, got %d: %+v" , len ( msgs ) , msgs )
}
if msgs [ 0 ] . Type != pb . MsgApp || msgs [ 0 ] . To != 2 {
2015-12-16 07:18:21 +03:00
t . Errorf ( "expected MsgApp to node 2, got %v to %d" , msgs [ 0 ] . Type , msgs [ 0 ] . To )
2015-01-28 01:40:14 +03:00
}
if len ( msgs [ 0 ] . Entries ) != 1 || msgs [ 0 ] . Entries [ 0 ] . Index != 2 {
t . Errorf ( "expected to send entry 2, but got %v" , msgs [ 0 ] . Entries )
}
// Now Node 3 acks the first entry. This releases the wait and entry 2 is sent.
sm . Step ( pb . Message {
From : 3 ,
Type : pb . MsgAppResp ,
Index : 1 ,
} )
msgs = sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "expected 1 message, got %d: %+v" , len ( msgs ) , msgs )
}
if msgs [ 0 ] . Type != pb . MsgApp || msgs [ 0 ] . To != 3 {
2015-12-16 07:18:21 +03:00
t . Errorf ( "expected MsgApp to node 3, got %v to %d" , msgs [ 0 ] . Type , msgs [ 0 ] . To )
2015-01-28 01:40:14 +03:00
}
if len ( msgs [ 0 ] . Entries ) != 1 || msgs [ 0 ] . Entries [ 0 ] . Index != 2 {
t . Errorf ( "expected to send entry 2, but got %v" , msgs [ 0 ] . Entries )
}
}
2014-06-11 19:58:26 +04:00
func TestRecvMsgVote ( t * testing . T ) {
2016-10-25 17:31:44 +03:00
testRecvMsgVote ( t , pb . MsgVote )
2017-09-01 17:45:47 +03:00
}
func TestRecvMsgPreVote ( t * testing . T ) {
testRecvMsgVote ( t , pb . MsgPreVote )
2016-10-25 17:31:44 +03:00
}
2014-05-25 08:08:06 +04:00
2016-10-25 17:31:44 +03:00
func testRecvMsgVote ( t * testing . T , msgType pb . MessageType ) {
tests := [ ] struct {
2017-07-20 23:46:05 +03:00
state StateType
index , logTerm uint64
voteFor uint64
wreject bool
2016-10-25 17:31:44 +03:00
} {
{ StateFollower , 0 , 0 , None , true } ,
{ StateFollower , 0 , 1 , None , true } ,
{ StateFollower , 0 , 2 , None , true } ,
{ StateFollower , 0 , 3 , None , false } ,
{ StateFollower , 1 , 0 , None , true } ,
{ StateFollower , 1 , 1 , None , true } ,
{ StateFollower , 1 , 2 , None , true } ,
{ StateFollower , 1 , 3 , None , false } ,
{ StateFollower , 2 , 0 , None , true } ,
{ StateFollower , 2 , 1 , None , true } ,
{ StateFollower , 2 , 2 , None , false } ,
{ StateFollower , 2 , 3 , None , false } ,
{ StateFollower , 3 , 0 , None , true } ,
{ StateFollower , 3 , 1 , None , true } ,
{ StateFollower , 3 , 2 , None , false } ,
{ StateFollower , 3 , 3 , None , false } ,
{ StateFollower , 3 , 2 , 2 , false } ,
{ StateFollower , 3 , 2 , 1 , true } ,
{ StateLeader , 3 , 3 , 1 , true } ,
{ StatePreCandidate , 3 , 3 , 1 , true } ,
{ StateCandidate , 3 , 3 , 1 , true } ,
}
2017-07-20 23:46:05 +03:00
max := func ( a , b uint64 ) uint64 {
if a > b {
return a
}
return b
}
2016-10-25 17:31:44 +03:00
for i , tt := range tests {
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
sm . state = tt . state
switch tt . state {
case StateFollower :
sm . step = stepFollower
case StateCandidate , StatePreCandidate :
sm . step = stepCandidate
case StateLeader :
sm . step = stepLeader
}
sm . Vote = tt . voteFor
sm . raftLog = & raftLog {
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 2 } , { Index : 2 , Term : 2 } } } ,
unstable : unstable { offset : 3 } ,
}
2017-07-20 23:46:05 +03:00
// raft.Term is greater than or equal to raft.raftLog.lastTerm. In this
// test we're only testing MsgVote responses when the campaigning node
// has a different raft log compared to the recipient node.
// Additionally we're verifying behaviour when the recipient node has
// already given out its vote for its current term. We're not testing
// what the recipient node does when receiving a message with a
// different term number, so we simply initialize both term numbers to
// be the same.
term := max ( sm . raftLog . lastTerm ( ) , tt . logTerm )
sm . Term = term
sm . Step ( pb . Message { Type : msgType , Term : term , From : 2 , Index : tt . index , LogTerm : tt . logTerm } )
2016-10-25 17:31:44 +03:00
msgs := sm . readMessages ( )
if g := len ( msgs ) ; g != 1 {
t . Fatalf ( "#%d: len(msgs) = %d, want 1" , i , g )
continue
}
if g := msgs [ 0 ] . Type ; g != voteRespMsgType ( msgType ) {
t . Errorf ( "#%d, m.Type = %v, want %v" , i , g , voteRespMsgType ( msgType ) )
}
if g := msgs [ 0 ] . Reject ; g != tt . wreject {
t . Errorf ( "#%d, m.Reject = %v, want %v" , i , g , tt . wreject )
2014-05-18 11:17:46 +04:00
}
}
}
2014-06-07 03:27:29 +04:00
func TestStateTransition ( t * testing . T ) {
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
from StateType
to StateType
2014-06-07 03:27:29 +04:00
wallow bool
2014-10-08 14:29:53 +04:00
wterm uint64
wlead uint64
2014-06-07 03:27:29 +04:00
} {
2014-09-16 04:35:02 +04:00
{ StateFollower , StateFollower , true , 1 , None } ,
2016-10-10 09:32:40 +03:00
{ StateFollower , StatePreCandidate , true , 0 , None } ,
2014-09-16 04:35:02 +04:00
{ StateFollower , StateCandidate , true , 1 , None } ,
2014-10-08 14:29:53 +04:00
{ StateFollower , StateLeader , false , 0 , None } ,
2014-06-07 03:27:29 +04:00
2016-10-10 09:32:40 +03:00
{ StatePreCandidate , StateFollower , true , 0 , None } ,
{ StatePreCandidate , StatePreCandidate , true , 0 , None } ,
{ StatePreCandidate , StateCandidate , true , 1 , None } ,
{ StatePreCandidate , StateLeader , true , 0 , 1 } ,
2014-09-16 04:35:02 +04:00
{ StateCandidate , StateFollower , true , 0 , None } ,
2016-10-10 09:32:40 +03:00
{ StateCandidate , StatePreCandidate , true , 0 , None } ,
2014-09-16 04:35:02 +04:00
{ StateCandidate , StateCandidate , true , 1 , None } ,
{ StateCandidate , StateLeader , true , 0 , 1 } ,
2014-06-07 03:27:29 +04:00
2014-09-16 04:35:02 +04:00
{ StateLeader , StateFollower , true , 1 , None } ,
2016-10-10 09:32:40 +03:00
{ StateLeader , StatePreCandidate , false , 0 , None } ,
2014-09-16 04:35:02 +04:00
{ StateLeader , StateCandidate , false , 1 , None } ,
{ StateLeader , StateLeader , true , 0 , 1 } ,
2014-06-07 03:27:29 +04:00
}
for i , tt := range tests {
func ( ) {
defer func ( ) {
if r := recover ( ) ; r != nil {
2016-04-03 04:27:54 +03:00
if tt . wallow {
2014-06-07 03:27:29 +04:00
t . Errorf ( "%d: allow = %v, want %v" , i , false , true )
}
}
} ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-06-07 03:27:29 +04:00
sm . state = tt . from
switch tt . to {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-06-07 03:27:29 +04:00
sm . becomeFollower ( tt . wterm , tt . wlead )
2016-10-10 09:32:40 +03:00
case StatePreCandidate :
sm . becomePreCandidate ( )
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-06-07 03:27:29 +04:00
sm . becomeCandidate ( )
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-06-07 03:27:29 +04:00
sm . becomeLeader ( )
}
2014-08-23 01:19:30 +04:00
if sm . Term != tt . wterm {
t . Errorf ( "%d: term = %d, want %d" , i , sm . Term , tt . wterm )
2014-06-07 03:27:29 +04:00
}
2014-08-24 04:40:25 +04:00
if sm . lead != tt . wlead {
2014-06-07 03:27:29 +04:00
t . Errorf ( "%d: lead = %d, want %d" , i , sm . lead , tt . wlead )
}
} ( )
}
}
2014-05-24 00:30:04 +04:00
func TestAllServerStepdown ( t * testing . T ) {
2014-06-26 22:55:56 +04:00
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
state StateType
2014-06-26 22:55:56 +04:00
2014-09-16 04:35:02 +04:00
wstate StateType
2014-10-08 14:29:53 +04:00
wterm uint64
windex uint64
2014-06-26 22:55:56 +04:00
} {
2014-11-18 00:37:46 +03:00
{ StateFollower , StateFollower , 3 , 0 } ,
2016-10-10 09:32:40 +03:00
{ StatePreCandidate , StateFollower , 3 , 0 } ,
2014-11-18 00:37:46 +03:00
{ StateCandidate , StateFollower , 3 , 0 } ,
{ StateLeader , StateFollower , 3 , 1 } ,
2014-06-26 22:55:56 +04:00
}
2014-05-24 00:30:04 +04:00
2014-10-12 11:34:22 +04:00
tmsgTypes := [ ... ] pb . MessageType { pb . MsgVote , pb . MsgApp }
2014-10-08 14:29:53 +04:00
tterm := uint64 ( 3 )
2014-05-24 00:30:04 +04:00
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-06-26 22:55:56 +04:00
switch tt . state {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-09-15 09:44:59 +04:00
sm . becomeFollower ( 1 , None )
2016-12-26 09:31:59 +03:00
case StatePreCandidate :
sm . becomePreCandidate ( )
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-05-24 00:30:04 +04:00
sm . becomeCandidate ( )
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-05-24 00:30:04 +04:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
}
for j , msgType := range tmsgTypes {
2014-09-12 06:23:05 +04:00
sm . Step ( pb . Message { From : 2 , Type : msgType , Term : tterm , LogTerm : tterm } )
2014-05-24 00:30:04 +04:00
2014-06-26 22:55:56 +04:00
if sm . state != tt . wstate {
t . Errorf ( "#%d.%d state = %v , want %v" , i , j , sm . state , tt . wstate )
2014-05-24 00:30:04 +04:00
}
2014-08-23 01:19:30 +04:00
if sm . Term != tt . wterm {
t . Errorf ( "#%d.%d term = %v , want %v" , i , j , sm . Term , tt . wterm )
2014-05-24 00:30:04 +04:00
}
2018-07-05 20:12:45 +03:00
if sm . raftLog . lastIndex ( ) != tt . windex {
2014-11-18 00:37:46 +03:00
t . Errorf ( "#%d.%d index = %v , want %v" , i , j , sm . raftLog . lastIndex ( ) , tt . windex )
}
2014-11-03 23:04:24 +03:00
if uint64 ( len ( sm . raftLog . allEntries ( ) ) ) != tt . windex {
2014-11-18 00:37:46 +03:00
t . Errorf ( "#%d.%d len(ents) = %v , want %v" , i , j , len ( sm . raftLog . allEntries ( ) ) , tt . windex )
2014-05-24 00:30:04 +04:00
}
2014-10-08 14:29:53 +04:00
wlead := uint64 ( 2 )
2014-10-12 11:34:22 +04:00
if msgType == pb . MsgVote {
2014-09-15 09:44:59 +04:00
wlead = None
2014-07-15 09:39:44 +04:00
}
2014-08-24 04:40:25 +04:00
if sm . lead != wlead {
2014-09-15 09:44:59 +04:00
t . Errorf ( "#%d, sm.lead = %d, want %d" , i , sm . lead , None )
2014-07-15 09:39:44 +04:00
}
2014-05-24 00:30:04 +04:00
}
}
}
2018-02-21 22:53:40 +03:00
func TestCandidateResetTermMsgHeartbeat ( t * testing . T ) {
testCandidateResetTerm ( t , pb . MsgHeartbeat )
}
func TestCandidateResetTermMsgApp ( t * testing . T ) {
testCandidateResetTerm ( t , pb . MsgApp )
}
// testCandidateResetTerm tests when a candidate receives a
// MsgHeartbeat or MsgApp from leader, "Step" resets the term
// with leader's and reverts back to follower.
func testCandidateResetTerm ( t * testing . T , mt pb . MessageType ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
nt := newNetwork ( a , b , c )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
// isolate 3 and increase term in rest
nt . isolate ( 3 )
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
// trigger campaign in isolated c
c . resetRandomizedElectionTimeout ( )
for i := 0 ; i < c . randomizedElectionTimeout ; i ++ {
c . tick ( )
}
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
nt . recover ( )
// leader sends to isolated candidate
// and expects candidate to revert to follower
nt . send ( pb . Message { From : 1 , To : 3 , Term : a . Term , Type : mt } )
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
// follower c term is reset with leader's
if a . Term != c . Term {
t . Errorf ( "follower term expected same term as leader's %d, got %d" , a . Term , c . Term )
}
}
2015-11-24 08:59:25 +03:00
func TestLeaderStepdownWhenQuorumActive ( t * testing . T ) {
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
sm . checkQuorum = true
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < sm . electionTimeout + 1 ; i ++ {
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp , Term : sm . Term } )
sm . tick ( )
}
if sm . state != StateLeader {
t . Errorf ( "state = %v, want %v" , sm . state , StateLeader )
}
}
func TestLeaderStepdownWhenQuorumLost ( t * testing . T ) {
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
sm . checkQuorum = true
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < sm . electionTimeout + 1 ; i ++ {
sm . tick ( )
}
if sm . state != StateFollower {
t . Errorf ( "state = %v, want %v" , sm . state , StateFollower )
}
}
2016-05-27 12:23:18 +03:00
func TestLeaderSupersedingWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
2016-05-27 12:23:18 +03:00
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// Peer b rejected c's vote since its electionElapsed had not reached to electionTimeout
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
// Letting b's electionElapsed reach to electionTimeout
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if c . state != StateLeader {
t . Errorf ( "state = %s, want %s" , c . state , StateLeader )
}
}
func TestLeaderElectionWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( a , a . electionTimeout + 1 )
setRandomizedElectionTimeout ( b , b . electionTimeout + 2 )
2016-05-27 12:23:18 +03:00
2016-08-29 00:35:19 +03:00
// Immediately after creation, votes are cast regardless of the
// election timeout.
2016-05-27 12:23:18 +03:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
2016-08-04 13:31:22 +03:00
// need to reset randomizedElectionTimeout larger than electionTimeout again,
// because the value might be reset to electionTimeout since the last state changes
setRandomizedElectionTimeout ( a , a . electionTimeout + 1 )
setRandomizedElectionTimeout ( b , b . electionTimeout + 2 )
2016-05-27 12:23:18 +03:00
for i := 0 ; i < a . electionTimeout ; i ++ {
a . tick ( )
}
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if a . state != StateFollower {
t . Errorf ( "state = %s, want %s" , a . state , StateFollower )
}
if c . state != StateLeader {
t . Errorf ( "state = %s, want %s" , c . state , StateLeader )
}
}
// TestFreeStuckCandidateWithCheckQuorum ensures that a candidate with a higher term
// can disrupt the leader even if the leader still "officially" holds the lease, The
// leader is expected to step down and adopt the candidate's term
func TestFreeStuckCandidateWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
2016-05-27 12:23:18 +03:00
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 1 )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
if c . Term != b . Term + 1 {
t . Errorf ( "term = %d, want %d" , c . Term , b . Term + 1 )
}
// Vote again for safety
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
if c . Term != b . Term + 2 {
t . Errorf ( "term = %d, want %d" , c . Term , b . Term + 2 )
}
nt . recover ( )
nt . send ( pb . Message { From : 1 , To : 3 , Type : pb . MsgHeartbeat , Term : a . Term } )
// Disrupt the leader so that the stuck peer is freed
if a . state != StateFollower {
t . Errorf ( "state = %s, want %s" , a . state , StateFollower )
}
if c . Term != a . Term {
t . Errorf ( "term = %d, want %d" , c . Term , a . Term )
}
2017-09-15 02:38:55 +03:00
// Vote again, should become leader this time
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if c . state != StateLeader {
t . Errorf ( "peer 3 state: %s, want %s" , c . state , StateLeader )
}
2016-05-27 12:23:18 +03:00
}
func TestNonPromotableVoterWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
nt := newNetwork ( a , b )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
2016-05-27 12:23:18 +03:00
// Need to remove 2 again to make it a non-promotable node since newNetwork overwritten some internal states
b . delProgress ( 2 )
if b . promotable ( ) {
t . Fatalf ( "promotable = %v, want false" , b . promotable ( ) )
}
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if b . lead != 1 {
t . Errorf ( "lead = %d, want 1" , b . lead )
}
}
2018-02-23 07:39:05 +03:00
// TestDisruptiveFollower tests isolated follower,
// with slow network incoming from leader, election times out
// to become a candidate with an increased term. Then, the
// candiate's response to late leader heartbeat forces the leader
// to step down.
func TestDisruptiveFollower ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . checkQuorum = true
n2 . checkQuorum = true
n3 . checkQuorum = true
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StateFollower
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateFollower {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateFollower )
}
// etcd server "advanceTicksForElection" on restart;
// this is to expedite campaign trigger when given larger
// election timeouts (e.g. multi-datacenter deploy)
// Or leader messages are being delayed while ticks elapse
setRandomizedElectionTimeout ( n3 , n3 . electionTimeout + 2 )
for i := 0 ; i < n3 . randomizedElectionTimeout - 1 ; i ++ {
n3 . tick ( )
}
// ideally, before last election tick elapses,
// the follower n3 receives "pb.MsgApp" or "pb.MsgHeartbeat"
// from leader n1, and then resets its "electionElapsed"
// however, last tick may elapse before receiving any
// messages from leader, thus triggering campaign
n3 . tick ( )
// n1 is still leader yet
// while its heartbeat to candidate n3 is being delayed
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StateCandidate
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateCandidate {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateCandidate )
}
// check term
// n1.Term == 2
// n2.Term == 2
// n3.Term == 3
if n1 . Term != 2 {
t . Fatalf ( "node 1 term: %d, want %d" , n1 . Term , 2 )
}
if n2 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n2 . Term , 2 )
}
if n3 . Term != 3 {
t . Fatalf ( "node 3 term: %d, want %d" , n3 . Term , 3 )
}
// while outgoing vote requests are still queued in n3,
// leader heartbeat finally arrives at candidate n3
// however, due to delayed network from leader, leader
// heartbeat was sent with lower term than candidate's
nt . send ( pb . Message { From : 1 , To : 3 , Term : n1 . Term , Type : pb . MsgHeartbeat } )
// then candidate n3 responds with "pb.MsgAppResp" of higher term
// and leader steps down from a message with higher term
// this is to disrupt the current leader, so that candidate
// with higher term can be freed with following election
// check state
// n1.state == StateFollower
// n2.state == StateFollower
// n3.state == StateCandidate
if n1 . state != StateFollower {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateFollower )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateCandidate {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateCandidate )
}
// check term
// n1.Term == 3
// n2.Term == 2
// n3.Term == 3
if n1 . Term != 3 {
t . Fatalf ( "node 1 term: %d, want %d" , n1 . Term , 3 )
}
if n2 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n2 . Term , 2 )
}
if n3 . Term != 3 {
t . Fatalf ( "node 3 term: %d, want %d" , n3 . Term , 3 )
}
}
// TestDisruptiveFollowerPreVote tests isolated follower,
// with slow network incoming from leader, election times out
// to become a pre-candidate with less log than current leader.
// Then pre-vote phase prevents this isolated node from forcing
// current leader to step down, thus less disruptions.
func TestDisruptiveFollowerPreVote ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . checkQuorum = true
n2 . checkQuorum = true
n3 . checkQuorum = true
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StateFollower
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateFollower {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateFollower )
}
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
n1 . preVote = true
n2 . preVote = true
n3 . preVote = true
nt . recover ( )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StatePreCandidate
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StatePreCandidate {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StatePreCandidate )
}
// check term
// n1.Term == 2
// n2.Term == 2
// n3.Term == 2
if n1 . Term != 2 {
t . Fatalf ( "node 1 term: %d, want %d" , n1 . Term , 2 )
}
if n2 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n2 . Term , 2 )
}
if n3 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n3 . Term , 2 )
}
// delayed leader heartbeat does not force current leader to step down
nt . send ( pb . Message { From : 1 , To : 3 , Term : n1 . Term , Type : pb . MsgHeartbeat } )
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
}
2016-08-27 02:03:06 +03:00
func TestReadOnlyOptionSafe ( t * testing . T ) {
2016-06-03 18:20:10 +03:00
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2016-08-27 02:03:06 +03:00
nt := newNetwork ( a , b , c )
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Fatalf ( "state = %s, want %s" , a . state , StateLeader )
}
tests := [ ] struct {
sm * raft
proposals int
wri uint64
wctx [ ] byte
} {
2016-09-28 05:19:30 +03:00
{ a , 10 , 11 , [ ] byte ( "ctx1" ) } ,
{ b , 10 , 21 , [ ] byte ( "ctx2" ) } ,
{ c , 10 , 31 , [ ] byte ( "ctx3" ) } ,
{ a , 10 , 41 , [ ] byte ( "ctx4" ) } ,
{ b , 10 , 51 , [ ] byte ( "ctx5" ) } ,
{ c , 10 , 61 , [ ] byte ( "ctx6" ) } ,
2016-08-27 02:03:06 +03:00
}
for i , tt := range tests {
for j := 0 ; j < tt . proposals ; j ++ {
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
}
nt . send ( pb . Message { From : tt . sm . id , To : tt . sm . id , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : tt . wctx } } } )
r := tt . sm
if len ( r . readStates ) == 0 {
t . Errorf ( "#%d: len(readStates) = 0, want non-zero" , i )
}
rs := r . readStates [ 0 ]
if rs . Index != tt . wri {
t . Errorf ( "#%d: readIndex = %d, want %d" , i , rs . Index , tt . wri )
}
if ! bytes . Equal ( rs . RequestCtx , tt . wctx ) {
t . Errorf ( "#%d: requestCtx = %v, want %v" , i , rs . RequestCtx , tt . wctx )
}
r . readStates = nil
}
}
func TestReadOnlyOptionLease ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . readOnly . option = ReadOnlyLeaseBased
b . readOnly . option = ReadOnlyLeaseBased
c . readOnly . option = ReadOnlyLeaseBased
2016-06-03 18:20:10 +03:00
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
2016-06-03 18:20:10 +03:00
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Fatalf ( "state = %s, want %s" , a . state , StateLeader )
}
tests := [ ] struct {
sm * raft
proposals int
wri uint64
wctx [ ] byte
} {
2016-09-28 05:19:30 +03:00
{ a , 10 , 11 , [ ] byte ( "ctx1" ) } ,
{ b , 10 , 21 , [ ] byte ( "ctx2" ) } ,
{ c , 10 , 31 , [ ] byte ( "ctx3" ) } ,
{ a , 10 , 41 , [ ] byte ( "ctx4" ) } ,
{ b , 10 , 51 , [ ] byte ( "ctx5" ) } ,
{ c , 10 , 61 , [ ] byte ( "ctx6" ) } ,
2016-06-03 18:20:10 +03:00
}
2016-08-27 02:03:06 +03:00
for i , tt := range tests {
2016-06-03 18:20:10 +03:00
for j := 0 ; j < tt . proposals ; j ++ {
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
}
nt . send ( pb . Message { From : tt . sm . id , To : tt . sm . id , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : tt . wctx } } } )
r := tt . sm
2016-08-27 02:03:06 +03:00
rs := r . readStates [ 0 ]
if rs . Index != tt . wri {
t . Errorf ( "#%d: readIndex = %d, want %d" , i , rs . Index , tt . wri )
2016-06-03 18:20:10 +03:00
}
2016-08-27 02:03:06 +03:00
if ! bytes . Equal ( rs . RequestCtx , tt . wctx ) {
t . Errorf ( "#%d: requestCtx = %v, want %v" , i , rs . RequestCtx , tt . wctx )
2016-06-03 18:20:10 +03:00
}
2016-08-27 02:03:06 +03:00
r . readStates = nil
2016-06-03 18:20:10 +03:00
}
}
2017-02-15 11:45:54 +03:00
// TestReadOnlyForNewLeader ensures that a leader only accepts MsgReadIndex message
// when it commits at least one log entry at it term.
func TestReadOnlyForNewLeader ( t * testing . T ) {
2017-02-21 10:23:42 +03:00
nodeConfigs := [ ] struct {
2018-07-05 20:25:47 +03:00
id uint64
committed uint64
applied uint64
compactIndex uint64
2017-02-21 10:23:42 +03:00
} {
{ 1 , 1 , 1 , 0 } ,
{ 2 , 2 , 2 , 2 } ,
{ 3 , 2 , 2 , 2 } ,
}
peers := make ( [ ] stateMachine , 0 )
for _ , c := range nodeConfigs {
storage := NewMemoryStorage ( )
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } )
storage . SetHardState ( pb . HardState { Term : 1 , Commit : c . committed } )
2018-07-05 20:25:47 +03:00
if c . compactIndex != 0 {
storage . Compact ( c . compactIndex )
2017-02-21 10:23:42 +03:00
}
cfg := newTestConfig ( c . id , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , storage )
cfg . Applied = c . applied
raft := newRaft ( cfg )
peers = append ( peers , raft )
}
nt := newNetwork ( peers ... )
2017-02-15 11:45:54 +03:00
// Drop MsgApp to forbid peer a to commit any log entry at its term after it becomes leader.
nt . ignore ( pb . MsgApp )
// Force peer a to become leader.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2017-02-21 10:23:42 +03:00
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Fatalf ( "state = %s, want %s" , sm . state , StateLeader )
2017-02-15 11:45:54 +03:00
}
// Ensure peer a drops read only request.
var windex uint64 = 4
wctx := [ ] byte ( "ctx" )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : wctx } } } )
2017-02-21 10:23:42 +03:00
if len ( sm . readStates ) != 0 {
t . Fatalf ( "len(readStates) = %d, want zero" , len ( sm . readStates ) )
2017-02-15 11:45:54 +03:00
}
nt . recover ( )
// Force peer a to commit a log entry at its term
2017-02-21 10:23:42 +03:00
for i := 0 ; i < sm . heartbeatTimeout ; i ++ {
sm . tick ( )
2017-02-15 11:45:54 +03:00
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2017-02-21 10:23:42 +03:00
if sm . raftLog . committed != 4 {
t . Fatalf ( "committed = %d, want 4" , sm . raftLog . committed )
2017-02-15 11:45:54 +03:00
}
2017-02-21 10:23:42 +03:00
lastLogTerm := sm . raftLog . zeroTermOnErrCompacted ( sm . raftLog . term ( sm . raftLog . committed ) )
if lastLogTerm != sm . Term {
t . Fatalf ( "last log term = %d, want %d" , lastLogTerm , sm . Term )
2017-02-15 11:45:54 +03:00
}
// Ensure peer a accepts read only request after it commits a entry at its term.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : wctx } } } )
2017-02-21 10:23:42 +03:00
if len ( sm . readStates ) != 1 {
t . Fatalf ( "len(readStates) = %d, want 1" , len ( sm . readStates ) )
2017-02-15 11:45:54 +03:00
}
2017-02-21 10:23:42 +03:00
rs := sm . readStates [ 0 ]
2017-02-15 11:45:54 +03:00
if rs . Index != windex {
t . Fatalf ( "readIndex = %d, want %d" , rs . Index , windex )
}
if ! bytes . Equal ( rs . RequestCtx , wctx ) {
t . Fatalf ( "requestCtx = %v, want %v" , rs . RequestCtx , wctx )
}
}
2014-05-25 10:03:13 +04:00
func TestLeaderAppResp ( t * testing . T ) {
2014-11-18 22:42:08 +03:00
// initial progress: match = 0; next = 3
2014-05-25 10:03:13 +04:00
tests := [ ] struct {
2014-10-24 22:50:21 +04:00
index uint64
reject bool
// progress
wmatch uint64
wnext uint64
// message
2014-05-25 10:03:13 +04:00
wmsgNum int
2014-10-08 14:29:53 +04:00
windex uint64
wcommitted uint64
2014-05-25 10:03:13 +04:00
} {
2014-10-24 22:50:21 +04:00
{ 3 , true , 0 , 3 , 0 , 0 , 0 } , // stale resp; no replies
2014-12-05 01:12:55 +03:00
{ 2 , true , 0 , 2 , 1 , 1 , 0 } , // denied resp; leader does not commit; decrease next and send probing msg
2014-11-18 22:42:08 +03:00
{ 2 , false , 2 , 4 , 2 , 2 , 2 } , // accept resp; leader commits; broadcast with commit index
2014-10-24 22:50:21 +04:00
{ 0 , false , 0 , 3 , 0 , 0 , 0 } , // ignore heartbeat replies
2014-05-25 10:03:13 +04:00
}
for i , tt := range tests {
// sm term is 1 after it becomes the leader.
// thus the last log term must be 1 to be committed.
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-11-03 23:04:24 +03:00
sm . raftLog = & raftLog {
2014-11-25 04:13:47 +03:00
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 0 } , { Index : 2 , Term : 1 } } } ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
}
2014-05-25 10:03:13 +04:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
2014-10-30 00:04:45 +03:00
sm . readMessages ( )
2014-12-31 06:42:59 +03:00
sm . Step ( pb . Message { From : 2 , Type : pb . MsgAppResp , Index : tt . index , Term : sm . Term , Reject : tt . reject , RejectHint : tt . index } )
2014-10-24 22:50:21 +04:00
p := sm . prs [ 2 ]
2015-01-20 21:26:22 +03:00
if p . Match != tt . wmatch {
t . Errorf ( "#%d match = %d, want %d" , i , p . Match , tt . wmatch )
2014-10-24 22:50:21 +04:00
}
2015-01-20 21:26:22 +03:00
if p . Next != tt . wnext {
t . Errorf ( "#%d next = %d, want %d" , i , p . Next , tt . wnext )
2014-10-24 22:50:21 +04:00
}
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-05-25 10:03:13 +04:00
if len ( msgs ) != tt . wmsgNum {
t . Errorf ( "#%d msgNum = %d, want %d" , i , len ( msgs ) , tt . wmsgNum )
}
for j , msg := range msgs {
if msg . Index != tt . windex {
t . Errorf ( "#%d.%d index = %d, want %d" , i , j , msg . Index , tt . windex )
}
if msg . Commit != tt . wcommitted {
t . Errorf ( "#%d.%d commit = %d, want %d" , i , j , msg . Commit , tt . wcommitted )
}
}
}
}
2014-09-15 20:58:22 +04:00
// When the leader receives a heartbeat tick, it should
2014-10-30 09:21:38 +03:00
// send a MsgApp with m.Index = 0, m.LogTerm=0 and empty entries.
2014-09-15 20:58:22 +04:00
func TestBcastBeat ( t * testing . T ) {
2014-10-08 14:29:53 +04:00
offset := uint64 ( 1000 )
2014-09-15 20:58:22 +04:00
// make a state machine with log.offset = 1000
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : offset ,
Term : 1 ,
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 , 3 } } ,
} ,
2014-09-15 20:58:22 +04:00
}
2014-11-20 00:17:50 +03:00
storage := NewMemoryStorage ( )
storage . ApplySnapshot ( s )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , nil , 10 , 1 , storage )
2014-09-15 20:58:22 +04:00
sm . Term = 1
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < 10 ; i ++ {
2014-11-25 04:13:47 +03:00
sm . appendEntry ( pb . Entry { Index : uint64 ( i ) + 1 } )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
// slow follower
2015-01-20 21:26:22 +03:00
sm . prs [ 2 ] . Match , sm . prs [ 2 ] . Next = 5 , 6
2014-11-18 02:44:57 +03:00
// normal follower
2015-01-20 21:26:22 +03:00
sm . prs [ 3 ] . Match , sm . prs [ 3 ] . Next = sm . raftLog . lastIndex ( ) , sm . raftLog . lastIndex ( ) + 1
2014-09-15 20:58:22 +04:00
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { Type : pb . MsgBeat } )
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-09-29 21:44:20 +04:00
if len ( msgs ) != 2 {
2014-10-30 09:21:38 +03:00
t . Fatalf ( "len(msgs) = %v, want 2" , len ( msgs ) )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
wantCommitMap := map [ uint64 ] uint64 {
2015-01-20 21:26:22 +03:00
2 : min ( sm . raftLog . committed , sm . prs [ 2 ] . Match ) ,
3 : min ( sm . raftLog . committed , sm . prs [ 3 ] . Match ) ,
2014-11-18 02:44:57 +03:00
}
2014-09-29 21:44:20 +04:00
for i , m := range msgs {
2014-12-04 08:32:12 +03:00
if m . Type != pb . MsgHeartbeat {
t . Fatalf ( "#%d: type = %v, want = %v" , i , m . Type , pb . MsgHeartbeat )
2014-09-15 20:58:22 +04:00
}
2014-09-29 21:44:20 +04:00
if m . Index != 0 {
t . Fatalf ( "#%d: prevIndex = %d, want %d" , i , m . Index , 0 )
2014-09-15 20:58:22 +04:00
}
2014-09-29 21:44:20 +04:00
if m . LogTerm != 0 {
t . Fatalf ( "#%d: prevTerm = %d, want %d" , i , m . LogTerm , 0 )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
if wantCommitMap [ m . To ] == 0 {
2014-09-29 21:44:20 +04:00
t . Fatalf ( "#%d: unexpected to %d" , i , m . To )
} else {
2014-11-18 02:44:57 +03:00
if m . Commit != wantCommitMap [ m . To ] {
t . Fatalf ( "#%d: commit = %d, want %d" , i , m . Commit , wantCommitMap [ m . To ] )
}
delete ( wantCommitMap , m . To )
2014-09-15 20:58:22 +04:00
}
if len ( m . Entries ) != 0 {
t . Fatalf ( "#%d: len(entries) = %d, want 0" , i , len ( m . Entries ) )
}
}
}
2016-01-31 05:15:56 +03:00
// tests the output of the state machine when receiving MsgBeat
2014-06-07 00:40:24 +04:00
func TestRecvMsgBeat ( t * testing . T ) {
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
state StateType
2014-06-07 00:40:24 +04:00
wMsg int
} {
2014-09-16 04:35:02 +04:00
{ StateLeader , 2 } ,
2014-10-30 09:21:38 +03:00
// candidate and follower should ignore MsgBeat
2014-09-16 04:35:02 +04:00
{ StateCandidate , 0 } ,
{ StateFollower , 0 } ,
2014-06-07 00:40:24 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-11-25 04:13:47 +03:00
sm . raftLog = & raftLog { storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 0 } , { Index : 2 , Term : 1 } } } }
2014-08-23 01:19:30 +04:00
sm . Term = 1
2014-06-07 00:40:24 +04:00
sm . state = tt . state
2014-09-03 21:16:33 +04:00
switch tt . state {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-09-03 21:16:33 +04:00
sm . step = stepFollower
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-09-03 21:16:33 +04:00
sm . step = stepCandidate
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-09-03 21:16:33 +04:00
sm . step = stepLeader
}
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
2014-06-07 00:40:24 +04:00
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-06-07 00:40:24 +04:00
if len ( msgs ) != tt . wMsg {
t . Errorf ( "%d: len(msgs) = %d, want %d" , i , len ( msgs ) , tt . wMsg )
}
for _ , m := range msgs {
2014-12-04 08:32:12 +03:00
if m . Type != pb . MsgHeartbeat {
t . Errorf ( "%d: msg.type = %v, want %v" , i , m . Type , pb . MsgHeartbeat )
2014-06-07 00:40:24 +04:00
}
}
}
}
2014-11-18 22:42:08 +03:00
func TestLeaderIncreaseNext ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
tests := [ ] struct {
// progress
2015-03-16 10:46:16 +03:00
state ProgressStateType
2014-11-18 22:42:08 +03:00
next uint64
wnext uint64
} {
2015-03-16 10:46:16 +03:00
// state replicate, optimistically increase next
2014-11-18 22:42:08 +03:00
// previous entries + noop entry + propose + 1
2015-03-16 10:46:16 +03:00
{ ProgressStateReplicate , 2 , uint64 ( len ( previousEnts ) + 1 + 1 + 1 ) } ,
// state probe, not optimistically increase next
{ ProgressStateProbe , 2 , 2 } ,
2014-11-18 22:42:08 +03:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-12-02 19:50:15 +03:00
sm . raftLog . append ( previousEnts ... )
2014-11-18 22:42:08 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
2015-03-16 10:46:16 +03:00
sm . prs [ 2 ] . State = tt . state
sm . prs [ 2 ] . Next = tt . next
2014-11-18 22:42:08 +03:00
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
p := sm . prs [ 2 ]
2015-01-20 21:26:22 +03:00
if p . Next != tt . wnext {
t . Errorf ( "#%d next = %d, want %d" , i , p . Next , tt . wnext )
2014-11-18 22:42:08 +03:00
}
}
2014-06-07 00:40:24 +04:00
}
2015-03-16 10:46:16 +03:00
func TestSendAppendForProgressProbe ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-02-05 01:41:14 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
2015-03-16 10:46:16 +03:00
r . prs [ 2 ] . becomeProbe ( )
2015-02-05 01:41:14 +03:00
2015-03-16 10:46:16 +03:00
// each round is a heartbeat
2015-02-05 01:41:14 +03:00
for i := 0 ; i < 3 ; i ++ {
2016-12-19 23:24:21 +03:00
if i == 0 {
// we expect that raft will only send out one msgAPP on the first
// loop. After that, the follower is paused until a heartbeat response is
// received.
r . appendEntry ( pb . Entry { Data : [ ] byte ( "somedata" ) } )
r . sendAppend ( 2 )
msg := r . readMessages ( )
if len ( msg ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msg ) , 1 )
}
if msg [ 0 ] . Index != 0 {
t . Errorf ( "index = %d, want %d" , msg [ 0 ] . Index , 0 )
}
2015-02-05 01:41:14 +03:00
}
2016-04-03 04:27:54 +03:00
if ! r . prs [ 2 ] . Paused {
2015-03-16 10:46:16 +03:00
t . Errorf ( "paused = %v, want true" , r . prs [ 2 ] . Paused )
2015-02-05 01:41:14 +03:00
}
2015-02-28 22:56:55 +03:00
for j := 0 ; j < 10 ; j ++ {
2015-03-16 10:46:16 +03:00
r . appendEntry ( pb . Entry { Data : [ ] byte ( "somedata" ) } )
r . sendAppend ( 2 )
2015-02-05 01:41:14 +03:00
if l := len ( r . readMessages ( ) ) ; l != 0 {
t . Errorf ( "len(msg) = %d, want %d" , l , 0 )
}
}
// do a heartbeat
2015-02-28 22:56:55 +03:00
for j := 0 ; j < r . heartbeatTimeout ; j ++ {
2015-02-05 01:41:14 +03:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
}
2016-12-19 23:24:21 +03:00
if ! r . prs [ 2 ] . Paused {
t . Errorf ( "paused = %v, want true" , r . prs [ 2 ] . Paused )
}
2015-02-05 01:41:14 +03:00
// consume the heartbeat
2016-12-19 23:24:21 +03:00
msg := r . readMessages ( )
2015-02-05 01:41:14 +03:00
if len ( msg ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msg ) , 1 )
}
if msg [ 0 ] . Type != pb . MsgHeartbeat {
2015-12-16 07:18:21 +03:00
t . Errorf ( "type = %v, want %v" , msg [ 0 ] . Type , pb . MsgHeartbeat )
2015-02-05 01:41:14 +03:00
}
}
2016-12-19 23:24:21 +03:00
// a heartbeat response will allow another message to be sent
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeatResp } )
msg := r . readMessages ( )
if len ( msg ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msg ) , 1 )
}
if msg [ 0 ] . Index != 0 {
t . Errorf ( "index = %d, want %d" , msg [ 0 ] . Index , 0 )
}
if ! r . prs [ 2 ] . Paused {
t . Errorf ( "paused = %v, want true" , r . prs [ 2 ] . Paused )
}
2015-03-16 10:46:16 +03:00
}
func TestSendAppendForProgressReplicate ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
r . prs [ 2 ] . becomeReplicate ( )
2015-02-05 01:41:14 +03:00
for i := 0 ; i < 10 ; i ++ {
2015-03-16 10:46:16 +03:00
r . appendEntry ( pb . Entry { Data : [ ] byte ( "somedata" ) } )
r . sendAppend ( 2 )
2015-02-05 01:41:14 +03:00
msgs := r . readMessages ( )
if len ( msgs ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msgs ) , 1 )
}
}
}
2015-03-16 10:46:16 +03:00
func TestSendAppendForProgressSnapshot ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
r . prs [ 2 ] . becomeSnapshot ( 10 )
for i := 0 ; i < 10 ; i ++ {
r . appendEntry ( pb . Entry { Data : [ ] byte ( "somedata" ) } )
r . sendAppend ( 2 )
msgs := r . readMessages ( )
if len ( msgs ) != 0 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msgs ) , 0 )
}
}
}
func TestRecvMsgUnreachable ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
s := NewMemoryStorage ( )
s . Append ( previousEnts )
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , s )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
// set node 2 to state replicate
r . prs [ 2 ] . Match = 3
r . prs [ 2 ] . becomeReplicate ( )
r . prs [ 2 ] . optimisticUpdate ( 5 )
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgUnreachable } )
if r . prs [ 2 ] . State != ProgressStateProbe {
t . Errorf ( "state = %s, want %s" , r . prs [ 2 ] . State , ProgressStateProbe )
}
if wnext := r . prs [ 2 ] . Match + 1 ; r . prs [ 2 ] . Next != wnext {
t . Errorf ( "next = %d, want %d" , r . prs [ 2 ] . Next , wnext )
}
}
2014-07-01 23:10:43 +04:00
func TestRestore ( t * testing . T ) {
2014-08-28 05:53:18 +04:00
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 , 3 } } ,
} ,
2014-07-01 23:10:43 +04:00
}
2014-11-20 00:17:50 +03:00
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , storage )
2014-08-01 00:03:23 +04:00
if ok := sm . restore ( s ) ; ! ok {
t . Fatal ( "restore fail, want succeed" )
}
2014-07-01 23:10:43 +04:00
2014-11-20 00:17:50 +03:00
if sm . raftLog . lastIndex ( ) != s . Metadata . Index {
t . Errorf ( "log.lastIndex = %d, want %d" , sm . raftLog . lastIndex ( ) , s . Metadata . Index )
2014-08-01 02:18:44 +04:00
}
2015-06-13 08:24:40 +03:00
if mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) != s . Metadata . Term {
t . Errorf ( "log.lastTerm = %d, want %d" , mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) , s . Metadata . Term )
2014-08-01 02:18:44 +04:00
}
2014-10-08 11:33:55 +04:00
sg := sm . nodes ( )
2014-11-20 00:17:50 +03:00
if ! reflect . DeepEqual ( sg , s . Metadata . ConfState . Nodes ) {
t . Errorf ( "sm.Nodes = %+v, want %+v" , sg , s . Metadata . ConfState . Nodes )
2014-07-01 23:10:43 +04:00
}
2014-08-01 00:03:23 +04:00
if ok := sm . restore ( s ) ; ok {
t . Fatal ( "restore succeed, want fail" )
}
2014-07-01 23:10:43 +04:00
}
2017-11-11 05:38:21 +03:00
// TestRestoreWithLearner restores a snapshot which contains learners.
func TestRestoreWithLearner ( t * testing . T ) {
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } , Learners : [ ] uint64 { 3 } } ,
} ,
}
storage := NewMemoryStorage ( )
2018-05-10 01:18:57 +03:00
sm := newTestLearnerRaft ( 3 , [ ] uint64 { 1 , 2 } , [ ] uint64 { 3 } , 8 , 2 , storage )
2017-11-11 05:38:21 +03:00
if ok := sm . restore ( s ) ; ! ok {
t . Error ( "restore fail, want succeed" )
}
if sm . raftLog . lastIndex ( ) != s . Metadata . Index {
t . Errorf ( "log.lastIndex = %d, want %d" , sm . raftLog . lastIndex ( ) , s . Metadata . Index )
}
if mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) != s . Metadata . Term {
t . Errorf ( "log.lastTerm = %d, want %d" , mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) , s . Metadata . Term )
}
sg := sm . nodes ( )
2018-01-08 18:43:04 +03:00
if len ( sg ) != len ( s . Metadata . ConfState . Nodes ) {
t . Errorf ( "sm.Nodes = %+v, length not equal with %+v" , sg , s . Metadata . ConfState . Nodes )
}
lns := sm . learnerNodes ( )
if len ( lns ) != len ( s . Metadata . ConfState . Learners ) {
t . Errorf ( "sm.LearnerNodes = %+v, length not equal with %+v" , sg , s . Metadata . ConfState . Learners )
2017-11-11 05:38:21 +03:00
}
for _ , n := range s . Metadata . ConfState . Nodes {
if sm . prs [ n ] . IsLearner {
t . Errorf ( "sm.Node %x isLearner = %s, want %t" , n , sm . prs [ n ] , false )
}
}
for _ , n := range s . Metadata . ConfState . Learners {
if ! sm . learnerPrs [ n ] . IsLearner {
t . Errorf ( "sm.Node %x isLearner = %s, want %t" , n , sm . prs [ n ] , true )
}
}
if ok := sm . restore ( s ) ; ok {
t . Error ( "restore succeed, want fail" )
}
}
// TestRestoreInvalidLearner verfies that a normal peer can't become learner again
// when restores snapshot.
func TestRestoreInvalidLearner ( t * testing . T ) {
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } , Learners : [ ] uint64 { 3 } } ,
} ,
}
storage := NewMemoryStorage ( )
sm := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , storage )
if sm . isLearner {
t . Errorf ( "%x is learner, want not" , sm . id )
}
if ok := sm . restore ( s ) ; ok {
t . Error ( "restore succeed, want fail" )
}
}
// TestRestoreLearnerPromotion checks that a learner can become to a follower after
// restoring snapshot.
func TestRestoreLearnerPromotion ( t * testing . T ) {
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 , 3 } } ,
} ,
}
storage := NewMemoryStorage ( )
sm := newTestLearnerRaft ( 3 , [ ] uint64 { 1 , 2 } , [ ] uint64 { 3 } , 10 , 1 , storage )
if ! sm . isLearner {
t . Errorf ( "%x is not learner, want yes" , sm . id )
}
if ok := sm . restore ( s ) ; ! ok {
t . Error ( "restore fail, want succeed" )
}
if sm . isLearner {
t . Errorf ( "%x is learner, want not" , sm . id )
}
}
// TestLearnerReceiveSnapshot tests that a learner can receive a snpahost from leader
func TestLearnerReceiveSnapshot ( t * testing . T ) {
// restore the state machine from a snapshot so it has a compacted log and a snapshot
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 } , Learners : [ ] uint64 { 2 } } ,
} ,
}
n1 := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . restore ( s )
// Force set n1 appplied index.
n1 . raftLog . appliedTo ( n1 . raftLog . committed )
nt := newNetwork ( n1 , n2 )
setRandomizedElectionTimeout ( n1 , n1 . electionTimeout )
for i := 0 ; i < n1 . electionTimeout ; i ++ {
n1 . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
if n2 . raftLog . committed != n1 . raftLog . committed {
t . Errorf ( "peer 2 must commit to %d, but %d" , n1 . raftLog . committed , n2 . raftLog . committed )
}
}
2014-12-03 08:34:14 +03:00
func TestRestoreIgnoreSnapshot ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
commit := uint64 ( 1 )
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , storage )
2014-12-03 08:34:14 +03:00
sm . raftLog . append ( previousEnts ... )
sm . raftLog . commitTo ( commit )
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : commit ,
Term : 1 ,
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } } ,
} ,
}
// ignore snapshot
if ok := sm . restore ( s ) ; ok {
t . Errorf ( "restore = %t, want %t" , ok , false )
}
if sm . raftLog . committed != commit {
t . Errorf ( "commit = %d, want %d" , sm . raftLog . committed , commit )
}
// ignore snapshot and fast forward commit
s . Metadata . Index = commit + 1
if ok := sm . restore ( s ) ; ok {
t . Errorf ( "restore = %t, want %t" , ok , false )
}
if sm . raftLog . committed != commit + 1 {
t . Errorf ( "commit = %d, want %d" , sm . raftLog . committed , commit + 1 )
}
}
2014-07-02 23:49:58 +04:00
func TestProvideSnap ( t * testing . T ) {
2015-12-10 23:17:30 +03:00
// restore the state machine from a snapshot so it has a compacted log and a snapshot
2014-11-20 00:17:50 +03:00
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } } ,
} ,
}
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
2014-07-02 23:49:58 +04:00
sm . restore ( s )
sm . becomeCandidate ( )
sm . becomeLeader ( )
2015-12-10 23:17:30 +03:00
// force set the next of node 2, so that node 2 needs a snapshot
2015-01-20 21:26:22 +03:00
sm . prs [ 2 ] . Next = sm . raftLog . firstIndex ( )
sm . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgAppResp , Index : sm . prs [ 2 ] . Next - 1 , Reject : true } )
2015-12-10 23:17:30 +03:00
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-07-02 23:49:58 +04:00
if len ( msgs ) != 1 {
2014-09-29 06:28:12 +04:00
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
2014-07-02 23:49:58 +04:00
}
2014-09-29 21:44:20 +04:00
m := msgs [ 0 ]
2014-10-12 11:34:22 +04:00
if m . Type != pb . MsgSnap {
t . Errorf ( "m.Type = %v, want %v" , m . Type , pb . MsgSnap )
2014-07-02 23:49:58 +04:00
}
}
2015-12-10 23:17:30 +03:00
func TestIgnoreProvidingSnap ( t * testing . T ) {
// restore the state machine from a snapshot so it has a compacted log and a snapshot
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } } ,
} ,
}
storage := NewMemoryStorage ( )
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
sm . restore ( s )
sm . becomeCandidate ( )
sm . becomeLeader ( )
// force set the next of node 2, so that node 2 needs a snapshot
// change node 2 to be inactive, expect node 1 ignore sending snapshot to 2
sm . prs [ 2 ] . Next = sm . raftLog . firstIndex ( ) - 1
sm . prs [ 2 ] . RecentActive = false
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
msgs := sm . readMessages ( )
if len ( msgs ) != 0 {
t . Errorf ( "len(msgs) = %d, want 0" , len ( msgs ) )
}
}
2014-07-02 23:49:58 +04:00
func TestRestoreFromSnapMsg ( t * testing . T ) {
2014-08-28 05:53:18 +04:00
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
ConfState : pb . ConfState { Nodes : [ ] uint64 { 1 , 2 } } ,
} ,
2014-07-02 23:49:58 +04:00
}
2014-10-12 11:34:22 +04:00
m := pb . Message { Type : pb . MsgSnap , From : 1 , Term : 2 , Snapshot : s }
2014-07-02 23:49:58 +04:00
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 2 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-07-02 23:49:58 +04:00
sm . Step ( m )
2016-07-12 02:37:31 +03:00
if sm . lead != uint64 ( 1 ) {
t . Errorf ( "sm.lead = %d, want 1" , sm . lead )
}
2014-11-20 00:17:50 +03:00
// TODO(bdarnell): what should this test?
2014-07-02 23:49:58 +04:00
}
func TestSlowNodeRestore ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-07-02 23:49:58 +04:00
2014-09-12 06:23:05 +04:00
nt . isolate ( 3 )
2014-10-17 02:32:57 +04:00
for j := 0 ; j <= 100 ; j ++ {
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2014-07-02 23:49:58 +04:00
}
2014-09-12 06:23:05 +04:00
lead := nt . peers [ 1 ] . ( * raft )
2014-11-13 00:23:42 +03:00
nextEnts ( lead , nt . storage [ 1 ] )
2015-01-23 21:50:43 +03:00
nt . storage [ 1 ] . CreateSnapshot ( lead . raftLog . applied , & pb . ConfState { Nodes : lead . nodes ( ) } , nil )
nt . storage [ 1 ] . Compact ( lead . raftLog . applied )
2014-07-02 23:49:58 +04:00
nt . recover ( )
2015-12-10 23:17:30 +03:00
// send heartbeats so that the leader can learn everyone is active.
// node 3 will only be considered as active when node 1 receives a reply from it.
for {
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
if lead . prs [ 3 ] . RecentActive {
break
}
}
2014-09-29 21:44:20 +04:00
// trigger a snapshot
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2015-12-10 23:17:30 +03:00
2014-09-12 06:23:05 +04:00
follower := nt . peers [ 3 ] . ( * raft )
2014-07-02 23:49:58 +04:00
2014-09-29 21:44:20 +04:00
// trigger a commit
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2014-09-29 21:44:20 +04:00
if follower . raftLog . committed != lead . raftLog . committed {
2014-12-05 01:12:55 +03:00
t . Errorf ( "follower.committed = %d, want %d" , follower . raftLog . committed , lead . raftLog . committed )
2014-07-02 23:49:58 +04:00
}
}
2014-09-23 23:02:44 +04:00
// TestStepConfig tests that when raft step msgProp in EntryConfChange type,
2014-09-19 23:35:56 +04:00
// it appends the entry to log and sets pendingConf to be true.
func TestStepConfig ( t * testing . T ) {
// a raft that cannot make progress
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
index := r . raftLog . lastIndex ( )
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-09-19 23:35:56 +04:00
if g := r . raftLog . lastIndex ( ) ; g != index + 1 {
t . Errorf ( "index = %d, want %d" , g , index + 1 )
}
2017-12-30 07:11:22 +03:00
if r . pendingConfIndex != index + 1 {
t . Errorf ( "pendingConfIndex = %d, want %d" , r . pendingConfIndex , index + 1 )
2014-09-19 23:35:56 +04:00
}
}
// TestStepIgnoreConfig tests that if raft step the second msgProp in
2014-12-10 09:13:42 +03:00
// EntryConfChange type when the first one is uncommitted, the node will set
// the proposal to noop and keep its original state.
2014-09-19 23:35:56 +04:00
func TestStepIgnoreConfig ( t * testing . T ) {
// a raft that cannot make progress
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-09-19 23:35:56 +04:00
index := r . raftLog . lastIndex ( )
2017-12-30 07:11:22 +03:00
pendingConfIndex := r . pendingConfIndex
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-12-10 09:13:42 +03:00
wents := [ ] pb . Entry { { Type : pb . EntryNormal , Term : 1 , Index : 3 , Data : nil } }
2015-06-13 08:24:40 +03:00
ents , err := r . raftLog . entries ( index + 1 , noLimit )
if err != nil {
t . Fatalf ( "unexpected error %v" , err )
}
if ! reflect . DeepEqual ( ents , wents ) {
2014-12-10 09:13:42 +03:00
t . Errorf ( "ents = %+v, want %+v" , ents , wents )
2014-09-19 23:35:56 +04:00
}
2017-12-30 07:11:22 +03:00
if r . pendingConfIndex != pendingConfIndex {
t . Errorf ( "pendingConfIndex = %d, want %d" , r . pendingConfIndex , pendingConfIndex )
2014-09-19 23:35:56 +04:00
}
}
2017-12-30 07:11:22 +03:00
// TestNewLeaderPendingConfig tests that new leader sets its pendingConfigIndex
2014-09-19 23:35:56 +04:00
// based on uncommitted entries.
2017-12-30 07:11:22 +03:00
func TestNewLeaderPendingConfig ( t * testing . T ) {
2014-09-19 23:35:56 +04:00
tests := [ ] struct {
2017-12-30 07:11:22 +03:00
addEntry bool
wpendingIndex uint64
2014-09-19 23:35:56 +04:00
} {
2017-12-30 07:11:22 +03:00
{ false , 0 } ,
{ true , 1 } ,
2014-09-19 23:35:56 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2017-12-30 07:11:22 +03:00
if tt . addEntry {
r . appendEntry ( pb . Entry { Type : pb . EntryNormal } )
}
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
2017-12-30 07:11:22 +03:00
if r . pendingConfIndex != tt . wpendingIndex {
t . Errorf ( "#%d: pendingConfIndex = %d, want %d" ,
i , r . pendingConfIndex , tt . wpendingIndex )
2014-09-19 23:35:56 +04:00
}
}
}
2017-12-30 07:11:22 +03:00
// TestAddNode tests that addNode could update nodes correctly.
2014-09-19 23:35:56 +04:00
func TestAddNode ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . addNode ( 2 )
nodes := r . nodes ( )
2014-10-08 14:29:53 +04:00
wnodes := [ ] uint64 { 1 , 2 }
2014-09-19 23:35:56 +04:00
if ! reflect . DeepEqual ( nodes , wnodes ) {
t . Errorf ( "nodes = %v, want %v" , nodes , wnodes )
}
}
2017-12-30 07:11:22 +03:00
// TestAddLearner tests that addLearner could update nodes correctly.
2017-11-11 05:38:21 +03:00
func TestAddLearner ( t * testing . T ) {
r := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
r . addLearner ( 2 )
2018-01-08 18:43:04 +03:00
nodes := r . learnerNodes ( )
wnodes := [ ] uint64 { 2 }
2017-11-11 05:38:21 +03:00
if ! reflect . DeepEqual ( nodes , wnodes ) {
t . Errorf ( "nodes = %v, want %v" , nodes , wnodes )
}
if ! r . learnerPrs [ 2 ] . IsLearner {
t . Errorf ( "node 2 is learner %t, want %t" , r . prs [ 2 ] . IsLearner , true )
}
}
2017-05-05 00:58:05 +03:00
// TestAddNodeCheckQuorum tests that addNode does not trigger a leader election
// immediately when checkQuorum is set.
func TestAddNodeCheckQuorum ( t * testing . T ) {
r := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
r . checkQuorum = true
r . becomeCandidate ( )
r . becomeLeader ( )
for i := 0 ; i < r . electionTimeout - 1 ; i ++ {
r . tick ( )
}
r . addNode ( 2 )
// This tick will reach electionTimeout, which triggers a quorum check.
r . tick ( )
// Node 1 should still be the leader after a single tick.
if r . state != StateLeader {
t . Errorf ( "state = %v, want %v" , r . state , StateLeader )
}
// After another electionTimeout ticks without hearing from node 2,
// node 1 should step down.
for i := 0 ; i < r . electionTimeout ; i ++ {
r . tick ( )
}
if r . state != StateFollower {
t . Errorf ( "state = %v, want %v" , r . state , StateFollower )
}
}
2017-12-30 07:11:22 +03:00
// TestRemoveNode tests that removeNode could update nodes and
2014-09-30 06:54:20 +04:00
// and removed list correctly.
2014-09-19 23:35:56 +04:00
func TestRemoveNode ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . removeNode ( 2 )
2014-10-08 14:29:53 +04:00
w := [ ] uint64 { 1 }
2014-09-19 23:35:56 +04:00
if g := r . nodes ( ) ; ! reflect . DeepEqual ( g , w ) {
t . Errorf ( "nodes = %v, want %v" , g , w )
}
2016-05-16 19:58:57 +03:00
// remove all nodes from cluster
r . removeNode ( 1 )
w = [ ] uint64 { }
if g := r . nodes ( ) ; ! reflect . DeepEqual ( g , w ) {
t . Errorf ( "nodes = %v, want %v" , g , w )
}
2014-09-19 23:35:56 +04:00
}
2017-12-30 07:11:22 +03:00
// TestRemoveLearner tests that removeNode could update nodes and
2017-11-11 05:38:21 +03:00
// and removed list correctly.
func TestRemoveLearner ( t * testing . T ) {
r := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
r . removeNode ( 2 )
w := [ ] uint64 { 1 }
if g := r . nodes ( ) ; ! reflect . DeepEqual ( g , w ) {
t . Errorf ( "nodes = %v, want %v" , g , w )
}
2018-01-08 18:43:04 +03:00
w = [ ] uint64 { }
if g := r . learnerNodes ( ) ; ! reflect . DeepEqual ( g , w ) {
t . Errorf ( "nodes = %v, want %v" , g , w )
}
2017-11-11 05:38:21 +03:00
// remove all nodes from cluster
r . removeNode ( 1 )
if g := r . nodes ( ) ; ! reflect . DeepEqual ( g , w ) {
t . Errorf ( "nodes = %v, want %v" , g , w )
}
}
2014-09-24 21:19:41 +04:00
func TestPromotable ( t * testing . T ) {
2014-10-08 14:29:53 +04:00
id := uint64 ( 1 )
2014-09-24 10:09:21 +04:00
tests := [ ] struct {
2014-10-08 14:29:53 +04:00
peers [ ] uint64
2014-09-24 21:19:41 +04:00
wp bool
2014-09-24 10:09:21 +04:00
} {
2014-10-08 14:29:53 +04:00
{ [ ] uint64 { 1 } , true } ,
{ [ ] uint64 { 1 , 2 , 3 } , true } ,
{ [ ] uint64 { } , false } ,
{ [ ] uint64 { 2 , 3 } , false } ,
2014-09-24 10:09:21 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( id , tt . peers , 5 , 1 , NewMemoryStorage ( ) )
2014-09-24 21:19:41 +04:00
if g := r . promotable ( ) ; g != tt . wp {
t . Errorf ( "#%d: promotable = %v, want %v" , i , g , tt . wp )
2014-09-24 10:09:21 +04:00
}
}
}
2014-11-12 08:11:09 +03:00
func TestRaftNodes ( t * testing . T ) {
tests := [ ] struct {
ids [ ] uint64
wids [ ] uint64
} {
{
[ ] uint64 { 1 , 2 , 3 } ,
[ ] uint64 { 1 , 2 , 3 } ,
} ,
{
[ ] uint64 { 3 , 2 , 1 } ,
[ ] uint64 { 1 , 2 , 3 } ,
} ,
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , tt . ids , 10 , 1 , NewMemoryStorage ( ) )
2014-11-12 08:11:09 +03:00
if ! reflect . DeepEqual ( r . nodes ( ) , tt . wids ) {
t . Errorf ( "#%d: nodes = %+v, want %+v" , i , r . nodes ( ) , tt . wids )
}
}
}
2015-11-17 03:32:12 +03:00
func TestCampaignWhileLeader ( t * testing . T ) {
2016-10-25 17:31:44 +03:00
testCampaignWhileLeader ( t , false )
}
func TestPreCampaignWhileLeader ( t * testing . T ) {
testCampaignWhileLeader ( t , true )
}
func testCampaignWhileLeader ( t * testing . T , preVote bool ) {
cfg := newTestConfig ( 1 , [ ] uint64 { 1 } , 5 , 1 , NewMemoryStorage ( ) )
cfg . PreVote = preVote
r := newRaft ( cfg )
if r . state != StateFollower {
t . Errorf ( "expected new node to be follower but got %s" , r . state )
}
// We don't call campaign() directly because it comes after the check
// for our current state.
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if r . state != StateLeader {
t . Errorf ( "expected single-node election to become leader but got %s" , r . state )
}
term := r . Term
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if r . state != StateLeader {
t . Errorf ( "expected to remain leader but got %s" , r . state )
}
if r . Term != term {
t . Errorf ( "expected to remain in term %v but got %v" , term , r . Term )
2015-11-17 03:32:12 +03:00
}
}
2016-01-20 18:03:12 +03:00
// TestCommitAfterRemoveNode verifies that pending commands can become
// committed when a config change reduces the quorum requirements.
func TestCommitAfterRemoveNode ( t * testing . T ) {
// Create a cluster with two nodes.
s := NewMemoryStorage ( )
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , s )
r . becomeCandidate ( )
r . becomeLeader ( )
// Begin to remove the second node.
cc := pb . ConfChange {
Type : pb . ConfChangeRemoveNode ,
NodeID : 2 ,
}
ccData , err := cc . Marshal ( )
if err != nil {
t . Fatal ( err )
}
r . Step ( pb . Message {
Type : pb . MsgProp ,
Entries : [ ] pb . Entry {
{ Type : pb . EntryConfChange , Data : ccData } ,
} ,
} )
// Stabilize the log and make sure nothing is committed yet.
if ents := nextEnts ( r , s ) ; len ( ents ) > 0 {
t . Fatalf ( "unexpected committed entries: %v" , ents )
}
ccIndex := r . raftLog . lastIndex ( )
// While the config change is pending, make another proposal.
r . Step ( pb . Message {
Type : pb . MsgProp ,
Entries : [ ] pb . Entry {
{ Type : pb . EntryNormal , Data : [ ] byte ( "hello" ) } ,
} ,
} )
// Node 2 acknowledges the config change, committing it.
r . Step ( pb . Message {
Type : pb . MsgAppResp ,
From : 2 ,
Index : ccIndex ,
} )
ents := nextEnts ( r , s )
if len ( ents ) != 2 {
t . Fatalf ( "expected two committed entries, got %v" , ents )
}
if ents [ 0 ] . Type != pb . EntryNormal || ents [ 0 ] . Data != nil {
t . Fatalf ( "expected ents[0] to be empty, but got %v" , ents [ 0 ] )
}
if ents [ 1 ] . Type != pb . EntryConfChange {
t . Fatalf ( "expected ents[1] to be EntryConfChange, got %v" , ents [ 1 ] )
}
// Apply the config change. This reduces quorum requirements so the
// pending command can now commit.
r . removeNode ( 2 )
ents = nextEnts ( r , s )
if len ( ents ) != 1 || ents [ 0 ] . Type != pb . EntryNormal ||
string ( ents [ 0 ] . Data ) != "hello" {
t . Fatalf ( "expected one committed EntryNormal, got %v" , ents )
}
}
2016-04-11 09:16:56 +03:00
// TestLeaderTransferToUpToDateNode verifies transferring should succeed
2016-09-08 11:56:02 +03:00
// if the transferee has the most up-to-date log entries when transfer starts.
2016-03-24 14:59:41 +03:00
func TestLeaderTransferToUpToDateNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
if lead . lead != 1 {
t . Fatalf ( "after election leader is %x, want 1" , lead . lead )
}
// Transfer leadership to 2.
nt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
// After some log replication, transfer leadership back to 1.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . send ( pb . Message { From : 1 , To : 2 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
2016-09-08 11:56:02 +03:00
// TestLeaderTransferToUpToDateNodeFromFollower verifies transferring should succeed
// if the transferee has the most up-to-date log entries when transfer starts.
// Not like TestLeaderTransferToUpToDateNode, where the leader transfer message
// is sent to the leader, in this test case every leader transfer message is sent
// to the follower.
func TestLeaderTransferToUpToDateNodeFromFollower ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
if lead . lead != 1 {
t . Fatalf ( "after election leader is %x, want 1" , lead . lead )
}
// Transfer leadership to 2.
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
// After some log replication, transfer leadership back to 1.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
2016-06-29 13:24:58 +03:00
// TestLeaderTransferWithCheckQuorum ensures transferring leader still works
// even the current leader is still under its leader lease
func TestLeaderTransferWithCheckQuorum ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
for i := 1 ; i < 4 ; i ++ {
r := nt . peers [ uint64 ( i ) ] . ( * raft )
r . checkQuorum = true
2016-08-04 13:31:22 +03:00
setRandomizedElectionTimeout ( r , r . electionTimeout + i )
2016-06-29 13:24:58 +03:00
}
2016-07-11 09:59:53 +03:00
// Letting peer 2 electionElapsed reach to timeout so that it can vote for peer 1
2016-06-29 13:24:58 +03:00
f := nt . peers [ 2 ] . ( * raft )
for i := 0 ; i < f . electionTimeout ; i ++ {
f . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
if lead . lead != 1 {
t . Fatalf ( "after election leader is %x, want 1" , lead . lead )
}
// Transfer leadership to 2.
nt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
// After some log replication, transfer leadership back to 1.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . send ( pb . Message { From : 1 , To : 2 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
2016-03-24 14:59:41 +03:00
func TestLeaderTransferToSlowFollower ( t * testing . T ) {
defaultLogger . EnableDebug ( )
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . recover ( )
lead := nt . peers [ 1 ] . ( * raft )
if lead . prs [ 3 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x for node 3, want %x" , lead . prs [ 3 ] . Match , 1 )
}
// Transfer leadership to 3 when node 3 is lack of log.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 3 )
}
func TestLeaderTransferAfterSnapshot ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
lead := nt . peers [ 1 ] . ( * raft )
nextEnts ( lead , nt . storage [ 1 ] )
nt . storage [ 1 ] . CreateSnapshot ( lead . raftLog . applied , & pb . ConfState { Nodes : lead . nodes ( ) } , nil )
nt . storage [ 1 ] . Compact ( lead . raftLog . applied )
nt . recover ( )
if lead . prs [ 3 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x for node 3, want %x" , lead . prs [ 3 ] . Match , 1 )
}
// Transfer leadership to 3 when node 3 is lack of snapshot.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
// Send pb.MsgHeartbeatResp to leader to trigger a snapshot for node 3.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgHeartbeatResp } )
checkLeaderTransferState ( t , lead , StateFollower , 3 )
}
func TestLeaderTransferToSelf ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to self, there will be noop.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferToNonExistingNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to non-existing node, there will be noop.
nt . send ( pb . Message { From : 4 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferTimeout ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node, wait for timeout.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . electionTimeout - lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferIgnoreProposal ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node to let transfer pending, then send proposal.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2018-01-11 07:43:55 +03:00
err := lead . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
if err != ErrProposalDropped {
t . Fatalf ( "should return drop proposal error while transferring" )
}
2016-03-24 14:59:41 +03:00
if lead . prs [ 1 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x, want %x" , lead . prs [ 1 ] . Match , 1 )
}
}
func TestLeaderTransferReceiveHigherTermVote ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node to let transfer pending.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup , Index : 1 , Term : 2 } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
}
func TestLeaderTransferRemoveNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . ignore ( pb . MsgTimeoutNow )
lead := nt . peers [ 1 ] . ( * raft )
2016-04-11 09:16:56 +03:00
// The leadTransferee is removed when leadship transferring.
2016-03-24 14:59:41 +03:00
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
lead . removeNode ( 3 )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
// TestLeaderTransferBack verifies leadership can transfer back to self when last transfer is pending.
func TestLeaderTransferBack ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
// Transfer leadership back to self.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
// TestLeaderTransferSecondTransferToAnotherNode verifies leader can transfer to another node
// when last transfer is pending.
func TestLeaderTransferSecondTransferToAnotherNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
// Transfer leadership to another node.
nt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
}
// TestLeaderTransferSecondTransferToSameNode verifies second transfer leader request
// to the same node should not extend the timeout while the first one is pending.
func TestLeaderTransferSecondTransferToSameNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
// Second transfer leadership request to the same node.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
for i := 0 ; i < lead . electionTimeout - lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func checkLeaderTransferState ( t * testing . T , r * raft , state StateType , lead uint64 ) {
if r . state != state || r . lead != lead {
2016-04-11 09:16:56 +03:00
t . Fatalf ( "after transferring, node has state %v lead %v, want state %v lead %v" , r . state , r . lead , state , lead )
2016-03-24 14:59:41 +03:00
}
if r . leadTransferee != None {
2016-04-11 09:16:56 +03:00
t . Fatalf ( "after transferring, node has leadTransferee %v, want leadTransferee %v" , r . leadTransferee , None )
2016-03-24 14:59:41 +03:00
}
}
2016-11-07 15:02:21 +03:00
// TestTransferNonMember verifies that when a MsgTimeoutNow arrives at
// a node that has been removed from the group, nothing happens.
// (previously, if the node also got votes, it would panic as it
// transitioned to StateLeader)
func TestTransferNonMember ( t * testing . T ) {
r := newTestRaft ( 1 , [ ] uint64 { 2 , 3 , 4 } , 5 , 1 , NewMemoryStorage ( ) )
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTimeoutNow } )
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgVoteResp } )
r . Step ( pb . Message { From : 3 , To : 1 , Type : pb . MsgVoteResp } )
if r . state != StateFollower {
t . Fatalf ( "state is %s, want StateFollower" , r . state )
}
}
2017-07-20 23:46:05 +03:00
// TestNodeWithSmallerTermCanCompleteElection tests the scenario where a node
// that has been partitioned away (and fallen behind) rejoins the cluster at
// about the same time the leader node gets partitioned away.
// Previously the cluster would come to a standstill when run with PreVote
// enabled.
func TestNodeWithSmallerTermCanCompleteElection ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
n1 . preVote = true
n2 . preVote = true
n3 . preVote = true
// cause a network partition to isolate node 3
nt := newNetwork ( n1 , n2 , n3 )
nt . cut ( 1 , 3 )
nt . cut ( 2 , 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "peer 1 state: %s, want %s" , sm . state , StateLeader )
}
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateFollower {
t . Errorf ( "peer 2 state: %s, want %s" , sm . state , StateFollower )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StatePreCandidate {
t . Errorf ( "peer 3 state: %s, want %s" , sm . state , StatePreCandidate )
}
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// check whether the term values are expected
// a.Term == 3
// b.Term == 3
// c.Term == 1
sm = nt . peers [ 1 ] . ( * raft )
if sm . Term != 3 {
t . Errorf ( "peer 1 term: %d, want %d" , sm . Term , 3 )
}
sm = nt . peers [ 2 ] . ( * raft )
if sm . Term != 3 {
t . Errorf ( "peer 2 term: %d, want %d" , sm . Term , 3 )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . Term != 1 {
t . Errorf ( "peer 3 term: %d, want %d" , sm . Term , 1 )
}
// check state
// a == follower
// b == leader
// c == pre-candidate
sm = nt . peers [ 1 ] . ( * raft )
if sm . state != StateFollower {
t . Errorf ( "peer 1 state: %s, want %s" , sm . state , StateFollower )
}
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "peer 2 state: %s, want %s" , sm . state , StateLeader )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StatePreCandidate {
t . Errorf ( "peer 3 state: %s, want %s" , sm . state , StatePreCandidate )
}
sm . logger . Infof ( "going to bring back peer 3 and kill peer 2" )
// recover the network then immediately isolate b which is currently
// the leader, this is to emulate the crash of b.
nt . recover ( )
nt . cut ( 2 , 1 )
nt . cut ( 2 , 3 )
// call for election
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// do we have a leader?
sma := nt . peers [ 1 ] . ( * raft )
smb := nt . peers [ 3 ] . ( * raft )
if sma . state != StateLeader && smb . state != StateLeader {
t . Errorf ( "no leader" )
}
}
2017-08-02 12:59:28 +03:00
// TestPreVoteWithSplitVote verifies that after split vote, cluster can complete
// election in next round.
func TestPreVoteWithSplitVote ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
n1 . preVote = true
n2 . preVote = true
n3 . preVote = true
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// simulate leader down. followers start split vote.
nt . isolate ( 1 )
nt . send ( [ ] pb . Message {
{ From : 2 , To : 2 , Type : pb . MsgHup } ,
{ From : 3 , To : 3 , Type : pb . MsgHup } ,
} ... )
// check whether the term values are expected
// n2.Term == 3
// n3.Term == 3
sm := nt . peers [ 2 ] . ( * raft )
if sm . Term != 3 {
t . Errorf ( "peer 2 term: %d, want %d" , sm . Term , 3 )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . Term != 3 {
t . Errorf ( "peer 3 term: %d, want %d" , sm . Term , 3 )
}
// check state
// n2 == candidate
// n3 == candidate
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateCandidate {
t . Errorf ( "peer 2 state: %s, want %s" , sm . state , StateCandidate )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateCandidate {
t . Errorf ( "peer 3 state: %s, want %s" , sm . state , StateCandidate )
}
// node 2 election timeout first
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// check whether the term values are expected
// n2.Term == 4
// n3.Term == 4
sm = nt . peers [ 2 ] . ( * raft )
if sm . Term != 4 {
t . Errorf ( "peer 2 term: %d, want %d" , sm . Term , 4 )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . Term != 4 {
t . Errorf ( "peer 3 term: %d, want %d" , sm . Term , 4 )
}
// check state
// n2 == leader
// n3 == follower
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "peer 2 state: %s, want %s" , sm . state , StateLeader )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateFollower {
t . Errorf ( "peer 3 state: %s, want %s" , sm . state , StateFollower )
}
}
2017-09-09 04:12:39 +03:00
// simulate rolling update a cluster for Pre-Vote. cluster has 3 nodes [n1, n2, n3].
// n1 is leader with term 2
// n2 is follower with term 2
// n3 is partitioned, with term 4 and less log, state is candidate
func newPreVoteMigrationCluster ( t * testing . T ) * network {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
n1 . preVote = true
n2 . preVote = true
// We intentionally do not enable PreVote for n3, this is done so in order
// to simulate a rolling restart process where it's possible to have a mixed
// version cluster with replicas with PreVote enabled, and replicas without.
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// Cause a network partition to isolate n3.
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StateCandidate
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateCandidate {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateCandidate )
}
// check term
// n1.Term == 2
// n2.Term == 2
// n3.Term == 4
if n1 . Term != 2 {
t . Fatalf ( "node 1 term: %d, want %d" , n1 . Term , 2 )
}
if n2 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n2 . Term , 2 )
}
if n3 . Term != 4 {
t . Fatalf ( "node 3 term: %d, want %d" , n3 . Term , 4 )
}
// Enable prevote on n3, then recover the network
n3 . preVote = true
nt . recover ( )
return nt
}
func TestPreVoteMigrationCanCompleteElection ( t * testing . T ) {
nt := newPreVoteMigrationCluster ( t )
// n1 is leader with term 2
// n2 is follower with term 2
// n3 is pre-candidate with term 4, and less log
n2 := nt . peers [ 2 ] . ( * raft )
n3 := nt . peers [ 3 ] . ( * raft )
// simulate leader down
nt . isolate ( 1 )
// Call for elections from both n2 and n3.
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// check state
// n2.state == Follower
// n3.state == PreCandidate
if n2 . state != StateFollower {
t . Errorf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StatePreCandidate {
t . Errorf ( "node 3 state: %s, want %s" , n3 . state , StatePreCandidate )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// Do we have a leader?
if n2 . state != StateLeader && n3 . state != StateFollower {
t . Errorf ( "no leader" )
}
}
func TestPreVoteMigrationWithFreeStuckPreCandidate ( t * testing . T ) {
nt := newPreVoteMigrationCluster ( t )
// n1 is leader with term 2
// n2 is follower with term 2
// n3 is pre-candidate with term 4, and less log
n1 := nt . peers [ 1 ] . ( * raft )
n2 := nt . peers [ 2 ] . ( * raft )
n3 := nt . peers [ 3 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if n1 . state != StateLeader {
t . Errorf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Errorf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StatePreCandidate {
t . Errorf ( "node 3 state: %s, want %s" , n3 . state , StatePreCandidate )
}
// Pre-Vote again for safety
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if n1 . state != StateLeader {
t . Errorf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Errorf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StatePreCandidate {
t . Errorf ( "node 3 state: %s, want %s" , n3 . state , StatePreCandidate )
}
nt . send ( pb . Message { From : 1 , To : 3 , Type : pb . MsgHeartbeat , Term : n1 . Term } )
// Disrupt the leader so that the stuck peer is freed
if n1 . state != StateFollower {
t . Errorf ( "state = %s, want %s" , n1 . state , StateFollower )
}
if n3 . Term != n1 . Term {
t . Errorf ( "term = %d, want %d" , n3 . Term , n1 . Term )
}
}
2016-12-03 12:29:15 +03:00
func entsWithConfig ( configFunc func ( * Config ) , terms ... uint64 ) * raft {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
2014-11-25 04:13:47 +03:00
for i , term := range terms {
2014-12-15 22:25:35 +03:00
storage . Append ( [ ] pb . Entry { { Index : uint64 ( i + 1 ) , Term : term } } )
2014-11-03 23:04:24 +03:00
}
2016-12-03 12:29:15 +03:00
cfg := newTestConfig ( 1 , [ ] uint64 { } , 5 , 1 , storage )
if configFunc != nil {
configFunc ( cfg )
}
sm := newRaft ( cfg )
2016-10-10 10:08:11 +03:00
sm . reset ( terms [ len ( terms ) - 1 ] )
2014-06-06 02:02:12 +04:00
return sm
2014-05-18 01:09:52 +04:00
}
2016-12-03 12:29:15 +03:00
// votedWithConfig creates a raft state machine with Vote and Term set
// to the given value but no log entries (indicating that it voted in
// the given term but has not received any logs).
func votedWithConfig ( configFunc func ( * Config ) , vote , term uint64 ) * raft {
storage := NewMemoryStorage ( )
storage . SetHardState ( pb . HardState { Vote : vote , Term : term } )
cfg := newTestConfig ( 1 , [ ] uint64 { } , 5 , 1 , storage )
if configFunc != nil {
configFunc ( cfg )
}
sm := newRaft ( cfg )
sm . reset ( term )
return sm
}
2014-05-16 21:11:21 +04:00
type network struct {
2016-02-01 08:42:39 +03:00
peers map [ uint64 ] stateMachine
2014-11-13 00:23:42 +03:00
storage map [ uint64 ] * MemoryStorage
2014-06-26 22:55:56 +04:00
dropm map [ connem ] float64
2014-10-12 11:34:22 +04:00
ignorem map [ pb . MessageType ] bool
2018-08-06 23:52:16 +03:00
// msgHook is called for each message sent. It may inspect the
// message and return true to send it or false to drop it.
msgHook func ( pb . Message ) bool
2014-05-16 21:11:21 +04:00
}
2014-05-06 10:28:14 +04:00
2014-06-18 04:35:19 +04:00
// newNetwork initializes a network from peers.
// A nil node will be replaced with a new *stateMachine.
// A *stateMachine will get its k, id.
2014-10-29 01:46:56 +03:00
// When using stateMachine, the address list is always [1, n].
2016-02-01 08:42:39 +03:00
func newNetwork ( peers ... stateMachine ) * network {
2016-10-10 09:32:40 +03:00
return newNetworkWithConfig ( nil , peers ... )
}
// newNetworkWithConfig is like newNetwork but calls the given func to
// modify the configuration of any state machines it creates.
func newNetworkWithConfig ( configFunc func ( * Config ) , peers ... stateMachine ) * network {
2014-06-18 04:35:19 +04:00
size := len ( peers )
2014-10-29 01:46:56 +03:00
peerAddrs := idsBySize ( size )
2014-06-10 03:45:42 +04:00
2016-02-01 08:42:39 +03:00
npeers := make ( map [ uint64 ] stateMachine , size )
2014-11-13 00:23:42 +03:00
nstorage := make ( map [ uint64 ] * MemoryStorage , size )
2014-06-18 04:35:19 +04:00
2015-02-18 00:15:48 +03:00
for j , p := range peers {
id := peerAddrs [ j ]
2014-06-06 02:02:12 +04:00
switch v := p . ( type ) {
2014-05-06 10:28:14 +04:00
case nil :
2014-11-13 00:23:42 +03:00
nstorage [ id ] = NewMemoryStorage ( )
2016-10-10 09:32:40 +03:00
cfg := newTestConfig ( id , peerAddrs , 10 , 1 , nstorage [ id ] )
if configFunc != nil {
configFunc ( cfg )
}
sm := newRaft ( cfg )
2014-09-10 03:33:09 +04:00
npeers [ id ] = sm
2014-08-23 00:24:33 +04:00
case * raft :
2017-11-11 05:38:21 +03:00
learners := make ( map [ uint64 ] bool , len ( v . learnerPrs ) )
for i := range v . learnerPrs {
learners [ i ] = true
}
2014-09-10 03:33:09 +04:00
v . id = id
2015-01-20 21:26:22 +03:00
v . prs = make ( map [ uint64 ] * Progress )
2017-11-11 05:38:21 +03:00
v . learnerPrs = make ( map [ uint64 ] * Progress )
2014-06-18 04:35:19 +04:00
for i := 0 ; i < size ; i ++ {
2017-11-11 05:38:21 +03:00
if _ , ok := learners [ peerAddrs [ i ] ] ; ok {
v . learnerPrs [ peerAddrs [ i ] ] = & Progress { IsLearner : true }
} else {
v . prs [ peerAddrs [ i ] ] = & Progress { }
}
2014-06-10 03:45:42 +04:00
}
2016-10-10 10:08:11 +03:00
v . reset ( v . Term )
2014-09-10 03:33:09 +04:00
npeers [ id ] = v
case * blackHole :
npeers [ id ] = v
2014-06-18 04:35:19 +04:00
default :
2014-09-12 10:18:51 +04:00
panic ( fmt . Sprintf ( "unexpected state machine type: %T" , p ) )
2014-05-06 10:28:14 +04:00
}
}
2014-06-26 22:55:56 +04:00
return & network {
peers : npeers ,
2014-11-13 00:23:42 +03:00
storage : nstorage ,
2014-06-26 22:55:56 +04:00
dropm : make ( map [ connem ] float64 ) ,
2014-10-12 11:34:22 +04:00
ignorem : make ( map [ pb . MessageType ] bool ) ,
2014-06-26 22:55:56 +04:00
}
2014-05-06 10:28:14 +04:00
}
2016-10-10 09:32:40 +03:00
func preVoteConfig ( c * Config ) {
c . PreVote = true
}
2014-08-28 05:53:18 +04:00
func ( nw * network ) send ( msgs ... pb . Message ) {
2014-06-06 02:02:12 +04:00
for len ( msgs ) > 0 {
m := msgs [ 0 ]
p := nw . peers [ m . To ]
p . Step ( m )
2014-10-30 00:04:45 +03:00
msgs = append ( msgs [ 1 : ] , nw . filter ( p . readMessages ( ) ) ... )
2014-05-16 21:11:21 +04:00
}
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) drop ( from , to uint64 , perc float64 ) {
2014-06-06 02:02:12 +04:00
nw . dropm [ connem { from , to } ] = perc
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) cut ( one , other uint64 ) {
2018-05-10 01:18:57 +03:00
nw . drop ( one , other , 2.0 ) // always drop
nw . drop ( other , one , 2.0 ) // always drop
2014-06-06 22:01:55 +04:00
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) isolate ( id uint64 ) {
2014-06-06 02:02:12 +04:00
for i := 0 ; i < len ( nw . peers ) ; i ++ {
2014-10-08 14:29:53 +04:00
nid := uint64 ( i ) + 1
2014-07-09 22:53:27 +04:00
if nid != id {
2018-05-10 01:18:57 +03:00
nw . drop ( id , nid , 1.0 ) // always drop
nw . drop ( nid , id , 1.0 ) // always drop
2014-05-06 10:28:14 +04:00
}
}
}
2014-10-12 11:34:22 +04:00
func ( nw * network ) ignore ( t pb . MessageType ) {
2014-06-26 22:55:56 +04:00
nw . ignorem [ t ] = true
}
2014-06-06 02:02:12 +04:00
func ( nw * network ) recover ( ) {
nw . dropm = make ( map [ connem ] float64 )
2014-10-12 11:34:22 +04:00
nw . ignorem = make ( map [ pb . MessageType ] bool )
2014-05-06 10:28:14 +04:00
}
2014-08-28 05:53:18 +04:00
func ( nw * network ) filter ( msgs [ ] pb . Message ) [ ] pb . Message {
2014-10-08 02:22:35 +04:00
mm := [ ] pb . Message { }
2014-06-06 02:02:12 +04:00
for _ , m := range msgs {
2014-06-26 22:55:56 +04:00
if nw . ignorem [ m . Type ] {
continue
}
2014-06-06 02:02:12 +04:00
switch m . Type {
2014-10-12 11:34:22 +04:00
case pb . MsgHup :
2014-06-06 02:02:12 +04:00
// hups never go over the network, so don't drop them but panic
panic ( "unexpected msgHup" )
2014-05-06 10:28:14 +04:00
default :
2014-06-06 02:02:12 +04:00
perc := nw . dropm [ connem { m . From , m . To } ]
if n := rand . Float64 ( ) ; n < perc {
2014-05-06 10:28:14 +04:00
continue
}
}
2018-08-06 23:52:16 +03:00
if nw . msgHook != nil {
if ! nw . msgHook ( m ) {
continue
}
}
2014-06-06 02:02:12 +04:00
mm = append ( mm , m )
2014-05-06 10:28:14 +04:00
}
2014-06-06 02:02:12 +04:00
return mm
2014-05-06 10:28:14 +04:00
}
2014-06-06 02:02:12 +04:00
type connem struct {
2014-10-08 14:29:53 +04:00
from , to uint64
2014-05-24 00:30:04 +04:00
}
2014-06-06 02:02:12 +04:00
type blackHole struct { }
2014-05-27 23:24:49 +04:00
2014-08-28 05:53:18 +04:00
func ( blackHole ) Step ( pb . Message ) error { return nil }
2014-10-30 00:04:45 +03:00
func ( blackHole ) readMessages ( ) [ ] pb . Message { return nil }
2014-06-06 02:02:12 +04:00
var nopStepper = & blackHole { }
2014-10-29 01:46:56 +03:00
func idsBySize ( size int ) [ ] uint64 {
ids := make ( [ ] uint64 , size )
for i := 0 ; i < size ; i ++ {
ids [ i ] = 1 + uint64 ( i )
}
return ids
}
2015-03-22 04:15:58 +03:00
2016-08-02 03:46:43 +03:00
// setRandomizedElectionTimeout set up the value by caller instead of choosing
// by system, in some test scenario we need to fill in some expected value to
// ensure the certainty
func setRandomizedElectionTimeout ( r * raft , v int ) {
r . randomizedElectionTimeout = v
}
2015-03-24 22:36:42 +03:00
func newTestConfig ( id uint64 , peers [ ] uint64 , election , heartbeat int , storage Storage ) * Config {
return & Config {
2015-03-22 04:15:58 +03:00
ID : id ,
2015-03-24 21:10:07 +03:00
peers : peers ,
2015-03-22 04:15:58 +03:00
ElectionTick : election ,
HeartbeatTick : heartbeat ,
Storage : storage ,
MaxSizePerMsg : noLimit ,
MaxInflightMsgs : 256 ,
}
2015-03-24 22:36:42 +03:00
}
2015-03-22 04:15:58 +03:00
2015-03-24 22:36:42 +03:00
func newTestRaft ( id uint64 , peers [ ] uint64 , election , heartbeat int , storage Storage ) * raft {
return newRaft ( newTestConfig ( id , peers , election , heartbeat , storage ) )
2015-03-22 04:15:58 +03:00
}
2017-11-11 05:38:21 +03:00
func newTestLearnerRaft ( id uint64 , peers [ ] uint64 , learners [ ] uint64 , election , heartbeat int , storage Storage ) * raft {
cfg := newTestConfig ( id , peers , election , heartbeat , storage )
cfg . learners = learners
return newRaft ( cfg )
}