2016-05-13 06:49:15 +03:00
// Copyright 2015 The etcd Authors
2015-01-25 06:19:16 +03:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2014-10-18 02:41:22 +04:00
2014-05-06 10:28:14 +04:00
package raft
import (
2014-05-24 20:53:01 +04:00
"bytes"
2014-09-12 10:18:51 +04:00
"fmt"
2014-10-07 16:34:15 +04:00
"math"
2014-06-06 02:02:12 +04:00
"math/rand"
2014-07-01 23:10:43 +04:00
"reflect"
2018-08-07 06:30:07 +03:00
"strings"
2014-05-06 10:28:14 +04:00
"testing"
2014-08-28 05:53:18 +04:00
2019-05-29 00:59:02 +03:00
pb "go.etcd.io/etcd/raft/raftpb"
2019-06-08 20:41:24 +03:00
"go.etcd.io/etcd/raft/tracker"
2014-05-06 10:28:14 +04:00
)
2014-08-24 04:56:06 +04:00
// nextEnts returns the appliable entries and updates the applied index
2014-11-13 00:23:42 +03:00
func nextEnts ( r * raft , s * MemoryStorage ) ( ents [ ] pb . Entry ) {
2014-11-03 23:04:24 +03:00
// Transfer all unstable entries to "stable" storage.
2014-11-13 00:23:42 +03:00
s . Append ( r . raftLog . unstableEntries ( ) )
2014-11-29 01:13:07 +03:00
r . raftLog . stableTo ( r . raftLog . lastIndex ( ) , r . raftLog . lastTerm ( ) )
2014-11-03 23:04:24 +03:00
2014-08-24 04:56:06 +04:00
ents = r . raftLog . nextEnts ( )
2014-11-07 01:13:35 +03:00
r . raftLog . appliedTo ( r . raftLog . committed )
2014-08-24 04:56:06 +04:00
return ents
}
2018-10-22 12:28:39 +03:00
func mustAppendEntry ( r * raft , ents ... pb . Entry ) {
if ! r . appendEntry ( ents ... ) {
panic ( "entry unexpectedly dropped" )
}
}
2016-02-01 08:42:39 +03:00
type stateMachine interface {
2014-08-28 05:53:18 +04:00
Step ( m pb . Message ) error
2014-10-30 00:04:45 +03:00
readMessages ( ) [ ] pb . Message
}
func ( r * raft ) readMessages ( ) [ ] pb . Message {
msgs := r . msgs
r . msgs = make ( [ ] pb . Message , 0 )
return msgs
2014-08-23 00:24:33 +04:00
}
2018-11-23 19:57:36 +03:00
func TestProgressLeader ( t * testing . T ) {
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
r . becomeCandidate ( )
r . becomeLeader ( )
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . BecomeReplicate ( )
2018-11-23 19:57:36 +03:00
// Send proposals to r1. The first 5 entries should be appended to the log.
propMsg := pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "foo" ) } } }
for i := 0 ; i < 5 ; i ++ {
2019-06-08 20:41:24 +03:00
if pr := r . prs . Progress [ r . id ] ; pr . State != tracker . StateReplicate || pr . Match != uint64 ( i + 1 ) || pr . Next != pr . Match + 1 {
2018-11-23 19:57:36 +03:00
t . Errorf ( "unexpected progress %v" , pr )
}
if err := r . Step ( propMsg ) ; err != nil {
t . Fatalf ( "proposal resulted in error: %v" , err )
}
}
}
2017-01-09 10:42:54 +03:00
// TestProgressResumeByHeartbeatResp ensures raft.heartbeat reset progress.paused by heartbeat response.
2016-12-19 23:24:21 +03:00
func TestProgressResumeByHeartbeatResp ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
2014-12-18 23:02:15 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
2018-11-23 19:57:36 +03:00
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . ProbeSent = true
2014-12-18 23:02:15 +03:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
2019-06-08 20:41:24 +03:00
if ! r . prs . Progress [ 2 ] . ProbeSent {
t . Errorf ( "paused = %v, want true" , r . prs . Progress [ 2 ] . ProbeSent )
2016-12-19 23:24:21 +03:00
}
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . BecomeReplicate ( )
2016-12-19 23:24:21 +03:00
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeatResp } )
2019-06-08 20:41:24 +03:00
if r . prs . Progress [ 2 ] . ProbeSent {
t . Errorf ( "paused = %v, want false" , r . prs . Progress [ 2 ] . ProbeSent )
2014-12-18 23:02:15 +03:00
}
}
2015-03-16 10:46:16 +03:00
func TestProgressPaused ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
2014-12-18 23:02:15 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
ms := r . readMessages ( )
if len ( ms ) != 1 {
t . Errorf ( "len(ms) = %d, want 1" , len ( ms ) )
}
}
2018-08-07 06:30:07 +03:00
func TestProgressFlowControl ( t * testing . T ) {
cfg := newTestConfig ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
cfg . MaxInflightMsgs = 3
cfg . MaxSizePerMsg = 2048
r := newRaft ( cfg )
r . becomeCandidate ( )
r . becomeLeader ( )
// Throw away all the messages relating to the initial election.
r . readMessages ( )
// While node 2 is in probe state, propose a bunch of entries.
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . BecomeProbe ( )
2018-08-07 06:30:07 +03:00
blob := [ ] byte ( strings . Repeat ( "a" , 1000 ) )
for i := 0 ; i < 10 ; i ++ {
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : blob } } } )
}
ms := r . readMessages ( )
// First append has two entries: the empty entry to confirm the
// election, and the first proposal (only one proposal gets sent
// because we're in probe state).
if len ( ms ) != 1 || ms [ 0 ] . Type != pb . MsgApp {
t . Fatalf ( "expected 1 MsgApp, got %v" , ms )
}
if len ( ms [ 0 ] . Entries ) != 2 {
t . Fatalf ( "expected 2 entries, got %d" , len ( ms [ 0 ] . Entries ) )
}
if len ( ms [ 0 ] . Entries [ 0 ] . Data ) != 0 || len ( ms [ 0 ] . Entries [ 1 ] . Data ) != 1000 {
t . Fatalf ( "unexpected entry sizes: %v" , ms [ 0 ] . Entries )
}
// When this append is acked, we change to replicate state and can
// send multiple messages at once.
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgAppResp , Index : ms [ 0 ] . Entries [ 1 ] . Index } )
ms = r . readMessages ( )
if len ( ms ) != 3 {
t . Fatalf ( "expected 3 messages, got %d" , len ( ms ) )
}
for i , m := range ms {
if m . Type != pb . MsgApp {
t . Errorf ( "%d: expected MsgApp, got %s" , i , m . Type )
}
if len ( m . Entries ) != 2 {
t . Errorf ( "%d: expected 2 entries, got %d" , i , len ( m . Entries ) )
}
}
// Ack all three of those messages together and get the last two
// messages (containing three entries).
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgAppResp , Index : ms [ 2 ] . Entries [ 1 ] . Index } )
ms = r . readMessages ( )
if len ( ms ) != 2 {
t . Fatalf ( "expected 2 messages, got %d" , len ( ms ) )
}
for i , m := range ms {
if m . Type != pb . MsgApp {
t . Errorf ( "%d: expected MsgApp, got %s" , i , m . Type )
}
}
if len ( ms [ 0 ] . Entries ) != 2 {
t . Errorf ( "%d: expected 2 entries, got %d" , 0 , len ( ms [ 0 ] . Entries ) )
}
if len ( ms [ 1 ] . Entries ) != 1 {
t . Errorf ( "%d: expected 1 entry, got %d" , 1 , len ( ms [ 1 ] . Entries ) )
}
}
2018-10-09 19:40:58 +03:00
func TestUncommittedEntryLimit ( t * testing . T ) {
2018-10-22 12:28:39 +03:00
// Use a relatively large number of entries here to prevent regression of a
// bug which computed the size before it was fixed. This test would fail
// with the bug, either because we'd get dropped proposals earlier than we
// expect them, or because the final tally ends up nonzero. (At the time of
// writing, the former).
const maxEntries = 1024
2018-10-09 19:40:58 +03:00
testEntry := pb . Entry { Data : [ ] byte ( "testdata" ) }
2018-10-22 12:28:39 +03:00
maxEntrySize := maxEntries * PayloadSize ( testEntry )
2018-10-09 19:40:58 +03:00
cfg := newTestConfig ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
cfg . MaxUncommittedEntriesSize = uint64 ( maxEntrySize )
2018-10-22 12:28:39 +03:00
cfg . MaxInflightMsgs = 2 * 1024 // avoid interference
2018-10-09 19:40:58 +03:00
r := newRaft ( cfg )
r . becomeCandidate ( )
r . becomeLeader ( )
2018-10-22 12:28:39 +03:00
if n := r . uncommittedSize ; n != 0 {
t . Fatalf ( "expected zero uncommitted size, got %d bytes" , n )
}
2018-10-09 19:40:58 +03:00
// Set the two followers to the replicate state. Commit to tail of log.
const numFollowers = 2
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . BecomeReplicate ( )
r . prs . Progress [ 3 ] . BecomeReplicate ( )
2018-10-09 19:40:58 +03:00
r . uncommittedSize = 0
// Send proposals to r1. The first 5 entries should be appended to the log.
propMsg := pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { testEntry } }
propEnts := make ( [ ] pb . Entry , maxEntries )
for i := 0 ; i < maxEntries ; i ++ {
if err := r . Step ( propMsg ) ; err != nil {
t . Fatalf ( "proposal resulted in error: %v" , err )
}
propEnts [ i ] = testEntry
}
// Send one more proposal to r1. It should be rejected.
if err := r . Step ( propMsg ) ; err != ErrProposalDropped {
t . Fatalf ( "proposal not dropped: %v" , err )
}
// Read messages and reduce the uncommitted size as if we had committed
// these entries.
ms := r . readMessages ( )
if e := maxEntries * numFollowers ; len ( ms ) != e {
t . Fatalf ( "expected %d messages, got %d" , e , len ( ms ) )
}
r . reduceUncommittedSize ( propEnts )
2018-10-22 12:28:39 +03:00
if r . uncommittedSize != 0 {
t . Fatalf ( "committed everything, but still tracking %d" , r . uncommittedSize )
}
2018-10-09 19:40:58 +03:00
// Send a single large proposal to r1. Should be accepted even though it
// pushes us above the limit because we were beneath it before the proposal.
propEnts = make ( [ ] pb . Entry , 2 * maxEntries )
for i := range propEnts {
propEnts [ i ] = testEntry
}
propMsgLarge := pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : propEnts }
if err := r . Step ( propMsgLarge ) ; err != nil {
t . Fatalf ( "proposal resulted in error: %v" , err )
}
// Send one more proposal to r1. It should be rejected, again.
if err := r . Step ( propMsg ) ; err != ErrProposalDropped {
t . Fatalf ( "proposal not dropped: %v" , err )
}
// Read messages and reduce the uncommitted size as if we had committed
// these entries.
ms = r . readMessages ( )
if e := 1 * numFollowers ; len ( ms ) != e {
t . Fatalf ( "expected %d messages, got %d" , e , len ( ms ) )
}
r . reduceUncommittedSize ( propEnts )
2018-10-22 12:28:39 +03:00
if n := r . uncommittedSize ; n != 0 {
t . Fatalf ( "expected zero uncommitted size, got %d" , n )
}
2018-10-09 19:40:58 +03:00
}
2014-05-06 10:28:14 +04:00
func TestLeaderElection ( t * testing . T ) {
2016-10-25 17:31:44 +03:00
testLeaderElection ( t , false )
}
func TestLeaderElectionPreVote ( t * testing . T ) {
testLeaderElection ( t , true )
}
func testLeaderElection ( t * testing . T , preVote bool ) {
var cfg func ( * Config )
2018-07-05 20:12:45 +03:00
candState := StateCandidate
2018-05-01 12:05:08 +03:00
candTerm := uint64 ( 1 )
2016-10-25 17:31:44 +03:00
if preVote {
cfg = preVoteConfig
2018-05-01 12:05:08 +03:00
// In pre-vote mode, an election that fails to complete
// leaves the node in pre-candidate state without advancing
// the term.
candState = StatePreCandidate
candTerm = 0
2016-10-25 17:31:44 +03:00
}
tests := [ ] struct {
* network
state StateType
expTerm uint64
} {
{ newNetworkWithConfig ( cfg , nil , nil , nil ) , StateLeader , 1 } ,
{ newNetworkWithConfig ( cfg , nil , nil , nopStepper ) , StateLeader , 1 } ,
2018-05-01 12:05:08 +03:00
{ newNetworkWithConfig ( cfg , nil , nopStepper , nopStepper ) , candState , candTerm } ,
{ newNetworkWithConfig ( cfg , nil , nopStepper , nopStepper , nil ) , candState , candTerm } ,
2016-10-25 17:31:44 +03:00
{ newNetworkWithConfig ( cfg , nil , nopStepper , nopStepper , nil , nil ) , StateLeader , 1 } ,
// three logs further along than 0, but in the same term so rejections
// are returned instead of the votes being ignored.
2016-12-03 12:29:15 +03:00
{ newNetworkWithConfig ( cfg ,
nil , entsWithConfig ( cfg , 1 ) , entsWithConfig ( cfg , 1 ) , entsWithConfig ( cfg , 1 , 1 ) , nil ) ,
StateFollower , 1 } ,
2016-10-25 17:31:44 +03:00
}
for i , tt := range tests {
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
sm := tt . network . peers [ 1 ] . ( * raft )
2018-05-01 12:05:08 +03:00
if sm . state != tt . state {
t . Errorf ( "#%d: state = %s, want %s" , i , sm . state , tt . state )
2016-10-25 17:31:44 +03:00
}
2018-05-01 12:05:08 +03:00
if g := sm . Term ; g != tt . expTerm {
t . Errorf ( "#%d: term = %d, want %d" , i , g , tt . expTerm )
2016-10-10 09:32:40 +03:00
}
2014-05-06 10:28:14 +04:00
}
2016-10-10 09:32:40 +03:00
}
2014-05-06 10:28:14 +04:00
2017-11-11 05:38:21 +03:00
// TestLearnerElectionTimeout verfies that the leader should not start election even
// when times out.
func TestLearnerElectionTimeout ( t * testing . T ) {
n1 := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
// n2 is learner. Learner should not start election even when times out.
setRandomizedElectionTimeout ( n2 , n2 . electionTimeout )
for i := 0 ; i < n2 . electionTimeout ; i ++ {
n2 . tick ( )
}
if n2 . state != StateFollower {
t . Errorf ( "peer 2 state: %s, want %s" , n2 . state , StateFollower )
}
}
2018-01-03 04:34:12 +03:00
// TestLearnerPromotion verifies that the learner should not election until
2017-11-11 05:38:21 +03:00
// it is promoted to a normal peer.
func TestLearnerPromotion ( t * testing . T ) {
n1 := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
nt := newNetwork ( n1 , n2 )
if n1 . state == StateLeader {
t . Error ( "peer 1 state is leader, want not" , n1 . state )
}
// n1 should become leader
setRandomizedElectionTimeout ( n1 , n1 . electionTimeout )
for i := 0 ; i < n1 . electionTimeout ; i ++ {
n1 . tick ( )
}
if n1 . state != StateLeader {
t . Errorf ( "peer 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Errorf ( "peer 2 state: %s, want %s" , n2 . state , StateFollower )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
2019-07-22 23:30:47 +03:00
n1 . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeAddNode } . AsV2 ( ) )
n2 . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeAddNode } . AsV2 ( ) )
2017-11-11 05:38:21 +03:00
if n2 . isLearner {
t . Error ( "peer 2 is learner, want not" )
}
// n2 start election, should become leader
setRandomizedElectionTimeout ( n2 , n2 . electionTimeout )
for i := 0 ; i < n2 . electionTimeout ; i ++ {
n2 . tick ( )
}
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgBeat } )
if n1 . state != StateFollower {
t . Errorf ( "peer 1 state: %s, want %s" , n1 . state , StateFollower )
}
if n2 . state != StateLeader {
t . Errorf ( "peer 2 state: %s, want %s" , n2 . state , StateLeader )
}
}
raft: let learners vote
It turns out that that learners must be allowed to cast votes.
This seems counter- intuitive but is necessary in the situation in which
a learner has been promoted (i.e. is now a voter) but has not learned
about this yet.
For example, consider a group in which id=1 is a learner and id=2 and
id=3 are voters. A configuration change promoting 1 can be committed on
the quorum `{2,3}` without the config change being appended to the
learner's log. If the leader (say 2) fails, there are de facto two
voters remaining. Only 3 can win an election (due to its log containing
all committed entries), but to do so it will need 1 to vote. But 1
considers itself a learner and will continue to do so until 3 has
stepped up as leader, replicates the conf change to 1, and 1 applies it.
Ultimately, by receiving a request to vote, the learner realizes that
the candidate believes it to be a voter, and that it should act
accordingly. The candidate's config may be stale, too; but in that case
it won't win the election, at least in the absence of the bug discussed
in:
https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263.
2019-08-07 13:03:18 +03:00
// TestLearnerCanVote checks that a learner can vote when it receives a valid Vote request.
// See (*raft).Step for why this is necessary and correct behavior.
func TestLearnerCanVote ( t * testing . T ) {
2017-11-11 05:38:21 +03:00
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 . becomeFollower ( 1 , None )
n2 . Step ( pb . Message { From : 1 , To : 2 , Term : 2 , Type : pb . MsgVote , LogTerm : 11 , Index : 11 } )
raft: let learners vote
It turns out that that learners must be allowed to cast votes.
This seems counter- intuitive but is necessary in the situation in which
a learner has been promoted (i.e. is now a voter) but has not learned
about this yet.
For example, consider a group in which id=1 is a learner and id=2 and
id=3 are voters. A configuration change promoting 1 can be committed on
the quorum `{2,3}` without the config change being appended to the
learner's log. If the leader (say 2) fails, there are de facto two
voters remaining. Only 3 can win an election (due to its log containing
all committed entries), but to do so it will need 1 to vote. But 1
considers itself a learner and will continue to do so until 3 has
stepped up as leader, replicates the conf change to 1, and 1 applies it.
Ultimately, by receiving a request to vote, the learner realizes that
the candidate believes it to be a voter, and that it should act
accordingly. The candidate's config may be stale, too; but in that case
it won't win the election, at least in the absence of the bug discussed
in:
https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263.
2019-08-07 13:03:18 +03:00
if len ( n2 . msgs ) != 1 {
t . Fatalf ( "expected exactly one message, not %+v" , n2 . msgs )
}
msg := n2 . msgs [ 0 ]
if msg . Type != pb . MsgVoteResp && ! msg . Reject {
t . Fatal ( "expected learner to not reject vote" )
2017-11-11 05:38:21 +03:00
}
}
2016-10-25 17:31:44 +03:00
func TestLeaderCycle ( t * testing . T ) {
testLeaderCycle ( t , false )
}
func TestLeaderCyclePreVote ( t * testing . T ) {
testLeaderCycle ( t , true )
}
// testLeaderCycle verifies that each node in a cluster can campaign
2016-10-10 09:32:40 +03:00
// and be elected in turn. This ensures that elections (including
// pre-vote) work when not starting from a clean slate (as they do in
// TestLeaderElection)
2016-10-25 17:31:44 +03:00
func testLeaderCycle ( t * testing . T , preVote bool ) {
var cfg func ( * Config )
if preVote {
cfg = preVoteConfig
}
n := newNetworkWithConfig ( cfg , nil , nil , nil )
for campaignerID := uint64 ( 1 ) ; campaignerID <= 3 ; campaignerID ++ {
n . send ( pb . Message { From : campaignerID , To : campaignerID , Type : pb . MsgHup } )
for _ , peer := range n . peers {
sm := peer . ( * raft )
if sm . id == campaignerID && sm . state != StateLeader {
t . Errorf ( "preVote=%v: campaigning node %d state = %v, want StateLeader" ,
preVote , sm . id , sm . state )
} else if sm . id != campaignerID && sm . state != StateFollower {
t . Errorf ( "preVote=%v: after campaign of node %d, " +
"node %d had state = %v, want StateFollower" ,
preVote , campaignerID , sm . id , sm . state )
2016-10-10 09:32:40 +03:00
}
2014-05-06 10:28:14 +04:00
}
}
}
2016-12-03 12:29:15 +03:00
// TestLeaderElectionOverwriteNewerLogs tests a scenario in which a
// newly-elected leader does *not* have the newest (i.e. highest term)
// log entries, and must overwrite higher-term log entries with
// lower-term ones.
func TestLeaderElectionOverwriteNewerLogs ( t * testing . T ) {
testLeaderElectionOverwriteNewerLogs ( t , false )
}
func TestLeaderElectionOverwriteNewerLogsPreVote ( t * testing . T ) {
testLeaderElectionOverwriteNewerLogs ( t , true )
}
func testLeaderElectionOverwriteNewerLogs ( t * testing . T , preVote bool ) {
var cfg func ( * Config )
if preVote {
cfg = preVoteConfig
}
// This network represents the results of the following sequence of
// events:
// - Node 1 won the election in term 1.
// - Node 1 replicated a log entry to node 2 but died before sending
// it to other nodes.
// - Node 3 won the second election in term 2.
// - Node 3 wrote an entry to its logs but died without sending it
// to any other nodes.
//
// At this point, nodes 1, 2, and 3 all have uncommitted entries in
// their logs and could win an election at term 3. The winner's log
// entry overwrites the losers'. (TestLeaderSyncFollowerLog tests
// the case where older log entries are overwritten, so this test
// focuses on the case where the newer entries are lost).
n := newNetworkWithConfig ( cfg ,
entsWithConfig ( cfg , 1 ) , // Node 1: Won first election
entsWithConfig ( cfg , 1 ) , // Node 2: Got logs from node 1
entsWithConfig ( cfg , 2 ) , // Node 3: Won second election
votedWithConfig ( cfg , 3 , 2 ) , // Node 4: Voted but didn't get logs
votedWithConfig ( cfg , 3 , 2 ) ) // Node 5: Voted but didn't get logs
// Node 1 campaigns. The election fails because a quorum of nodes
// know about the election that already happened at term 2. Node 1's
// term is pushed ahead to 2.
n . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
sm1 := n . peers [ 1 ] . ( * raft )
if sm1 . state != StateFollower {
t . Errorf ( "state = %s, want StateFollower" , sm1 . state )
}
if sm1 . Term != 2 {
t . Errorf ( "term = %d, want 2" , sm1 . Term )
}
// Node 1 campaigns again with a higher term. This time it succeeds.
n . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if sm1 . state != StateLeader {
t . Errorf ( "state = %s, want StateLeader" , sm1 . state )
}
if sm1 . Term != 3 {
t . Errorf ( "term = %d, want 3" , sm1 . Term )
}
// Now all nodes agree on a log entry with term 1 at index 1 (and
// term 3 at index 2).
for i := range n . peers {
sm := n . peers [ i ] . ( * raft )
entries := sm . raftLog . allEntries ( )
if len ( entries ) != 2 {
t . Fatalf ( "node %d: len(entries) == %d, want 2" , i , len ( entries ) )
}
if entries [ 0 ] . Term != 1 {
t . Errorf ( "node %d: term at index 1 == %d, want 1" , i , entries [ 0 ] . Term )
}
if entries [ 1 ] . Term != 3 {
t . Errorf ( "node %d: term at index 2 == %d, want 3" , i , entries [ 1 ] . Term )
}
}
}
2016-10-19 06:07:43 +03:00
func TestVoteFromAnyState ( t * testing . T ) {
2016-10-25 17:31:44 +03:00
testVoteFromAnyState ( t , pb . MsgVote )
}
2016-10-19 06:07:43 +03:00
2016-10-25 17:31:44 +03:00
func TestPreVoteFromAnyState ( t * testing . T ) {
testVoteFromAnyState ( t , pb . MsgPreVote )
}
func testVoteFromAnyState ( t * testing . T , vt pb . MessageType ) {
for st := StateType ( 0 ) ; st < numStates ; st ++ {
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
r . Term = 1
switch st {
case StateFollower :
r . becomeFollower ( r . Term , 3 )
case StatePreCandidate :
r . becomePreCandidate ( )
case StateCandidate :
r . becomeCandidate ( )
case StateLeader :
r . becomeCandidate ( )
r . becomeLeader ( )
}
// Note that setting our state above may have advanced r.Term
// past its initial value.
origTerm := r . Term
newTerm := r . Term + 1
msg := pb . Message {
From : 2 ,
To : 1 ,
Type : vt ,
Term : newTerm ,
LogTerm : newTerm ,
Index : 42 ,
}
if err := r . Step ( msg ) ; err != nil {
t . Errorf ( "%s,%s: Step failed: %s" , vt , st , err )
}
if len ( r . msgs ) != 1 {
t . Errorf ( "%s,%s: %d response messages, want 1: %+v" , vt , st , len ( r . msgs ) , r . msgs )
} else {
resp := r . msgs [ 0 ]
if resp . Type != voteRespMsgType ( vt ) {
t . Errorf ( "%s,%s: response message is %s, want %s" ,
vt , st , resp . Type , voteRespMsgType ( vt ) )
}
if resp . Reject {
t . Errorf ( "%s,%s: unexpected rejection" , vt , st )
2016-10-19 06:07:43 +03:00
}
2016-10-25 17:31:44 +03:00
}
2016-10-19 06:07:43 +03:00
2016-10-25 17:31:44 +03:00
// If this was a real vote, we reset our state and term.
if vt == pb . MsgVote {
if r . state != StateFollower {
2016-12-26 09:31:59 +03:00
t . Errorf ( "%s,%s: state %s, want %s" , vt , st , r . state , StateFollower )
2016-10-19 06:07:43 +03:00
}
2016-10-25 17:31:44 +03:00
if r . Term != newTerm {
t . Errorf ( "%s,%s: term %d, want %d" , vt , st , r . Term , newTerm )
2016-10-19 06:07:43 +03:00
}
2016-10-25 17:31:44 +03:00
if r . Vote != 2 {
t . Errorf ( "%s,%s: vote %d, want 2" , vt , st , r . Vote )
2016-10-19 06:07:43 +03:00
}
2016-10-25 17:31:44 +03:00
} else {
// In a prevote, nothing changes.
if r . state != st {
t . Errorf ( "%s,%s: state %s, want %s" , vt , st , r . state , st )
}
if r . Term != origTerm {
t . Errorf ( "%s,%s: term %d, want %d" , vt , st , r . Term , origTerm )
}
// if st == StateFollower or StatePreCandidate, r hasn't voted yet.
// In StateCandidate or StateLeader, it's voted for itself.
if r . Vote != None && r . Vote != 1 {
t . Errorf ( "%s,%s: vote %d, want %d or 1" , vt , st , r . Vote , None )
2016-10-19 06:07:43 +03:00
}
}
}
}
2014-05-24 20:53:01 +04:00
func TestLogReplication ( t * testing . T ) {
tests := [ ] struct {
* network
2014-08-28 05:53:18 +04:00
msgs [ ] pb . Message
2014-10-08 14:29:53 +04:00
wcommitted uint64
2014-05-24 20:53:01 +04:00
} {
{
newNetwork ( nil , nil , nil ) ,
2014-08-28 05:53:18 +04:00
[ ] pb . Message {
2014-10-12 11:34:22 +04:00
{ From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
2014-05-24 20:53:01 +04:00
} ,
2014-06-26 22:55:56 +04:00
2 ,
2014-05-24 20:53:01 +04:00
} ,
{
newNetwork ( nil , nil , nil ) ,
2014-08-28 05:53:18 +04:00
[ ] pb . Message {
2014-10-12 11:34:22 +04:00
{ From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
{ From : 1 , To : 2 , Type : pb . MsgHup } ,
{ From : 1 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } ,
2014-05-24 20:53:01 +04:00
} ,
2014-06-26 22:55:56 +04:00
4 ,
2014-05-24 20:53:01 +04:00
} ,
}
for i , tt := range tests {
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-24 20:53:01 +04:00
for _ , m := range tt . msgs {
2014-06-06 02:02:12 +04:00
tt . send ( m )
2014-05-24 20:53:01 +04:00
}
2014-06-06 02:02:12 +04:00
for j , x := range tt . network . peers {
2014-08-23 00:24:33 +04:00
sm := x . ( * raft )
2014-05-24 20:53:01 +04:00
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != tt . wcommitted {
t . Errorf ( "#%d.%d: committed = %d, want %d" , i , j , sm . raftLog . committed , tt . wcommitted )
2014-05-24 20:53:01 +04:00
}
2014-10-08 02:22:35 +04:00
ents := [ ] pb . Entry { }
2014-11-13 00:23:42 +03:00
for _ , e := range nextEnts ( sm , tt . network . storage [ j ] ) {
2014-06-26 22:55:56 +04:00
if e . Data != nil {
ents = append ( ents , e )
}
}
2014-10-08 02:22:35 +04:00
props := [ ] pb . Message { }
2014-05-24 20:53:01 +04:00
for _ , m := range tt . msgs {
2014-10-12 11:34:22 +04:00
if m . Type == pb . MsgProp {
2014-05-24 20:53:01 +04:00
props = append ( props , m )
}
}
for k , m := range props {
2014-06-06 00:00:18 +04:00
if ! bytes . Equal ( ents [ k ] . Data , m . Entries [ 0 ] . Data ) {
t . Errorf ( "#%d.%d: data = %d, want %d" , i , j , ents [ k ] . Data , m . Entries [ 0 ] . Data )
2014-05-24 20:53:01 +04:00
}
}
}
}
}
2017-11-11 05:38:21 +03:00
// TestLearnerLogReplication tests that a learner can receive entries from the leader.
func TestLearnerLogReplication ( t * testing . T ) {
n1 := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
nt := newNetwork ( n1 , n2 )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
setRandomizedElectionTimeout ( n1 , n1 . electionTimeout )
for i := 0 ; i < n1 . electionTimeout ; i ++ {
n1 . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
// n1 is leader and n2 is learner
if n1 . state != StateLeader {
t . Errorf ( "peer 1 state: %s, want %s" , n1 . state , StateLeader )
}
if ! n2 . isLearner {
t . Error ( "peer 2 state: not learner, want yes" )
}
nextCommitted := n1 . raftLog . committed + 1
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
if n1 . raftLog . committed != nextCommitted {
t . Errorf ( "peer 1 wants committed to %d, but still %d" , nextCommitted , n1 . raftLog . committed )
}
if n1 . raftLog . committed != n2 . raftLog . committed {
t . Errorf ( "peer 2 wants committed to %d, but still %d" , n1 . raftLog . committed , n2 . raftLog . committed )
}
2019-06-08 20:41:24 +03:00
match := n1 . prs . Progress [ 2 ] . Match
2017-11-11 05:38:21 +03:00
if match != n2 . raftLog . committed {
2018-02-26 21:03:25 +03:00
t . Errorf ( "progress 2 of leader 1 wants match %d, but got %d" , n2 . raftLog . committed , match )
2017-11-11 05:38:21 +03:00
}
}
2014-05-29 02:30:01 +04:00
func TestSingleNodeCommit ( t * testing . T ) {
tt := newNetwork ( nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-05-29 02:30:01 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 3 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 3 )
2014-05-29 02:30:01 +04:00
}
}
2014-06-26 22:55:56 +04:00
// TestCannotCommitWithoutNewTermEntry tests the entries cannot be committed
// when leader changes, no new proposal comes in and ChangeTerm proposal is
// filtered.
2014-06-06 21:26:44 +04:00
func TestCannotCommitWithoutNewTermEntry ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil , nil , nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-06-06 21:26:44 +04:00
// 0 cannot reach 2,3,4
2014-09-12 06:23:05 +04:00
tt . cut ( 1 , 3 )
tt . cut ( 1 , 4 )
tt . cut ( 1 , 5 )
2014-06-06 21:26:44 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-06-06 21:26:44 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-06 21:26:44 +04:00
}
// network recovery
tt . recover ( )
2014-06-26 22:55:56 +04:00
// avoid committing ChangeTerm proposal
2014-10-12 11:34:22 +04:00
tt . ignore ( pb . MsgApp )
2014-06-06 21:26:44 +04:00
2014-12-18 23:02:15 +03:00
// elect 2 as the new leader with term 2
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
2014-06-06 21:26:44 +04:00
// no log entries from previous term should be committed
2014-09-12 06:23:05 +04:00
sm = tt . peers [ 2 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-06 21:26:44 +04:00
}
2014-06-26 22:55:56 +04:00
tt . recover ( )
2014-12-18 23:02:15 +03:00
// send heartbeat; reset wait
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgBeat } )
// append an entry at current term
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-12-18 23:02:15 +03:00
// expect the committed to be advanced
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 5 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 5 )
2014-06-26 22:55:56 +04:00
}
}
// TestCommitWithoutNewTermEntry tests the entries could be committed
// when leader changes, no new proposal comes in.
func TestCommitWithoutNewTermEntry ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil , nil , nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-06-26 22:55:56 +04:00
// 0 cannot reach 2,3,4
2014-09-12 06:23:05 +04:00
tt . cut ( 1 , 3 )
tt . cut ( 1 , 4 )
tt . cut ( 1 , 5 )
2014-06-26 22:55:56 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
2014-06-26 22:55:56 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 1 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 1 )
2014-06-26 22:55:56 +04:00
}
// network recovery
tt . recover ( )
2018-10-22 08:59:50 +03:00
// elect 2 as the new leader with term 2
2014-06-26 22:55:56 +04:00
// after append a ChangeTerm entry from the current term, all entries
// should be committed
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
2014-06-26 22:55:56 +04:00
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != 4 {
t . Errorf ( "committed = %d, want %d" , sm . raftLog . committed , 4 )
2014-06-06 21:26:44 +04:00
}
}
2014-06-06 02:02:12 +04:00
func TestDuelingCandidates ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-05-19 00:59:10 +04:00
2014-06-26 22:55:56 +04:00
nt := newNetwork ( a , b , c )
2014-09-12 06:23:05 +04:00
nt . cut ( 1 , 3 )
2014-05-19 00:59:10 +04:00
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-16 21:11:21 +04:00
2016-05-19 05:38:10 +03:00
// 1 becomes leader since it receives votes from 1 and 2
2016-05-18 09:20:02 +03:00
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "state = %s, want %s" , sm . state , StateLeader )
}
2016-05-19 05:38:10 +03:00
// 3 stays as candidate since it receives a vote from 3 and a rejection from 2
2016-05-18 09:20:02 +03:00
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , sm . state , StateCandidate )
}
2014-06-26 22:55:56 +04:00
nt . recover ( )
2016-05-19 05:38:10 +03:00
// candidate 3 now increases its term and tries to vote again
// we expect it to disrupt the leader 1 since it has a higher term
// 3 will be follower again since both 1 and 2 rejects its vote request since 3 does not have a long enough log
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-16 21:53:56 +04:00
2014-11-03 23:04:24 +03:00
wlog := & raftLog {
2015-08-22 04:52:16 +03:00
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } } } ,
2014-11-03 23:04:24 +03:00
committed : 1 ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 2 } ,
2014-11-03 23:04:24 +03:00
}
2014-05-16 21:53:56 +04:00
tests := [ ] struct {
2014-08-23 00:24:33 +04:00
sm * raft
2014-09-16 04:35:02 +04:00
state StateType
2014-10-08 14:29:53 +04:00
term uint64
2014-07-24 03:15:25 +04:00
raftLog * raftLog
2014-05-16 21:53:56 +04:00
} {
2014-09-16 04:35:02 +04:00
{ a , StateFollower , 2 , wlog } ,
{ b , StateFollower , 2 , wlog } ,
2015-08-10 18:04:38 +03:00
{ c , StateFollower , 2 , newLog ( NewMemoryStorage ( ) , raftLogger ) } ,
2014-05-16 21:53:56 +04:00
}
for i , tt := range tests {
if g := tt . sm . state ; g != tt . state {
t . Errorf ( "#%d: state = %s, want %s" , i , g , tt . state )
}
2014-08-23 01:19:30 +04:00
if g := tt . sm . Term ; g != tt . term {
2014-05-16 21:53:56 +04:00
t . Errorf ( "#%d: term = %d, want %d" , i , g , tt . term )
}
2014-07-24 03:15:25 +04:00
base := ltoa ( tt . raftLog )
2014-10-08 14:29:53 +04:00
if sm , ok := nt . peers [ 1 + uint64 ( i ) ] . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-19 04:51:45 +04:00
}
}
}
2016-10-10 09:32:40 +03:00
func TestDuelingPreCandidates ( t * testing . T ) {
cfgA := newTestConfig ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
cfgB := newTestConfig ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
cfgC := newTestConfig ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
cfgA . PreVote = true
cfgB . PreVote = true
cfgC . PreVote = true
a := newRaft ( cfgA )
b := newRaft ( cfgB )
c := newRaft ( cfgC )
nt := newNetwork ( a , b , c )
nt . cut ( 1 , 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// 1 becomes leader since it receives votes from 1 and 2
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "state = %s, want %s" , sm . state , StateLeader )
}
// 3 campaigns then reverts to follower when its PreVote is rejected
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateFollower {
t . Errorf ( "state = %s, want %s" , sm . state , StateFollower )
}
nt . recover ( )
// Candidate 3 now increases its term and tries to vote again.
// With PreVote, it does not disrupt the leader.
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
wlog := & raftLog {
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } } } ,
committed : 1 ,
unstable : unstable { offset : 2 } ,
}
tests := [ ] struct {
sm * raft
state StateType
term uint64
raftLog * raftLog
} {
{ a , StateLeader , 1 , wlog } ,
{ b , StateFollower , 1 , wlog } ,
{ c , StateFollower , 1 , newLog ( NewMemoryStorage ( ) , raftLogger ) } ,
}
for i , tt := range tests {
if g := tt . sm . state ; g != tt . state {
t . Errorf ( "#%d: state = %s, want %s" , i , g , tt . state )
}
if g := tt . sm . Term ; g != tt . term {
t . Errorf ( "#%d: term = %d, want %d" , i , g , tt . term )
}
base := ltoa ( tt . raftLog )
if sm , ok := nt . peers [ 1 + uint64 ( i ) ] . ( * raft ) ; ok {
l := ltoa ( sm . raftLog )
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
}
}
}
2014-05-19 04:51:45 +04:00
func TestCandidateConcede ( t * testing . T ) {
2014-06-06 02:02:12 +04:00
tt := newNetwork ( nil , nil , nil )
2014-09-12 06:23:05 +04:00
tt . isolate ( 1 )
2014-05-19 04:51:45 +04:00
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
2014-05-19 04:51:45 +04:00
// heal the partition
2014-06-06 02:02:12 +04:00
tt . recover ( )
2014-12-18 23:02:15 +03:00
// send heartbeat; reset wait
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgBeat } )
2014-05-19 04:51:45 +04:00
data := [ ] byte ( "force follower" )
2014-12-19 04:10:37 +03:00
// send a proposal to 3 to flush out a MsgApp to 1
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : data } } } )
2014-12-19 04:10:37 +03:00
// send heartbeat; flush out commit
tt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgBeat } )
2014-05-19 04:51:45 +04:00
2014-09-12 06:23:05 +04:00
a := tt . peers [ 1 ] . ( * raft )
2014-09-16 04:35:02 +04:00
if g := a . state ; g != StateFollower {
t . Errorf ( "state = %s, want %s" , g , StateFollower )
2014-05-19 04:51:45 +04:00
}
2014-08-23 01:19:30 +04:00
if g := a . Term ; g != 1 {
2014-05-19 04:51:45 +04:00
t . Errorf ( "term = %d, want %d" , g , 1 )
}
2014-11-03 23:04:24 +03:00
wantLog := ltoa ( & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Index : 2 , Data : data } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 ,
} )
2014-06-06 02:02:12 +04:00
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( wantLog , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-16 21:53:56 +04:00
}
2014-05-16 21:11:21 +04:00
}
2014-05-16 21:53:56 +04:00
}
2014-05-28 22:59:01 +04:00
func TestSingleNodeCandidate ( t * testing . T ) {
tt := newNetwork ( nil )
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-28 22:59:01 +04:00
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-09-16 04:35:02 +04:00
if sm . state != StateLeader {
t . Errorf ( "state = %d, want %d" , sm . state , StateLeader )
2014-05-28 22:59:01 +04:00
}
}
2016-10-10 09:32:40 +03:00
func TestSingleNodePreCandidate ( t * testing . T ) {
tt := newNetworkWithConfig ( preVoteConfig , nil )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
sm := tt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "state = %d, want %d" , sm . state , StateLeader )
}
}
2014-05-16 21:53:56 +04:00
func TestOldMessages ( t * testing . T ) {
tt := newNetwork ( nil , nil , nil )
// make 0 leader @ term 3
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-12-18 23:02:15 +03:00
// pretend we're an old leader trying to make progress; this entry is expected to be ignored.
tt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgApp , Term : 2 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } } } )
// commit a new entry
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
2014-06-06 02:02:12 +04:00
2015-02-18 00:15:48 +03:00
ilog := & raftLog {
2014-11-03 23:04:24 +03:00
storage : & MemoryStorage {
ents : [ ] pb . Entry {
{ } , { Data : nil , Term : 1 , Index : 1 } ,
{ Data : nil , Term : 2 , Index : 2 } , { Data : nil , Term : 3 , Index : 3 } ,
2014-12-18 23:02:15 +03:00
{ Data : [ ] byte ( "somedata" ) , Term : 3 , Index : 4 } ,
2014-11-03 23:04:24 +03:00
} ,
2014-06-26 22:55:56 +04:00
} ,
2014-12-18 23:02:15 +03:00
unstable : unstable { offset : 5 } ,
committed : 4 ,
2014-06-26 22:55:56 +04:00
}
2015-02-18 00:15:48 +03:00
base := ltoa ( ilog )
2014-06-06 02:02:12 +04:00
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-16 21:11:21 +04:00
}
}
}
2014-05-18 04:18:58 +04:00
// TestOldMessagesReply - optimization - reply with new term.
2014-05-06 10:28:14 +04:00
func TestProposal ( t * testing . T ) {
tests := [ ] struct {
2014-05-16 21:11:21 +04:00
* network
2014-05-18 01:09:52 +04:00
success bool
2014-05-06 10:28:14 +04:00
} {
2014-05-18 01:09:52 +04:00
{ newNetwork ( nil , nil , nil ) , true } ,
{ newNetwork ( nil , nil , nopStepper ) , true } ,
{ newNetwork ( nil , nopStepper , nopStepper ) , false } ,
{ newNetwork ( nil , nopStepper , nopStepper , nil ) , false } ,
{ newNetwork ( nil , nopStepper , nopStepper , nil , nil ) , true } ,
2014-05-06 10:28:14 +04:00
}
2015-02-18 00:15:48 +03:00
for j , tt := range tests {
2014-08-28 05:53:18 +04:00
send := func ( m pb . Message ) {
2014-05-06 10:28:14 +04:00
defer func ( ) {
2018-10-25 09:23:32 +03:00
// only recover if we expect it to panic (success==false)
2014-05-18 01:09:52 +04:00
if ! tt . success {
2014-05-06 10:28:14 +04:00
e := recover ( )
if e != nil {
2015-02-18 00:15:48 +03:00
t . Logf ( "#%d: err: %s" , j , e )
2014-05-06 10:28:14 +04:00
}
}
} ( )
2014-06-06 02:02:12 +04:00
tt . send ( m )
}
2014-05-06 10:28:14 +04:00
2014-05-18 01:09:52 +04:00
data := [ ] byte ( "somedata" )
2014-05-06 10:28:14 +04:00
2018-10-25 09:23:32 +03:00
// promote 1 to become leader
2014-10-12 11:34:22 +04:00
send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : data } } } )
2014-05-06 10:28:14 +04:00
2015-08-10 18:04:38 +03:00
wantLog := newLog ( NewMemoryStorage ( ) , raftLogger )
2014-05-18 01:09:52 +04:00
if tt . success {
2014-11-03 23:04:24 +03:00
wantLog = & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Index : 2 , Data : data } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 }
2014-05-18 01:09:52 +04:00
}
2014-06-06 02:02:12 +04:00
base := ltoa ( wantLog )
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-06 10:28:14 +04:00
}
}
2014-09-12 06:23:05 +04:00
sm := tt . network . peers [ 1 ] . ( * raft )
2014-08-23 01:19:30 +04:00
if g := sm . Term ; g != 1 {
2015-02-18 00:15:48 +03:00
t . Errorf ( "#%d: term = %d, want %d" , j , g , 1 )
2014-05-06 10:28:14 +04:00
}
}
}
func TestProposalByProxy ( t * testing . T ) {
2014-05-16 11:55:17 +04:00
data := [ ] byte ( "somedata" )
2014-05-18 01:09:52 +04:00
tests := [ ] * network {
newNetwork ( nil , nil , nil ) ,
newNetwork ( nil , nil , nopStepper ) ,
2014-05-06 10:28:14 +04:00
}
2015-02-18 00:15:48 +03:00
for j , tt := range tests {
2014-05-06 10:28:14 +04:00
// promote 0 the leader
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-05-06 10:28:14 +04:00
// propose via follower
2014-10-12 11:34:22 +04:00
tt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
2014-06-06 02:02:12 +04:00
2014-11-03 23:04:24 +03:00
wantLog := & raftLog {
storage : & MemoryStorage {
ents : [ ] pb . Entry { { } , { Data : nil , Term : 1 , Index : 1 } , { Term : 1 , Data : data , Index : 2 } } ,
} ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
committed : 2 }
2014-06-06 02:02:12 +04:00
base := ltoa ( wantLog )
for i , p := range tt . peers {
2014-08-23 00:24:33 +04:00
if sm , ok := p . ( * raft ) ; ok {
2014-07-24 03:15:25 +04:00
l := ltoa ( sm . raftLog )
2014-06-06 02:02:12 +04:00
if g := diffu ( base , l ) ; g != "" {
t . Errorf ( "#%d: diff:\n%s" , i , g )
}
} else {
t . Logf ( "#%d: empty log" , i )
2014-05-06 10:28:14 +04:00
}
}
2014-09-12 06:23:05 +04:00
sm := tt . peers [ 1 ] . ( * raft )
2014-08-23 01:19:30 +04:00
if g := sm . Term ; g != 1 {
2015-02-18 00:15:48 +03:00
t . Errorf ( "#%d: term = %d, want %d" , j , g , 1 )
2014-05-06 10:28:14 +04:00
}
}
}
2014-05-24 20:53:01 +04:00
func TestCommit ( t * testing . T ) {
2014-05-22 03:02:15 +04:00
tests := [ ] struct {
2014-10-08 14:29:53 +04:00
matches [ ] uint64
2014-08-28 05:53:18 +04:00
logs [ ] pb . Entry
2014-10-08 14:29:53 +04:00
smTerm uint64
w uint64
2014-05-22 03:02:15 +04:00
} {
2014-06-06 22:21:26 +04:00
// single
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } } , 1 , 1 } ,
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 1 } , [ ] pb . Entry { { Index : 1 , Term : 2 } } , 2 , 1 } ,
2014-06-06 22:21:26 +04:00
2014-05-22 03:02:15 +04:00
// odd
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 2 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 2 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
2014-05-22 03:02:15 +04:00
// even
2014-12-15 22:25:35 +03:00
{ [ ] uint64 { 2 , 1 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 , 1 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 1 , 1 } ,
{ [ ] uint64 { 2 , 1 , 1 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
{ [ ] uint64 { 2 , 1 , 2 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } , 2 , 2 } ,
{ [ ] uint64 { 2 , 1 , 2 , 2 } , [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } , 2 , 0 } ,
2014-05-22 03:02:15 +04:00
}
for i , tt := range tests {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
storage . Append ( tt . logs )
storage . hardState = pb . HardState { Term : tt . smTerm }
2018-05-10 01:18:57 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 2 , storage )
2014-06-10 03:45:42 +04:00
for j := 0 ; j < len ( tt . matches ) ; j ++ {
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
id := uint64 ( j ) + 1
if id > 1 {
2019-07-22 23:30:47 +03:00
sm . applyConfChange ( pb . ConfChange { Type : pb . ConfChangeAddNode , NodeID : id } . AsV2 ( ) )
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
}
pr := sm . prs . Progress [ id ]
pr . Match , pr . Next = tt . matches [ j ] , tt . matches [ j ] + 1
2014-11-03 23:04:24 +03:00
}
2014-05-24 20:53:01 +04:00
sm . maybeCommit ( )
2014-07-24 03:15:25 +04:00
if g := sm . raftLog . committed ; g != tt . w {
2014-05-28 22:08:32 +04:00
t . Errorf ( "#%d: committed = %d, want %d" , i , g , tt . w )
2014-05-22 03:02:15 +04:00
}
}
}
2016-04-06 15:41:46 +03:00
func TestPastElectionTimeout ( t * testing . T ) {
2014-10-07 16:34:15 +04:00
tests := [ ] struct {
elapse int
2014-10-08 03:41:17 +04:00
wprobability float64
2014-10-07 16:34:15 +04:00
round bool
} {
{ 5 , 0 , false } ,
2016-04-01 20:09:09 +03:00
{ 10 , 0.1 , true } ,
{ 13 , 0.4 , true } ,
{ 15 , 0.6 , true } ,
{ 18 , 0.9 , true } ,
2014-10-07 16:34:15 +04:00
{ 20 , 1 , false } ,
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2015-11-24 08:59:25 +03:00
sm . electionElapsed = tt . elapse
2014-10-07 16:34:15 +04:00
c := 0
for j := 0 ; j < 10000 ; j ++ {
2016-04-01 20:09:09 +03:00
sm . resetRandomizedElectionTimeout ( )
if sm . pastElectionTimeout ( ) {
2014-10-07 16:34:15 +04:00
c ++
}
}
got := float64 ( c ) / 10000.0
if tt . round {
got = math . Floor ( got * 10 + 0.5 ) / 10.0
}
2014-10-08 03:41:17 +04:00
if got != tt . wprobability {
2016-04-06 15:41:46 +03:00
t . Errorf ( "#%d: probability = %v, want %v" , i , got , tt . wprobability )
2014-10-07 16:34:15 +04:00
}
}
}
2014-09-04 21:58:22 +04:00
// ensure that the Step function ignores the message from old term and does not pass it to the
2015-09-11 01:06:56 +03:00
// actual stepX function.
2014-09-04 21:58:22 +04:00
func TestStepIgnoreOldTermMsg ( t * testing . T ) {
called := false
2018-01-11 07:43:55 +03:00
fakeStep := func ( r * raft , m pb . Message ) error {
2014-09-04 21:58:22 +04:00
called = true
2018-01-11 07:43:55 +03:00
return nil
2014-09-04 21:58:22 +04:00
}
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-04 21:58:22 +04:00
sm . step = fakeStep
sm . Term = 2
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { Type : pb . MsgApp , Term : sm . Term - 1 } )
2016-04-03 04:27:54 +03:00
if called {
2014-09-04 21:58:22 +04:00
t . Errorf ( "stepFunc called = %v , want %v" , called , false )
}
}
2014-07-14 21:58:41 +04:00
// TestHandleMsgApp ensures:
// 1. Reply false if log doesn’ t contain an entry at prevLogIndex whose term matches prevLogTerm.
// 2. If an existing entry conflicts with a new one (same index but different terms),
// delete the existing entry and all that follow it; append any new entries not already in the log.
// 3. If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry).
func TestHandleMsgApp ( t * testing . T ) {
tests := [ ] struct {
2014-08-28 05:53:18 +04:00
m pb . Message
2014-10-08 14:29:53 +04:00
wIndex uint64
wCommit uint64
2014-10-01 23:59:30 +04:00
wReject bool
2014-07-14 21:58:41 +04:00
} {
// Ensure 1
2014-10-12 11:34:22 +04:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 3 , Index : 2 , Commit : 3 } , 2 , 0 , true } , // previous log mismatch
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 3 , Index : 3 , Commit : 3 } , 2 , 0 , true } , // previous log non-exist
2014-07-14 21:58:41 +04:00
// Ensure 2
2014-10-12 11:34:22 +04:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 1 , Index : 1 , Commit : 1 } , 2 , 1 , false } ,
2014-12-02 19:50:15 +03:00
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 0 , Index : 0 , Commit : 1 , Entries : [ ] pb . Entry { { Index : 1 , Term : 2 } } } , 1 , 1 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 3 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } , { Index : 4 , Term : 2 } } } , 4 , 3 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 4 , Entries : [ ] pb . Entry { { Index : 3 , Term : 2 } } } , 3 , 3 , false } ,
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 1 , Index : 1 , Commit : 4 , Entries : [ ] pb . Entry { { Index : 2 , Term : 2 } } } , 2 , 2 , false } ,
2014-07-14 21:58:41 +04:00
// Ensure 3
2014-12-05 01:12:55 +03:00
{ pb . Message { Type : pb . MsgApp , Term : 1 , LogTerm : 1 , Index : 1 , Commit : 3 } , 2 , 1 , false } , // match entry 1, commit up to last new entry 1
{ pb . Message { Type : pb . MsgApp , Term : 1 , LogTerm : 1 , Index : 1 , Commit : 3 , Entries : [ ] pb . Entry { { Index : 2 , Term : 2 } } } , 2 , 2 , false } , // match entry 1, commit up to last new entry 2
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 3 } , 2 , 2 , false } , // match entry 2, commit up to last new entry 2
{ pb . Message { Type : pb . MsgApp , Term : 2 , LogTerm : 2 , Index : 2 , Commit : 4 } , 2 , 2 , false } , // commit up to log.last()
2014-07-14 21:58:41 +04:00
}
for i , tt := range tests {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
2014-12-15 22:25:35 +03:00
sm . becomeFollower ( 2 , None )
2014-07-14 21:58:41 +04:00
sm . handleAppendEntries ( tt . m )
2014-07-24 03:15:25 +04:00
if sm . raftLog . lastIndex ( ) != tt . wIndex {
t . Errorf ( "#%d: lastIndex = %d, want %d" , i , sm . raftLog . lastIndex ( ) , tt . wIndex )
2014-07-14 21:58:41 +04:00
}
2014-07-24 03:15:25 +04:00
if sm . raftLog . committed != tt . wCommit {
t . Errorf ( "#%d: committed = %d, want %d" , i , sm . raftLog . committed , tt . wCommit )
2014-07-14 21:58:41 +04:00
}
2014-10-30 00:04:45 +03:00
m := sm . readMessages ( )
2014-07-14 21:58:41 +04:00
if len ( m ) != 1 {
2014-09-29 06:38:37 +04:00
t . Fatalf ( "#%d: msg = nil, want 1" , i )
2014-07-14 21:58:41 +04:00
}
2014-10-01 23:59:30 +04:00
if m [ 0 ] . Reject != tt . wReject {
t . Errorf ( "#%d: reject = %v, want %v" , i , m [ 0 ] . Reject , tt . wReject )
2014-07-14 21:58:41 +04:00
}
}
}
2014-11-18 03:58:49 +03:00
// TestHandleHeartbeat ensures that the follower commits to the commit in the message.
func TestHandleHeartbeat ( t * testing . T ) {
commit := uint64 ( 2 )
tests := [ ] struct {
m pb . Message
wCommit uint64
} {
2016-05-20 09:04:08 +03:00
{ pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeat , Term : 2 , Commit : commit + 1 } , commit + 1 } ,
{ pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeat , Term : 2 , Commit : commit - 1 } , commit } , // do not decrease commit
2014-11-18 03:58:49 +03:00
}
for i , tt := range tests {
2014-11-20 01:05:16 +03:00
storage := NewMemoryStorage ( )
2014-11-25 04:13:47 +03:00
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } , { Index : 3 , Term : 3 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , storage )
2014-12-15 22:25:35 +03:00
sm . becomeFollower ( 2 , 2 )
2014-11-18 03:58:49 +03:00
sm . raftLog . commitTo ( commit )
sm . handleHeartbeat ( tt . m )
if sm . raftLog . committed != tt . wCommit {
t . Errorf ( "#%d: committed = %d, want %d" , i , sm . raftLog . committed , tt . wCommit )
}
m := sm . readMessages ( )
2015-01-14 22:42:30 +03:00
if len ( m ) != 1 {
t . Fatalf ( "#%d: msg = nil, want 1" , i )
}
if m [ 0 ] . Type != pb . MsgHeartbeatResp {
t . Errorf ( "#%d: type = %v, want MsgHeartbeatResp" , i , m [ 0 ] . Type )
2014-11-18 03:58:49 +03:00
}
}
}
2015-01-14 22:42:30 +03:00
// TestHandleHeartbeatResp ensures that we re-send log entries when we get a heartbeat response.
func TestHandleHeartbeatResp ( t * testing . T ) {
storage := NewMemoryStorage ( )
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 2 } , { Index : 3 , Term : 3 } } )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , storage )
2015-01-14 22:42:30 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
sm . raftLog . commitTo ( sm . raftLog . lastIndex ( ) )
// A heartbeat response from a node that is behind; re-send MsgApp
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs := sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
}
if msgs [ 0 ] . Type != pb . MsgApp {
t . Errorf ( "type = %v, want MsgApp" , msgs [ 0 ] . Type )
}
2016-12-19 23:24:21 +03:00
// A second heartbeat response generates another MsgApp re-send
2015-01-14 22:42:30 +03:00
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs = sm . readMessages ( )
2016-12-19 23:24:21 +03:00
if len ( msgs ) != 1 {
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
2015-01-14 22:42:30 +03:00
}
2016-12-19 23:24:21 +03:00
if msgs [ 0 ] . Type != pb . MsgApp {
t . Errorf ( "type = %v, want MsgApp" , msgs [ 0 ] . Type )
2015-01-14 22:42:30 +03:00
}
// Once we have an MsgAppResp, heartbeats no longer send MsgApp.
sm . Step ( pb . Message {
From : 2 ,
Type : pb . MsgAppResp ,
2016-12-19 23:24:21 +03:00
Index : msgs [ 0 ] . Index + uint64 ( len ( msgs [ 0 ] . Entries ) ) ,
2015-01-14 22:42:30 +03:00
} )
2015-01-28 01:40:14 +03:00
// Consume the message sent in response to MsgAppResp
sm . readMessages ( )
2015-01-14 22:42:30 +03:00
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp } )
msgs = sm . readMessages ( )
2016-12-19 23:24:21 +03:00
if len ( msgs ) != 0 {
t . Fatalf ( "len(msgs) = %d, want 0: %+v" , len ( msgs ) , msgs )
2015-01-14 22:42:30 +03:00
}
}
2017-03-22 23:59:02 +03:00
// TestRaftFreesReadOnlyMem ensures raft will free read request from
// readOnly readIndexQueue and pendingReadIndex map.
2018-11-10 05:10:12 +03:00
// related issue: https://github.com/etcd-io/etcd/issues/7571
2017-03-22 23:59:02 +03:00
func TestRaftFreesReadOnlyMem ( t * testing . T ) {
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , NewMemoryStorage ( ) )
sm . becomeCandidate ( )
sm . becomeLeader ( )
sm . raftLog . commitTo ( sm . raftLog . lastIndex ( ) )
ctx := [ ] byte ( "ctx" )
// leader starts linearizable read request.
// more info: raft dissertation 6.4, step 2.
sm . Step ( pb . Message { From : 2 , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : ctx } } } )
msgs := sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
}
if msgs [ 0 ] . Type != pb . MsgHeartbeat {
t . Fatalf ( "type = %v, want MsgHeartbeat" , msgs [ 0 ] . Type )
}
if ! bytes . Equal ( msgs [ 0 ] . Context , ctx ) {
t . Fatalf ( "Context = %v, want %v" , msgs [ 0 ] . Context , ctx )
}
if len ( sm . readOnly . readIndexQueue ) != 1 {
t . Fatalf ( "len(readIndexQueue) = %v, want 1" , len ( sm . readOnly . readIndexQueue ) )
}
if len ( sm . readOnly . pendingReadIndex ) != 1 {
t . Fatalf ( "len(pendingReadIndex) = %v, want 1" , len ( sm . readOnly . pendingReadIndex ) )
}
if _ , ok := sm . readOnly . pendingReadIndex [ string ( ctx ) ] ; ! ok {
t . Fatalf ( "can't find context %v in pendingReadIndex " , ctx )
}
// heartbeat responses from majority of followers (1 in this case)
// acknowledge the authority of the leader.
// more info: raft dissertation 6.4, step 3.
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp , Context : ctx } )
if len ( sm . readOnly . readIndexQueue ) != 0 {
t . Fatalf ( "len(readIndexQueue) = %v, want 0" , len ( sm . readOnly . readIndexQueue ) )
}
if len ( sm . readOnly . pendingReadIndex ) != 0 {
t . Fatalf ( "len(pendingReadIndex) = %v, want 0" , len ( sm . readOnly . pendingReadIndex ) )
}
if _ , ok := sm . readOnly . pendingReadIndex [ string ( ctx ) ] ; ok {
t . Fatalf ( "found context %v in pendingReadIndex, want none" , ctx )
}
}
2016-02-01 08:42:39 +03:00
// TestMsgAppRespWaitReset verifies the resume behavior of a leader
2015-01-28 01:40:14 +03:00
// MsgAppResp.
func TestMsgAppRespWaitReset ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
2015-01-28 01:40:14 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
// The new leader has just emitted a new Term 4 entry; consume those messages
// from the outgoing queue.
sm . bcastAppend ( )
sm . readMessages ( )
// Node 2 acks the first entry, making it committed.
sm . Step ( pb . Message {
From : 2 ,
Type : pb . MsgAppResp ,
Index : 1 ,
} )
2016-01-26 23:18:55 +03:00
if sm . raftLog . committed != 1 {
t . Fatalf ( "expected committed to be 1, got %d" , sm . raftLog . committed )
2015-01-28 01:40:14 +03:00
}
// Also consume the MsgApp messages that update Commit on the followers.
sm . readMessages ( )
// A new command is now proposed on node 1.
sm . Step ( pb . Message {
From : 1 ,
Type : pb . MsgProp ,
Entries : [ ] pb . Entry { { } } ,
} )
// The command is broadcast to all nodes not in the wait state.
// Node 2 left the wait state due to its MsgAppResp, but node 3 is still waiting.
msgs := sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "expected 1 message, got %d: %+v" , len ( msgs ) , msgs )
}
if msgs [ 0 ] . Type != pb . MsgApp || msgs [ 0 ] . To != 2 {
2015-12-16 07:18:21 +03:00
t . Errorf ( "expected MsgApp to node 2, got %v to %d" , msgs [ 0 ] . Type , msgs [ 0 ] . To )
2015-01-28 01:40:14 +03:00
}
if len ( msgs [ 0 ] . Entries ) != 1 || msgs [ 0 ] . Entries [ 0 ] . Index != 2 {
t . Errorf ( "expected to send entry 2, but got %v" , msgs [ 0 ] . Entries )
}
// Now Node 3 acks the first entry. This releases the wait and entry 2 is sent.
sm . Step ( pb . Message {
From : 3 ,
Type : pb . MsgAppResp ,
Index : 1 ,
} )
msgs = sm . readMessages ( )
if len ( msgs ) != 1 {
t . Fatalf ( "expected 1 message, got %d: %+v" , len ( msgs ) , msgs )
}
if msgs [ 0 ] . Type != pb . MsgApp || msgs [ 0 ] . To != 3 {
2015-12-16 07:18:21 +03:00
t . Errorf ( "expected MsgApp to node 3, got %v to %d" , msgs [ 0 ] . Type , msgs [ 0 ] . To )
2015-01-28 01:40:14 +03:00
}
if len ( msgs [ 0 ] . Entries ) != 1 || msgs [ 0 ] . Entries [ 0 ] . Index != 2 {
t . Errorf ( "expected to send entry 2, but got %v" , msgs [ 0 ] . Entries )
}
}
2014-06-11 19:58:26 +04:00
func TestRecvMsgVote ( t * testing . T ) {
2016-10-25 17:31:44 +03:00
testRecvMsgVote ( t , pb . MsgVote )
2017-09-01 17:45:47 +03:00
}
func TestRecvMsgPreVote ( t * testing . T ) {
testRecvMsgVote ( t , pb . MsgPreVote )
2016-10-25 17:31:44 +03:00
}
2014-05-25 08:08:06 +04:00
2016-10-25 17:31:44 +03:00
func testRecvMsgVote ( t * testing . T , msgType pb . MessageType ) {
tests := [ ] struct {
2017-07-20 23:46:05 +03:00
state StateType
index , logTerm uint64
voteFor uint64
wreject bool
2016-10-25 17:31:44 +03:00
} {
{ StateFollower , 0 , 0 , None , true } ,
{ StateFollower , 0 , 1 , None , true } ,
{ StateFollower , 0 , 2 , None , true } ,
{ StateFollower , 0 , 3 , None , false } ,
{ StateFollower , 1 , 0 , None , true } ,
{ StateFollower , 1 , 1 , None , true } ,
{ StateFollower , 1 , 2 , None , true } ,
{ StateFollower , 1 , 3 , None , false } ,
{ StateFollower , 2 , 0 , None , true } ,
{ StateFollower , 2 , 1 , None , true } ,
{ StateFollower , 2 , 2 , None , false } ,
{ StateFollower , 2 , 3 , None , false } ,
{ StateFollower , 3 , 0 , None , true } ,
{ StateFollower , 3 , 1 , None , true } ,
{ StateFollower , 3 , 2 , None , false } ,
{ StateFollower , 3 , 3 , None , false } ,
{ StateFollower , 3 , 2 , 2 , false } ,
{ StateFollower , 3 , 2 , 1 , true } ,
{ StateLeader , 3 , 3 , 1 , true } ,
{ StatePreCandidate , 3 , 3 , 1 , true } ,
{ StateCandidate , 3 , 3 , 1 , true } ,
}
2017-07-20 23:46:05 +03:00
max := func ( a , b uint64 ) uint64 {
if a > b {
return a
}
return b
}
2016-10-25 17:31:44 +03:00
for i , tt := range tests {
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
sm . state = tt . state
switch tt . state {
case StateFollower :
sm . step = stepFollower
case StateCandidate , StatePreCandidate :
sm . step = stepCandidate
case StateLeader :
sm . step = stepLeader
}
sm . Vote = tt . voteFor
sm . raftLog = & raftLog {
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 2 } , { Index : 2 , Term : 2 } } } ,
unstable : unstable { offset : 3 } ,
}
2017-07-20 23:46:05 +03:00
// raft.Term is greater than or equal to raft.raftLog.lastTerm. In this
// test we're only testing MsgVote responses when the campaigning node
// has a different raft log compared to the recipient node.
// Additionally we're verifying behaviour when the recipient node has
// already given out its vote for its current term. We're not testing
// what the recipient node does when receiving a message with a
// different term number, so we simply initialize both term numbers to
// be the same.
term := max ( sm . raftLog . lastTerm ( ) , tt . logTerm )
sm . Term = term
sm . Step ( pb . Message { Type : msgType , Term : term , From : 2 , Index : tt . index , LogTerm : tt . logTerm } )
2016-10-25 17:31:44 +03:00
msgs := sm . readMessages ( )
if g := len ( msgs ) ; g != 1 {
t . Fatalf ( "#%d: len(msgs) = %d, want 1" , i , g )
continue
}
if g := msgs [ 0 ] . Type ; g != voteRespMsgType ( msgType ) {
t . Errorf ( "#%d, m.Type = %v, want %v" , i , g , voteRespMsgType ( msgType ) )
}
if g := msgs [ 0 ] . Reject ; g != tt . wreject {
t . Errorf ( "#%d, m.Reject = %v, want %v" , i , g , tt . wreject )
2014-05-18 11:17:46 +04:00
}
}
}
2014-06-07 03:27:29 +04:00
func TestStateTransition ( t * testing . T ) {
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
from StateType
to StateType
2014-06-07 03:27:29 +04:00
wallow bool
2014-10-08 14:29:53 +04:00
wterm uint64
wlead uint64
2014-06-07 03:27:29 +04:00
} {
2014-09-16 04:35:02 +04:00
{ StateFollower , StateFollower , true , 1 , None } ,
2016-10-10 09:32:40 +03:00
{ StateFollower , StatePreCandidate , true , 0 , None } ,
2014-09-16 04:35:02 +04:00
{ StateFollower , StateCandidate , true , 1 , None } ,
2014-10-08 14:29:53 +04:00
{ StateFollower , StateLeader , false , 0 , None } ,
2014-06-07 03:27:29 +04:00
2016-10-10 09:32:40 +03:00
{ StatePreCandidate , StateFollower , true , 0 , None } ,
{ StatePreCandidate , StatePreCandidate , true , 0 , None } ,
{ StatePreCandidate , StateCandidate , true , 1 , None } ,
{ StatePreCandidate , StateLeader , true , 0 , 1 } ,
2014-09-16 04:35:02 +04:00
{ StateCandidate , StateFollower , true , 0 , None } ,
2016-10-10 09:32:40 +03:00
{ StateCandidate , StatePreCandidate , true , 0 , None } ,
2014-09-16 04:35:02 +04:00
{ StateCandidate , StateCandidate , true , 1 , None } ,
{ StateCandidate , StateLeader , true , 0 , 1 } ,
2014-06-07 03:27:29 +04:00
2014-09-16 04:35:02 +04:00
{ StateLeader , StateFollower , true , 1 , None } ,
2016-10-10 09:32:40 +03:00
{ StateLeader , StatePreCandidate , false , 0 , None } ,
2014-09-16 04:35:02 +04:00
{ StateLeader , StateCandidate , false , 1 , None } ,
{ StateLeader , StateLeader , true , 0 , 1 } ,
2014-06-07 03:27:29 +04:00
}
for i , tt := range tests {
func ( ) {
defer func ( ) {
if r := recover ( ) ; r != nil {
2016-04-03 04:27:54 +03:00
if tt . wallow {
2014-06-07 03:27:29 +04:00
t . Errorf ( "%d: allow = %v, want %v" , i , false , true )
}
}
} ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2014-06-07 03:27:29 +04:00
sm . state = tt . from
switch tt . to {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-06-07 03:27:29 +04:00
sm . becomeFollower ( tt . wterm , tt . wlead )
2016-10-10 09:32:40 +03:00
case StatePreCandidate :
sm . becomePreCandidate ( )
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-06-07 03:27:29 +04:00
sm . becomeCandidate ( )
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-06-07 03:27:29 +04:00
sm . becomeLeader ( )
}
2014-08-23 01:19:30 +04:00
if sm . Term != tt . wterm {
t . Errorf ( "%d: term = %d, want %d" , i , sm . Term , tt . wterm )
2014-06-07 03:27:29 +04:00
}
2014-08-24 04:40:25 +04:00
if sm . lead != tt . wlead {
2014-06-07 03:27:29 +04:00
t . Errorf ( "%d: lead = %d, want %d" , i , sm . lead , tt . wlead )
}
} ( )
}
}
2014-05-24 00:30:04 +04:00
func TestAllServerStepdown ( t * testing . T ) {
2014-06-26 22:55:56 +04:00
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
state StateType
2014-06-26 22:55:56 +04:00
2014-09-16 04:35:02 +04:00
wstate StateType
2014-10-08 14:29:53 +04:00
wterm uint64
windex uint64
2014-06-26 22:55:56 +04:00
} {
2014-11-18 00:37:46 +03:00
{ StateFollower , StateFollower , 3 , 0 } ,
2016-10-10 09:32:40 +03:00
{ StatePreCandidate , StateFollower , 3 , 0 } ,
2014-11-18 00:37:46 +03:00
{ StateCandidate , StateFollower , 3 , 0 } ,
{ StateLeader , StateFollower , 3 , 1 } ,
2014-06-26 22:55:56 +04:00
}
2014-05-24 00:30:04 +04:00
2014-10-12 11:34:22 +04:00
tmsgTypes := [ ... ] pb . MessageType { pb . MsgVote , pb . MsgApp }
2014-10-08 14:29:53 +04:00
tterm := uint64 ( 3 )
2014-05-24 00:30:04 +04:00
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-06-26 22:55:56 +04:00
switch tt . state {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-09-15 09:44:59 +04:00
sm . becomeFollower ( 1 , None )
2016-12-26 09:31:59 +03:00
case StatePreCandidate :
sm . becomePreCandidate ( )
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-05-24 00:30:04 +04:00
sm . becomeCandidate ( )
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-05-24 00:30:04 +04:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
}
for j , msgType := range tmsgTypes {
2014-09-12 06:23:05 +04:00
sm . Step ( pb . Message { From : 2 , Type : msgType , Term : tterm , LogTerm : tterm } )
2014-05-24 00:30:04 +04:00
2014-06-26 22:55:56 +04:00
if sm . state != tt . wstate {
t . Errorf ( "#%d.%d state = %v , want %v" , i , j , sm . state , tt . wstate )
2014-05-24 00:30:04 +04:00
}
2014-08-23 01:19:30 +04:00
if sm . Term != tt . wterm {
t . Errorf ( "#%d.%d term = %v , want %v" , i , j , sm . Term , tt . wterm )
2014-05-24 00:30:04 +04:00
}
2018-07-05 20:12:45 +03:00
if sm . raftLog . lastIndex ( ) != tt . windex {
2014-11-18 00:37:46 +03:00
t . Errorf ( "#%d.%d index = %v , want %v" , i , j , sm . raftLog . lastIndex ( ) , tt . windex )
}
2014-11-03 23:04:24 +03:00
if uint64 ( len ( sm . raftLog . allEntries ( ) ) ) != tt . windex {
2014-11-18 00:37:46 +03:00
t . Errorf ( "#%d.%d len(ents) = %v , want %v" , i , j , len ( sm . raftLog . allEntries ( ) ) , tt . windex )
2014-05-24 00:30:04 +04:00
}
2014-10-08 14:29:53 +04:00
wlead := uint64 ( 2 )
2014-10-12 11:34:22 +04:00
if msgType == pb . MsgVote {
2014-09-15 09:44:59 +04:00
wlead = None
2014-07-15 09:39:44 +04:00
}
2014-08-24 04:40:25 +04:00
if sm . lead != wlead {
2014-09-15 09:44:59 +04:00
t . Errorf ( "#%d, sm.lead = %d, want %d" , i , sm . lead , None )
2014-07-15 09:39:44 +04:00
}
2014-05-24 00:30:04 +04:00
}
}
}
2018-02-21 22:53:40 +03:00
func TestCandidateResetTermMsgHeartbeat ( t * testing . T ) {
testCandidateResetTerm ( t , pb . MsgHeartbeat )
}
func TestCandidateResetTermMsgApp ( t * testing . T ) {
testCandidateResetTerm ( t , pb . MsgApp )
}
// testCandidateResetTerm tests when a candidate receives a
// MsgHeartbeat or MsgApp from leader, "Step" resets the term
// with leader's and reverts back to follower.
func testCandidateResetTerm ( t * testing . T , mt pb . MessageType ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
nt := newNetwork ( a , b , c )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
// isolate 3 and increase term in rest
nt . isolate ( 3 )
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
// trigger campaign in isolated c
c . resetRandomizedElectionTimeout ( )
for i := 0 ; i < c . randomizedElectionTimeout ; i ++ {
c . tick ( )
}
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
nt . recover ( )
// leader sends to isolated candidate
// and expects candidate to revert to follower
nt . send ( pb . Message { From : 1 , To : 3 , Term : a . Term , Type : mt } )
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
// follower c term is reset with leader's
if a . Term != c . Term {
t . Errorf ( "follower term expected same term as leader's %d, got %d" , a . Term , c . Term )
}
}
2015-11-24 08:59:25 +03:00
func TestLeaderStepdownWhenQuorumActive ( t * testing . T ) {
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
sm . checkQuorum = true
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < sm . electionTimeout + 1 ; i ++ {
sm . Step ( pb . Message { From : 2 , Type : pb . MsgHeartbeatResp , Term : sm . Term } )
sm . tick ( )
}
if sm . state != StateLeader {
t . Errorf ( "state = %v, want %v" , sm . state , StateLeader )
}
}
func TestLeaderStepdownWhenQuorumLost ( t * testing . T ) {
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 5 , 1 , NewMemoryStorage ( ) )
sm . checkQuorum = true
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < sm . electionTimeout + 1 ; i ++ {
sm . tick ( )
}
if sm . state != StateFollower {
t . Errorf ( "state = %v, want %v" , sm . state , StateFollower )
}
}
2016-05-27 12:23:18 +03:00
func TestLeaderSupersedingWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
2016-05-27 12:23:18 +03:00
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// Peer b rejected c's vote since its electionElapsed had not reached to electionTimeout
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
// Letting b's electionElapsed reach to electionTimeout
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if c . state != StateLeader {
t . Errorf ( "state = %s, want %s" , c . state , StateLeader )
}
}
func TestLeaderElectionWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( a , a . electionTimeout + 1 )
setRandomizedElectionTimeout ( b , b . electionTimeout + 2 )
2016-05-27 12:23:18 +03:00
2016-08-29 00:35:19 +03:00
// Immediately after creation, votes are cast regardless of the
// election timeout.
2016-05-27 12:23:18 +03:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if c . state != StateFollower {
t . Errorf ( "state = %s, want %s" , c . state , StateFollower )
}
2016-08-04 13:31:22 +03:00
// need to reset randomizedElectionTimeout larger than electionTimeout again,
// because the value might be reset to electionTimeout since the last state changes
setRandomizedElectionTimeout ( a , a . electionTimeout + 1 )
setRandomizedElectionTimeout ( b , b . electionTimeout + 2 )
2016-05-27 12:23:18 +03:00
for i := 0 ; i < a . electionTimeout ; i ++ {
a . tick ( )
}
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if a . state != StateFollower {
t . Errorf ( "state = %s, want %s" , a . state , StateFollower )
}
if c . state != StateLeader {
t . Errorf ( "state = %s, want %s" , c . state , StateLeader )
}
}
// TestFreeStuckCandidateWithCheckQuorum ensures that a candidate with a higher term
// can disrupt the leader even if the leader still "officially" holds the lease, The
// leader is expected to step down and adopt the candidate's term
func TestFreeStuckCandidateWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
2016-05-27 12:23:18 +03:00
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 1 )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
if c . Term != b . Term + 1 {
t . Errorf ( "term = %d, want %d" , c . Term , b . Term + 1 )
}
// Vote again for safety
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if c . state != StateCandidate {
t . Errorf ( "state = %s, want %s" , c . state , StateCandidate )
}
if c . Term != b . Term + 2 {
t . Errorf ( "term = %d, want %d" , c . Term , b . Term + 2 )
}
nt . recover ( )
nt . send ( pb . Message { From : 1 , To : 3 , Type : pb . MsgHeartbeat , Term : a . Term } )
// Disrupt the leader so that the stuck peer is freed
if a . state != StateFollower {
t . Errorf ( "state = %s, want %s" , a . state , StateFollower )
}
if c . Term != a . Term {
t . Errorf ( "term = %d, want %d" , c . Term , a . Term )
}
2017-09-15 02:38:55 +03:00
// Vote again, should become leader this time
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if c . state != StateLeader {
t . Errorf ( "peer 3 state: %s, want %s" , c . state , StateLeader )
}
2016-05-27 12:23:18 +03:00
}
func TestNonPromotableVoterWithCheckQuorum ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
a . checkQuorum = true
b . checkQuorum = true
nt := newNetwork ( a , b )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
2016-05-27 12:23:18 +03:00
// Need to remove 2 again to make it a non-promotable node since newNetwork overwritten some internal states
2019-07-22 23:30:47 +03:00
b . applyConfChange ( pb . ConfChange { Type : pb . ConfChangeRemoveNode , NodeID : 2 } . AsV2 ( ) )
2016-05-27 12:23:18 +03:00
if b . promotable ( ) {
t . Fatalf ( "promotable = %v, want false" , b . promotable ( ) )
}
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Errorf ( "state = %s, want %s" , a . state , StateLeader )
}
if b . state != StateFollower {
t . Errorf ( "state = %s, want %s" , b . state , StateFollower )
}
if b . lead != 1 {
t . Errorf ( "lead = %d, want 1" , b . lead )
}
}
2018-02-23 07:39:05 +03:00
// TestDisruptiveFollower tests isolated follower,
// with slow network incoming from leader, election times out
// to become a candidate with an increased term. Then, the
// candiate's response to late leader heartbeat forces the leader
// to step down.
func TestDisruptiveFollower ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . checkQuorum = true
n2 . checkQuorum = true
n3 . checkQuorum = true
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StateFollower
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateFollower {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateFollower )
}
// etcd server "advanceTicksForElection" on restart;
// this is to expedite campaign trigger when given larger
// election timeouts (e.g. multi-datacenter deploy)
// Or leader messages are being delayed while ticks elapse
setRandomizedElectionTimeout ( n3 , n3 . electionTimeout + 2 )
for i := 0 ; i < n3 . randomizedElectionTimeout - 1 ; i ++ {
n3 . tick ( )
}
// ideally, before last election tick elapses,
// the follower n3 receives "pb.MsgApp" or "pb.MsgHeartbeat"
// from leader n1, and then resets its "electionElapsed"
// however, last tick may elapse before receiving any
// messages from leader, thus triggering campaign
n3 . tick ( )
// n1 is still leader yet
// while its heartbeat to candidate n3 is being delayed
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StateCandidate
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateCandidate {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateCandidate )
}
// check term
// n1.Term == 2
// n2.Term == 2
// n3.Term == 3
if n1 . Term != 2 {
t . Fatalf ( "node 1 term: %d, want %d" , n1 . Term , 2 )
}
if n2 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n2 . Term , 2 )
}
if n3 . Term != 3 {
t . Fatalf ( "node 3 term: %d, want %d" , n3 . Term , 3 )
}
// while outgoing vote requests are still queued in n3,
// leader heartbeat finally arrives at candidate n3
// however, due to delayed network from leader, leader
// heartbeat was sent with lower term than candidate's
nt . send ( pb . Message { From : 1 , To : 3 , Term : n1 . Term , Type : pb . MsgHeartbeat } )
// then candidate n3 responds with "pb.MsgAppResp" of higher term
// and leader steps down from a message with higher term
// this is to disrupt the current leader, so that candidate
// with higher term can be freed with following election
// check state
// n1.state == StateFollower
// n2.state == StateFollower
// n3.state == StateCandidate
if n1 . state != StateFollower {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateFollower )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateCandidate {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateCandidate )
}
// check term
// n1.Term == 3
// n2.Term == 2
// n3.Term == 3
if n1 . Term != 3 {
t . Fatalf ( "node 1 term: %d, want %d" , n1 . Term , 3 )
}
if n2 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n2 . Term , 2 )
}
if n3 . Term != 3 {
t . Fatalf ( "node 3 term: %d, want %d" , n3 . Term , 3 )
}
}
// TestDisruptiveFollowerPreVote tests isolated follower,
// with slow network incoming from leader, election times out
// to become a pre-candidate with less log than current leader.
// Then pre-vote phase prevents this isolated node from forcing
// current leader to step down, thus less disruptions.
func TestDisruptiveFollowerPreVote ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . checkQuorum = true
n2 . checkQuorum = true
n3 . checkQuorum = true
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StateFollower
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateFollower {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateFollower )
}
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
n1 . preVote = true
n2 . preVote = true
n3 . preVote = true
nt . recover ( )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StatePreCandidate
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StatePreCandidate {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StatePreCandidate )
}
// check term
// n1.Term == 2
// n2.Term == 2
// n3.Term == 2
if n1 . Term != 2 {
t . Fatalf ( "node 1 term: %d, want %d" , n1 . Term , 2 )
}
if n2 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n2 . Term , 2 )
}
if n3 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n3 . Term , 2 )
}
// delayed leader heartbeat does not force current leader to step down
nt . send ( pb . Message { From : 1 , To : 3 , Term : n1 . Term , Type : pb . MsgHeartbeat } )
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
}
2016-08-27 02:03:06 +03:00
func TestReadOnlyOptionSafe ( t * testing . T ) {
2016-06-03 18:20:10 +03:00
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2016-08-27 02:03:06 +03:00
nt := newNetwork ( a , b , c )
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Fatalf ( "state = %s, want %s" , a . state , StateLeader )
}
tests := [ ] struct {
sm * raft
proposals int
wri uint64
wctx [ ] byte
} {
2016-09-28 05:19:30 +03:00
{ a , 10 , 11 , [ ] byte ( "ctx1" ) } ,
{ b , 10 , 21 , [ ] byte ( "ctx2" ) } ,
{ c , 10 , 31 , [ ] byte ( "ctx3" ) } ,
{ a , 10 , 41 , [ ] byte ( "ctx4" ) } ,
{ b , 10 , 51 , [ ] byte ( "ctx5" ) } ,
{ c , 10 , 61 , [ ] byte ( "ctx6" ) } ,
2016-08-27 02:03:06 +03:00
}
for i , tt := range tests {
for j := 0 ; j < tt . proposals ; j ++ {
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
}
nt . send ( pb . Message { From : tt . sm . id , To : tt . sm . id , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : tt . wctx } } } )
r := tt . sm
if len ( r . readStates ) == 0 {
t . Errorf ( "#%d: len(readStates) = 0, want non-zero" , i )
}
rs := r . readStates [ 0 ]
if rs . Index != tt . wri {
t . Errorf ( "#%d: readIndex = %d, want %d" , i , rs . Index , tt . wri )
}
2019-03-27 09:15:23 +03:00
if ! bytes . Equal ( rs . RequestCtx , tt . wctx ) {
t . Errorf ( "#%d: requestCtx = %v, want %v" , i , rs . RequestCtx , tt . wctx )
}
r . readStates = nil
}
}
func TestReadOnlyWithLearner ( t * testing . T ) {
a := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
nt := newNetwork ( a , b )
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Fatalf ( "state = %s, want %s" , a . state , StateLeader )
}
tests := [ ] struct {
sm * raft
proposals int
wri uint64
wctx [ ] byte
} {
{ a , 10 , 11 , [ ] byte ( "ctx1" ) } ,
{ b , 10 , 21 , [ ] byte ( "ctx2" ) } ,
{ a , 10 , 31 , [ ] byte ( "ctx3" ) } ,
{ b , 10 , 41 , [ ] byte ( "ctx4" ) } ,
}
for i , tt := range tests {
for j := 0 ; j < tt . proposals ; j ++ {
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
}
nt . send ( pb . Message { From : tt . sm . id , To : tt . sm . id , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : tt . wctx } } } )
r := tt . sm
if len ( r . readStates ) == 0 {
t . Fatalf ( "#%d: len(readStates) = 0, want non-zero" , i )
}
rs := r . readStates [ 0 ]
if rs . Index != tt . wri {
t . Errorf ( "#%d: readIndex = %d, want %d" , i , rs . Index , tt . wri )
}
2016-08-27 02:03:06 +03:00
if ! bytes . Equal ( rs . RequestCtx , tt . wctx ) {
t . Errorf ( "#%d: requestCtx = %v, want %v" , i , rs . RequestCtx , tt . wctx )
}
r . readStates = nil
}
}
func TestReadOnlyOptionLease ( t * testing . T ) {
a := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
b := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
c := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
a . readOnly . option = ReadOnlyLeaseBased
b . readOnly . option = ReadOnlyLeaseBased
c . readOnly . option = ReadOnlyLeaseBased
2016-06-03 18:20:10 +03:00
a . checkQuorum = true
b . checkQuorum = true
c . checkQuorum = true
nt := newNetwork ( a , b , c )
2016-08-02 03:46:43 +03:00
setRandomizedElectionTimeout ( b , b . electionTimeout + 1 )
2016-06-03 18:20:10 +03:00
for i := 0 ; i < b . electionTimeout ; i ++ {
b . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if a . state != StateLeader {
t . Fatalf ( "state = %s, want %s" , a . state , StateLeader )
}
tests := [ ] struct {
sm * raft
proposals int
wri uint64
wctx [ ] byte
} {
2016-09-28 05:19:30 +03:00
{ a , 10 , 11 , [ ] byte ( "ctx1" ) } ,
{ b , 10 , 21 , [ ] byte ( "ctx2" ) } ,
{ c , 10 , 31 , [ ] byte ( "ctx3" ) } ,
{ a , 10 , 41 , [ ] byte ( "ctx4" ) } ,
{ b , 10 , 51 , [ ] byte ( "ctx5" ) } ,
{ c , 10 , 61 , [ ] byte ( "ctx6" ) } ,
2016-06-03 18:20:10 +03:00
}
2016-08-27 02:03:06 +03:00
for i , tt := range tests {
2016-06-03 18:20:10 +03:00
for j := 0 ; j < tt . proposals ; j ++ {
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
}
nt . send ( pb . Message { From : tt . sm . id , To : tt . sm . id , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : tt . wctx } } } )
r := tt . sm
2016-08-27 02:03:06 +03:00
rs := r . readStates [ 0 ]
if rs . Index != tt . wri {
t . Errorf ( "#%d: readIndex = %d, want %d" , i , rs . Index , tt . wri )
2016-06-03 18:20:10 +03:00
}
2016-08-27 02:03:06 +03:00
if ! bytes . Equal ( rs . RequestCtx , tt . wctx ) {
t . Errorf ( "#%d: requestCtx = %v, want %v" , i , rs . RequestCtx , tt . wctx )
2016-06-03 18:20:10 +03:00
}
2016-08-27 02:03:06 +03:00
r . readStates = nil
2016-06-03 18:20:10 +03:00
}
}
2017-02-15 11:45:54 +03:00
// TestReadOnlyForNewLeader ensures that a leader only accepts MsgReadIndex message
// when it commits at least one log entry at it term.
func TestReadOnlyForNewLeader ( t * testing . T ) {
2017-02-21 10:23:42 +03:00
nodeConfigs := [ ] struct {
2018-07-05 20:25:47 +03:00
id uint64
committed uint64
applied uint64
compactIndex uint64
2017-02-21 10:23:42 +03:00
} {
{ 1 , 1 , 1 , 0 } ,
{ 2 , 2 , 2 , 2 } ,
{ 3 , 2 , 2 , 2 } ,
}
peers := make ( [ ] stateMachine , 0 )
for _ , c := range nodeConfigs {
storage := NewMemoryStorage ( )
storage . Append ( [ ] pb . Entry { { Index : 1 , Term : 1 } , { Index : 2 , Term : 1 } } )
storage . SetHardState ( pb . HardState { Term : 1 , Commit : c . committed } )
2018-07-05 20:25:47 +03:00
if c . compactIndex != 0 {
storage . Compact ( c . compactIndex )
2017-02-21 10:23:42 +03:00
}
cfg := newTestConfig ( c . id , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , storage )
cfg . Applied = c . applied
raft := newRaft ( cfg )
peers = append ( peers , raft )
}
nt := newNetwork ( peers ... )
2017-02-15 11:45:54 +03:00
// Drop MsgApp to forbid peer a to commit any log entry at its term after it becomes leader.
nt . ignore ( pb . MsgApp )
// Force peer a to become leader.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2017-02-21 10:23:42 +03:00
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Fatalf ( "state = %s, want %s" , sm . state , StateLeader )
2017-02-15 11:45:54 +03:00
}
// Ensure peer a drops read only request.
var windex uint64 = 4
wctx := [ ] byte ( "ctx" )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : wctx } } } )
2017-02-21 10:23:42 +03:00
if len ( sm . readStates ) != 0 {
t . Fatalf ( "len(readStates) = %d, want zero" , len ( sm . readStates ) )
2017-02-15 11:45:54 +03:00
}
nt . recover ( )
// Force peer a to commit a log entry at its term
2017-02-21 10:23:42 +03:00
for i := 0 ; i < sm . heartbeatTimeout ; i ++ {
sm . tick ( )
2017-02-15 11:45:54 +03:00
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2017-02-21 10:23:42 +03:00
if sm . raftLog . committed != 4 {
t . Fatalf ( "committed = %d, want 4" , sm . raftLog . committed )
2017-02-15 11:45:54 +03:00
}
2017-02-21 10:23:42 +03:00
lastLogTerm := sm . raftLog . zeroTermOnErrCompacted ( sm . raftLog . term ( sm . raftLog . committed ) )
if lastLogTerm != sm . Term {
t . Fatalf ( "last log term = %d, want %d" , lastLogTerm , sm . Term )
2017-02-15 11:45:54 +03:00
}
// Ensure peer a accepts read only request after it commits a entry at its term.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgReadIndex , Entries : [ ] pb . Entry { { Data : wctx } } } )
2017-02-21 10:23:42 +03:00
if len ( sm . readStates ) != 1 {
t . Fatalf ( "len(readStates) = %d, want 1" , len ( sm . readStates ) )
2017-02-15 11:45:54 +03:00
}
2017-02-21 10:23:42 +03:00
rs := sm . readStates [ 0 ]
2017-02-15 11:45:54 +03:00
if rs . Index != windex {
t . Fatalf ( "readIndex = %d, want %d" , rs . Index , windex )
}
if ! bytes . Equal ( rs . RequestCtx , wctx ) {
t . Fatalf ( "requestCtx = %v, want %v" , rs . RequestCtx , wctx )
}
}
2014-05-25 10:03:13 +04:00
func TestLeaderAppResp ( t * testing . T ) {
2014-11-18 22:42:08 +03:00
// initial progress: match = 0; next = 3
2014-05-25 10:03:13 +04:00
tests := [ ] struct {
2014-10-24 22:50:21 +04:00
index uint64
reject bool
// progress
wmatch uint64
wnext uint64
// message
2014-05-25 10:03:13 +04:00
wmsgNum int
2014-10-08 14:29:53 +04:00
windex uint64
wcommitted uint64
2014-05-25 10:03:13 +04:00
} {
2014-10-24 22:50:21 +04:00
{ 3 , true , 0 , 3 , 0 , 0 , 0 } , // stale resp; no replies
2014-12-05 01:12:55 +03:00
{ 2 , true , 0 , 2 , 1 , 1 , 0 } , // denied resp; leader does not commit; decrease next and send probing msg
2014-11-18 22:42:08 +03:00
{ 2 , false , 2 , 4 , 2 , 2 , 2 } , // accept resp; leader commits; broadcast with commit index
2014-10-24 22:50:21 +04:00
{ 0 , false , 0 , 3 , 0 , 0 , 0 } , // ignore heartbeat replies
2014-05-25 10:03:13 +04:00
}
for i , tt := range tests {
// sm term is 1 after it becomes the leader.
// thus the last log term must be 1 to be committed.
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-11-03 23:04:24 +03:00
sm . raftLog = & raftLog {
2014-11-25 04:13:47 +03:00
storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 0 } , { Index : 2 , Term : 1 } } } ,
2014-11-27 00:36:17 +03:00
unstable : unstable { offset : 3 } ,
2014-11-03 23:04:24 +03:00
}
2014-05-25 10:03:13 +04:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
2014-10-30 00:04:45 +03:00
sm . readMessages ( )
2014-12-31 06:42:59 +03:00
sm . Step ( pb . Message { From : 2 , Type : pb . MsgAppResp , Index : tt . index , Term : sm . Term , Reject : tt . reject , RejectHint : tt . index } )
2014-10-24 22:50:21 +04:00
2019-06-08 20:41:24 +03:00
p := sm . prs . Progress [ 2 ]
2015-01-20 21:26:22 +03:00
if p . Match != tt . wmatch {
t . Errorf ( "#%d match = %d, want %d" , i , p . Match , tt . wmatch )
2014-10-24 22:50:21 +04:00
}
2015-01-20 21:26:22 +03:00
if p . Next != tt . wnext {
t . Errorf ( "#%d next = %d, want %d" , i , p . Next , tt . wnext )
2014-10-24 22:50:21 +04:00
}
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-05-25 10:03:13 +04:00
if len ( msgs ) != tt . wmsgNum {
t . Errorf ( "#%d msgNum = %d, want %d" , i , len ( msgs ) , tt . wmsgNum )
}
for j , msg := range msgs {
if msg . Index != tt . windex {
t . Errorf ( "#%d.%d index = %d, want %d" , i , j , msg . Index , tt . windex )
}
if msg . Commit != tt . wcommitted {
t . Errorf ( "#%d.%d commit = %d, want %d" , i , j , msg . Commit , tt . wcommitted )
}
}
}
}
2014-09-15 20:58:22 +04:00
// When the leader receives a heartbeat tick, it should
2018-10-22 08:59:50 +03:00
// send a MsgHeartbeat with m.Index = 0, m.LogTerm=0 and empty entries.
2014-09-15 20:58:22 +04:00
func TestBcastBeat ( t * testing . T ) {
2014-10-08 14:29:53 +04:00
offset := uint64 ( 1000 )
2014-09-15 20:58:22 +04:00
// make a state machine with log.offset = 1000
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : offset ,
Term : 1 ,
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 , 3 } } ,
2014-11-20 00:17:50 +03:00
} ,
2014-09-15 20:58:22 +04:00
}
2014-11-20 00:17:50 +03:00
storage := NewMemoryStorage ( )
storage . ApplySnapshot ( s )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , nil , 10 , 1 , storage )
2014-09-15 20:58:22 +04:00
sm . Term = 1
sm . becomeCandidate ( )
sm . becomeLeader ( )
for i := 0 ; i < 10 ; i ++ {
2018-10-22 12:28:39 +03:00
mustAppendEntry ( sm , pb . Entry { Index : uint64 ( i ) + 1 } )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
// slow follower
2019-06-08 20:41:24 +03:00
sm . prs . Progress [ 2 ] . Match , sm . prs . Progress [ 2 ] . Next = 5 , 6
2014-11-18 02:44:57 +03:00
// normal follower
2019-06-08 20:41:24 +03:00
sm . prs . Progress [ 3 ] . Match , sm . prs . Progress [ 3 ] . Next = sm . raftLog . lastIndex ( ) , sm . raftLog . lastIndex ( ) + 1
2014-09-15 20:58:22 +04:00
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { Type : pb . MsgBeat } )
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-09-29 21:44:20 +04:00
if len ( msgs ) != 2 {
2014-10-30 09:21:38 +03:00
t . Fatalf ( "len(msgs) = %v, want 2" , len ( msgs ) )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
wantCommitMap := map [ uint64 ] uint64 {
2019-06-08 20:41:24 +03:00
2 : min ( sm . raftLog . committed , sm . prs . Progress [ 2 ] . Match ) ,
3 : min ( sm . raftLog . committed , sm . prs . Progress [ 3 ] . Match ) ,
2014-11-18 02:44:57 +03:00
}
2014-09-29 21:44:20 +04:00
for i , m := range msgs {
2014-12-04 08:32:12 +03:00
if m . Type != pb . MsgHeartbeat {
t . Fatalf ( "#%d: type = %v, want = %v" , i , m . Type , pb . MsgHeartbeat )
2014-09-15 20:58:22 +04:00
}
2014-09-29 21:44:20 +04:00
if m . Index != 0 {
t . Fatalf ( "#%d: prevIndex = %d, want %d" , i , m . Index , 0 )
2014-09-15 20:58:22 +04:00
}
2014-09-29 21:44:20 +04:00
if m . LogTerm != 0 {
t . Fatalf ( "#%d: prevTerm = %d, want %d" , i , m . LogTerm , 0 )
2014-09-15 20:58:22 +04:00
}
2014-11-18 02:44:57 +03:00
if wantCommitMap [ m . To ] == 0 {
2014-09-29 21:44:20 +04:00
t . Fatalf ( "#%d: unexpected to %d" , i , m . To )
} else {
2014-11-18 02:44:57 +03:00
if m . Commit != wantCommitMap [ m . To ] {
t . Fatalf ( "#%d: commit = %d, want %d" , i , m . Commit , wantCommitMap [ m . To ] )
}
delete ( wantCommitMap , m . To )
2014-09-15 20:58:22 +04:00
}
if len ( m . Entries ) != 0 {
t . Fatalf ( "#%d: len(entries) = %d, want 0" , i , len ( m . Entries ) )
}
}
}
2016-01-31 05:15:56 +03:00
// tests the output of the state machine when receiving MsgBeat
2014-06-07 00:40:24 +04:00
func TestRecvMsgBeat ( t * testing . T ) {
tests := [ ] struct {
2014-09-16 04:35:02 +04:00
state StateType
2014-06-07 00:40:24 +04:00
wMsg int
} {
2014-09-16 04:35:02 +04:00
{ StateLeader , 2 } ,
2014-10-30 09:21:38 +03:00
// candidate and follower should ignore MsgBeat
2014-09-16 04:35:02 +04:00
{ StateCandidate , 0 } ,
{ StateFollower , 0 } ,
2014-06-07 00:40:24 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
2014-11-25 04:13:47 +03:00
sm . raftLog = & raftLog { storage : & MemoryStorage { ents : [ ] pb . Entry { { } , { Index : 1 , Term : 0 } , { Index : 2 , Term : 1 } } } }
2014-08-23 01:19:30 +04:00
sm . Term = 1
2014-06-07 00:40:24 +04:00
sm . state = tt . state
2014-09-03 21:16:33 +04:00
switch tt . state {
2014-09-16 04:35:02 +04:00
case StateFollower :
2014-09-03 21:16:33 +04:00
sm . step = stepFollower
2014-09-16 04:35:02 +04:00
case StateCandidate :
2014-09-03 21:16:33 +04:00
sm . step = stepCandidate
2014-09-16 04:35:02 +04:00
case StateLeader :
2014-09-03 21:16:33 +04:00
sm . step = stepLeader
}
2014-10-12 11:34:22 +04:00
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
2014-06-07 00:40:24 +04:00
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-06-07 00:40:24 +04:00
if len ( msgs ) != tt . wMsg {
t . Errorf ( "%d: len(msgs) = %d, want %d" , i , len ( msgs ) , tt . wMsg )
}
for _ , m := range msgs {
2014-12-04 08:32:12 +03:00
if m . Type != pb . MsgHeartbeat {
t . Errorf ( "%d: msg.type = %v, want %v" , i , m . Type , pb . MsgHeartbeat )
2014-06-07 00:40:24 +04:00
}
}
}
}
2014-11-18 22:42:08 +03:00
func TestLeaderIncreaseNext ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
tests := [ ] struct {
// progress
2019-06-08 20:41:24 +03:00
state tracker . StateType
2014-11-18 22:42:08 +03:00
next uint64
wnext uint64
} {
2015-03-16 10:46:16 +03:00
// state replicate, optimistically increase next
2014-11-18 22:42:08 +03:00
// previous entries + noop entry + propose + 1
2019-06-08 20:41:24 +03:00
{ tracker . StateReplicate , 2 , uint64 ( len ( previousEnts ) + 1 + 1 + 1 ) } ,
2015-03-16 10:46:16 +03:00
// state probe, not optimistically increase next
2019-06-08 20:41:24 +03:00
{ tracker . StateProbe , 2 , 2 } ,
2014-11-18 22:42:08 +03:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-12-02 19:50:15 +03:00
sm . raftLog . append ( previousEnts ... )
2014-11-18 22:42:08 +03:00
sm . becomeCandidate ( )
sm . becomeLeader ( )
2019-06-08 20:41:24 +03:00
sm . prs . Progress [ 2 ] . State = tt . state
sm . prs . Progress [ 2 ] . Next = tt . next
2014-11-18 22:42:08 +03:00
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
2019-06-08 20:41:24 +03:00
p := sm . prs . Progress [ 2 ]
2015-01-20 21:26:22 +03:00
if p . Next != tt . wnext {
t . Errorf ( "#%d next = %d, want %d" , i , p . Next , tt . wnext )
2014-11-18 22:42:08 +03:00
}
}
2014-06-07 00:40:24 +04:00
}
2015-03-16 10:46:16 +03:00
func TestSendAppendForProgressProbe ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-02-05 01:41:14 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . BecomeProbe ( )
2015-02-05 01:41:14 +03:00
2015-03-16 10:46:16 +03:00
// each round is a heartbeat
2015-02-05 01:41:14 +03:00
for i := 0 ; i < 3 ; i ++ {
2016-12-19 23:24:21 +03:00
if i == 0 {
// we expect that raft will only send out one msgAPP on the first
// loop. After that, the follower is paused until a heartbeat response is
// received.
2018-10-22 12:28:39 +03:00
mustAppendEntry ( r , pb . Entry { Data : [ ] byte ( "somedata" ) } )
2016-12-19 23:24:21 +03:00
r . sendAppend ( 2 )
msg := r . readMessages ( )
if len ( msg ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msg ) , 1 )
}
if msg [ 0 ] . Index != 0 {
t . Errorf ( "index = %d, want %d" , msg [ 0 ] . Index , 0 )
}
2015-02-05 01:41:14 +03:00
}
2019-06-08 20:41:24 +03:00
if ! r . prs . Progress [ 2 ] . ProbeSent {
t . Errorf ( "paused = %v, want true" , r . prs . Progress [ 2 ] . ProbeSent )
2015-02-05 01:41:14 +03:00
}
2015-02-28 22:56:55 +03:00
for j := 0 ; j < 10 ; j ++ {
2018-10-22 12:28:39 +03:00
mustAppendEntry ( r , pb . Entry { Data : [ ] byte ( "somedata" ) } )
2015-03-16 10:46:16 +03:00
r . sendAppend ( 2 )
2015-02-05 01:41:14 +03:00
if l := len ( r . readMessages ( ) ) ; l != 0 {
t . Errorf ( "len(msg) = %d, want %d" , l , 0 )
}
}
// do a heartbeat
2015-02-28 22:56:55 +03:00
for j := 0 ; j < r . heartbeatTimeout ; j ++ {
2015-02-05 01:41:14 +03:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
}
2019-06-08 20:41:24 +03:00
if ! r . prs . Progress [ 2 ] . ProbeSent {
t . Errorf ( "paused = %v, want true" , r . prs . Progress [ 2 ] . ProbeSent )
2016-12-19 23:24:21 +03:00
}
2015-02-05 01:41:14 +03:00
// consume the heartbeat
2016-12-19 23:24:21 +03:00
msg := r . readMessages ( )
2015-02-05 01:41:14 +03:00
if len ( msg ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msg ) , 1 )
}
if msg [ 0 ] . Type != pb . MsgHeartbeat {
2015-12-16 07:18:21 +03:00
t . Errorf ( "type = %v, want %v" , msg [ 0 ] . Type , pb . MsgHeartbeat )
2015-02-05 01:41:14 +03:00
}
}
2016-12-19 23:24:21 +03:00
// a heartbeat response will allow another message to be sent
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgHeartbeatResp } )
msg := r . readMessages ( )
if len ( msg ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msg ) , 1 )
}
if msg [ 0 ] . Index != 0 {
t . Errorf ( "index = %d, want %d" , msg [ 0 ] . Index , 0 )
}
2019-06-08 20:41:24 +03:00
if ! r . prs . Progress [ 2 ] . ProbeSent {
t . Errorf ( "paused = %v, want true" , r . prs . Progress [ 2 ] . ProbeSent )
2016-12-19 23:24:21 +03:00
}
2015-03-16 10:46:16 +03:00
}
func TestSendAppendForProgressReplicate ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . BecomeReplicate ( )
2015-02-05 01:41:14 +03:00
for i := 0 ; i < 10 ; i ++ {
2018-10-22 12:28:39 +03:00
mustAppendEntry ( r , pb . Entry { Data : [ ] byte ( "somedata" ) } )
2015-03-16 10:46:16 +03:00
r . sendAppend ( 2 )
2015-02-05 01:41:14 +03:00
msgs := r . readMessages ( )
if len ( msgs ) != 1 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msgs ) , 1 )
}
}
}
2015-03-16 10:46:16 +03:00
func TestSendAppendForProgressSnapshot ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . BecomeSnapshot ( 10 )
2015-03-16 10:46:16 +03:00
for i := 0 ; i < 10 ; i ++ {
2018-10-22 12:28:39 +03:00
mustAppendEntry ( r , pb . Entry { Data : [ ] byte ( "somedata" ) } )
2015-03-16 10:46:16 +03:00
r . sendAppend ( 2 )
msgs := r . readMessages ( )
if len ( msgs ) != 0 {
t . Errorf ( "len(msg) = %d, want %d" , len ( msgs ) , 0 )
}
}
}
func TestRecvMsgUnreachable ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
s := NewMemoryStorage ( )
s . Append ( previousEnts )
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , s )
2015-03-16 10:46:16 +03:00
r . becomeCandidate ( )
r . becomeLeader ( )
r . readMessages ( )
// set node 2 to state replicate
2019-06-08 20:41:24 +03:00
r . prs . Progress [ 2 ] . Match = 3
r . prs . Progress [ 2 ] . BecomeReplicate ( )
r . prs . Progress [ 2 ] . OptimisticUpdate ( 5 )
2015-03-16 10:46:16 +03:00
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgUnreachable } )
2019-06-08 20:41:24 +03:00
if r . prs . Progress [ 2 ] . State != tracker . StateProbe {
t . Errorf ( "state = %s, want %s" , r . prs . Progress [ 2 ] . State , tracker . StateProbe )
2015-03-16 10:46:16 +03:00
}
2019-06-08 20:41:24 +03:00
if wnext := r . prs . Progress [ 2 ] . Match + 1 ; r . prs . Progress [ 2 ] . Next != wnext {
t . Errorf ( "next = %d, want %d" , r . prs . Progress [ 2 ] . Next , wnext )
2015-03-16 10:46:16 +03:00
}
}
2014-07-01 23:10:43 +04:00
func TestRestore ( t * testing . T ) {
2014-08-28 05:53:18 +04:00
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 , 3 } } ,
2014-11-20 00:17:50 +03:00
} ,
2014-07-01 23:10:43 +04:00
}
2014-11-20 00:17:50 +03:00
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , storage )
2014-08-01 00:03:23 +04:00
if ok := sm . restore ( s ) ; ! ok {
t . Fatal ( "restore fail, want succeed" )
}
2014-07-01 23:10:43 +04:00
2014-11-20 00:17:50 +03:00
if sm . raftLog . lastIndex ( ) != s . Metadata . Index {
t . Errorf ( "log.lastIndex = %d, want %d" , sm . raftLog . lastIndex ( ) , s . Metadata . Index )
2014-08-01 02:18:44 +04:00
}
2015-06-13 08:24:40 +03:00
if mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) != s . Metadata . Term {
t . Errorf ( "log.lastTerm = %d, want %d" , mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) , s . Metadata . Term )
2014-08-01 02:18:44 +04:00
}
2019-06-08 20:41:24 +03:00
sg := sm . prs . VoterNodes ( )
2019-07-23 01:34:46 +03:00
if ! reflect . DeepEqual ( sg , s . Metadata . ConfState . Voters ) {
t . Errorf ( "sm.Voters = %+v, want %+v" , sg , s . Metadata . ConfState . Voters )
2014-07-01 23:10:43 +04:00
}
2014-08-01 00:03:23 +04:00
if ok := sm . restore ( s ) ; ok {
t . Fatal ( "restore succeed, want fail" )
}
2014-07-01 23:10:43 +04:00
}
2017-11-11 05:38:21 +03:00
// TestRestoreWithLearner restores a snapshot which contains learners.
func TestRestoreWithLearner ( t * testing . T ) {
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 } , Learners : [ ] uint64 { 3 } } ,
2017-11-11 05:38:21 +03:00
} ,
}
storage := NewMemoryStorage ( )
2018-05-10 01:18:57 +03:00
sm := newTestLearnerRaft ( 3 , [ ] uint64 { 1 , 2 } , [ ] uint64 { 3 } , 8 , 2 , storage )
2017-11-11 05:38:21 +03:00
if ok := sm . restore ( s ) ; ! ok {
t . Error ( "restore fail, want succeed" )
}
if sm . raftLog . lastIndex ( ) != s . Metadata . Index {
t . Errorf ( "log.lastIndex = %d, want %d" , sm . raftLog . lastIndex ( ) , s . Metadata . Index )
}
if mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) != s . Metadata . Term {
t . Errorf ( "log.lastTerm = %d, want %d" , mustTerm ( sm . raftLog . term ( s . Metadata . Index ) ) , s . Metadata . Term )
}
2019-06-08 20:41:24 +03:00
sg := sm . prs . VoterNodes ( )
2019-07-23 01:34:46 +03:00
if len ( sg ) != len ( s . Metadata . ConfState . Voters ) {
t . Errorf ( "sm.Voters = %+v, length not equal with %+v" , sg , s . Metadata . ConfState . Voters )
2018-01-08 18:43:04 +03:00
}
2019-06-08 20:41:24 +03:00
lns := sm . prs . LearnerNodes ( )
2018-01-08 18:43:04 +03:00
if len ( lns ) != len ( s . Metadata . ConfState . Learners ) {
t . Errorf ( "sm.LearnerNodes = %+v, length not equal with %+v" , sg , s . Metadata . ConfState . Learners )
2017-11-11 05:38:21 +03:00
}
2019-07-23 01:34:46 +03:00
for _ , n := range s . Metadata . ConfState . Voters {
2019-06-08 20:41:24 +03:00
if sm . prs . Progress [ n ] . IsLearner {
t . Errorf ( "sm.Node %x isLearner = %s, want %t" , n , sm . prs . Progress [ n ] , false )
2017-11-11 05:38:21 +03:00
}
}
for _ , n := range s . Metadata . ConfState . Learners {
2019-06-08 20:41:24 +03:00
if ! sm . prs . Progress [ n ] . IsLearner {
t . Errorf ( "sm.Node %x isLearner = %s, want %t" , n , sm . prs . Progress [ n ] , true )
2017-11-11 05:38:21 +03:00
}
}
if ok := sm . restore ( s ) ; ok {
t . Error ( "restore succeed, want fail" )
}
}
2019-07-08 10:32:24 +03:00
// TestRestoreVoterToLearner verifies that a normal peer can be downgraded to a
// learner through a snapshot. At the time of writing, we don't allow
// configuration changes to do this directly, but note that the snapshot may
// compress multiple changes to the configuration into one: the voter could have
// been removed, then readded as a learner and the snapshot reflects both
// changes. In that case, a voter receives a snapshot telling it that it is now
// a learner. In fact, the node has to accept that snapshot, or it is
// permanently cut off from the Raft log.
func TestRestoreVoterToLearner ( t * testing . T ) {
2017-11-11 05:38:21 +03:00
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 } , Learners : [ ] uint64 { 3 } } ,
2017-11-11 05:38:21 +03:00
} ,
}
storage := NewMemoryStorage ( )
sm := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , storage )
if sm . isLearner {
t . Errorf ( "%x is learner, want not" , sm . id )
}
2019-07-08 10:32:24 +03:00
if ok := sm . restore ( s ) ; ! ok {
t . Error ( "restore failed unexpectedly" )
2017-11-11 05:38:21 +03:00
}
}
// TestRestoreLearnerPromotion checks that a learner can become to a follower after
// restoring snapshot.
func TestRestoreLearnerPromotion ( t * testing . T ) {
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 , 3 } } ,
2017-11-11 05:38:21 +03:00
} ,
}
storage := NewMemoryStorage ( )
sm := newTestLearnerRaft ( 3 , [ ] uint64 { 1 , 2 } , [ ] uint64 { 3 } , 10 , 1 , storage )
if ! sm . isLearner {
t . Errorf ( "%x is not learner, want yes" , sm . id )
}
if ok := sm . restore ( s ) ; ! ok {
t . Error ( "restore fail, want succeed" )
}
if sm . isLearner {
t . Errorf ( "%x is learner, want not" , sm . id )
}
}
// TestLearnerReceiveSnapshot tests that a learner can receive a snpahost from leader
func TestLearnerReceiveSnapshot ( t * testing . T ) {
// restore the state machine from a snapshot so it has a compacted log and a snapshot
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 } , Learners : [ ] uint64 { 2 } } ,
2017-11-11 05:38:21 +03:00
} ,
}
n1 := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestLearnerRaft ( 2 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . restore ( s )
// Force set n1 appplied index.
n1 . raftLog . appliedTo ( n1 . raftLog . committed )
nt := newNetwork ( n1 , n2 )
setRandomizedElectionTimeout ( n1 , n1 . electionTimeout )
for i := 0 ; i < n1 . electionTimeout ; i ++ {
n1 . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
if n2 . raftLog . committed != n1 . raftLog . committed {
t . Errorf ( "peer 2 must commit to %d, but %d" , n1 . raftLog . committed , n2 . raftLog . committed )
}
}
2014-12-03 08:34:14 +03:00
func TestRestoreIgnoreSnapshot ( t * testing . T ) {
previousEnts := [ ] pb . Entry { { Term : 1 , Index : 1 } , { Term : 1 , Index : 2 } , { Term : 1 , Index : 3 } }
commit := uint64 ( 1 )
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , storage )
2014-12-03 08:34:14 +03:00
sm . raftLog . append ( previousEnts ... )
sm . raftLog . commitTo ( commit )
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : commit ,
Term : 1 ,
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 } } ,
2014-12-03 08:34:14 +03:00
} ,
}
// ignore snapshot
if ok := sm . restore ( s ) ; ok {
t . Errorf ( "restore = %t, want %t" , ok , false )
}
if sm . raftLog . committed != commit {
t . Errorf ( "commit = %d, want %d" , sm . raftLog . committed , commit )
}
// ignore snapshot and fast forward commit
s . Metadata . Index = commit + 1
if ok := sm . restore ( s ) ; ok {
t . Errorf ( "restore = %t, want %t" , ok , false )
}
if sm . raftLog . committed != commit + 1 {
t . Errorf ( "commit = %d, want %d" , sm . raftLog . committed , commit + 1 )
}
}
2014-07-02 23:49:58 +04:00
func TestProvideSnap ( t * testing . T ) {
2015-12-10 23:17:30 +03:00
// restore the state machine from a snapshot so it has a compacted log and a snapshot
2014-11-20 00:17:50 +03:00
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 } } ,
2014-11-20 00:17:50 +03:00
} ,
}
storage := NewMemoryStorage ( )
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
2014-07-02 23:49:58 +04:00
sm . restore ( s )
sm . becomeCandidate ( )
sm . becomeLeader ( )
2015-12-10 23:17:30 +03:00
// force set the next of node 2, so that node 2 needs a snapshot
2019-06-08 20:41:24 +03:00
sm . prs . Progress [ 2 ] . Next = sm . raftLog . firstIndex ( )
sm . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgAppResp , Index : sm . prs . Progress [ 2 ] . Next - 1 , Reject : true } )
2015-12-10 23:17:30 +03:00
2014-10-30 00:04:45 +03:00
msgs := sm . readMessages ( )
2014-07-02 23:49:58 +04:00
if len ( msgs ) != 1 {
2014-09-29 06:28:12 +04:00
t . Fatalf ( "len(msgs) = %d, want 1" , len ( msgs ) )
2014-07-02 23:49:58 +04:00
}
2014-09-29 21:44:20 +04:00
m := msgs [ 0 ]
2014-10-12 11:34:22 +04:00
if m . Type != pb . MsgSnap {
t . Errorf ( "m.Type = %v, want %v" , m . Type , pb . MsgSnap )
2014-07-02 23:49:58 +04:00
}
}
2015-12-10 23:17:30 +03:00
func TestIgnoreProvidingSnap ( t * testing . T ) {
// restore the state machine from a snapshot so it has a compacted log and a snapshot
s := pb . Snapshot {
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 } } ,
2015-12-10 23:17:30 +03:00
} ,
}
storage := NewMemoryStorage ( )
sm := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , storage )
sm . restore ( s )
sm . becomeCandidate ( )
sm . becomeLeader ( )
// force set the next of node 2, so that node 2 needs a snapshot
// change node 2 to be inactive, expect node 1 ignore sending snapshot to 2
2019-06-08 20:41:24 +03:00
sm . prs . Progress [ 2 ] . Next = sm . raftLog . firstIndex ( ) - 1
sm . prs . Progress [ 2 ] . RecentActive = false
2015-12-10 23:17:30 +03:00
sm . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "somedata" ) } } } )
msgs := sm . readMessages ( )
if len ( msgs ) != 0 {
t . Errorf ( "len(msgs) = %d, want 0" , len ( msgs ) )
}
}
2014-07-02 23:49:58 +04:00
func TestRestoreFromSnapMsg ( t * testing . T ) {
2014-08-28 05:53:18 +04:00
s := pb . Snapshot {
2014-11-20 00:17:50 +03:00
Metadata : pb . SnapshotMetadata {
Index : 11 , // magic number
Term : 11 , // magic number
2019-07-23 01:34:46 +03:00
ConfState : pb . ConfState { Voters : [ ] uint64 { 1 , 2 } } ,
2014-11-20 00:17:50 +03:00
} ,
2014-07-02 23:49:58 +04:00
}
2014-10-12 11:34:22 +04:00
m := pb . Message { Type : pb . MsgSnap , From : 1 , Term : 2 , Snapshot : s }
2014-07-02 23:49:58 +04:00
2015-03-22 04:15:58 +03:00
sm := newTestRaft ( 2 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-07-02 23:49:58 +04:00
sm . Step ( m )
2016-07-12 02:37:31 +03:00
if sm . lead != uint64 ( 1 ) {
t . Errorf ( "sm.lead = %d, want 1" , sm . lead )
}
2014-11-20 00:17:50 +03:00
// TODO(bdarnell): what should this test?
2014-07-02 23:49:58 +04:00
}
func TestSlowNodeRestore ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
2014-07-02 23:49:58 +04:00
2014-09-12 06:23:05 +04:00
nt . isolate ( 3 )
2014-10-17 02:32:57 +04:00
for j := 0 ; j <= 100 ; j ++ {
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2014-07-02 23:49:58 +04:00
}
2014-09-12 06:23:05 +04:00
lead := nt . peers [ 1 ] . ( * raft )
2014-11-13 00:23:42 +03:00
nextEnts ( lead , nt . storage [ 1 ] )
2019-07-23 01:34:46 +03:00
nt . storage [ 1 ] . CreateSnapshot ( lead . raftLog . applied , & pb . ConfState { Voters : lead . prs . VoterNodes ( ) } , nil )
2015-01-23 21:50:43 +03:00
nt . storage [ 1 ] . Compact ( lead . raftLog . applied )
2014-07-02 23:49:58 +04:00
nt . recover ( )
2015-12-10 23:17:30 +03:00
// send heartbeats so that the leader can learn everyone is active.
// node 3 will only be considered as active when node 1 receives a reply from it.
for {
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgBeat } )
2019-06-08 20:41:24 +03:00
if lead . prs . Progress [ 3 ] . RecentActive {
2015-12-10 23:17:30 +03:00
break
}
}
2014-09-29 21:44:20 +04:00
// trigger a snapshot
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2015-12-10 23:17:30 +03:00
2014-09-12 06:23:05 +04:00
follower := nt . peers [ 3 ] . ( * raft )
2014-07-02 23:49:58 +04:00
2014-09-29 21:44:20 +04:00
// trigger a commit
2014-10-12 11:34:22 +04:00
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2014-09-29 21:44:20 +04:00
if follower . raftLog . committed != lead . raftLog . committed {
2014-12-05 01:12:55 +03:00
t . Errorf ( "follower.committed = %d, want %d" , follower . raftLog . committed , lead . raftLog . committed )
2014-07-02 23:49:58 +04:00
}
}
2014-09-23 23:02:44 +04:00
// TestStepConfig tests that when raft step msgProp in EntryConfChange type,
2014-09-19 23:35:56 +04:00
// it appends the entry to log and sets pendingConf to be true.
func TestStepConfig ( t * testing . T ) {
// a raft that cannot make progress
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
index := r . raftLog . lastIndex ( )
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-09-19 23:35:56 +04:00
if g := r . raftLog . lastIndex ( ) ; g != index + 1 {
t . Errorf ( "index = %d, want %d" , g , index + 1 )
}
2017-12-30 07:11:22 +03:00
if r . pendingConfIndex != index + 1 {
t . Errorf ( "pendingConfIndex = %d, want %d" , r . pendingConfIndex , index + 1 )
2014-09-19 23:35:56 +04:00
}
}
// TestStepIgnoreConfig tests that if raft step the second msgProp in
2014-12-10 09:13:42 +03:00
// EntryConfChange type when the first one is uncommitted, the node will set
// the proposal to noop and keep its original state.
2014-09-19 23:35:56 +04:00
func TestStepIgnoreConfig ( t * testing . T ) {
// a raft that cannot make progress
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-09-19 23:35:56 +04:00
index := r . raftLog . lastIndex ( )
2017-12-30 07:11:22 +03:00
pendingConfIndex := r . pendingConfIndex
2014-10-12 11:34:22 +04:00
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Type : pb . EntryConfChange } } } )
2014-12-10 09:13:42 +03:00
wents := [ ] pb . Entry { { Type : pb . EntryNormal , Term : 1 , Index : 3 , Data : nil } }
2015-06-13 08:24:40 +03:00
ents , err := r . raftLog . entries ( index + 1 , noLimit )
if err != nil {
t . Fatalf ( "unexpected error %v" , err )
}
if ! reflect . DeepEqual ( ents , wents ) {
2014-12-10 09:13:42 +03:00
t . Errorf ( "ents = %+v, want %+v" , ents , wents )
2014-09-19 23:35:56 +04:00
}
2017-12-30 07:11:22 +03:00
if r . pendingConfIndex != pendingConfIndex {
t . Errorf ( "pendingConfIndex = %d, want %d" , r . pendingConfIndex , pendingConfIndex )
2014-09-19 23:35:56 +04:00
}
}
2017-12-30 07:11:22 +03:00
// TestNewLeaderPendingConfig tests that new leader sets its pendingConfigIndex
2014-09-19 23:35:56 +04:00
// based on uncommitted entries.
2017-12-30 07:11:22 +03:00
func TestNewLeaderPendingConfig ( t * testing . T ) {
2014-09-19 23:35:56 +04:00
tests := [ ] struct {
2017-12-30 07:11:22 +03:00
addEntry bool
wpendingIndex uint64
2014-09-19 23:35:56 +04:00
} {
2017-12-30 07:11:22 +03:00
{ false , 0 } ,
{ true , 1 } ,
2014-09-19 23:35:56 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2017-12-30 07:11:22 +03:00
if tt . addEntry {
2018-10-22 12:28:39 +03:00
mustAppendEntry ( r , pb . Entry { Type : pb . EntryNormal } )
2017-12-30 07:11:22 +03:00
}
2014-09-19 23:35:56 +04:00
r . becomeCandidate ( )
r . becomeLeader ( )
2017-12-30 07:11:22 +03:00
if r . pendingConfIndex != tt . wpendingIndex {
t . Errorf ( "#%d: pendingConfIndex = %d, want %d" ,
i , r . pendingConfIndex , tt . wpendingIndex )
2014-09-19 23:35:56 +04:00
}
}
}
2017-12-30 07:11:22 +03:00
// TestAddNode tests that addNode could update nodes correctly.
2014-09-19 23:35:56 +04:00
func TestAddNode ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeAddNode } . AsV2 ( ) )
2019-06-08 20:41:24 +03:00
nodes := r . prs . VoterNodes ( )
2014-10-08 14:29:53 +04:00
wnodes := [ ] uint64 { 1 , 2 }
2014-09-19 23:35:56 +04:00
if ! reflect . DeepEqual ( nodes , wnodes ) {
t . Errorf ( "nodes = %v, want %v" , nodes , wnodes )
}
}
2017-12-30 07:11:22 +03:00
// TestAddLearner tests that addLearner could update nodes correctly.
2017-11-11 05:38:21 +03:00
func TestAddLearner ( t * testing . T ) {
r := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
// Add new learner peer.
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeAddLearnerNode } . AsV2 ( ) )
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
if r . isLearner {
t . Fatal ( "expected 1 to be voter" )
}
2019-06-08 20:41:24 +03:00
nodes := r . prs . LearnerNodes ( )
2018-01-08 18:43:04 +03:00
wnodes := [ ] uint64 { 2 }
2017-11-11 05:38:21 +03:00
if ! reflect . DeepEqual ( nodes , wnodes ) {
t . Errorf ( "nodes = %v, want %v" , nodes , wnodes )
}
2019-06-08 20:41:24 +03:00
if ! r . prs . Progress [ 2 ] . IsLearner {
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
t . Fatal ( "expected 2 to be learner" )
}
// Promote peer to voter.
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeAddNode } . AsV2 ( ) )
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
if r . prs . Progress [ 2 ] . IsLearner {
t . Fatal ( "expected 2 to be voter" )
}
// Demote r.
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 1 , Type : pb . ConfChangeAddLearnerNode } . AsV2 ( ) )
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
if ! r . prs . Progress [ 1 ] . IsLearner {
t . Fatal ( "expected 1 to be learner" )
}
if ! r . isLearner {
t . Fatal ( "expected 1 to be learner" )
}
// Promote r again.
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 1 , Type : pb . ConfChangeAddNode } . AsV2 ( ) )
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
if r . prs . Progress [ 1 ] . IsLearner {
t . Fatal ( "expected 1 to be voter" )
}
if r . isLearner {
t . Fatal ( "expected 1 to be voter" )
2017-11-11 05:38:21 +03:00
}
}
2017-05-05 00:58:05 +03:00
// TestAddNodeCheckQuorum tests that addNode does not trigger a leader election
// immediately when checkQuorum is set.
func TestAddNodeCheckQuorum ( t * testing . T ) {
r := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
r . checkQuorum = true
r . becomeCandidate ( )
r . becomeLeader ( )
for i := 0 ; i < r . electionTimeout - 1 ; i ++ {
r . tick ( )
}
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeAddNode } . AsV2 ( ) )
2017-05-05 00:58:05 +03:00
// This tick will reach electionTimeout, which triggers a quorum check.
r . tick ( )
// Node 1 should still be the leader after a single tick.
if r . state != StateLeader {
t . Errorf ( "state = %v, want %v" , r . state , StateLeader )
}
// After another electionTimeout ticks without hearing from node 2,
// node 1 should step down.
for i := 0 ; i < r . electionTimeout ; i ++ {
r . tick ( )
}
if r . state != StateFollower {
t . Errorf ( "state = %v, want %v" , r . state , StateFollower )
}
}
2017-12-30 07:11:22 +03:00
// TestRemoveNode tests that removeNode could update nodes and
2014-09-30 06:54:20 +04:00
// and removed list correctly.
2014-09-19 23:35:56 +04:00
func TestRemoveNode ( t * testing . T ) {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 10 , 1 , NewMemoryStorage ( ) )
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeRemoveNode } . AsV2 ( ) )
2014-10-08 14:29:53 +04:00
w := [ ] uint64 { 1 }
2019-06-08 20:41:24 +03:00
if g := r . prs . VoterNodes ( ) ; ! reflect . DeepEqual ( g , w ) {
2014-09-19 23:35:56 +04:00
t . Errorf ( "nodes = %v, want %v" , g , w )
}
2016-05-16 19:58:57 +03:00
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
// Removing the remaining voter will panic.
defer func ( ) {
if r := recover ( ) ; r == nil {
t . Error ( "did not panic" )
}
} ( )
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 1 , Type : pb . ConfChangeRemoveNode } . AsV2 ( ) )
2014-09-19 23:35:56 +04:00
}
2017-12-30 07:11:22 +03:00
// TestRemoveLearner tests that removeNode could update nodes and
2017-11-11 05:38:21 +03:00
// and removed list correctly.
func TestRemoveLearner ( t * testing . T ) {
r := newTestLearnerRaft ( 1 , [ ] uint64 { 1 } , [ ] uint64 { 2 } , 10 , 1 , NewMemoryStorage ( ) )
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeRemoveNode } . AsV2 ( ) )
2017-11-11 05:38:21 +03:00
w := [ ] uint64 { 1 }
2019-06-08 20:41:24 +03:00
if g := r . prs . VoterNodes ( ) ; ! reflect . DeepEqual ( g , w ) {
2017-11-11 05:38:21 +03:00
t . Errorf ( "nodes = %v, want %v" , g , w )
}
2019-07-22 23:30:47 +03:00
w = nil
2019-06-08 20:41:24 +03:00
if g := r . prs . LearnerNodes ( ) ; ! reflect . DeepEqual ( g , w ) {
2018-01-08 18:43:04 +03:00
t . Errorf ( "nodes = %v, want %v" , g , w )
}
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
// Removing the remaining voter will panic.
defer func ( ) {
if r := recover ( ) ; r == nil {
t . Error ( "did not panic" )
}
} ( )
2019-07-22 23:30:47 +03:00
r . applyConfChange ( pb . ConfChange { NodeID : 1 , Type : pb . ConfChangeRemoveNode } . AsV2 ( ) )
2017-11-11 05:38:21 +03:00
}
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
2014-09-24 21:19:41 +04:00
func TestPromotable ( t * testing . T ) {
2014-10-08 14:29:53 +04:00
id := uint64 ( 1 )
2014-09-24 10:09:21 +04:00
tests := [ ] struct {
2014-10-08 14:29:53 +04:00
peers [ ] uint64
2014-09-24 21:19:41 +04:00
wp bool
2014-09-24 10:09:21 +04:00
} {
2014-10-08 14:29:53 +04:00
{ [ ] uint64 { 1 } , true } ,
{ [ ] uint64 { 1 , 2 , 3 } , true } ,
{ [ ] uint64 { } , false } ,
{ [ ] uint64 { 2 , 3 } , false } ,
2014-09-24 10:09:21 +04:00
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( id , tt . peers , 5 , 1 , NewMemoryStorage ( ) )
2014-09-24 21:19:41 +04:00
if g := r . promotable ( ) ; g != tt . wp {
t . Errorf ( "#%d: promotable = %v, want %v" , i , g , tt . wp )
2014-09-24 10:09:21 +04:00
}
}
}
2014-11-12 08:11:09 +03:00
func TestRaftNodes ( t * testing . T ) {
tests := [ ] struct {
ids [ ] uint64
wids [ ] uint64
} {
{
[ ] uint64 { 1 , 2 , 3 } ,
[ ] uint64 { 1 , 2 , 3 } ,
} ,
{
[ ] uint64 { 3 , 2 , 1 } ,
[ ] uint64 { 1 , 2 , 3 } ,
} ,
}
for i , tt := range tests {
2015-03-22 04:15:58 +03:00
r := newTestRaft ( 1 , tt . ids , 10 , 1 , NewMemoryStorage ( ) )
2019-06-08 20:41:24 +03:00
if ! reflect . DeepEqual ( r . prs . VoterNodes ( ) , tt . wids ) {
t . Errorf ( "#%d: nodes = %+v, want %+v" , i , r . prs . VoterNodes ( ) , tt . wids )
2014-11-12 08:11:09 +03:00
}
}
}
2015-11-17 03:32:12 +03:00
func TestCampaignWhileLeader ( t * testing . T ) {
2016-10-25 17:31:44 +03:00
testCampaignWhileLeader ( t , false )
}
func TestPreCampaignWhileLeader ( t * testing . T ) {
testCampaignWhileLeader ( t , true )
}
func testCampaignWhileLeader ( t * testing . T , preVote bool ) {
cfg := newTestConfig ( 1 , [ ] uint64 { 1 } , 5 , 1 , NewMemoryStorage ( ) )
cfg . PreVote = preVote
r := newRaft ( cfg )
if r . state != StateFollower {
t . Errorf ( "expected new node to be follower but got %s" , r . state )
}
// We don't call campaign() directly because it comes after the check
// for our current state.
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if r . state != StateLeader {
t . Errorf ( "expected single-node election to become leader but got %s" , r . state )
}
term := r . Term
r . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if r . state != StateLeader {
t . Errorf ( "expected to remain leader but got %s" , r . state )
}
if r . Term != term {
t . Errorf ( "expected to remain in term %v but got %v" , term , r . Term )
2015-11-17 03:32:12 +03:00
}
}
2016-01-20 18:03:12 +03:00
// TestCommitAfterRemoveNode verifies that pending commands can become
// committed when a config change reduces the quorum requirements.
func TestCommitAfterRemoveNode ( t * testing . T ) {
// Create a cluster with two nodes.
s := NewMemoryStorage ( )
r := newTestRaft ( 1 , [ ] uint64 { 1 , 2 } , 5 , 1 , s )
r . becomeCandidate ( )
r . becomeLeader ( )
// Begin to remove the second node.
cc := pb . ConfChange {
Type : pb . ConfChangeRemoveNode ,
NodeID : 2 ,
}
ccData , err := cc . Marshal ( )
if err != nil {
t . Fatal ( err )
}
r . Step ( pb . Message {
Type : pb . MsgProp ,
Entries : [ ] pb . Entry {
{ Type : pb . EntryConfChange , Data : ccData } ,
} ,
} )
// Stabilize the log and make sure nothing is committed yet.
if ents := nextEnts ( r , s ) ; len ( ents ) > 0 {
t . Fatalf ( "unexpected committed entries: %v" , ents )
}
ccIndex := r . raftLog . lastIndex ( )
// While the config change is pending, make another proposal.
r . Step ( pb . Message {
Type : pb . MsgProp ,
Entries : [ ] pb . Entry {
{ Type : pb . EntryNormal , Data : [ ] byte ( "hello" ) } ,
} ,
} )
// Node 2 acknowledges the config change, committing it.
r . Step ( pb . Message {
Type : pb . MsgAppResp ,
From : 2 ,
Index : ccIndex ,
} )
ents := nextEnts ( r , s )
if len ( ents ) != 2 {
t . Fatalf ( "expected two committed entries, got %v" , ents )
}
if ents [ 0 ] . Type != pb . EntryNormal || ents [ 0 ] . Data != nil {
t . Fatalf ( "expected ents[0] to be empty, but got %v" , ents [ 0 ] )
}
if ents [ 1 ] . Type != pb . EntryConfChange {
t . Fatalf ( "expected ents[1] to be EntryConfChange, got %v" , ents [ 1 ] )
}
// Apply the config change. This reduces quorum requirements so the
// pending command can now commit.
2019-07-22 23:30:47 +03:00
r . applyConfChange ( cc . AsV2 ( ) )
2016-01-20 18:03:12 +03:00
ents = nextEnts ( r , s )
if len ( ents ) != 1 || ents [ 0 ] . Type != pb . EntryNormal ||
string ( ents [ 0 ] . Data ) != "hello" {
t . Fatalf ( "expected one committed EntryNormal, got %v" , ents )
}
}
2016-04-11 09:16:56 +03:00
// TestLeaderTransferToUpToDateNode verifies transferring should succeed
2016-09-08 11:56:02 +03:00
// if the transferee has the most up-to-date log entries when transfer starts.
2016-03-24 14:59:41 +03:00
func TestLeaderTransferToUpToDateNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
if lead . lead != 1 {
t . Fatalf ( "after election leader is %x, want 1" , lead . lead )
}
// Transfer leadership to 2.
nt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
// After some log replication, transfer leadership back to 1.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . send ( pb . Message { From : 1 , To : 2 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
2016-09-08 11:56:02 +03:00
// TestLeaderTransferToUpToDateNodeFromFollower verifies transferring should succeed
// if the transferee has the most up-to-date log entries when transfer starts.
// Not like TestLeaderTransferToUpToDateNode, where the leader transfer message
// is sent to the leader, in this test case every leader transfer message is sent
// to the follower.
func TestLeaderTransferToUpToDateNodeFromFollower ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
if lead . lead != 1 {
t . Fatalf ( "after election leader is %x, want 1" , lead . lead )
}
// Transfer leadership to 2.
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
// After some log replication, transfer leadership back to 1.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
2016-06-29 13:24:58 +03:00
// TestLeaderTransferWithCheckQuorum ensures transferring leader still works
// even the current leader is still under its leader lease
func TestLeaderTransferWithCheckQuorum ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
for i := 1 ; i < 4 ; i ++ {
r := nt . peers [ uint64 ( i ) ] . ( * raft )
r . checkQuorum = true
2016-08-04 13:31:22 +03:00
setRandomizedElectionTimeout ( r , r . electionTimeout + i )
2016-06-29 13:24:58 +03:00
}
2016-07-11 09:59:53 +03:00
// Letting peer 2 electionElapsed reach to timeout so that it can vote for peer 1
2016-06-29 13:24:58 +03:00
f := nt . peers [ 2 ] . ( * raft )
for i := 0 ; i < f . electionTimeout ; i ++ {
f . tick ( )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
if lead . lead != 1 {
t . Fatalf ( "after election leader is %x, want 1" , lead . lead )
}
// Transfer leadership to 2.
nt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
// After some log replication, transfer leadership back to 1.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . send ( pb . Message { From : 1 , To : 2 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
2016-03-24 14:59:41 +03:00
func TestLeaderTransferToSlowFollower ( t * testing . T ) {
defaultLogger . EnableDebug ( )
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
nt . recover ( )
lead := nt . peers [ 1 ] . ( * raft )
2019-06-08 20:41:24 +03:00
if lead . prs . Progress [ 3 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x for node 3, want %x" , lead . prs . Progress [ 3 ] . Match , 1 )
2016-03-24 14:59:41 +03:00
}
// Transfer leadership to 3 when node 3 is lack of log.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 3 )
}
func TestLeaderTransferAfterSnapshot ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
lead := nt . peers [ 1 ] . ( * raft )
nextEnts ( lead , nt . storage [ 1 ] )
2019-07-23 01:34:46 +03:00
nt . storage [ 1 ] . CreateSnapshot ( lead . raftLog . applied , & pb . ConfState { Voters : lead . prs . VoterNodes ( ) } , nil )
2016-03-24 14:59:41 +03:00
nt . storage [ 1 ] . Compact ( lead . raftLog . applied )
nt . recover ( )
2019-06-08 20:41:24 +03:00
if lead . prs . Progress [ 3 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x for node 3, want %x" , lead . prs . Progress [ 3 ] . Match , 1 )
2016-03-24 14:59:41 +03:00
}
// Transfer leadership to 3 when node 3 is lack of snapshot.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
// Send pb.MsgHeartbeatResp to leader to trigger a snapshot for node 3.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgHeartbeatResp } )
checkLeaderTransferState ( t , lead , StateFollower , 3 )
}
func TestLeaderTransferToSelf ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to self, there will be noop.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferToNonExistingNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to non-existing node, there will be noop.
nt . send ( pb . Message { From : 4 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferTimeout ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node, wait for timeout.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . electionTimeout - lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func TestLeaderTransferIgnoreProposal ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node to let transfer pending, then send proposal.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
2018-01-11 07:43:55 +03:00
err := lead . Step ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { } } } )
if err != ErrProposalDropped {
t . Fatalf ( "should return drop proposal error while transferring" )
}
2016-03-24 14:59:41 +03:00
2019-06-08 20:41:24 +03:00
if lead . prs . Progress [ 1 ] . Match != 1 {
t . Fatalf ( "node 1 has match %x, want %x" , lead . prs . Progress [ 1 ] . Match , 1 )
2016-03-24 14:59:41 +03:00
}
}
func TestLeaderTransferReceiveHigherTermVote ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
// Transfer leadership to isolated node to let transfer pending.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup , Index : 1 , Term : 2 } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
}
func TestLeaderTransferRemoveNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . ignore ( pb . MsgTimeoutNow )
lead := nt . peers [ 1 ] . ( * raft )
2016-04-11 09:16:56 +03:00
// The leadTransferee is removed when leadship transferring.
2016-03-24 14:59:41 +03:00
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
2019-07-22 23:30:47 +03:00
lead . applyConfChange ( pb . ConfChange { NodeID : 3 , Type : pb . ConfChangeRemoveNode } . AsV2 ( ) )
2016-03-24 14:59:41 +03:00
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
// TestLeaderTransferBack verifies leadership can transfer back to self when last transfer is pending.
func TestLeaderTransferBack ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
// Transfer leadership back to self.
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
// TestLeaderTransferSecondTransferToAnotherNode verifies leader can transfer to another node
// when last transfer is pending.
func TestLeaderTransferSecondTransferToAnotherNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
// Transfer leadership to another node.
nt . send ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTransferLeader } )
checkLeaderTransferState ( t , lead , StateFollower , 2 )
}
// TestLeaderTransferSecondTransferToSameNode verifies second transfer leader request
// to the same node should not extend the timeout while the first one is pending.
func TestLeaderTransferSecondTransferToSameNode ( t * testing . T ) {
nt := newNetwork ( nil , nil , nil )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
nt . isolate ( 3 )
lead := nt . peers [ 1 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
if lead . leadTransferee != 3 {
t . Fatalf ( "wait transferring, leadTransferee = %v, want %v" , lead . leadTransferee , 3 )
}
for i := 0 ; i < lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
// Second transfer leadership request to the same node.
nt . send ( pb . Message { From : 3 , To : 1 , Type : pb . MsgTransferLeader } )
for i := 0 ; i < lead . electionTimeout - lead . heartbeatTimeout ; i ++ {
lead . tick ( )
}
checkLeaderTransferState ( t , lead , StateLeader , 1 )
}
func checkLeaderTransferState ( t * testing . T , r * raft , state StateType , lead uint64 ) {
if r . state != state || r . lead != lead {
2016-04-11 09:16:56 +03:00
t . Fatalf ( "after transferring, node has state %v lead %v, want state %v lead %v" , r . state , r . lead , state , lead )
2016-03-24 14:59:41 +03:00
}
if r . leadTransferee != None {
2016-04-11 09:16:56 +03:00
t . Fatalf ( "after transferring, node has leadTransferee %v, want leadTransferee %v" , r . leadTransferee , None )
2016-03-24 14:59:41 +03:00
}
}
2016-11-07 15:02:21 +03:00
// TestTransferNonMember verifies that when a MsgTimeoutNow arrives at
// a node that has been removed from the group, nothing happens.
// (previously, if the node also got votes, it would panic as it
// transitioned to StateLeader)
func TestTransferNonMember ( t * testing . T ) {
r := newTestRaft ( 1 , [ ] uint64 { 2 , 3 , 4 } , 5 , 1 , NewMemoryStorage ( ) )
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgTimeoutNow } )
r . Step ( pb . Message { From : 2 , To : 1 , Type : pb . MsgVoteResp } )
r . Step ( pb . Message { From : 3 , To : 1 , Type : pb . MsgVoteResp } )
if r . state != StateFollower {
t . Fatalf ( "state is %s, want StateFollower" , r . state )
}
}
2017-07-20 23:46:05 +03:00
// TestNodeWithSmallerTermCanCompleteElection tests the scenario where a node
// that has been partitioned away (and fallen behind) rejoins the cluster at
// about the same time the leader node gets partitioned away.
// Previously the cluster would come to a standstill when run with PreVote
// enabled.
func TestNodeWithSmallerTermCanCompleteElection ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
n1 . preVote = true
n2 . preVote = true
n3 . preVote = true
// cause a network partition to isolate node 3
nt := newNetwork ( n1 , n2 , n3 )
nt . cut ( 1 , 3 )
nt . cut ( 2 , 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "peer 1 state: %s, want %s" , sm . state , StateLeader )
}
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateFollower {
t . Errorf ( "peer 2 state: %s, want %s" , sm . state , StateFollower )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StatePreCandidate {
t . Errorf ( "peer 3 state: %s, want %s" , sm . state , StatePreCandidate )
}
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// check whether the term values are expected
// a.Term == 3
// b.Term == 3
// c.Term == 1
sm = nt . peers [ 1 ] . ( * raft )
if sm . Term != 3 {
t . Errorf ( "peer 1 term: %d, want %d" , sm . Term , 3 )
}
sm = nt . peers [ 2 ] . ( * raft )
if sm . Term != 3 {
t . Errorf ( "peer 2 term: %d, want %d" , sm . Term , 3 )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . Term != 1 {
t . Errorf ( "peer 3 term: %d, want %d" , sm . Term , 1 )
}
// check state
// a == follower
// b == leader
// c == pre-candidate
sm = nt . peers [ 1 ] . ( * raft )
if sm . state != StateFollower {
t . Errorf ( "peer 1 state: %s, want %s" , sm . state , StateFollower )
}
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "peer 2 state: %s, want %s" , sm . state , StateLeader )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StatePreCandidate {
t . Errorf ( "peer 3 state: %s, want %s" , sm . state , StatePreCandidate )
}
sm . logger . Infof ( "going to bring back peer 3 and kill peer 2" )
// recover the network then immediately isolate b which is currently
// the leader, this is to emulate the crash of b.
nt . recover ( )
nt . cut ( 2 , 1 )
nt . cut ( 2 , 3 )
// call for election
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// do we have a leader?
sma := nt . peers [ 1 ] . ( * raft )
smb := nt . peers [ 3 ] . ( * raft )
if sma . state != StateLeader && smb . state != StateLeader {
t . Errorf ( "no leader" )
}
}
2017-08-02 12:59:28 +03:00
// TestPreVoteWithSplitVote verifies that after split vote, cluster can complete
// election in next round.
func TestPreVoteWithSplitVote ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
n1 . preVote = true
n2 . preVote = true
n3 . preVote = true
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// simulate leader down. followers start split vote.
nt . isolate ( 1 )
nt . send ( [ ] pb . Message {
{ From : 2 , To : 2 , Type : pb . MsgHup } ,
{ From : 3 , To : 3 , Type : pb . MsgHup } ,
} ... )
// check whether the term values are expected
// n2.Term == 3
// n3.Term == 3
sm := nt . peers [ 2 ] . ( * raft )
if sm . Term != 3 {
t . Errorf ( "peer 2 term: %d, want %d" , sm . Term , 3 )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . Term != 3 {
t . Errorf ( "peer 3 term: %d, want %d" , sm . Term , 3 )
}
// check state
// n2 == candidate
// n3 == candidate
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateCandidate {
t . Errorf ( "peer 2 state: %s, want %s" , sm . state , StateCandidate )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateCandidate {
t . Errorf ( "peer 3 state: %s, want %s" , sm . state , StateCandidate )
}
// node 2 election timeout first
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// check whether the term values are expected
// n2.Term == 4
// n3.Term == 4
sm = nt . peers [ 2 ] . ( * raft )
if sm . Term != 4 {
t . Errorf ( "peer 2 term: %d, want %d" , sm . Term , 4 )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . Term != 4 {
t . Errorf ( "peer 3 term: %d, want %d" , sm . Term , 4 )
}
// check state
// n2 == leader
// n3 == follower
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateLeader {
t . Errorf ( "peer 2 state: %s, want %s" , sm . state , StateLeader )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateFollower {
t . Errorf ( "peer 3 state: %s, want %s" , sm . state , StateFollower )
}
}
2018-12-28 14:57:26 +03:00
// TestPreVoteWithCheckQuorum ensures that after a node become pre-candidate,
// it will checkQuorum correctly.
func TestPreVoteWithCheckQuorum ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
n1 . preVote = true
n2 . preVote = true
n3 . preVote = true
n1 . checkQuorum = true
n2 . checkQuorum = true
n3 . checkQuorum = true
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// isolate node 1. node 2 and node 3 have leader info
nt . isolate ( 1 )
// check state
sm := nt . peers [ 1 ] . ( * raft )
if sm . state != StateLeader {
t . Fatalf ( "peer 1 state: %s, want %s" , sm . state , StateLeader )
}
sm = nt . peers [ 2 ] . ( * raft )
if sm . state != StateFollower {
t . Fatalf ( "peer 2 state: %s, want %s" , sm . state , StateFollower )
}
sm = nt . peers [ 3 ] . ( * raft )
if sm . state != StateFollower {
t . Fatalf ( "peer 3 state: %s, want %s" , sm . state , StateFollower )
}
// node 2 will ignore node 3's PreVote
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// Do we have a leader?
if n2 . state != StateLeader && n3 . state != StateFollower {
t . Errorf ( "no leader" )
}
}
2019-06-17 10:20:45 +03:00
// TestLearnerCampaign verifies that a learner won't campaign even if it receives
// a MsgHup or MsgTimeoutNow.
func TestLearnerCampaign ( t * testing . T ) {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2019-07-22 23:30:47 +03:00
n1 . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeAddLearnerNode } . AsV2 ( ) )
2019-06-17 10:20:45 +03:00
n2 := newTestRaft ( 2 , [ ] uint64 { 1 } , 10 , 1 , NewMemoryStorage ( ) )
2019-07-22 23:30:47 +03:00
n2 . applyConfChange ( pb . ConfChange { NodeID : 2 , Type : pb . ConfChangeAddLearnerNode } . AsV2 ( ) )
2019-06-17 10:20:45 +03:00
nt := newNetwork ( n1 , n2 )
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
if ! n2 . isLearner {
t . Fatalf ( "failed to make n2 a learner" )
}
if n2 . state != StateFollower {
t . Fatalf ( "n2 campaigned despite being learner" )
}
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
if n1 . state != StateLeader || n1 . lead != 1 {
t . Fatalf ( "n1 did not become leader" )
}
// NB: TransferLeader already checks that the recipient is not a learner, but
// the check could have happened by the time the recipient becomes a learner,
// in which case it will receive MsgTimeoutNow as in this test case and we
// verify that it's ignored.
nt . send ( pb . Message { From : 1 , To : 2 , Type : pb . MsgTimeoutNow } )
if n2 . state != StateFollower {
t . Fatalf ( "n2 accepted leadership transfer despite being learner" )
}
}
2017-09-09 04:12:39 +03:00
// simulate rolling update a cluster for Pre-Vote. cluster has 3 nodes [n1, n2, n3].
// n1 is leader with term 2
// n2 is follower with term 2
// n3 is partitioned, with term 4 and less log, state is candidate
func newPreVoteMigrationCluster ( t * testing . T ) * network {
n1 := newTestRaft ( 1 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n2 := newTestRaft ( 2 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n3 := newTestRaft ( 3 , [ ] uint64 { 1 , 2 , 3 } , 10 , 1 , NewMemoryStorage ( ) )
n1 . becomeFollower ( 1 , None )
n2 . becomeFollower ( 1 , None )
n3 . becomeFollower ( 1 , None )
n1 . preVote = true
n2 . preVote = true
// We intentionally do not enable PreVote for n3, this is done so in order
// to simulate a rolling restart process where it's possible to have a mixed
// version cluster with replicas with PreVote enabled, and replicas without.
nt := newNetwork ( n1 , n2 , n3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgHup } )
// Cause a network partition to isolate n3.
nt . isolate ( 3 )
nt . send ( pb . Message { From : 1 , To : 1 , Type : pb . MsgProp , Entries : [ ] pb . Entry { { Data : [ ] byte ( "some data" ) } } } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
// check state
// n1.state == StateLeader
// n2.state == StateFollower
// n3.state == StateCandidate
if n1 . state != StateLeader {
t . Fatalf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Fatalf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StateCandidate {
t . Fatalf ( "node 3 state: %s, want %s" , n3 . state , StateCandidate )
}
// check term
// n1.Term == 2
// n2.Term == 2
// n3.Term == 4
if n1 . Term != 2 {
t . Fatalf ( "node 1 term: %d, want %d" , n1 . Term , 2 )
}
if n2 . Term != 2 {
t . Fatalf ( "node 2 term: %d, want %d" , n2 . Term , 2 )
}
if n3 . Term != 4 {
t . Fatalf ( "node 3 term: %d, want %d" , n3 . Term , 4 )
}
// Enable prevote on n3, then recover the network
n3 . preVote = true
nt . recover ( )
return nt
}
func TestPreVoteMigrationCanCompleteElection ( t * testing . T ) {
nt := newPreVoteMigrationCluster ( t )
// n1 is leader with term 2
// n2 is follower with term 2
// n3 is pre-candidate with term 4, and less log
n2 := nt . peers [ 2 ] . ( * raft )
n3 := nt . peers [ 3 ] . ( * raft )
// simulate leader down
nt . isolate ( 1 )
// Call for elections from both n2 and n3.
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// check state
// n2.state == Follower
// n3.state == PreCandidate
if n2 . state != StateFollower {
t . Errorf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StatePreCandidate {
t . Errorf ( "node 3 state: %s, want %s" , n3 . state , StatePreCandidate )
}
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
nt . send ( pb . Message { From : 2 , To : 2 , Type : pb . MsgHup } )
// Do we have a leader?
if n2 . state != StateLeader && n3 . state != StateFollower {
t . Errorf ( "no leader" )
}
}
func TestPreVoteMigrationWithFreeStuckPreCandidate ( t * testing . T ) {
nt := newPreVoteMigrationCluster ( t )
// n1 is leader with term 2
// n2 is follower with term 2
// n3 is pre-candidate with term 4, and less log
n1 := nt . peers [ 1 ] . ( * raft )
n2 := nt . peers [ 2 ] . ( * raft )
n3 := nt . peers [ 3 ] . ( * raft )
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if n1 . state != StateLeader {
t . Errorf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Errorf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StatePreCandidate {
t . Errorf ( "node 3 state: %s, want %s" , n3 . state , StatePreCandidate )
}
// Pre-Vote again for safety
nt . send ( pb . Message { From : 3 , To : 3 , Type : pb . MsgHup } )
if n1 . state != StateLeader {
t . Errorf ( "node 1 state: %s, want %s" , n1 . state , StateLeader )
}
if n2 . state != StateFollower {
t . Errorf ( "node 2 state: %s, want %s" , n2 . state , StateFollower )
}
if n3 . state != StatePreCandidate {
t . Errorf ( "node 3 state: %s, want %s" , n3 . state , StatePreCandidate )
}
nt . send ( pb . Message { From : 1 , To : 3 , Type : pb . MsgHeartbeat , Term : n1 . Term } )
// Disrupt the leader so that the stuck peer is freed
if n1 . state != StateFollower {
t . Errorf ( "state = %s, want %s" , n1 . state , StateFollower )
}
if n3 . Term != n1 . Term {
t . Errorf ( "term = %d, want %d" , n3 . Term , n1 . Term )
}
}
2016-12-03 12:29:15 +03:00
func entsWithConfig ( configFunc func ( * Config ) , terms ... uint64 ) * raft {
2014-12-15 22:25:35 +03:00
storage := NewMemoryStorage ( )
2014-11-25 04:13:47 +03:00
for i , term := range terms {
2014-12-15 22:25:35 +03:00
storage . Append ( [ ] pb . Entry { { Index : uint64 ( i + 1 ) , Term : term } } )
2014-11-03 23:04:24 +03:00
}
2016-12-03 12:29:15 +03:00
cfg := newTestConfig ( 1 , [ ] uint64 { } , 5 , 1 , storage )
if configFunc != nil {
configFunc ( cfg )
}
sm := newRaft ( cfg )
2016-10-10 10:08:11 +03:00
sm . reset ( terms [ len ( terms ) - 1 ] )
2014-06-06 02:02:12 +04:00
return sm
2014-05-18 01:09:52 +04:00
}
2016-12-03 12:29:15 +03:00
// votedWithConfig creates a raft state machine with Vote and Term set
// to the given value but no log entries (indicating that it voted in
// the given term but has not received any logs).
func votedWithConfig ( configFunc func ( * Config ) , vote , term uint64 ) * raft {
storage := NewMemoryStorage ( )
storage . SetHardState ( pb . HardState { Vote : vote , Term : term } )
cfg := newTestConfig ( 1 , [ ] uint64 { } , 5 , 1 , storage )
if configFunc != nil {
configFunc ( cfg )
}
sm := newRaft ( cfg )
sm . reset ( term )
return sm
}
2014-05-16 21:11:21 +04:00
type network struct {
2016-02-01 08:42:39 +03:00
peers map [ uint64 ] stateMachine
2014-11-13 00:23:42 +03:00
storage map [ uint64 ] * MemoryStorage
2014-06-26 22:55:56 +04:00
dropm map [ connem ] float64
2014-10-12 11:34:22 +04:00
ignorem map [ pb . MessageType ] bool
2018-08-06 23:52:16 +03:00
// msgHook is called for each message sent. It may inspect the
// message and return true to send it or false to drop it.
msgHook func ( pb . Message ) bool
2014-05-16 21:11:21 +04:00
}
2014-05-06 10:28:14 +04:00
2014-06-18 04:35:19 +04:00
// newNetwork initializes a network from peers.
// A nil node will be replaced with a new *stateMachine.
// A *stateMachine will get its k, id.
2014-10-29 01:46:56 +03:00
// When using stateMachine, the address list is always [1, n].
2016-02-01 08:42:39 +03:00
func newNetwork ( peers ... stateMachine ) * network {
2016-10-10 09:32:40 +03:00
return newNetworkWithConfig ( nil , peers ... )
}
// newNetworkWithConfig is like newNetwork but calls the given func to
// modify the configuration of any state machines it creates.
func newNetworkWithConfig ( configFunc func ( * Config ) , peers ... stateMachine ) * network {
2014-06-18 04:35:19 +04:00
size := len ( peers )
2014-10-29 01:46:56 +03:00
peerAddrs := idsBySize ( size )
2014-06-10 03:45:42 +04:00
2016-02-01 08:42:39 +03:00
npeers := make ( map [ uint64 ] stateMachine , size )
2014-11-13 00:23:42 +03:00
nstorage := make ( map [ uint64 ] * MemoryStorage , size )
2014-06-18 04:35:19 +04:00
2015-02-18 00:15:48 +03:00
for j , p := range peers {
id := peerAddrs [ j ]
2014-06-06 02:02:12 +04:00
switch v := p . ( type ) {
2014-05-06 10:28:14 +04:00
case nil :
2014-11-13 00:23:42 +03:00
nstorage [ id ] = NewMemoryStorage ( )
2016-10-10 09:32:40 +03:00
cfg := newTestConfig ( id , peerAddrs , 10 , 1 , nstorage [ id ] )
if configFunc != nil {
configFunc ( cfg )
}
sm := newRaft ( cfg )
2014-09-10 03:33:09 +04:00
npeers [ id ] = sm
2014-08-23 00:24:33 +04:00
case * raft :
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
// TODO(tbg): this is all pretty confused. Clean this up.
2019-06-08 20:41:24 +03:00
learners := make ( map [ uint64 ] bool , len ( v . prs . Learners ) )
for i := range v . prs . Learners {
2017-11-11 05:38:21 +03:00
learners [ i ] = true
}
2014-09-10 03:33:09 +04:00
v . id = id
2019-06-08 20:41:24 +03:00
v . prs = tracker . MakeProgressTracker ( v . prs . MaxInflight )
raft: internally support joint consensus
This commit introduces machinery to safely apply joint consensus
configuration changes to Raft.
The main contribution is the new package, `confchange`, which offers
the primitives `Simple`, `EnterJoint`, and `LeaveJoint`.
The first two take a list of configuration changes. `Simple` only
declares success if these configuration changes (applied atomically)
change the set of voters by at most one (i.e. it's fine to add or
remove any number of learners, but change only one voter). `EnterJoint`
makes the configuration joint and then applies the changes to it, in
preparation of the caller returning later and transitioning out of the
joint config into the final desired configuration via `LeaveJoint()`.
This commit streamlines the conversion between voters and learners, which
is now generally allowed whenever the above conditions are upheld (i.e.
it's not possible to demote a voter and add a new voter in the context
of a Simple configuration change, but it is possible via EnterJoint).
Previously, we had the artificial restriction that a voter could not be
demoted to a learner, but had to be removed first.
Even though demoting a learner is generally less useful than promoting
a learner (the latter is used to catch up future voters), demotions
could see use in improved handling of temporary node unavailability,
where it is desired to remove voting power from a down node, but to
preserve its data should it return.
An additional change that was made in this commit is to prevent the use
of empty commit quorums, which was previously possible but for no good
reason; this:
Closes #10884.
The work left to do in a future PR is to actually expose joint
configurations to the applications using Raft. This will entail mostly
API design and the addition of suitable testing, which to be carried
out ergonomically is likely to motivate a larger refactor.
Touches #7625.
2019-07-16 16:35:17 +03:00
if len ( learners ) > 0 {
v . prs . Learners = map [ uint64 ] struct { } { }
}
2014-06-18 04:35:19 +04:00
for i := 0 ; i < size ; i ++ {
2019-06-08 20:41:24 +03:00
pr := & tracker . Progress { }
2017-11-11 05:38:21 +03:00
if _ , ok := learners [ peerAddrs [ i ] ] ; ok {
2019-06-03 14:16:28 +03:00
pr . IsLearner = true
2019-06-08 20:41:24 +03:00
v . prs . Learners [ peerAddrs [ i ] ] = struct { } { }
2017-11-11 05:38:21 +03:00
} else {
2019-06-08 20:41:24 +03:00
v . prs . Voters [ 0 ] [ peerAddrs [ i ] ] = struct { } { }
2017-11-11 05:38:21 +03:00
}
2019-06-08 20:41:24 +03:00
v . prs . Progress [ peerAddrs [ i ] ] = pr
2014-06-10 03:45:42 +04:00
}
2016-10-10 10:08:11 +03:00
v . reset ( v . Term )
2014-09-10 03:33:09 +04:00
npeers [ id ] = v
case * blackHole :
npeers [ id ] = v
2014-06-18 04:35:19 +04:00
default :
2014-09-12 10:18:51 +04:00
panic ( fmt . Sprintf ( "unexpected state machine type: %T" , p ) )
2014-05-06 10:28:14 +04:00
}
}
2014-06-26 22:55:56 +04:00
return & network {
peers : npeers ,
2014-11-13 00:23:42 +03:00
storage : nstorage ,
2014-06-26 22:55:56 +04:00
dropm : make ( map [ connem ] float64 ) ,
2014-10-12 11:34:22 +04:00
ignorem : make ( map [ pb . MessageType ] bool ) ,
2014-06-26 22:55:56 +04:00
}
2014-05-06 10:28:14 +04:00
}
2016-10-10 09:32:40 +03:00
func preVoteConfig ( c * Config ) {
c . PreVote = true
}
2014-08-28 05:53:18 +04:00
func ( nw * network ) send ( msgs ... pb . Message ) {
2014-06-06 02:02:12 +04:00
for len ( msgs ) > 0 {
m := msgs [ 0 ]
p := nw . peers [ m . To ]
p . Step ( m )
2014-10-30 00:04:45 +03:00
msgs = append ( msgs [ 1 : ] , nw . filter ( p . readMessages ( ) ) ... )
2014-05-16 21:11:21 +04:00
}
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) drop ( from , to uint64 , perc float64 ) {
2014-06-06 02:02:12 +04:00
nw . dropm [ connem { from , to } ] = perc
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) cut ( one , other uint64 ) {
2018-05-10 01:18:57 +03:00
nw . drop ( one , other , 2.0 ) // always drop
nw . drop ( other , one , 2.0 ) // always drop
2014-06-06 22:01:55 +04:00
}
2014-10-08 14:29:53 +04:00
func ( nw * network ) isolate ( id uint64 ) {
2014-06-06 02:02:12 +04:00
for i := 0 ; i < len ( nw . peers ) ; i ++ {
2014-10-08 14:29:53 +04:00
nid := uint64 ( i ) + 1
2014-07-09 22:53:27 +04:00
if nid != id {
2018-05-10 01:18:57 +03:00
nw . drop ( id , nid , 1.0 ) // always drop
nw . drop ( nid , id , 1.0 ) // always drop
2014-05-06 10:28:14 +04:00
}
}
}
2014-10-12 11:34:22 +04:00
func ( nw * network ) ignore ( t pb . MessageType ) {
2014-06-26 22:55:56 +04:00
nw . ignorem [ t ] = true
}
2014-06-06 02:02:12 +04:00
func ( nw * network ) recover ( ) {
nw . dropm = make ( map [ connem ] float64 )
2014-10-12 11:34:22 +04:00
nw . ignorem = make ( map [ pb . MessageType ] bool )
2014-05-06 10:28:14 +04:00
}
2014-08-28 05:53:18 +04:00
func ( nw * network ) filter ( msgs [ ] pb . Message ) [ ] pb . Message {
2014-10-08 02:22:35 +04:00
mm := [ ] pb . Message { }
2014-06-06 02:02:12 +04:00
for _ , m := range msgs {
2014-06-26 22:55:56 +04:00
if nw . ignorem [ m . Type ] {
continue
}
2014-06-06 02:02:12 +04:00
switch m . Type {
2014-10-12 11:34:22 +04:00
case pb . MsgHup :
2014-06-06 02:02:12 +04:00
// hups never go over the network, so don't drop them but panic
panic ( "unexpected msgHup" )
2014-05-06 10:28:14 +04:00
default :
2014-06-06 02:02:12 +04:00
perc := nw . dropm [ connem { m . From , m . To } ]
if n := rand . Float64 ( ) ; n < perc {
2014-05-06 10:28:14 +04:00
continue
}
}
2018-08-06 23:52:16 +03:00
if nw . msgHook != nil {
if ! nw . msgHook ( m ) {
continue
}
}
2014-06-06 02:02:12 +04:00
mm = append ( mm , m )
2014-05-06 10:28:14 +04:00
}
2014-06-06 02:02:12 +04:00
return mm
2014-05-06 10:28:14 +04:00
}
2014-06-06 02:02:12 +04:00
type connem struct {
2014-10-08 14:29:53 +04:00
from , to uint64
2014-05-24 00:30:04 +04:00
}
2014-06-06 02:02:12 +04:00
type blackHole struct { }
2014-05-27 23:24:49 +04:00
2014-08-28 05:53:18 +04:00
func ( blackHole ) Step ( pb . Message ) error { return nil }
2014-10-30 00:04:45 +03:00
func ( blackHole ) readMessages ( ) [ ] pb . Message { return nil }
2014-06-06 02:02:12 +04:00
var nopStepper = & blackHole { }
2014-10-29 01:46:56 +03:00
func idsBySize ( size int ) [ ] uint64 {
ids := make ( [ ] uint64 , size )
for i := 0 ; i < size ; i ++ {
ids [ i ] = 1 + uint64 ( i )
}
return ids
}
2015-03-22 04:15:58 +03:00
2016-08-02 03:46:43 +03:00
// setRandomizedElectionTimeout set up the value by caller instead of choosing
// by system, in some test scenario we need to fill in some expected value to
// ensure the certainty
func setRandomizedElectionTimeout ( r * raft , v int ) {
r . randomizedElectionTimeout = v
}
2015-03-24 22:36:42 +03:00
func newTestConfig ( id uint64 , peers [ ] uint64 , election , heartbeat int , storage Storage ) * Config {
return & Config {
2015-03-22 04:15:58 +03:00
ID : id ,
2015-03-24 21:10:07 +03:00
peers : peers ,
2015-03-22 04:15:58 +03:00
ElectionTick : election ,
HeartbeatTick : heartbeat ,
Storage : storage ,
MaxSizePerMsg : noLimit ,
MaxInflightMsgs : 256 ,
}
2015-03-24 22:36:42 +03:00
}
2015-03-22 04:15:58 +03:00
2015-03-24 22:36:42 +03:00
func newTestRaft ( id uint64 , peers [ ] uint64 , election , heartbeat int , storage Storage ) * raft {
return newRaft ( newTestConfig ( id , peers , election , heartbeat , storage ) )
2015-03-22 04:15:58 +03:00
}
2017-11-11 05:38:21 +03:00
func newTestLearnerRaft ( id uint64 , peers [ ] uint64 , learners [ ] uint64 , election , heartbeat int , storage Storage ) * raft {
cfg := newTestConfig ( id , peers , election , heartbeat , storage )
cfg . learners = learners
return newRaft ( cfg )
}
2019-07-15 18:18:56 +03:00
raft: clean up bootstrap
This is the first (maybe not last) step in cleaning up the bootstrap
code around StartNode.
Initializing a Raft group for the first time is awkward, since a
configuration has to be pulled from thin air. The way this is solved
today is unclean: The app is supposed to pass peers to StartNode(),
we add configuration changes for them to the log, immediately pretend
that they are applied, but actually leave them unapplied (to give the
app a chance to observe them, though if the app did decide to not apply
them things would really go off the rails), and then return control to
the app. The app will then process the initial Readys and as a result
the configuration will be persisted to disk; restarts of the node then
use RestartNode which doesn't take any peers.
The code that did this lived awkwardly in two places fairly deep down
the callstack, though it was really only necessary in StartNode(). This
commit refactors things to make this more obvious: only StartNode does
this dance now. In particular, RawNode does not support this at all any
more; it expects the app to set up its Storage correctly.
Future work may provide helpers to make this "preseeding" of the Storage
more user-friendly. It isn't entirely straightforward to do so since
the Storage interface doesn't provide the right accessors for this
purpose. Briefly speaking, we want to make sure that a non-bootstrapped
node can never catch up via the log so that we can implicitly use one
of the "skipped" log entries to represent the configuration change into
the bootstrap configuration. This is an invasive change that affects
all consumers of raft, and it is of lower urgency since the code (post
this commit) already encapsulates the complexity sufficiently.
2019-07-18 14:41:52 +03:00
// newTestRawNode sets up a RawNode with the given peers. The configuration will
// not be reflected in the Storage.
2019-07-15 18:18:56 +03:00
func newTestRawNode ( id uint64 , peers [ ] uint64 , election , heartbeat int , storage Storage ) * RawNode {
cfg := newTestConfig ( id , peers , election , heartbeat , storage )
raft: clean up bootstrap
This is the first (maybe not last) step in cleaning up the bootstrap
code around StartNode.
Initializing a Raft group for the first time is awkward, since a
configuration has to be pulled from thin air. The way this is solved
today is unclean: The app is supposed to pass peers to StartNode(),
we add configuration changes for them to the log, immediately pretend
that they are applied, but actually leave them unapplied (to give the
app a chance to observe them, though if the app did decide to not apply
them things would really go off the rails), and then return control to
the app. The app will then process the initial Readys and as a result
the configuration will be persisted to disk; restarts of the node then
use RestartNode which doesn't take any peers.
The code that did this lived awkwardly in two places fairly deep down
the callstack, though it was really only necessary in StartNode(). This
commit refactors things to make this more obvious: only StartNode does
this dance now. In particular, RawNode does not support this at all any
more; it expects the app to set up its Storage correctly.
Future work may provide helpers to make this "preseeding" of the Storage
more user-friendly. It isn't entirely straightforward to do so since
the Storage interface doesn't provide the right accessors for this
purpose. Briefly speaking, we want to make sure that a non-bootstrapped
node can never catch up via the log so that we can implicitly use one
of the "skipped" log entries to represent the configuration change into
the bootstrap configuration. This is an invasive change that affects
all consumers of raft, and it is of lower urgency since the code (post
this commit) already encapsulates the complexity sufficiently.
2019-07-18 14:41:52 +03:00
rn , err := NewRawNode ( cfg )
2019-07-15 18:18:56 +03:00
if err != nil {
panic ( err )
}
return rn
}