2016-05-13 06:49:15 +03:00
|
|
|
// Copyright 2015 The etcd Authors
|
2015-01-25 06:19:16 +03:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2014-10-18 02:41:22 +04:00
|
|
|
|
2014-05-19 10:34:55 +04:00
|
|
|
package raft
|
|
|
|
|
2014-08-25 23:30:44 +04:00
|
|
|
import (
|
2017-09-07 00:57:25 +03:00
|
|
|
"context"
|
2014-09-01 05:49:07 +04:00
|
|
|
"errors"
|
|
|
|
|
2014-08-28 05:53:18 +04:00
|
|
|
pb "github.com/coreos/etcd/raft/raftpb"
|
2014-08-25 23:30:44 +04:00
|
|
|
)
|
2014-06-05 21:49:34 +04:00
|
|
|
|
2015-02-25 08:38:18 +03:00
|
|
|
type SnapshotStatus int
|
|
|
|
|
|
|
|
const (
|
|
|
|
SnapshotFinish SnapshotStatus = 1
|
|
|
|
SnapshotFailure SnapshotStatus = 2
|
|
|
|
)
|
|
|
|
|
2014-09-09 08:15:35 +04:00
|
|
|
var (
|
2014-09-16 04:35:02 +04:00
|
|
|
emptyState = pb.HardState{}
|
2014-10-08 02:22:35 +04:00
|
|
|
|
|
|
|
// ErrStopped is returned by methods on Nodes that have been stopped.
|
2014-09-09 08:15:35 +04:00
|
|
|
ErrStopped = errors.New("raft: stopped")
|
|
|
|
)
|
2014-09-01 05:49:07 +04:00
|
|
|
|
2014-09-16 04:35:02 +04:00
|
|
|
// SoftState provides state that is useful for logging and debugging.
|
|
|
|
// The state is volatile and does not need to be persisted to the WAL.
|
|
|
|
type SoftState struct {
|
2016-07-08 21:05:41 +03:00
|
|
|
Lead uint64 // must use atomic operations to access; keep 64-bit aligned.
|
2014-10-21 02:07:20 +04:00
|
|
|
RaftState StateType
|
2014-09-16 04:35:02 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func (a *SoftState) equal(b *SoftState) bool {
|
2014-12-09 08:17:04 +03:00
|
|
|
return a.Lead == b.Lead && a.RaftState == b.RaftState
|
2014-09-16 04:35:02 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
// Ready encapsulates the entries and messages that are ready to read,
|
|
|
|
// be saved to stable storage, committed or sent to other peers.
|
|
|
|
// All fields in Ready are read-only.
|
2014-08-27 02:28:21 +04:00
|
|
|
type Ready struct {
|
2014-09-16 04:35:02 +04:00
|
|
|
// The current volatile state of a Node.
|
|
|
|
// SoftState will be nil if there is no update.
|
|
|
|
// It is not required to consume or store SoftState.
|
|
|
|
*SoftState
|
|
|
|
|
|
|
|
// The current state of a Node to be saved to stable storage BEFORE
|
|
|
|
// Messages are sent.
|
|
|
|
// HardState will be equal to empty state if there is no update.
|
|
|
|
pb.HardState
|
2014-08-27 02:28:21 +04:00
|
|
|
|
2016-08-27 02:03:06 +03:00
|
|
|
// ReadStates can be used for node to serve linearizable read requests locally
|
2016-06-03 18:20:10 +03:00
|
|
|
// when its applied index is greater than the index in ReadState.
|
|
|
|
// Note that the readState will be returned when raft receives msgReadIndex.
|
|
|
|
// The returned is only valid for the request that requested to read.
|
2016-08-27 02:03:06 +03:00
|
|
|
ReadStates []ReadState
|
2016-06-03 18:20:10 +03:00
|
|
|
|
2014-08-27 02:28:21 +04:00
|
|
|
// Entries specifies entries to be saved to stable storage BEFORE
|
|
|
|
// Messages are sent.
|
2014-08-28 05:53:18 +04:00
|
|
|
Entries []pb.Entry
|
2014-08-27 02:28:21 +04:00
|
|
|
|
2014-09-17 05:18:45 +04:00
|
|
|
// Snapshot specifies the snapshot to be saved to stable storage.
|
|
|
|
Snapshot pb.Snapshot
|
|
|
|
|
2014-08-27 02:28:21 +04:00
|
|
|
// CommittedEntries specifies entries to be committed to a
|
|
|
|
// store/state-machine. These have previously been committed to stable
|
|
|
|
// store.
|
2014-08-28 05:53:18 +04:00
|
|
|
CommittedEntries []pb.Entry
|
2014-08-27 02:28:21 +04:00
|
|
|
|
|
|
|
// Messages specifies outbound messages to be sent AFTER Entries are
|
|
|
|
// committed to stable storage.
|
2015-02-25 08:38:18 +03:00
|
|
|
// If it contains a MsgSnap message, the application MUST report back to raft
|
|
|
|
// when the snapshot has been received or has failed by calling ReportSnapshot.
|
2014-08-28 05:53:18 +04:00
|
|
|
Messages []pb.Message
|
2017-02-13 23:09:54 +03:00
|
|
|
|
|
|
|
// MustSync indicates whether the HardState and Entries must be synchronously
|
|
|
|
// written to disk or if an asynchronous write is permissible.
|
|
|
|
MustSync bool
|
2014-05-19 10:34:55 +04:00
|
|
|
}
|
|
|
|
|
2014-09-17 00:55:00 +04:00
|
|
|
func isHardStateEqual(a, b pb.HardState) bool {
|
2014-09-16 01:34:23 +04:00
|
|
|
return a.Term == b.Term && a.Vote == b.Vote && a.Commit == b.Commit
|
2014-08-25 07:09:06 +04:00
|
|
|
}
|
2014-05-29 00:53:26 +04:00
|
|
|
|
2014-10-08 02:22:35 +04:00
|
|
|
// IsEmptyHardState returns true if the given HardState is empty.
|
2014-09-17 00:55:00 +04:00
|
|
|
func IsEmptyHardState(st pb.HardState) bool {
|
|
|
|
return isHardStateEqual(st, emptyState)
|
2014-09-09 08:45:10 +04:00
|
|
|
}
|
|
|
|
|
2014-10-08 02:22:35 +04:00
|
|
|
// IsEmptySnap returns true if the given Snapshot is empty.
|
2014-09-17 05:18:45 +04:00
|
|
|
func IsEmptySnap(sp pb.Snapshot) bool {
|
2014-11-20 00:17:50 +03:00
|
|
|
return sp.Metadata.Index == 0
|
2014-09-17 05:18:45 +04:00
|
|
|
}
|
|
|
|
|
2014-09-09 08:40:02 +04:00
|
|
|
func (rd Ready) containsUpdates() bool {
|
2014-11-20 00:17:50 +03:00
|
|
|
return rd.SoftState != nil || !IsEmptyHardState(rd.HardState) ||
|
|
|
|
!IsEmptySnap(rd.Snapshot) || len(rd.Entries) > 0 ||
|
2016-08-27 02:03:06 +03:00
|
|
|
len(rd.CommittedEntries) > 0 || len(rd.Messages) > 0 || len(rd.ReadStates) != 0
|
2014-06-05 21:49:34 +04:00
|
|
|
}
|
|
|
|
|
2014-10-08 02:22:35 +04:00
|
|
|
// Node represents a node in a raft cluster.
|
2014-09-17 23:23:44 +04:00
|
|
|
type Node interface {
|
|
|
|
// Tick increments the internal logical clock for the Node by a single tick. Election
|
|
|
|
// timeouts and heartbeat timeouts are in units of ticks.
|
|
|
|
Tick()
|
2014-09-19 23:35:56 +04:00
|
|
|
// Campaign causes the Node to transition to candidate state and start campaigning to become leader.
|
2014-09-17 23:23:44 +04:00
|
|
|
Campaign(ctx context.Context) error
|
|
|
|
// Propose proposes that data be appended to the log.
|
|
|
|
Propose(ctx context.Context, data []byte) error
|
2014-09-23 23:02:44 +04:00
|
|
|
// ProposeConfChange proposes config change.
|
|
|
|
// At most one ConfChange can be in the process of going through consensus.
|
|
|
|
// Application needs to call ApplyConfChange when applying EntryConfChange type entry.
|
|
|
|
ProposeConfChange(ctx context.Context, cc pb.ConfChange) error
|
2014-09-17 23:23:44 +04:00
|
|
|
// Step advances the state machine using the given message. ctx.Err() will be returned, if any.
|
|
|
|
Step(ctx context.Context, msg pb.Message) error
|
2015-12-23 02:51:27 +03:00
|
|
|
|
2015-09-21 07:13:33 +03:00
|
|
|
// Ready returns a channel that returns the current point-in-time state.
|
2015-12-23 02:51:27 +03:00
|
|
|
// Users of the Node must call Advance after retrieving the state returned by Ready.
|
|
|
|
//
|
|
|
|
// NOTE: No committed entries from the next Ready may be applied until all committed entries
|
|
|
|
// and snapshots from the previous one have finished.
|
2014-09-17 23:23:44 +04:00
|
|
|
Ready() <-chan Ready
|
2015-12-23 02:51:27 +03:00
|
|
|
|
|
|
|
// Advance notifies the Node that the application has saved progress up to the last Ready.
|
2014-11-03 23:26:13 +03:00
|
|
|
// It prepares the node to return the next available Ready.
|
2015-12-23 02:51:27 +03:00
|
|
|
//
|
|
|
|
// The application should generally call Advance after it applies the entries in last Ready.
|
|
|
|
//
|
|
|
|
// However, as an optimization, the application may call Advance while it is applying the
|
|
|
|
// commands. For example. when the last Ready contains a snapshot, the application might take
|
|
|
|
// a long time to apply the snapshot data. To continue receiving Ready without blocking raft
|
2016-07-26 01:06:57 +03:00
|
|
|
// progress, it can call Advance before finishing applying the last ready.
|
2014-11-03 23:26:13 +03:00
|
|
|
Advance()
|
2014-09-23 23:02:44 +04:00
|
|
|
// ApplyConfChange applies config change to the local node.
|
2014-11-20 00:17:50 +03:00
|
|
|
// Returns an opaque ConfState protobuf which must be recorded
|
|
|
|
// in snapshots. Will never return nil; it returns a pointer only
|
|
|
|
// to match MemoryStorage.Compact.
|
|
|
|
ApplyConfChange(cc pb.ConfChange) *pb.ConfState
|
2016-07-20 00:19:13 +03:00
|
|
|
|
2016-08-11 02:25:22 +03:00
|
|
|
// TransferLeadership attempts to transfer leadership to the given transferee.
|
|
|
|
TransferLeadership(ctx context.Context, lead, transferee uint64)
|
|
|
|
|
2016-07-20 00:19:13 +03:00
|
|
|
// ReadIndex request a read state. The read state will be set in the ready.
|
|
|
|
// Read state has a read index. Once the application advances further than the read
|
|
|
|
// index, any linearizable read requests issued before the read request can be
|
|
|
|
// processed safely. The read state will have the same rctx attached.
|
|
|
|
ReadIndex(ctx context.Context, rctx []byte) error
|
|
|
|
|
2014-12-26 04:20:49 +03:00
|
|
|
// Status returns the current status of the raft state machine.
|
|
|
|
Status() Status
|
2016-02-24 11:45:40 +03:00
|
|
|
// ReportUnreachable reports the given node is not reachable for the last send.
|
2015-02-05 01:41:14 +03:00
|
|
|
ReportUnreachable(id uint64)
|
2015-09-21 07:13:33 +03:00
|
|
|
// ReportSnapshot reports the status of the sent snapshot.
|
2015-02-25 08:38:18 +03:00
|
|
|
ReportSnapshot(id uint64, status SnapshotStatus)
|
2015-09-21 07:13:33 +03:00
|
|
|
// Stop performs any necessary termination of the Node.
|
2014-09-17 23:23:44 +04:00
|
|
|
Stop()
|
2014-05-19 10:34:55 +04:00
|
|
|
}
|
|
|
|
|
2014-10-08 04:38:58 +04:00
|
|
|
type Peer struct {
|
|
|
|
ID uint64
|
|
|
|
Context []byte
|
|
|
|
}
|
|
|
|
|
2015-03-24 07:19:03 +03:00
|
|
|
// StartNode returns a new Node given configuration and a list of raft peers.
|
2014-11-12 22:05:25 +03:00
|
|
|
// It appends a ConfChangeAddNode entry for each given peer to the initial log.
|
2015-03-24 07:19:03 +03:00
|
|
|
func StartNode(c *Config, peers []Peer) Node {
|
2015-03-22 04:15:58 +03:00
|
|
|
r := newRaft(c)
|
2014-12-09 03:32:36 +03:00
|
|
|
// become the follower at term 1 and apply initial configuration
|
2016-02-01 08:42:39 +03:00
|
|
|
// entries of term 1
|
2014-12-09 03:32:36 +03:00
|
|
|
r.becomeFollower(1, None)
|
2014-10-11 15:20:14 +04:00
|
|
|
for _, peer := range peers {
|
2014-10-08 04:38:58 +04:00
|
|
|
cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context}
|
2014-10-11 15:20:14 +04:00
|
|
|
d, err := cc.Marshal()
|
2014-10-07 01:15:50 +04:00
|
|
|
if err != nil {
|
|
|
|
panic("unexpected marshal error")
|
|
|
|
}
|
2014-10-11 15:20:14 +04:00
|
|
|
e := pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: r.raftLog.lastIndex() + 1, Data: d}
|
2014-12-02 19:50:15 +03:00
|
|
|
r.raftLog.append(e)
|
2014-10-07 01:15:50 +04:00
|
|
|
}
|
2014-11-20 00:17:50 +03:00
|
|
|
// Mark these initial entries as committed.
|
|
|
|
// TODO(bdarnell): These entries are still unstable; do we need to preserve
|
|
|
|
// the invariant that committed < unstable?
|
2014-10-11 15:20:14 +04:00
|
|
|
r.raftLog.committed = r.raftLog.lastIndex()
|
2014-12-05 23:35:39 +03:00
|
|
|
// Now apply them, mainly so that the application can call Campaign
|
|
|
|
// immediately after StartNode in tests. Note that these nodes will
|
|
|
|
// be added to raft twice: here and when the application's Ready
|
|
|
|
// loop calls ApplyConfChange. The calls to addNode must come after
|
|
|
|
// all calls to raftLog.append so progress.next is set after these
|
|
|
|
// bootstrapping entries (it is an error if we try to append these
|
|
|
|
// entries since they have already been committed).
|
|
|
|
// We do not set raftLog.applied so the application will be able
|
|
|
|
// to observe all conf changes via Ready.CommittedEntries.
|
|
|
|
for _, peer := range peers {
|
|
|
|
r.addNode(peer.ID)
|
|
|
|
}
|
2014-10-07 02:22:35 +04:00
|
|
|
|
2015-03-22 04:15:58 +03:00
|
|
|
n := newNode()
|
2016-06-16 17:47:19 +03:00
|
|
|
n.logger = c.Logger
|
2014-09-05 08:15:39 +04:00
|
|
|
go n.run(r)
|
2014-09-17 23:23:44 +04:00
|
|
|
return &n
|
2014-09-05 08:15:39 +04:00
|
|
|
}
|
|
|
|
|
2015-01-22 19:37:02 +03:00
|
|
|
// RestartNode is similar to StartNode but does not take a list of peers.
|
2014-11-20 00:17:50 +03:00
|
|
|
// The current membership of the cluster will be restored from the Storage.
|
2015-01-22 19:37:02 +03:00
|
|
|
// If the caller has an existing state machine, pass in the last log index that
|
|
|
|
// has been applied to it; otherwise use zero.
|
2015-03-24 07:19:03 +03:00
|
|
|
func RestartNode(c *Config) Node {
|
2015-03-22 04:15:58 +03:00
|
|
|
r := newRaft(c)
|
2014-11-20 00:17:50 +03:00
|
|
|
|
2015-03-22 04:15:58 +03:00
|
|
|
n := newNode()
|
2016-06-16 17:47:19 +03:00
|
|
|
n.logger = c.Logger
|
2014-09-05 08:15:39 +04:00
|
|
|
go n.run(r)
|
2014-09-17 23:23:44 +04:00
|
|
|
return &n
|
|
|
|
}
|
|
|
|
|
2018-01-12 10:41:59 +03:00
|
|
|
type msgWithResult struct {
|
|
|
|
m pb.Message
|
|
|
|
result chan error
|
|
|
|
}
|
|
|
|
|
2014-09-17 23:23:44 +04:00
|
|
|
// node is the canonical implementation of the Node interface
|
|
|
|
type node struct {
|
2018-01-12 10:41:59 +03:00
|
|
|
propc chan msgWithResult
|
2014-11-20 00:17:50 +03:00
|
|
|
recvc chan pb.Message
|
|
|
|
confc chan pb.ConfChange
|
|
|
|
confstatec chan pb.ConfState
|
|
|
|
readyc chan Ready
|
|
|
|
advancec chan struct{}
|
|
|
|
tickc chan struct{}
|
|
|
|
done chan struct{}
|
|
|
|
stop chan struct{}
|
2015-01-19 02:23:50 +03:00
|
|
|
status chan chan Status
|
2016-06-16 17:47:19 +03:00
|
|
|
|
|
|
|
logger Logger
|
2014-09-05 08:15:39 +04:00
|
|
|
}
|
|
|
|
|
2014-09-17 23:23:44 +04:00
|
|
|
func newNode() node {
|
|
|
|
return node{
|
2018-01-12 10:41:59 +03:00
|
|
|
propc: make(chan msgWithResult),
|
2014-11-20 00:17:50 +03:00
|
|
|
recvc: make(chan pb.Message),
|
|
|
|
confc: make(chan pb.ConfChange),
|
|
|
|
confstatec: make(chan pb.ConfState),
|
|
|
|
readyc: make(chan Ready),
|
|
|
|
advancec: make(chan struct{}),
|
2016-06-16 17:47:19 +03:00
|
|
|
// make tickc a buffered chan, so raft node can buffer some ticks when the node
|
|
|
|
// is busy processing raft messages. Raft node will resume process buffered
|
|
|
|
// ticks when it becomes idle.
|
|
|
|
tickc: make(chan struct{}, 128),
|
|
|
|
done: make(chan struct{}),
|
|
|
|
stop: make(chan struct{}),
|
|
|
|
status: make(chan chan Status),
|
2014-08-25 07:09:06 +04:00
|
|
|
}
|
2014-05-19 10:34:55 +04:00
|
|
|
}
|
|
|
|
|
2014-09-17 23:23:44 +04:00
|
|
|
func (n *node) Stop() {
|
2014-11-13 02:02:48 +03:00
|
|
|
select {
|
|
|
|
case n.stop <- struct{}{}:
|
|
|
|
// Not already stopped, so trigger it
|
|
|
|
case <-n.done:
|
|
|
|
// Node has already been stopped - no need to do anything
|
|
|
|
return
|
|
|
|
}
|
|
|
|
// Block until the stop has been acknowledged by run()
|
|
|
|
<-n.done
|
2014-09-01 05:49:07 +04:00
|
|
|
}
|
|
|
|
|
2014-09-17 23:23:44 +04:00
|
|
|
func (n *node) run(r *raft) {
|
2018-01-12 10:41:59 +03:00
|
|
|
var propc chan msgWithResult
|
2014-09-12 23:57:50 +04:00
|
|
|
var readyc chan Ready
|
2014-11-03 23:26:13 +03:00
|
|
|
var advancec chan struct{}
|
2014-12-26 04:20:49 +03:00
|
|
|
var prevLastUnstablei, prevLastUnstablet uint64
|
2014-11-13 22:57:01 +03:00
|
|
|
var havePrevLastUnstablei bool
|
2014-11-25 11:22:26 +03:00
|
|
|
var prevSnapi uint64
|
2014-11-03 23:26:13 +03:00
|
|
|
var rd Ready
|
2014-06-20 01:39:17 +04:00
|
|
|
|
2014-09-16 04:35:02 +04:00
|
|
|
lead := None
|
|
|
|
prevSoftSt := r.softState()
|
2015-05-27 23:30:08 +03:00
|
|
|
prevHardSt := emptyState
|
2014-09-05 08:15:39 +04:00
|
|
|
|
2014-08-25 07:09:06 +04:00
|
|
|
for {
|
2014-11-03 23:26:13 +03:00
|
|
|
if advancec != nil {
|
2014-09-16 04:35:02 +04:00
|
|
|
readyc = nil
|
2014-11-03 23:26:13 +03:00
|
|
|
} else {
|
2014-11-20 00:17:50 +03:00
|
|
|
rd = newReady(r, prevSoftSt, prevHardSt)
|
2014-11-03 23:26:13 +03:00
|
|
|
if rd.containsUpdates() {
|
|
|
|
readyc = n.readyc
|
|
|
|
} else {
|
|
|
|
readyc = nil
|
|
|
|
}
|
2014-11-25 06:07:10 +03:00
|
|
|
}
|
2014-09-16 04:35:02 +04:00
|
|
|
|
2014-12-26 04:04:33 +03:00
|
|
|
if lead != r.lead {
|
2014-11-25 06:07:10 +03:00
|
|
|
if r.hasLeader() {
|
|
|
|
if lead == None {
|
2016-01-24 16:15:44 +03:00
|
|
|
r.logger.Infof("raft.node: %x elected leader %x at term %d", r.id, r.lead, r.Term)
|
2014-10-30 18:38:16 +03:00
|
|
|
} else {
|
2016-01-24 16:15:44 +03:00
|
|
|
r.logger.Infof("raft.node: %x changed leader from %x to %x at term %d", r.id, lead, r.lead, r.Term)
|
2014-10-30 18:38:16 +03:00
|
|
|
}
|
2014-11-25 06:07:10 +03:00
|
|
|
propc = n.propc
|
|
|
|
} else {
|
2016-01-24 16:15:44 +03:00
|
|
|
r.logger.Infof("raft.node: %x lost leader %x at term %d", r.id, lead, r.Term)
|
2014-11-25 06:07:10 +03:00
|
|
|
propc = nil
|
2014-09-03 07:21:58 +04:00
|
|
|
}
|
2014-12-26 04:04:33 +03:00
|
|
|
lead = r.lead
|
2014-07-11 13:35:31 +04:00
|
|
|
}
|
|
|
|
|
2014-08-25 07:09:06 +04:00
|
|
|
select {
|
2014-09-23 08:59:13 +04:00
|
|
|
// TODO: maybe buffer the config propose if there exists one (the way
|
|
|
|
// described in raft dissertation)
|
|
|
|
// Currently it is dropped in Step silently.
|
2018-01-12 10:41:59 +03:00
|
|
|
case pm := <-propc:
|
|
|
|
m := pm.m
|
2014-08-25 23:10:10 +04:00
|
|
|
m.From = r.id
|
2018-01-12 10:41:59 +03:00
|
|
|
err := r.Step(m)
|
|
|
|
if pm.result != nil {
|
|
|
|
pm.result <- err
|
|
|
|
close(pm.result)
|
|
|
|
}
|
2014-08-25 07:09:06 +04:00
|
|
|
case m := <-n.recvc:
|
2015-02-05 01:41:14 +03:00
|
|
|
// filter out response message from unknown From.
|
2017-11-11 05:38:21 +03:00
|
|
|
if pr := r.getProgress(m.From); pr != nil || !IsResponseMsg(m.Type) {
|
2018-01-12 10:41:59 +03:00
|
|
|
r.Step(m)
|
2014-12-05 21:45:05 +03:00
|
|
|
}
|
2014-09-23 08:59:13 +04:00
|
|
|
case cc := <-n.confc:
|
2014-10-16 20:49:26 +04:00
|
|
|
if cc.NodeID == None {
|
2014-11-20 00:17:50 +03:00
|
|
|
select {
|
2018-01-08 18:43:04 +03:00
|
|
|
case n.confstatec <- pb.ConfState{
|
|
|
|
Nodes: r.nodes(),
|
|
|
|
Learners: r.learnerNodes()}:
|
2014-11-20 00:17:50 +03:00
|
|
|
case <-n.done:
|
|
|
|
}
|
2014-10-16 20:49:26 +04:00
|
|
|
break
|
|
|
|
}
|
2014-09-23 08:59:13 +04:00
|
|
|
switch cc.Type {
|
2014-09-23 23:02:44 +04:00
|
|
|
case pb.ConfChangeAddNode:
|
2014-09-23 08:59:13 +04:00
|
|
|
r.addNode(cc.NodeID)
|
2017-11-11 05:38:21 +03:00
|
|
|
case pb.ConfChangeAddLearnerNode:
|
|
|
|
r.addLearner(cc.NodeID)
|
2014-09-23 23:02:44 +04:00
|
|
|
case pb.ConfChangeRemoveNode:
|
2014-12-24 02:47:41 +03:00
|
|
|
// block incoming proposal when local node is
|
|
|
|
// removed
|
|
|
|
if cc.NodeID == r.id {
|
2016-04-23 01:45:33 +03:00
|
|
|
propc = nil
|
2014-12-24 02:47:41 +03:00
|
|
|
}
|
2014-09-23 08:59:13 +04:00
|
|
|
r.removeNode(cc.NodeID)
|
2014-11-11 22:46:49 +03:00
|
|
|
case pb.ConfChangeUpdateNode:
|
2014-09-19 23:35:56 +04:00
|
|
|
default:
|
|
|
|
panic("unexpected conf type")
|
|
|
|
}
|
2014-11-20 00:17:50 +03:00
|
|
|
select {
|
2018-01-08 18:43:04 +03:00
|
|
|
case n.confstatec <- pb.ConfState{
|
|
|
|
Nodes: r.nodes(),
|
|
|
|
Learners: r.learnerNodes()}:
|
2014-11-20 00:17:50 +03:00
|
|
|
case <-n.done:
|
|
|
|
}
|
2014-08-25 07:09:06 +04:00
|
|
|
case <-n.tickc:
|
2014-09-03 03:59:29 +04:00
|
|
|
r.tick()
|
2014-08-27 02:28:21 +04:00
|
|
|
case readyc <- rd:
|
2014-09-16 04:35:02 +04:00
|
|
|
if rd.SoftState != nil {
|
|
|
|
prevSoftSt = rd.SoftState
|
|
|
|
}
|
2014-11-03 23:26:13 +03:00
|
|
|
if len(rd.Entries) > 0 {
|
|
|
|
prevLastUnstablei = rd.Entries[len(rd.Entries)-1].Index
|
2014-11-29 01:13:07 +03:00
|
|
|
prevLastUnstablet = rd.Entries[len(rd.Entries)-1].Term
|
2014-11-13 22:57:01 +03:00
|
|
|
havePrevLastUnstablei = true
|
2014-11-03 23:26:13 +03:00
|
|
|
}
|
2014-09-17 00:55:00 +04:00
|
|
|
if !IsEmptyHardState(rd.HardState) {
|
2014-09-16 04:35:02 +04:00
|
|
|
prevHardSt = rd.HardState
|
|
|
|
}
|
2014-09-17 05:18:45 +04:00
|
|
|
if !IsEmptySnap(rd.Snapshot) {
|
2014-11-25 11:22:26 +03:00
|
|
|
prevSnapi = rd.Snapshot.Metadata.Index
|
2014-09-17 05:18:45 +04:00
|
|
|
}
|
2016-06-03 18:20:10 +03:00
|
|
|
|
2014-08-25 07:09:06 +04:00
|
|
|
r.msgs = nil
|
2016-08-27 02:03:06 +03:00
|
|
|
r.readStates = nil
|
2014-11-03 23:26:13 +03:00
|
|
|
advancec = n.advancec
|
|
|
|
case <-advancec:
|
|
|
|
if prevHardSt.Commit != 0 {
|
|
|
|
r.raftLog.appliedTo(prevHardSt.Commit)
|
|
|
|
}
|
2014-11-13 22:57:01 +03:00
|
|
|
if havePrevLastUnstablei {
|
2014-11-29 01:13:07 +03:00
|
|
|
r.raftLog.stableTo(prevLastUnstablei, prevLastUnstablet)
|
2014-11-13 22:57:01 +03:00
|
|
|
havePrevLastUnstablei = false
|
|
|
|
}
|
2014-11-27 01:40:58 +03:00
|
|
|
r.raftLog.stableSnapTo(prevSnapi)
|
2014-11-03 23:26:13 +03:00
|
|
|
advancec = nil
|
2015-01-19 02:23:50 +03:00
|
|
|
case c := <-n.status:
|
|
|
|
c <- getStatus(r)
|
2014-11-12 23:32:20 +03:00
|
|
|
case <-n.stop:
|
|
|
|
close(n.done)
|
2014-08-25 07:09:06 +04:00
|
|
|
return
|
2014-07-15 10:41:19 +04:00
|
|
|
}
|
2014-05-29 00:53:26 +04:00
|
|
|
}
|
|
|
|
}
|
2014-06-05 21:49:34 +04:00
|
|
|
|
2014-09-12 10:32:46 +04:00
|
|
|
// Tick increments the internal logical clock for this Node. Election timeouts
|
|
|
|
// and heartbeat timeouts are in units of ticks.
|
2014-09-17 23:23:44 +04:00
|
|
|
func (n *node) Tick() {
|
2014-08-25 07:09:06 +04:00
|
|
|
select {
|
|
|
|
case n.tickc <- struct{}{}:
|
2014-09-01 05:49:07 +04:00
|
|
|
case <-n.done:
|
2016-06-16 17:47:19 +03:00
|
|
|
default:
|
|
|
|
n.logger.Warningf("A tick missed to fire. Node blocks too long!")
|
2014-06-08 13:50:39 +04:00
|
|
|
}
|
2014-06-05 21:49:34 +04:00
|
|
|
}
|
2014-07-24 08:45:57 +04:00
|
|
|
|
2014-12-05 21:45:05 +03:00
|
|
|
func (n *node) Campaign(ctx context.Context) error { return n.step(ctx, pb.Message{Type: pb.MsgHup}) }
|
2014-08-29 03:41:42 +04:00
|
|
|
|
2014-09-17 23:23:44 +04:00
|
|
|
func (n *node) Propose(ctx context.Context, data []byte) error {
|
2018-01-12 10:41:59 +03:00
|
|
|
return n.stepWait(ctx, pb.Message{Type: pb.MsgProp, Entries: []pb.Entry{{Data: data}}})
|
2014-09-24 00:50:43 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func (n *node) Step(ctx context.Context, m pb.Message) error {
|
|
|
|
// ignore unexpected local messages receiving over network
|
2016-03-24 14:59:41 +03:00
|
|
|
if IsLocalMsg(m.Type) {
|
2014-09-24 00:50:43 +04:00
|
|
|
// TODO: return an error?
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return n.step(ctx, m)
|
2014-07-25 01:11:53 +04:00
|
|
|
}
|
2014-07-31 04:21:27 +04:00
|
|
|
|
2014-09-23 23:02:44 +04:00
|
|
|
func (n *node) ProposeConfChange(ctx context.Context, cc pb.ConfChange) error {
|
2014-09-23 08:59:13 +04:00
|
|
|
data, err := cc.Marshal()
|
2014-09-23 01:38:21 +04:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2014-10-12 11:34:22 +04:00
|
|
|
return n.Step(ctx, pb.Message{Type: pb.MsgProp, Entries: []pb.Entry{{Type: pb.EntryConfChange, Data: data}}})
|
2014-09-19 23:35:56 +04:00
|
|
|
}
|
|
|
|
|
2018-01-12 10:41:59 +03:00
|
|
|
func (n *node) step(ctx context.Context, m pb.Message) error {
|
|
|
|
return n.stepWithWaitOption(ctx, m, false)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (n *node) stepWait(ctx context.Context, m pb.Message) error {
|
|
|
|
return n.stepWithWaitOption(ctx, m, true)
|
|
|
|
}
|
|
|
|
|
2014-08-28 04:22:34 +04:00
|
|
|
// Step advances the state machine using msgs. The ctx.Err() will be returned,
|
2014-08-25 23:49:14 +04:00
|
|
|
// if any.
|
2018-01-12 10:41:59 +03:00
|
|
|
func (n *node) stepWithWaitOption(ctx context.Context, m pb.Message, wait bool) error {
|
|
|
|
if m.Type != pb.MsgProp {
|
|
|
|
select {
|
|
|
|
case n.recvc <- m:
|
|
|
|
return nil
|
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
|
|
|
case <-n.done:
|
|
|
|
return ErrStopped
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ch := n.propc
|
|
|
|
pm := msgWithResult{m: m}
|
|
|
|
if wait {
|
|
|
|
pm.result = make(chan error, 1)
|
2014-08-28 04:22:34 +04:00
|
|
|
}
|
|
|
|
select {
|
2018-01-12 10:41:59 +03:00
|
|
|
case ch <- pm:
|
|
|
|
if !wait {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
|
|
|
case <-n.done:
|
|
|
|
return ErrStopped
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case rsp := <-pm.result:
|
|
|
|
if rsp != nil {
|
|
|
|
return rsp
|
|
|
|
}
|
2014-08-28 04:22:34 +04:00
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
2014-09-01 05:49:07 +04:00
|
|
|
case <-n.done:
|
|
|
|
return ErrStopped
|
2014-08-01 02:18:44 +04:00
|
|
|
}
|
2018-01-12 10:41:59 +03:00
|
|
|
return nil
|
2014-07-31 04:21:27 +04:00
|
|
|
}
|
|
|
|
|
2014-12-05 21:45:05 +03:00
|
|
|
func (n *node) Ready() <-chan Ready { return n.readyc }
|
2014-09-09 08:50:04 +04:00
|
|
|
|
2014-11-03 23:26:13 +03:00
|
|
|
func (n *node) Advance() {
|
|
|
|
select {
|
|
|
|
case n.advancec <- struct{}{}:
|
|
|
|
case <-n.done:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-11-20 00:17:50 +03:00
|
|
|
func (n *node) ApplyConfChange(cc pb.ConfChange) *pb.ConfState {
|
|
|
|
var cs pb.ConfState
|
2014-09-17 05:18:45 +04:00
|
|
|
select {
|
2014-09-23 08:59:13 +04:00
|
|
|
case n.confc <- cc:
|
2014-09-17 05:18:45 +04:00
|
|
|
case <-n.done:
|
|
|
|
}
|
2014-09-19 23:35:56 +04:00
|
|
|
select {
|
2014-11-20 00:17:50 +03:00
|
|
|
case cs = <-n.confstatec:
|
2014-09-19 23:35:56 +04:00
|
|
|
case <-n.done:
|
|
|
|
}
|
2014-11-20 00:17:50 +03:00
|
|
|
return &cs
|
2014-09-19 23:35:56 +04:00
|
|
|
}
|
|
|
|
|
2015-01-19 02:23:50 +03:00
|
|
|
func (n *node) Status() Status {
|
|
|
|
c := make(chan Status)
|
2016-11-16 02:38:03 +03:00
|
|
|
select {
|
|
|
|
case n.status <- c:
|
|
|
|
return <-c
|
|
|
|
case <-n.done:
|
|
|
|
return Status{}
|
|
|
|
}
|
2015-01-19 02:23:50 +03:00
|
|
|
}
|
2014-12-26 04:20:49 +03:00
|
|
|
|
2015-02-05 01:41:14 +03:00
|
|
|
func (n *node) ReportUnreachable(id uint64) {
|
|
|
|
select {
|
|
|
|
case n.recvc <- pb.Message{Type: pb.MsgUnreachable, From: id}:
|
|
|
|
case <-n.done:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-25 08:38:18 +03:00
|
|
|
func (n *node) ReportSnapshot(id uint64, status SnapshotStatus) {
|
|
|
|
rej := status == SnapshotFailure
|
|
|
|
|
|
|
|
select {
|
|
|
|
case n.recvc <- pb.Message{Type: pb.MsgSnapStatus, From: id, Reject: rej}:
|
|
|
|
case <-n.done:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-11 02:25:22 +03:00
|
|
|
func (n *node) TransferLeadership(ctx context.Context, lead, transferee uint64) {
|
|
|
|
select {
|
|
|
|
// manually set 'from' and 'to', so that leader can voluntarily transfers its leadership
|
|
|
|
case n.recvc <- pb.Message{Type: pb.MsgTransferLeader, From: transferee, To: lead}:
|
|
|
|
case <-n.done:
|
|
|
|
case <-ctx.Done():
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-07-20 00:19:13 +03:00
|
|
|
func (n *node) ReadIndex(ctx context.Context, rctx []byte) error {
|
|
|
|
return n.step(ctx, pb.Message{Type: pb.MsgReadIndex, Entries: []pb.Entry{{Data: rctx}}})
|
2016-06-03 18:20:10 +03:00
|
|
|
}
|
|
|
|
|
2014-11-20 00:17:50 +03:00
|
|
|
func newReady(r *raft, prevSoftSt *SoftState, prevHardSt pb.HardState) Ready {
|
2014-09-09 08:50:04 +04:00
|
|
|
rd := Ready{
|
2014-11-03 23:04:24 +03:00
|
|
|
Entries: r.raftLog.unstableEntries(),
|
2014-09-09 08:50:04 +04:00
|
|
|
CommittedEntries: r.raftLog.nextEnts(),
|
|
|
|
Messages: r.msgs,
|
|
|
|
}
|
2014-09-16 04:35:02 +04:00
|
|
|
if softSt := r.softState(); !softSt.equal(prevSoftSt) {
|
|
|
|
rd.SoftState = softSt
|
|
|
|
}
|
2016-01-26 23:18:55 +03:00
|
|
|
if hardSt := r.hardState(); !isHardStateEqual(hardSt, prevHardSt) {
|
|
|
|
rd.HardState = hardSt
|
2014-09-09 08:50:04 +04:00
|
|
|
}
|
2014-11-27 00:36:17 +03:00
|
|
|
if r.raftLog.unstable.snapshot != nil {
|
|
|
|
rd.Snapshot = *r.raftLog.unstable.snapshot
|
2014-09-17 05:18:45 +04:00
|
|
|
}
|
2016-08-27 02:03:06 +03:00
|
|
|
if len(r.readStates) != 0 {
|
|
|
|
rd.ReadStates = r.readStates
|
2016-06-03 18:20:10 +03:00
|
|
|
}
|
2017-02-13 23:09:54 +03:00
|
|
|
rd.MustSync = MustSync(rd.HardState, prevHardSt, len(rd.Entries))
|
2014-09-09 08:50:04 +04:00
|
|
|
return rd
|
|
|
|
}
|
2017-02-13 23:09:54 +03:00
|
|
|
|
|
|
|
// MustSync returns true if the hard state and count of Raft entries indicate
|
|
|
|
// that a synchronous write to persistent storage is required.
|
|
|
|
func MustSync(st, prevst pb.HardState, entsnum int) bool {
|
|
|
|
// Persistent state on all servers:
|
|
|
|
// (Updated on stable storage before responding to RPCs)
|
|
|
|
// currentTerm
|
|
|
|
// votedFor
|
|
|
|
// log entries[]
|
|
|
|
return entsnum != 0 || st.Vote != prevst.Vote || st.Term != prevst.Term
|
|
|
|
}
|