Compare commits
5 Commits
dfa8262e4c
...
0bbc7a2568
Author | SHA1 | Date |
---|---|---|
Vitaliy Filippov | 0bbc7a2568 | |
Vitaliy Filippov | e95e9d1feb | |
Vitaliy Filippov | cbb528fb30 | |
Vitaliy Filippov | e12367bc82 | |
Vitaliy Filippov | 6b61e16877 |
|
@ -0,0 +1,103 @@
|
||||||
|
# TinyRaft
|
||||||
|
|
||||||
|
Raft leader election isolated from the rest of the algorithm.
|
||||||
|
|
||||||
|
TinyRaft doesn't know anything about replication and doesn't require you to
|
||||||
|
implement it.
|
||||||
|
|
||||||
|
Actual network communication is also abstracted away and hidden behind a simple
|
||||||
|
callback interface.
|
||||||
|
|
||||||
|
The only task of TinyRaft is to elect the leader and guarantee that there is
|
||||||
|
only one leader at each moment.
|
||||||
|
|
||||||
|
TinyRaft can be used:
|
||||||
|
- As a simple leader election algorithm without replication at all
|
||||||
|
- As a building block for the standard Raft algorithm if you add log replication
|
||||||
|
- For other variations of "weaker consensus" if you add another method of replication
|
||||||
|
|
||||||
|
Some replication ideas for you:
|
||||||
|
- Log-less replication: Add a version number for each key in the database
|
||||||
|
and make the leader synchronize follower databases by simply dumping all
|
||||||
|
followers' databases with the newest term (followers with older terms should
|
||||||
|
be ignored), comparing version numbers and making the newest version of each
|
||||||
|
key win.
|
||||||
|
- Erasure coding: Suppose you store large values. You can split each value into
|
||||||
|
N parts, add K parity parts to it using Reed-Solomon codes (ISA-L/jerasure)
|
||||||
|
and store them on different nodes in the form of Raft-like logs or similar
|
||||||
|
to the log-less replication with version numbers, and make master synchronize
|
||||||
|
followers by reconstructing every original value.
|
||||||
|
|
||||||
|
## Example Application
|
||||||
|
|
||||||
|
TODO: Extract and describe Antietcd.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```js
|
||||||
|
const node = new TinyRaft({
|
||||||
|
nodes: [ 1, 2, 3 ],
|
||||||
|
nodeId: 1,
|
||||||
|
electionTimeout: 5000,
|
||||||
|
heartbeatTimeout: 1000,
|
||||||
|
leadershipTimeout: 10000,
|
||||||
|
initialTerm: 0,
|
||||||
|
leaderPriority: 0,
|
||||||
|
send: function(to, msg)
|
||||||
|
{
|
||||||
|
// Function to send message <msg> to node with ID <to>
|
||||||
|
// msg.type is one of TinyRaft.VOTE_REQUEST, TinyRaft.VOTE, TinyRaft.PING, TinyRaft.PONG
|
||||||
|
// msg.leader is the leader ID or null
|
||||||
|
// msg.term is the term number
|
||||||
|
// msg.priority is the optional leadership priority if set in config
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Watch for election state
|
||||||
|
node.on('change', (st) =>
|
||||||
|
{
|
||||||
|
console.log(
|
||||||
|
'node '+node.nodeId+': '+(st.state == TinyRaft.FOLLOWER ? 'following '+st.leader : st.state)+
|
||||||
|
', term '+st.term+(st.state == TinyRaft.LEADER ? ', followers: '+st.followers.join(', ') : '')
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Start Raft node or start a new election round
|
||||||
|
node.start();
|
||||||
|
|
||||||
|
// Optional; may be called for a follower when it receives a message from a live leader,
|
||||||
|
// for example, a replication message, and causes the follower to move its round expiration forward
|
||||||
|
node.markAlive();
|
||||||
|
|
||||||
|
// Update cluster node list
|
||||||
|
node.setNodes([ 1, 2, 3 ]);
|
||||||
|
|
||||||
|
// Incoming messages should be fed to TinyRaft like this (from = ID of the sender):
|
||||||
|
node.onReceive(from, msg);
|
||||||
|
|
||||||
|
// Stop Raft node
|
||||||
|
node.stop();
|
||||||
|
```
|
||||||
|
|
||||||
|
## Additional features
|
||||||
|
|
||||||
|
### Leader expiration
|
||||||
|
|
||||||
|
Supports leader expiration like in NuRaft:
|
||||||
|
https://github.com/eBay/NuRaft/blob/master/docs/leadership_expiration.md
|
||||||
|
|
||||||
|
When leader expiration is enabled, followers respond to leader heartbeats
|
||||||
|
(pings) with "pong" messages and if the leader doesn't receive a quorum of
|
||||||
|
replies in leadershipTimeout - it starts a new round of voting.
|
||||||
|
|
||||||
|
### Leadership priorities
|
||||||
|
|
||||||
|
Also supports leader priorities, similar to NuRaft but even simpler:
|
||||||
|
if a node receives a VoteRequest message with larger term but with smaller
|
||||||
|
priority than its own, it immediately starts a new voting round.
|
||||||
|
|
||||||
|
It guarantees that a node with non-maximum priority can't become leader
|
||||||
|
without being re-elected.
|
||||||
|
|
||||||
|
If all priorities are equal (or just zero), the election algorithm
|
||||||
|
becomes identical to the basic algorithm without priorities.
|
|
@ -0,0 +1,44 @@
|
||||||
|
// Fake timer for node.js which _guarantees_ that all asynchronous timer
|
||||||
|
// callbacks are called during a tested period
|
||||||
|
|
||||||
|
function FakeTimer(startTime)
|
||||||
|
{
|
||||||
|
this.time = Number(startTime)||0;
|
||||||
|
this.timers = [];
|
||||||
|
this.id = 0;
|
||||||
|
this.setTimeout = (func, ms) =>
|
||||||
|
{
|
||||||
|
this.timers.push({ id: ++this.id, func, at: this.time+ms });
|
||||||
|
return this.id;
|
||||||
|
};
|
||||||
|
this.setInterval = (func, ms) =>
|
||||||
|
{
|
||||||
|
this.timers.push({ id: ++this.id, func, at: this.time+ms, repeat: ms });
|
||||||
|
return this.id;
|
||||||
|
};
|
||||||
|
this.clearInterval = this.clearTimeout = (id) =>
|
||||||
|
{
|
||||||
|
this.timers = this.timers.filter(t => t.id != id);
|
||||||
|
};
|
||||||
|
this.setImmediate = (func) => this.setTimeout(func, 0);
|
||||||
|
this.runFor = (ms) =>
|
||||||
|
{
|
||||||
|
const end = this.time+ms;
|
||||||
|
this.timers.sort((a, b) => a.at - b.at);
|
||||||
|
while (this.timers.length && this.timers[0].at <= end)
|
||||||
|
{
|
||||||
|
const { func, at, repeat } = this.timers[0];
|
||||||
|
if (!repeat)
|
||||||
|
this.timers.shift();
|
||||||
|
else
|
||||||
|
this.timers[0].at += repeat;
|
||||||
|
if (at > this.time)
|
||||||
|
this.time = at;
|
||||||
|
func();
|
||||||
|
this.timers.sort((a, b) => a.at - b.at);
|
||||||
|
}
|
||||||
|
this.time = end;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = FakeTimer;
|
34
tinyraft.js
34
tinyraft.js
|
@ -62,18 +62,19 @@ class TinyRaft extends EventEmitter
|
||||||
this.term = 0;
|
this.term = 0;
|
||||||
this.state = null;
|
this.state = null;
|
||||||
this.leader = null;
|
this.leader = null;
|
||||||
|
this.confirmed = {};
|
||||||
}
|
}
|
||||||
|
|
||||||
_nextTerm(after)
|
_nextTerm(after)
|
||||||
{
|
{
|
||||||
if (this.electionTimer)
|
if (this.electionTimer)
|
||||||
{
|
{
|
||||||
clearTimeout(this.electionTimer);
|
TinyRaft.clearTimeout(this.electionTimer);
|
||||||
this.electionTimer = null;
|
this.electionTimer = null;
|
||||||
}
|
}
|
||||||
if (after >= 0)
|
if (after >= 0)
|
||||||
{
|
{
|
||||||
this.electionTimer = setTimeout(() => this.start(), after + this.randomTimeout * Math.random());
|
this.electionTimer = TinyRaft.setTimeout(() => this.start(), after + this.randomTimeout * Math.random());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -86,10 +87,11 @@ class TinyRaft extends EventEmitter
|
||||||
this.votes = { [this.nodeId]: [ this.nodeId ] };
|
this.votes = { [this.nodeId]: [ this.nodeId ] };
|
||||||
this.state = CANDIDATE;
|
this.state = CANDIDATE;
|
||||||
this.followers = null;
|
this.followers = null;
|
||||||
|
this.confirmed = {};
|
||||||
this.leader = this.nodeId;
|
this.leader = this.nodeId;
|
||||||
if (!this.heartbeatTimer)
|
if (!this.heartbeatTimer)
|
||||||
{
|
{
|
||||||
this.heartbeatTimer = setInterval(() => this._heartbeat(), this.heartbeatTimeout);
|
this.heartbeatTimer = TinyRaft.setInterval(() => this._heartbeat(), this.heartbeatTimeout);
|
||||||
}
|
}
|
||||||
for (const node of this.nodes)
|
for (const node of this.nodes)
|
||||||
{
|
{
|
||||||
|
@ -107,12 +109,12 @@ class TinyRaft extends EventEmitter
|
||||||
{
|
{
|
||||||
if (this.electionTimer)
|
if (this.electionTimer)
|
||||||
{
|
{
|
||||||
clearTimeout(this.electionTimer);
|
TinyRaft.clearTimeout(this.electionTimer);
|
||||||
this.electionTimer = null;
|
this.electionTimer = null;
|
||||||
}
|
}
|
||||||
if (this.heartbeatTimer)
|
if (this.heartbeatTimer)
|
||||||
{
|
{
|
||||||
clearTimeout(this.heartbeatTimer);
|
TinyRaft.clearInterval(this.heartbeatTimer);
|
||||||
this.heartbeatTimer = null;
|
this.heartbeatTimer = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -121,6 +123,7 @@ class TinyRaft extends EventEmitter
|
||||||
{
|
{
|
||||||
if (this.state == LEADER)
|
if (this.state == LEADER)
|
||||||
{
|
{
|
||||||
|
this.confirmed = {};
|
||||||
for (const node of (this.leadershipTimeout ? this.followers : this.nodes))
|
for (const node of (this.leadershipTimeout ? this.followers : this.nodes))
|
||||||
{
|
{
|
||||||
if (node != this.nodeId)
|
if (node != this.nodeId)
|
||||||
|
@ -165,6 +168,8 @@ class TinyRaft extends EventEmitter
|
||||||
this.leader = msg.leader;
|
this.leader = msg.leader;
|
||||||
this.term = msg.term;
|
this.term = msg.term;
|
||||||
this.state = CANDIDATE;
|
this.state = CANDIDATE;
|
||||||
|
this.followers = null;
|
||||||
|
this.confirmed = {};
|
||||||
this._nextTerm(this.heartbeatTimeout*2 + this.electionTimeout);
|
this._nextTerm(this.heartbeatTimeout*2 + this.electionTimeout);
|
||||||
this.emit('change', { state: this.state, term: this.term, leader: this.leader });
|
this.emit('change', { state: this.state, term: this.term, leader: this.leader });
|
||||||
}
|
}
|
||||||
|
@ -184,6 +189,7 @@ class TinyRaft extends EventEmitter
|
||||||
this.term = msg.term;
|
this.term = msg.term;
|
||||||
this.state = FOLLOWER;
|
this.state = FOLLOWER;
|
||||||
this.followers = null;
|
this.followers = null;
|
||||||
|
this.confirmed = {};
|
||||||
this.leader = msg.leader;
|
this.leader = msg.leader;
|
||||||
this.votes = {};
|
this.votes = {};
|
||||||
this.emit('change', { state: this.state, term: this.term, leader: this.leader });
|
this.emit('change', { state: this.state, term: this.term, leader: this.leader });
|
||||||
|
@ -214,6 +220,7 @@ class TinyRaft extends EventEmitter
|
||||||
{
|
{
|
||||||
this.leader = msg.leader;
|
this.leader = msg.leader;
|
||||||
this.state = LEADER;
|
this.state = LEADER;
|
||||||
|
this.confirmed = {};
|
||||||
this._nextTerm(this.leadershipTimeout > 0 ? this.leadershipTimeout : -1);
|
this._nextTerm(this.leadershipTimeout > 0 ? this.leadershipTimeout : -1);
|
||||||
this.followers = this.votes[this.nodeId];
|
this.followers = this.votes[this.nodeId];
|
||||||
// Send a heartbeat to confirm leadership
|
// Send a heartbeat to confirm leadership
|
||||||
|
@ -249,7 +256,7 @@ class TinyRaft extends EventEmitter
|
||||||
{
|
{
|
||||||
this.markAlive();
|
this.markAlive();
|
||||||
}
|
}
|
||||||
if (this.leadershipTimeout > 0)
|
if (this.leadershipTimeout > 0 || this.state == FOLLOWER && from != this.leader)
|
||||||
{
|
{
|
||||||
this.send(from, { type: PONG, term: this.term, leader: this.leader });
|
this.send(from, { type: PONG, term: this.term, leader: this.leader });
|
||||||
}
|
}
|
||||||
|
@ -261,13 +268,18 @@ class TinyRaft extends EventEmitter
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (msg.leader != this.nodeId)
|
if (msg.leader != this.nodeId || msg.term != this.term)
|
||||||
{
|
{
|
||||||
this.start();
|
this.start();
|
||||||
}
|
}
|
||||||
else
|
else if (this.leadershipTimeout > 0)
|
||||||
{
|
{
|
||||||
this._nextTerm(this.leadershipTimeout > 0 ? this.leadershipTimeout : -1);
|
this.confirmed[from] = true;
|
||||||
|
if (Object.keys(this.confirmed).length >= (1 + (0 | this.nodes.length/2)))
|
||||||
|
{
|
||||||
|
// We have quorum
|
||||||
|
this._nextTerm(this.leadershipTimeout);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -311,6 +323,10 @@ class TinyRaft extends EventEmitter
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TinyRaft.setTimeout = setTimeout;
|
||||||
|
TinyRaft.clearTimeout = clearTimeout;
|
||||||
|
TinyRaft.setInterval = setInterval;
|
||||||
|
TinyRaft.clearInterval = clearInterval;
|
||||||
TinyRaft.VOTE_REQUEST = VOTE_REQUEST;
|
TinyRaft.VOTE_REQUEST = VOTE_REQUEST;
|
||||||
TinyRaft.VOTE = VOTE;
|
TinyRaft.VOTE = VOTE;
|
||||||
TinyRaft.PING = PING;
|
TinyRaft.PING = PING;
|
||||||
|
|
102
tinyraft.spec.js
102
tinyraft.spec.js
|
@ -1,4 +1,11 @@
|
||||||
const TinyRaft = require('./tinyraft.js');
|
const TinyRaft = require('./tinyraft.js');
|
||||||
|
const FakeTimer = require('./faketimer.js');
|
||||||
|
|
||||||
|
const fake = new FakeTimer();
|
||||||
|
TinyRaft.setTimeout = fake.setTimeout;
|
||||||
|
TinyRaft.clearTimeout = fake.clearTimeout;
|
||||||
|
TinyRaft.setInterval = fake.setInterval;
|
||||||
|
TinyRaft.clearInterval = fake.clearInterval;
|
||||||
|
|
||||||
function newNode(id, nodes, partitions, mod)
|
function newNode(id, nodes, partitions, mod)
|
||||||
{
|
{
|
||||||
|
@ -12,8 +19,8 @@ function newNode(id, nodes, partitions, mod)
|
||||||
{
|
{
|
||||||
if (!partitions[n.nodeId+'-'+to] && !partitions[to+'-'+n.nodeId] && nodes[to])
|
if (!partitions[n.nodeId+'-'+to] && !partitions[to+'-'+n.nodeId] && nodes[to])
|
||||||
{
|
{
|
||||||
console.log('received from '+n.nodeId+' to '+to+': '+JSON.stringify(msg));
|
console.log('['+fake.time+'] received from '+n.nodeId+' to '+to+': '+JSON.stringify(msg));
|
||||||
setImmediate(function() { nodes[to].onReceive(n.nodeId, msg); });
|
fake.setImmediate(function() { nodes[to].onReceive(n.nodeId, msg); });
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
@ -23,7 +30,7 @@ function newNode(id, nodes, partitions, mod)
|
||||||
n.on('change', (st) =>
|
n.on('change', (st) =>
|
||||||
{
|
{
|
||||||
console.log(
|
console.log(
|
||||||
'node '+n.nodeId+': '+(st.state == TinyRaft.FOLLOWER ? 'following '+st.leader : st.state)+
|
'['+fake.time+'] node '+n.nodeId+': '+(st.state == TinyRaft.FOLLOWER ? 'following '+st.leader : st.state)+
|
||||||
', term '+st.term+(st.state == TinyRaft.LEADER ? ', followers: '+st.followers.join(', ') : '')
|
', term '+st.term+(st.state == TinyRaft.LEADER ? ', followers: '+st.followers.join(', ') : '')
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
@ -48,6 +55,7 @@ function newNodes(count, partitions, mod)
|
||||||
function checkQuorum(nodes, count)
|
function checkQuorum(nodes, count)
|
||||||
{
|
{
|
||||||
let leaders = 0;
|
let leaders = 0;
|
||||||
|
let out = 0;
|
||||||
for (const i in nodes)
|
for (const i in nodes)
|
||||||
{
|
{
|
||||||
if (nodes[i].state == TinyRaft.LEADER)
|
if (nodes[i].state == TinyRaft.LEADER)
|
||||||
|
@ -58,14 +66,16 @@ function checkQuorum(nodes, count)
|
||||||
}
|
}
|
||||||
else if (nodes[i].state != TinyRaft.FOLLOWER)
|
else if (nodes[i].state != TinyRaft.FOLLOWER)
|
||||||
{
|
{
|
||||||
throw new Error('node '+i+' is not in quorum: state '+nodes[i].state);
|
out++;
|
||||||
|
if (out > nodes.length-count)
|
||||||
|
throw new Error('node '+i+' is not in quorum: state '+nodes[i].state);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (leaders != 1)
|
if (leaders != 1)
|
||||||
{
|
{
|
||||||
throw new Error('we have '+leaders+' leaders instead of exactly 1');
|
throw new Error('we have '+leaders+' leaders instead of exactly 1');
|
||||||
}
|
}
|
||||||
console.log('OK: '+count+' nodes in quorum');
|
console.log('['+fake.time+'] OK: '+count+' nodes in quorum');
|
||||||
}
|
}
|
||||||
|
|
||||||
function checkNoQuorum(nodes)
|
function checkNoQuorum(nodes)
|
||||||
|
@ -75,11 +85,12 @@ function checkNoQuorum(nodes)
|
||||||
{
|
{
|
||||||
throw new Error('we have non-candidates ('+nc.map(n => n.nodeId).join(', ')+'), but we should not');
|
throw new Error('we have non-candidates ('+nc.map(n => n.nodeId).join(', ')+'), but we should not');
|
||||||
}
|
}
|
||||||
console.log('OK: '+Object.keys(nodes).length+' candidates, no quorum');
|
console.log('['+fake.time+'] OK: '+Object.keys(nodes).length+' candidates, no quorum');
|
||||||
}
|
}
|
||||||
|
|
||||||
async function testStartThenRemoveNode()
|
async function testStartThenRemoveNode()
|
||||||
{
|
{
|
||||||
|
console.log('--------------------------------------------------------------------------------');
|
||||||
console.log('testStartThenRemoveNode');
|
console.log('testStartThenRemoveNode');
|
||||||
const nodes = newNodes(5);
|
const nodes = newNodes(5);
|
||||||
let leaderChanges = 0, prevLeader = null;
|
let leaderChanges = 0, prevLeader = null;
|
||||||
|
@ -91,7 +102,7 @@ async function testStartThenRemoveNode()
|
||||||
leaderChanges++;
|
leaderChanges++;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 5);
|
checkQuorum(nodes, 5);
|
||||||
if (leaderChanges >= 3)
|
if (leaderChanges >= 3)
|
||||||
{
|
{
|
||||||
|
@ -103,7 +114,7 @@ async function testStartThenRemoveNode()
|
||||||
nodes[leader].stop();
|
nodes[leader].stop();
|
||||||
delete nodes[leader];
|
delete nodes[leader];
|
||||||
// Check quorum after 2000ms
|
// Check quorum after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 4);
|
checkQuorum(nodes, 4);
|
||||||
// Stop the leader again
|
// Stop the leader again
|
||||||
leader = nodes[Object.keys(nodes)[0]].leader;
|
leader = nodes[Object.keys(nodes)[0]].leader;
|
||||||
|
@ -111,7 +122,7 @@ async function testStartThenRemoveNode()
|
||||||
nodes[leader].stop();
|
nodes[leader].stop();
|
||||||
delete nodes[leader];
|
delete nodes[leader];
|
||||||
// Check quorum after 2000ms
|
// Check quorum after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 3);
|
checkQuorum(nodes, 3);
|
||||||
// Stop the leader again
|
// Stop the leader again
|
||||||
leader = nodes[Object.keys(nodes)[0]].leader;
|
leader = nodes[Object.keys(nodes)[0]].leader;
|
||||||
|
@ -119,7 +130,7 @@ async function testStartThenRemoveNode()
|
||||||
nodes[leader].stop();
|
nodes[leader].stop();
|
||||||
delete nodes[leader];
|
delete nodes[leader];
|
||||||
// Check that no quorum exists
|
// Check that no quorum exists
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkNoQuorum(nodes);
|
checkNoQuorum(nodes);
|
||||||
// Clean up
|
// Clean up
|
||||||
for (const id in nodes)
|
for (const id in nodes)
|
||||||
|
@ -131,9 +142,10 @@ async function testStartThenRemoveNode()
|
||||||
|
|
||||||
async function testAddNode()
|
async function testAddNode()
|
||||||
{
|
{
|
||||||
|
console.log('--------------------------------------------------------------------------------');
|
||||||
console.log('testAddNode');
|
console.log('testAddNode');
|
||||||
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 5);
|
checkQuorum(nodes, 5);
|
||||||
// Add node
|
// Add node
|
||||||
newNode(6, nodes);
|
newNode(6, nodes);
|
||||||
|
@ -141,7 +153,7 @@ async function testAddNode()
|
||||||
nodes[i].setNodes([ 1, 2, 3, 4, 5, 6 ]);
|
nodes[i].setNodes([ 1, 2, 3, 4, 5, 6 ]);
|
||||||
nodes[6].start();
|
nodes[6].start();
|
||||||
// Check quorum after 2000ms
|
// Check quorum after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 6);
|
checkQuorum(nodes, 6);
|
||||||
// Clean up
|
// Clean up
|
||||||
for (const id in nodes)
|
for (const id in nodes)
|
||||||
|
@ -153,22 +165,23 @@ async function testAddNode()
|
||||||
|
|
||||||
async function testLeadershipExpiration()
|
async function testLeadershipExpiration()
|
||||||
{
|
{
|
||||||
|
console.log('--------------------------------------------------------------------------------');
|
||||||
console.log('testLeadershipExpiration');
|
console.log('testLeadershipExpiration');
|
||||||
const partitions = {};
|
const partitions = {};
|
||||||
const nodes = newNodes(5, partitions, cfg => cfg.leadershipTimeout = 1500);
|
const nodes = newNodes(5, partitions, cfg => cfg.leadershipTimeout = 1500);
|
||||||
// Check that 5 nodes are in quorum after 2000ms
|
// Check that 5 nodes are in quorum after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 5);
|
checkQuorum(nodes, 5);
|
||||||
// Break network on the leader
|
// Break network on the leader
|
||||||
let leader = nodes[1].leader;
|
let leader = nodes[1].leader;
|
||||||
console.log("stopping the leader's ("+leader+") network");
|
console.log("["+fake.time+"] --> stopping the leader's ("+leader+") network");
|
||||||
for (let i = 1; i <= 5; i++)
|
for (let i = 1; i <= 5; i++)
|
||||||
{
|
{
|
||||||
partitions[i+'-'+leader] = true;
|
partitions[i+'-'+leader] = true;
|
||||||
partitions[leader+'-'+i] = true;
|
partitions[leader+'-'+i] = true;
|
||||||
}
|
}
|
||||||
// Check that the leader loses leadership after 2 * leadershipTimeout
|
// Check that the leader loses leadership after 2 * leadershipTimeout
|
||||||
await new Promise(ok => setTimeout(ok, 3000));
|
fake.runFor(3000);
|
||||||
if (nodes[leader].state != TinyRaft.CANDIDATE)
|
if (nodes[leader].state != TinyRaft.CANDIDATE)
|
||||||
{
|
{
|
||||||
throw new Error("leadership expiration doesn't work");
|
throw new Error("leadership expiration doesn't work");
|
||||||
|
@ -183,6 +196,7 @@ async function testLeadershipExpiration()
|
||||||
|
|
||||||
async function testRestart()
|
async function testRestart()
|
||||||
{
|
{
|
||||||
|
console.log('--------------------------------------------------------------------------------');
|
||||||
console.log('testRestart');
|
console.log('testRestart');
|
||||||
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
||||||
let leaderChanges = 0, prevLeader = null;
|
let leaderChanges = 0, prevLeader = null;
|
||||||
|
@ -196,7 +210,7 @@ async function testRestart()
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
// Check that 5 nodes are in quorum after 2000ms
|
// Check that 5 nodes are in quorum after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 5);
|
checkQuorum(nodes, 5);
|
||||||
if (leaderChanges >= 3)
|
if (leaderChanges >= 3)
|
||||||
{
|
{
|
||||||
|
@ -208,18 +222,18 @@ async function testRestart()
|
||||||
{
|
{
|
||||||
restarted = 1 + (prevLeader + 1) % 5;
|
restarted = 1 + (prevLeader + 1) % 5;
|
||||||
}
|
}
|
||||||
console.log("stopping a follower (node "+restarted+")");
|
console.log("["+fake.time+"] --> stopping a follower (node "+restarted+")");
|
||||||
nodes[restarted].stop();
|
nodes[restarted].stop();
|
||||||
delete nodes[restarted];
|
delete nodes[restarted];
|
||||||
// Wait 2000ms
|
// Wait 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
// Restart a follower
|
// Restart a follower
|
||||||
console.log("restarting a follower (node "+restarted+")");
|
console.log("["+fake.time+"] --> restarting a follower (node "+restarted+")");
|
||||||
leaderChanges = 0;
|
leaderChanges = 0;
|
||||||
newNode(restarted, nodes, {}, null);
|
newNode(restarted, nodes, {}, null);
|
||||||
nodes[restarted].start();
|
nodes[restarted].start();
|
||||||
// Check quorum and the fact that the leader didn't change after 2000ms
|
// Check quorum and the fact that the leader didn't change after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 5);
|
checkQuorum(nodes, 5);
|
||||||
if (leaderChanges > 0)
|
if (leaderChanges > 0)
|
||||||
{
|
{
|
||||||
|
@ -235,20 +249,21 @@ async function testRestart()
|
||||||
|
|
||||||
async function testChangeNodes()
|
async function testChangeNodes()
|
||||||
{
|
{
|
||||||
|
console.log('--------------------------------------------------------------------------------');
|
||||||
console.log('testChangeNodes');
|
console.log('testChangeNodes');
|
||||||
console.log('starting nodes 1-5');
|
console.log('['+fake.time+'] --> starting nodes 1-5');
|
||||||
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
||||||
// Check that 5 nodes are in quorum after 2000ms
|
// Check that 5 nodes are in quorum after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 5);
|
checkQuorum(nodes, 5);
|
||||||
// Stop node 4
|
// Stop node 4
|
||||||
console.log('stopping node 4');
|
console.log('['+fake.time+'] --> stopping node 4');
|
||||||
nodes[4].stop();
|
nodes[4].stop();
|
||||||
delete nodes[4];
|
delete nodes[4];
|
||||||
// Wait 1000ms
|
// Wait 1000ms
|
||||||
await new Promise(ok => setTimeout(ok, 1000));
|
fake.runFor(1000);
|
||||||
// Change nodes from 1 2 3 4 5 to 1 2 3 5 6
|
// Change nodes from 1 2 3 4 5 to 1 2 3 5 6
|
||||||
console.log('starting node 6');
|
console.log('['+fake.time+'] --> starting node 6');
|
||||||
newNode(6, nodes);
|
newNode(6, nodes);
|
||||||
nodes[6].start();
|
nodes[6].start();
|
||||||
nodes[1].setNodes([ 1, 2, 3, 5, 6 ]);
|
nodes[1].setNodes([ 1, 2, 3, 5, 6 ]);
|
||||||
|
@ -257,7 +272,7 @@ async function testChangeNodes()
|
||||||
nodes[5].setNodes([ 1, 2, 3, 5, 6 ]);
|
nodes[5].setNodes([ 1, 2, 3, 5, 6 ]);
|
||||||
nodes[6].setNodes([ 1, 2, 3, 5, 6 ]);
|
nodes[6].setNodes([ 1, 2, 3, 5, 6 ]);
|
||||||
// Check that 5 nodes are in quorum after 2000ms
|
// Check that 5 nodes are in quorum after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 5);
|
checkQuorum(nodes, 5);
|
||||||
// Clean up
|
// Clean up
|
||||||
for (const id in nodes)
|
for (const id in nodes)
|
||||||
|
@ -270,33 +285,34 @@ async function testChangeNodes()
|
||||||
|
|
||||||
async function testLeaderPriority()
|
async function testLeaderPriority()
|
||||||
{
|
{
|
||||||
|
console.log('--------------------------------------------------------------------------------');
|
||||||
console.log('testLeaderPriority');
|
console.log('testLeaderPriority');
|
||||||
console.log('starting nodes 1-5');
|
console.log('['+fake.time+'] --> starting nodes 1-5');
|
||||||
const nodes = newNodes(5, {}, cfg => cfg.leaderPriority = cfg.nodeId+1);
|
const nodes = newNodes(5, {}, cfg => cfg.leaderPriority = cfg.nodeId+1);
|
||||||
// Check that 5 nodes are in quorum after 2000ms
|
// Check that 5 nodes are in quorum after 2000ms
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 5);
|
checkQuorum(nodes, 5);
|
||||||
if (nodes[1].leader != 5)
|
if (nodes[1].leader != 5)
|
||||||
{
|
{
|
||||||
throw new Error('leader is not 5');
|
throw new Error('leader is not 5');
|
||||||
}
|
}
|
||||||
// Stop node 5
|
// Stop node 5
|
||||||
console.log('stopping node 5');
|
console.log('['+fake.time+'] --> stopping node 5');
|
||||||
nodes[5].stop();
|
nodes[5].stop();
|
||||||
delete nodes[5];
|
delete nodes[5];
|
||||||
// Wait 2000ms and check that the leader is now 4
|
// Wait 2000ms and check that the leader is now 4
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 4);
|
checkQuorum(nodes, 4);
|
||||||
if (nodes[1].leader != 4)
|
if (nodes[1].leader != 4)
|
||||||
{
|
{
|
||||||
throw new Error('leader is not 4');
|
throw new Error('leader is not 4');
|
||||||
}
|
}
|
||||||
// Stop node 4
|
// Stop node 4
|
||||||
console.log('stopping node 4');
|
console.log('['+fake.time+'] --> stopping node 4');
|
||||||
nodes[4].stop();
|
nodes[4].stop();
|
||||||
delete nodes[4];
|
delete nodes[4];
|
||||||
// Wait 2000ms and check that the leader is now 3
|
// Wait 2000ms and check that the leader is now 3
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
fake.runFor(2000);
|
||||||
checkQuorum(nodes, 3);
|
checkQuorum(nodes, 3);
|
||||||
if (nodes[1].leader != 3)
|
if (nodes[1].leader != 3)
|
||||||
{
|
{
|
||||||
|
@ -313,6 +329,7 @@ async function testLeaderPriority()
|
||||||
|
|
||||||
async function testPartition1_3()
|
async function testPartition1_3()
|
||||||
{
|
{
|
||||||
|
console.log('--------------------------------------------------------------------------------');
|
||||||
console.log('testPartition1_3');
|
console.log('testPartition1_3');
|
||||||
const partitions = { '1-3': true, '3-1': true };
|
const partitions = { '1-3': true, '3-1': true };
|
||||||
const nodes = newNodes(3, partitions, cfg => cfg.nodes = [ 1, 2, 3 ]);
|
const nodes = newNodes(3, partitions, cfg => cfg.nodes = [ 1, 2, 3 ]);
|
||||||
|
@ -325,24 +342,19 @@ async function testPartition1_3()
|
||||||
leaderChanges++;
|
leaderChanges++;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
// Check that 3 nodes are in quorum after 2000ms
|
// Check that 2 or 3 nodes are in quorum after 5000ms
|
||||||
// 2 is always elected as leader because it's the first node that can determine
|
// This situation should be fixed by "prevote protocol", but it breaks other things
|
||||||
// that the first voting round has failed: it receives a vote for 1 from 1 and
|
fake.runFor(5000);
|
||||||
// a vote for 3 from 3. So it knows that no node can get 2 votes. 1 and 3, at
|
if (nodes[2].leader == 2)
|
||||||
// the same time, don't know it for sure because they don't receive failures
|
checkQuorum(nodes, 3);
|
||||||
// from each other due to the network partition.
|
else
|
||||||
await new Promise(ok => setTimeout(ok, 2000));
|
checkQuorum(nodes, 2);
|
||||||
checkQuorum(nodes, 3);
|
|
||||||
if (leaderChanges >= 3)
|
|
||||||
{
|
|
||||||
throw new Error('More than 3 leader changes in 2000ms: '+leaderChanges);
|
|
||||||
}
|
|
||||||
// Clean up
|
// Clean up
|
||||||
for (const id in nodes)
|
for (const id in nodes)
|
||||||
{
|
{
|
||||||
nodes[id].stop();
|
nodes[id].stop();
|
||||||
}
|
}
|
||||||
console.log('testLeadershipExpiration: OK');
|
console.log('testPartition1_3: OK');
|
||||||
}
|
}
|
||||||
|
|
||||||
async function run()
|
async function run()
|
||||||
|
|
Loading…
Reference in New Issue