|
|
@ -102,7 +102,7 @@ async function testStartThenRemoveNode()
|
|
|
|
leaderChanges++;
|
|
|
|
leaderChanges++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
if (leaderChanges >= 3)
|
|
|
|
if (leaderChanges >= 3)
|
|
|
|
{
|
|
|
|
{
|
|
|
@ -114,7 +114,7 @@ async function testStartThenRemoveNode()
|
|
|
|
nodes[leader].stop();
|
|
|
|
nodes[leader].stop();
|
|
|
|
delete nodes[leader];
|
|
|
|
delete nodes[leader];
|
|
|
|
// Check quorum after 2000ms
|
|
|
|
// Check quorum after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 4);
|
|
|
|
checkQuorum(nodes, 4);
|
|
|
|
// Stop the leader again
|
|
|
|
// Stop the leader again
|
|
|
|
leader = nodes[Object.keys(nodes)[0]].leader;
|
|
|
|
leader = nodes[Object.keys(nodes)[0]].leader;
|
|
|
@ -122,7 +122,7 @@ async function testStartThenRemoveNode()
|
|
|
|
nodes[leader].stop();
|
|
|
|
nodes[leader].stop();
|
|
|
|
delete nodes[leader];
|
|
|
|
delete nodes[leader];
|
|
|
|
// Check quorum after 2000ms
|
|
|
|
// Check quorum after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 3);
|
|
|
|
checkQuorum(nodes, 3);
|
|
|
|
// Stop the leader again
|
|
|
|
// Stop the leader again
|
|
|
|
leader = nodes[Object.keys(nodes)[0]].leader;
|
|
|
|
leader = nodes[Object.keys(nodes)[0]].leader;
|
|
|
@ -130,7 +130,7 @@ async function testStartThenRemoveNode()
|
|
|
|
nodes[leader].stop();
|
|
|
|
nodes[leader].stop();
|
|
|
|
delete nodes[leader];
|
|
|
|
delete nodes[leader];
|
|
|
|
// Check that no quorum exists
|
|
|
|
// Check that no quorum exists
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkNoQuorum(nodes);
|
|
|
|
checkNoQuorum(nodes);
|
|
|
|
// Clean up
|
|
|
|
// Clean up
|
|
|
|
for (const id in nodes)
|
|
|
|
for (const id in nodes)
|
|
|
@ -145,7 +145,7 @@ async function testAddNode()
|
|
|
|
console.log('--------------------------------------------------------------------------------');
|
|
|
|
console.log('--------------------------------------------------------------------------------');
|
|
|
|
console.log('testAddNode');
|
|
|
|
console.log('testAddNode');
|
|
|
|
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
|
|
|
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
// Add node
|
|
|
|
// Add node
|
|
|
|
newNode(6, nodes);
|
|
|
|
newNode(6, nodes);
|
|
|
@ -153,7 +153,7 @@ async function testAddNode()
|
|
|
|
nodes[i].setNodes([ 1, 2, 3, 4, 5, 6 ]);
|
|
|
|
nodes[i].setNodes([ 1, 2, 3, 4, 5, 6 ]);
|
|
|
|
nodes[6].start();
|
|
|
|
nodes[6].start();
|
|
|
|
// Check quorum after 2000ms
|
|
|
|
// Check quorum after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 6);
|
|
|
|
checkQuorum(nodes, 6);
|
|
|
|
// Clean up
|
|
|
|
// Clean up
|
|
|
|
for (const id in nodes)
|
|
|
|
for (const id in nodes)
|
|
|
@ -170,7 +170,7 @@ async function testLeadershipExpiration()
|
|
|
|
const partitions = {};
|
|
|
|
const partitions = {};
|
|
|
|
const nodes = newNodes(5, partitions, cfg => cfg.leadershipTimeout = 1500);
|
|
|
|
const nodes = newNodes(5, partitions, cfg => cfg.leadershipTimeout = 1500);
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
// Break network on the leader
|
|
|
|
// Break network on the leader
|
|
|
|
let leader = nodes[1].leader;
|
|
|
|
let leader = nodes[1].leader;
|
|
|
@ -181,7 +181,7 @@ async function testLeadershipExpiration()
|
|
|
|
partitions[leader+'-'+i] = true;
|
|
|
|
partitions[leader+'-'+i] = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Check that the leader loses leadership after 2 * leadershipTimeout
|
|
|
|
// Check that the leader loses leadership after 2 * leadershipTimeout
|
|
|
|
fake.runFor(3000);
|
|
|
|
await fake.runFor(3000);
|
|
|
|
if (nodes[leader].state != TinyRaft.CANDIDATE)
|
|
|
|
if (nodes[leader].state != TinyRaft.CANDIDATE)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
throw new Error("leadership expiration doesn't work");
|
|
|
|
throw new Error("leadership expiration doesn't work");
|
|
|
@ -210,7 +210,7 @@ async function testRestart()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
if (leaderChanges >= 3)
|
|
|
|
if (leaderChanges >= 3)
|
|
|
|
{
|
|
|
|
{
|
|
|
@ -226,14 +226,14 @@ async function testRestart()
|
|
|
|
nodes[restarted].stop();
|
|
|
|
nodes[restarted].stop();
|
|
|
|
delete nodes[restarted];
|
|
|
|
delete nodes[restarted];
|
|
|
|
// Wait 2000ms
|
|
|
|
// Wait 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
// Restart a follower
|
|
|
|
// Restart a follower
|
|
|
|
console.log("["+fake.time+"] --> restarting a follower (node "+restarted+")");
|
|
|
|
console.log("["+fake.time+"] --> restarting a follower (node "+restarted+")");
|
|
|
|
leaderChanges = 0;
|
|
|
|
leaderChanges = 0;
|
|
|
|
newNode(restarted, nodes, {}, null);
|
|
|
|
newNode(restarted, nodes, {}, null);
|
|
|
|
nodes[restarted].start();
|
|
|
|
nodes[restarted].start();
|
|
|
|
// Check quorum and the fact that the leader didn't change after 2000ms
|
|
|
|
// Check quorum and the fact that the leader didn't change after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
if (leaderChanges > 0)
|
|
|
|
if (leaderChanges > 0)
|
|
|
|
{
|
|
|
|
{
|
|
|
@ -254,14 +254,14 @@ async function testChangeNodes()
|
|
|
|
console.log('['+fake.time+'] --> starting nodes 1-5');
|
|
|
|
console.log('['+fake.time+'] --> starting nodes 1-5');
|
|
|
|
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
|
|
|
const nodes = newNodes(5, {}, cfg => cfg.initialTerm = 1000);
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
// Stop node 4
|
|
|
|
// Stop node 4
|
|
|
|
console.log('['+fake.time+'] --> stopping node 4');
|
|
|
|
console.log('['+fake.time+'] --> stopping node 4');
|
|
|
|
nodes[4].stop();
|
|
|
|
nodes[4].stop();
|
|
|
|
delete nodes[4];
|
|
|
|
delete nodes[4];
|
|
|
|
// Wait 1000ms
|
|
|
|
// Wait 1000ms
|
|
|
|
fake.runFor(1000);
|
|
|
|
await fake.runFor(1000);
|
|
|
|
// Change nodes from 1 2 3 4 5 to 1 2 3 5 6
|
|
|
|
// Change nodes from 1 2 3 4 5 to 1 2 3 5 6
|
|
|
|
console.log('['+fake.time+'] --> starting node 6');
|
|
|
|
console.log('['+fake.time+'] --> starting node 6');
|
|
|
|
newNode(6, nodes);
|
|
|
|
newNode(6, nodes);
|
|
|
@ -272,7 +272,7 @@ async function testChangeNodes()
|
|
|
|
nodes[5].setNodes([ 1, 2, 3, 5, 6 ]);
|
|
|
|
nodes[5].setNodes([ 1, 2, 3, 5, 6 ]);
|
|
|
|
nodes[6].setNodes([ 1, 2, 3, 5, 6 ]);
|
|
|
|
nodes[6].setNodes([ 1, 2, 3, 5, 6 ]);
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
// Clean up
|
|
|
|
// Clean up
|
|
|
|
for (const id in nodes)
|
|
|
|
for (const id in nodes)
|
|
|
@ -290,7 +290,7 @@ async function testLeaderPriority()
|
|
|
|
console.log('['+fake.time+'] --> starting nodes 1-5');
|
|
|
|
console.log('['+fake.time+'] --> starting nodes 1-5');
|
|
|
|
const nodes = newNodes(5, {}, cfg => cfg.leaderPriority = cfg.nodeId+1);
|
|
|
|
const nodes = newNodes(5, {}, cfg => cfg.leaderPriority = cfg.nodeId+1);
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
// Check that 5 nodes are in quorum after 2000ms
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
checkQuorum(nodes, 5);
|
|
|
|
if (nodes[1].leader != 5)
|
|
|
|
if (nodes[1].leader != 5)
|
|
|
|
{
|
|
|
|
{
|
|
|
@ -301,7 +301,7 @@ async function testLeaderPriority()
|
|
|
|
nodes[5].stop();
|
|
|
|
nodes[5].stop();
|
|
|
|
delete nodes[5];
|
|
|
|
delete nodes[5];
|
|
|
|
// Wait 2000ms and check that the leader is now 4
|
|
|
|
// Wait 2000ms and check that the leader is now 4
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 4);
|
|
|
|
checkQuorum(nodes, 4);
|
|
|
|
if (nodes[1].leader != 4)
|
|
|
|
if (nodes[1].leader != 4)
|
|
|
|
{
|
|
|
|
{
|
|
|
@ -312,7 +312,7 @@ async function testLeaderPriority()
|
|
|
|
nodes[4].stop();
|
|
|
|
nodes[4].stop();
|
|
|
|
delete nodes[4];
|
|
|
|
delete nodes[4];
|
|
|
|
// Wait 2000ms and check that the leader is now 3
|
|
|
|
// Wait 2000ms and check that the leader is now 3
|
|
|
|
fake.runFor(2000);
|
|
|
|
await fake.runFor(2000);
|
|
|
|
checkQuorum(nodes, 3);
|
|
|
|
checkQuorum(nodes, 3);
|
|
|
|
if (nodes[1].leader != 3)
|
|
|
|
if (nodes[1].leader != 3)
|
|
|
|
{
|
|
|
|
{
|
|
|
@ -344,7 +344,7 @@ async function testPartition1_3()
|
|
|
|
});
|
|
|
|
});
|
|
|
|
// Check that 2 or 3 nodes are in quorum after 5000ms
|
|
|
|
// Check that 2 or 3 nodes are in quorum after 5000ms
|
|
|
|
// This situation should be fixed by "prevote protocol", but it breaks other things
|
|
|
|
// This situation should be fixed by "prevote protocol", but it breaks other things
|
|
|
|
fake.runFor(5000);
|
|
|
|
await fake.runFor(5000);
|
|
|
|
if (nodes[2].leader == 2)
|
|
|
|
if (nodes[2].leader == 2)
|
|
|
|
checkQuorum(nodes, 3);
|
|
|
|
checkQuorum(nodes, 3);
|
|
|
|
else
|
|
|
|
else
|
|
|
|