2016-05-13 06:49:15 +03:00
|
|
|
// Copyright 2015 The etcd Authors
|
2015-09-15 23:54:11 +03:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2015-02-01 07:51:50 +03:00
|
|
|
package rafttest
|
|
|
|
|
|
|
|
import (
|
2015-02-06 21:50:41 +03:00
|
|
|
"math/rand"
|
2015-02-03 07:02:06 +03:00
|
|
|
"sync"
|
2015-02-01 07:51:50 +03:00
|
|
|
"time"
|
|
|
|
|
2019-05-29 00:59:02 +03:00
|
|
|
"go.etcd.io/etcd/raft/raftpb"
|
2015-02-01 07:51:50 +03:00
|
|
|
)
|
|
|
|
|
2015-02-04 09:47:16 +03:00
|
|
|
// a network interface
|
|
|
|
type iface interface {
|
2015-02-01 07:51:50 +03:00
|
|
|
send(m raftpb.Message)
|
|
|
|
recv() chan raftpb.Message
|
2015-02-04 09:47:16 +03:00
|
|
|
disconnect()
|
|
|
|
connect()
|
|
|
|
}
|
|
|
|
|
|
|
|
// a network
|
|
|
|
type network interface {
|
2015-02-01 07:51:50 +03:00
|
|
|
// drop message at given rate (1.0 drops all messages)
|
|
|
|
drop(from, to uint64, rate float64)
|
|
|
|
// delay message for (0, d] randomly at given rate (1.0 delay all messages)
|
|
|
|
// do we need rate here?
|
|
|
|
delay(from, to uint64, d time.Duration, rate float64)
|
2015-02-03 07:02:06 +03:00
|
|
|
disconnect(id uint64)
|
|
|
|
connect(id uint64)
|
2015-02-04 09:47:16 +03:00
|
|
|
// heal heals the network
|
|
|
|
heal()
|
2015-02-01 07:51:50 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
type raftNetwork struct {
|
raft/rafttest: introduce datadriven testing
It has often been tedious to test the interactions between multi-member
Raft groups, especially when many steps were required to reach a certain
scenario. Often, this boilerplate was as boring as it is hard to write
and hard to maintain, making it attractive to resort to shortcuts
whenever possible, which in turn tended to undercut how meaningful and
maintainable the tests ended up being - that is, if the tests were even
written, which sometimes they weren't.
This change introduces a datadriven framework specifically for testing
deterministically the interaction between multiple members of a raft group
with the goal of reducing the friction for writing these tests to near
zero.
In the near term, this will be used to add thorough testing for joint
consensus (which is already available today, but wildly undertested),
but just converting an existing test into this framework has shown that
the concise representation and built-in inspection of log messages
highlights unexpected behavior much more readily than the previous unit
tests did (the test in question is `snapshot_succeed_via_app_resp`; the
reader is invited to compare the old and new version of it).
The main building block is `InteractionEnv`, which holds on to the state
of the whole system and exposes various relevant methods for
manipulating it, including but not limited to adding nodes, delivering
and dropping messages, and proposing configuration changes. All of this
is extensible so that in the future I hope to use it to explore the
phenomena discussed in
https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263
which requires injecting appropriate "crash points" in the Ready
handling loop. Discussions of the "what if X happened in state Y"
can quickly be made concrete by "scripting up an interaction test".
Additionally, this framework is intentionally not kept internal to the
raft package.. Though this is in its infancy, a goal is that it should
be possible for a suite of interaction tests to allow applications to
validate that their Storage implementation behaves accordingly, simply
by running a raft-provided interaction suite against their Storage.
2019-08-12 12:13:51 +03:00
|
|
|
rand *rand.Rand
|
2015-02-03 07:02:06 +03:00
|
|
|
mu sync.Mutex
|
|
|
|
disconnected map[uint64]bool
|
2015-02-06 21:50:41 +03:00
|
|
|
dropmap map[conn]float64
|
2015-02-07 02:01:07 +03:00
|
|
|
delaymap map[conn]delay
|
2015-02-03 07:02:06 +03:00
|
|
|
recvQueues map[uint64]chan raftpb.Message
|
2015-02-01 07:51:50 +03:00
|
|
|
}
|
|
|
|
|
2015-02-06 21:50:41 +03:00
|
|
|
type conn struct {
|
|
|
|
from, to uint64
|
|
|
|
}
|
|
|
|
|
2015-02-07 02:01:07 +03:00
|
|
|
type delay struct {
|
|
|
|
d time.Duration
|
|
|
|
rate float64
|
|
|
|
}
|
|
|
|
|
2015-02-01 07:51:50 +03:00
|
|
|
func newRaftNetwork(nodes ...uint64) *raftNetwork {
|
|
|
|
pn := &raftNetwork{
|
raft/rafttest: introduce datadriven testing
It has often been tedious to test the interactions between multi-member
Raft groups, especially when many steps were required to reach a certain
scenario. Often, this boilerplate was as boring as it is hard to write
and hard to maintain, making it attractive to resort to shortcuts
whenever possible, which in turn tended to undercut how meaningful and
maintainable the tests ended up being - that is, if the tests were even
written, which sometimes they weren't.
This change introduces a datadriven framework specifically for testing
deterministically the interaction between multiple members of a raft group
with the goal of reducing the friction for writing these tests to near
zero.
In the near term, this will be used to add thorough testing for joint
consensus (which is already available today, but wildly undertested),
but just converting an existing test into this framework has shown that
the concise representation and built-in inspection of log messages
highlights unexpected behavior much more readily than the previous unit
tests did (the test in question is `snapshot_succeed_via_app_resp`; the
reader is invited to compare the old and new version of it).
The main building block is `InteractionEnv`, which holds on to the state
of the whole system and exposes various relevant methods for
manipulating it, including but not limited to adding nodes, delivering
and dropping messages, and proposing configuration changes. All of this
is extensible so that in the future I hope to use it to explore the
phenomena discussed in
https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263
which requires injecting appropriate "crash points" in the Ready
handling loop. Discussions of the "what if X happened in state Y"
can quickly be made concrete by "scripting up an interaction test".
Additionally, this framework is intentionally not kept internal to the
raft package.. Though this is in its infancy, a goal is that it should
be possible for a suite of interaction tests to allow applications to
validate that their Storage implementation behaves accordingly, simply
by running a raft-provided interaction suite against their Storage.
2019-08-12 12:13:51 +03:00
|
|
|
rand: rand.New(rand.NewSource(1)),
|
2015-02-03 07:02:06 +03:00
|
|
|
recvQueues: make(map[uint64]chan raftpb.Message),
|
2015-02-06 21:50:41 +03:00
|
|
|
dropmap: make(map[conn]float64),
|
2015-02-07 02:01:07 +03:00
|
|
|
delaymap: make(map[conn]delay),
|
2015-02-03 07:02:06 +03:00
|
|
|
disconnected: make(map[uint64]bool),
|
2015-02-01 07:51:50 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, n := range nodes {
|
|
|
|
pn.recvQueues[n] = make(chan raftpb.Message, 1024)
|
|
|
|
}
|
|
|
|
return pn
|
|
|
|
}
|
|
|
|
|
2015-02-04 09:47:16 +03:00
|
|
|
func (rn *raftNetwork) nodeNetwork(id uint64) iface {
|
2015-02-01 07:51:50 +03:00
|
|
|
return &nodeNetwork{id: id, raftNetwork: rn}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (rn *raftNetwork) send(m raftpb.Message) {
|
2015-02-03 07:02:06 +03:00
|
|
|
rn.mu.Lock()
|
2015-02-01 07:51:50 +03:00
|
|
|
to := rn.recvQueues[m.To]
|
2015-02-03 07:02:06 +03:00
|
|
|
if rn.disconnected[m.To] {
|
|
|
|
to = nil
|
|
|
|
}
|
2015-02-06 21:50:41 +03:00
|
|
|
drop := rn.dropmap[conn{m.From, m.To}]
|
2015-12-12 16:54:02 +03:00
|
|
|
dl := rn.delaymap[conn{m.From, m.To}]
|
2015-02-03 07:02:06 +03:00
|
|
|
rn.mu.Unlock()
|
|
|
|
|
2015-02-01 07:51:50 +03:00
|
|
|
if to == nil {
|
2015-02-03 07:02:06 +03:00
|
|
|
return
|
2015-02-01 07:51:50 +03:00
|
|
|
}
|
raft/rafttest: introduce datadriven testing
It has often been tedious to test the interactions between multi-member
Raft groups, especially when many steps were required to reach a certain
scenario. Often, this boilerplate was as boring as it is hard to write
and hard to maintain, making it attractive to resort to shortcuts
whenever possible, which in turn tended to undercut how meaningful and
maintainable the tests ended up being - that is, if the tests were even
written, which sometimes they weren't.
This change introduces a datadriven framework specifically for testing
deterministically the interaction between multiple members of a raft group
with the goal of reducing the friction for writing these tests to near
zero.
In the near term, this will be used to add thorough testing for joint
consensus (which is already available today, but wildly undertested),
but just converting an existing test into this framework has shown that
the concise representation and built-in inspection of log messages
highlights unexpected behavior much more readily than the previous unit
tests did (the test in question is `snapshot_succeed_via_app_resp`; the
reader is invited to compare the old and new version of it).
The main building block is `InteractionEnv`, which holds on to the state
of the whole system and exposes various relevant methods for
manipulating it, including but not limited to adding nodes, delivering
and dropping messages, and proposing configuration changes. All of this
is extensible so that in the future I hope to use it to explore the
phenomena discussed in
https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263
which requires injecting appropriate "crash points" in the Ready
handling loop. Discussions of the "what if X happened in state Y"
can quickly be made concrete by "scripting up an interaction test".
Additionally, this framework is intentionally not kept internal to the
raft package.. Though this is in its infancy, a goal is that it should
be possible for a suite of interaction tests to allow applications to
validate that their Storage implementation behaves accordingly, simply
by running a raft-provided interaction suite against their Storage.
2019-08-12 12:13:51 +03:00
|
|
|
if drop != 0 && rn.rand.Float64() < drop {
|
2015-02-06 21:50:41 +03:00
|
|
|
return
|
|
|
|
}
|
2015-12-12 16:54:02 +03:00
|
|
|
// TODO: shall we dl without blocking the send call?
|
raft/rafttest: introduce datadriven testing
It has often been tedious to test the interactions between multi-member
Raft groups, especially when many steps were required to reach a certain
scenario. Often, this boilerplate was as boring as it is hard to write
and hard to maintain, making it attractive to resort to shortcuts
whenever possible, which in turn tended to undercut how meaningful and
maintainable the tests ended up being - that is, if the tests were even
written, which sometimes they weren't.
This change introduces a datadriven framework specifically for testing
deterministically the interaction between multiple members of a raft group
with the goal of reducing the friction for writing these tests to near
zero.
In the near term, this will be used to add thorough testing for joint
consensus (which is already available today, but wildly undertested),
but just converting an existing test into this framework has shown that
the concise representation and built-in inspection of log messages
highlights unexpected behavior much more readily than the previous unit
tests did (the test in question is `snapshot_succeed_via_app_resp`; the
reader is invited to compare the old and new version of it).
The main building block is `InteractionEnv`, which holds on to the state
of the whole system and exposes various relevant methods for
manipulating it, including but not limited to adding nodes, delivering
and dropping messages, and proposing configuration changes. All of this
is extensible so that in the future I hope to use it to explore the
phenomena discussed in
https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263
which requires injecting appropriate "crash points" in the Ready
handling loop. Discussions of the "what if X happened in state Y"
can quickly be made concrete by "scripting up an interaction test".
Additionally, this framework is intentionally not kept internal to the
raft package.. Though this is in its infancy, a goal is that it should
be possible for a suite of interaction tests to allow applications to
validate that their Storage implementation behaves accordingly, simply
by running a raft-provided interaction suite against their Storage.
2019-08-12 12:13:51 +03:00
|
|
|
if dl.d != 0 && rn.rand.Float64() < dl.rate {
|
|
|
|
rd := rn.rand.Int63n(int64(dl.d))
|
2015-02-07 02:01:07 +03:00
|
|
|
time.Sleep(time.Duration(rd))
|
|
|
|
}
|
2015-02-06 21:50:41 +03:00
|
|
|
|
2016-11-05 01:40:50 +03:00
|
|
|
// use marshal/unmarshal to copy message to avoid data race.
|
|
|
|
b, err := m.Marshal()
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
var cm raftpb.Message
|
|
|
|
err = cm.Unmarshal(b)
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
|
2015-02-06 21:03:52 +03:00
|
|
|
select {
|
2016-11-05 01:40:50 +03:00
|
|
|
case to <- cm:
|
2015-02-06 21:03:52 +03:00
|
|
|
default:
|
|
|
|
// drop messages when the receiver queue is full.
|
|
|
|
}
|
2015-02-01 07:51:50 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
func (rn *raftNetwork) recvFrom(from uint64) chan raftpb.Message {
|
2015-02-03 07:02:06 +03:00
|
|
|
rn.mu.Lock()
|
2015-02-01 07:51:50 +03:00
|
|
|
fromc := rn.recvQueues[from]
|
2015-02-03 07:02:06 +03:00
|
|
|
if rn.disconnected[from] {
|
|
|
|
fromc = nil
|
2015-02-01 07:51:50 +03:00
|
|
|
}
|
2015-02-03 07:02:06 +03:00
|
|
|
rn.mu.Unlock()
|
|
|
|
|
2015-02-01 07:51:50 +03:00
|
|
|
return fromc
|
|
|
|
}
|
|
|
|
|
|
|
|
func (rn *raftNetwork) drop(from, to uint64, rate float64) {
|
2015-02-06 21:50:41 +03:00
|
|
|
rn.mu.Lock()
|
|
|
|
defer rn.mu.Unlock()
|
|
|
|
rn.dropmap[conn{from, to}] = rate
|
2015-02-01 07:51:50 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
func (rn *raftNetwork) delay(from, to uint64, d time.Duration, rate float64) {
|
2015-02-07 02:01:07 +03:00
|
|
|
rn.mu.Lock()
|
|
|
|
defer rn.mu.Unlock()
|
|
|
|
rn.delaymap[conn{from, to}] = delay{d, rate}
|
2015-02-01 07:51:50 +03:00
|
|
|
}
|
|
|
|
|
2015-02-06 21:50:41 +03:00
|
|
|
func (rn *raftNetwork) heal() {
|
|
|
|
rn.mu.Lock()
|
|
|
|
defer rn.mu.Unlock()
|
|
|
|
rn.dropmap = make(map[conn]float64)
|
2015-02-07 02:01:07 +03:00
|
|
|
rn.delaymap = make(map[conn]delay)
|
2015-02-06 21:50:41 +03:00
|
|
|
}
|
2015-02-04 09:47:16 +03:00
|
|
|
|
2015-02-03 07:02:06 +03:00
|
|
|
func (rn *raftNetwork) disconnect(id uint64) {
|
|
|
|
rn.mu.Lock()
|
|
|
|
defer rn.mu.Unlock()
|
|
|
|
rn.disconnected[id] = true
|
|
|
|
}
|
|
|
|
|
|
|
|
func (rn *raftNetwork) connect(id uint64) {
|
|
|
|
rn.mu.Lock()
|
|
|
|
defer rn.mu.Unlock()
|
|
|
|
rn.disconnected[id] = false
|
|
|
|
}
|
|
|
|
|
2015-02-01 07:51:50 +03:00
|
|
|
type nodeNetwork struct {
|
|
|
|
id uint64
|
|
|
|
*raftNetwork
|
|
|
|
}
|
|
|
|
|
2015-02-04 09:47:16 +03:00
|
|
|
func (nt *nodeNetwork) connect() {
|
|
|
|
nt.raftNetwork.connect(nt.id)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (nt *nodeNetwork) disconnect() {
|
|
|
|
nt.raftNetwork.disconnect(nt.id)
|
|
|
|
}
|
|
|
|
|
2015-02-01 07:51:50 +03:00
|
|
|
func (nt *nodeNetwork) send(m raftpb.Message) {
|
|
|
|
nt.raftNetwork.send(m)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (nt *nodeNetwork) recv() chan raftpb.Message {
|
|
|
|
return nt.recvFrom(nt.id)
|
|
|
|
}
|