tests/robustness: Limit model to start only from fresh state

It is just to complicated to support starting from non-empty etcd.
Existing implementation was very naive to assume that we can build
full state from just one request. We might consider implementing
validation of non-empty history in future, but for now settting
this limit should clean up the code and speed up development.

Signed-off-by: Marek Siarkowicz <siarkowicz@google.com>
dependabot/go_modules/github.com/prometheus/procfs-0.11.0
Marek Siarkowicz 2023-06-16 13:50:15 +02:00
parent 90cbadc660
commit ea3255b477
4 changed files with 161 additions and 262 deletions

View File

@ -39,8 +39,7 @@ import (
// whole change history as real etcd does.
var DeterministicModel = porcupine.Model{
Init: func() interface{} {
var s etcdState
data, err := json.Marshal(s)
data, err := json.Marshal(freshEtcdState())
if err != nil {
panic(err)
}
@ -72,67 +71,11 @@ type etcdState struct {
}
func (s etcdState) Step(request EtcdRequest, response EtcdResponse) (bool, etcdState) {
if s.Revision == 0 {
return true, initState(request, response)
}
newState, modelResponse := s.step(request)
return Match(MaybeEtcdResponse{EtcdResponse: response}, modelResponse), newState
}
// initState tries to create etcd state based on the first request.
func initState(request EtcdRequest, response EtcdResponse) etcdState {
state := emptyState()
state.Revision = response.Revision
switch request.Type {
case Range:
for _, kv := range response.Range.KVs {
state.KeyValues[kv.Key] = ValueRevision{
Value: kv.Value,
ModRevision: kv.ModRevision,
}
}
case Txn:
if response.Txn.Failure {
return state
}
if len(request.Txn.OperationsOnSuccess) != len(response.Txn.Results) {
panic(fmt.Sprintf("Incorrect request %s, response %+v", describeEtcdRequest(request), describeEtcdResponse(request, MaybeEtcdResponse{EtcdResponse: response})))
}
for i, op := range request.Txn.OperationsOnSuccess {
opResp := response.Txn.Results[i]
switch op.Type {
case RangeOperation:
for _, kv := range opResp.KVs {
state.KeyValues[kv.Key] = ValueRevision{
Value: kv.Value,
ModRevision: kv.ModRevision,
}
}
case PutOperation:
state.KeyValues[op.Key] = ValueRevision{
Value: op.Value,
ModRevision: response.Revision,
}
case DeleteOperation:
default:
panic("Unknown operation")
}
}
case LeaseGrant:
lease := EtcdLease{
LeaseID: request.LeaseGrant.LeaseID,
Keys: map[string]struct{}{},
}
state.Leases[request.LeaseGrant.LeaseID] = lease
case LeaseRevoke:
case Defragment:
default:
panic(fmt.Sprintf("Unknown request type: %v", request.Type))
}
return state
}
func emptyState() etcdState {
func freshEtcdState() etcdState {
return etcdState{
Revision: 1,
KeyValues: map[string]ValueRevision{},

View File

@ -32,7 +32,7 @@ func TestModelDeterministic(t *testing.T) {
ok, newState := DeterministicModel.Step(state, op.req, op.resp.EtcdResponse)
if op.expectFailure == ok {
t.Logf("state: %v", state)
t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, DeterministicModel.DescribeOperation(op.req, op.resp))
t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.expectFailure, ok, DeterministicModel.DescribeOperation(op.req, op.resp.EtcdResponse))
var loadedState etcdState
err := json.Unmarshal([]byte(state.(string)), &loadedState)
if err != nil {
@ -63,125 +63,92 @@ type testOperation struct {
}
var commonTestScenarios = []modelTestCase{
{
name: "First Get can start from non-empty value and non-zero revision",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42)},
{req: getRequest("key"), resp: getResponse("key", "1", 42, 42)},
},
},
{
name: "First Range can start from non-empty value and non-zero revision",
operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key"), Value: []byte("1")}}, 1, 42)},
},
},
{
name: "First Range can start from non-zero revision",
operations: []testOperation{
{req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1)},
{req: rangeRequest("key", true, 0), resp: rangeResponse(nil, 0, 1)},
},
},
{
name: "First Put can start from non-zero revision",
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(42)},
},
},
{
name: "First delete can start from non-zero revision",
operations: []testOperation{
{req: deleteRequest("key"), resp: deleteResponse(0, 42)},
},
},
{
name: "First Txn can start from non-zero revision",
operations: []testOperation{
{req: compareRevisionAndPutRequest("key", 0, "42"), resp: compareRevisionAndPutResponse(false, 42)},
},
},
{
name: "Get response data should match put",
operations: []testOperation{
{req: putRequest("key1", "11"), resp: putResponse(1)},
{req: putRequest("key2", "12"), resp: putResponse(2)},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 1), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 1, 1), expectFailure: true},
{req: putRequest("key1", "11"), resp: putResponse(2)},
{req: putRequest("key2", "12"), resp: putResponse(3)},
{req: getRequest("key1"), resp: getResponse("key1", "11", 2, 2), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "12", 2, 2), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "11", 1, 2)},
{req: getRequest("key1"), resp: getResponse("key1", "12", 3, 3), expectFailure: true},
{req: getRequest("key1"), resp: getResponse("key1", "11", 2, 3)},
{req: getRequest("key2"), resp: getResponse("key2", "11", 3, 3), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 2, 2), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "11", 2, 2), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 1, 1), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "11", 1, 1), expectFailure: true},
{req: getRequest("key2"), resp: getResponse("key2", "12", 2, 2)},
{req: getRequest("key2"), resp: getResponse("key2", "12", 3, 3)},
},
},
{
name: "Range response data should match put",
operations: []testOperation{
{req: putRequest("key1", "1"), resp: putResponse(1)},
{req: putRequest("key2", "2"), resp: putResponse(2)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 2}}, 2, 2)},
{req: putRequest("key1", "1"), resp: putResponse(2)},
{req: putRequest("key2", "2"), resp: putResponse(3)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3}}, 2, 3)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2}, {Key: []byte("key2"), Value: []byte("2"), ModRevision: 3}}, 2, 3)},
},
},
{
name: "Range limit should reduce number of kvs, but maintain count",
operations: []testOperation{
{req: putRequest("key1", "1"), resp: putResponse(2)},
{req: putRequest("key2", "2"), resp: putResponse(3)},
{req: putRequest("key3", "3"), resp: putResponse(4)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3)},
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 4},
}, 3, 4)},
{req: rangeRequest("key", true, 4), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3)},
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 4},
}, 3, 4)},
{req: rangeRequest("key", true, 3), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 3},
}, 3, 3)},
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 4},
}, 3, 4)},
{req: rangeRequest("key", true, 2), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 2},
}, 3, 3)},
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key2"), Value: []byte("2"), ModRevision: 3},
}, 3, 4)},
{req: rangeRequest("key", true, 1), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 1},
}, 3, 3)},
{Key: []byte("key1"), Value: []byte("1"), ModRevision: 2},
}, 3, 4)},
},
},
{
name: "Range response should be ordered by key",
operations: []testOperation{
{req: putRequest("key3", "3"), resp: putResponse(2)},
{req: putRequest("key2", "1"), resp: putResponse(3)},
{req: putRequest("key1", "2"), resp: putResponse(4)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
}, 3, 3)},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 4},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 3},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 2},
}, 3, 4)},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
}, 3, 3), expectFailure: true},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 3},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 4},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 2},
}, 3, 4), expectFailure: true},
{req: rangeRequest("key", true, 0), resp: rangeResponse([]*mvccpb.KeyValue{
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 1},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 2},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 3},
}, 3, 3), expectFailure: true},
{Key: []byte("key3"), Value: []byte("3"), ModRevision: 2},
{Key: []byte("key2"), Value: []byte("1"), ModRevision: 3},
{Key: []byte("key1"), Value: []byte("2"), ModRevision: 4},
}, 3, 4), expectFailure: true},
},
},
{
name: "Range response data should match large put",
operations: []testOperation{
{req: putRequest("key", "012345678901234567890"), resp: putResponse(1)},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 1, 1)},
{req: putRequest("key", "123456789012345678901"), resp: putResponse(2)},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 2, 2)},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 2, 2), expectFailure: true},
{req: putRequest("key", "012345678901234567890"), resp: putResponse(2)},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 2, 2)},
{req: putRequest("key", "123456789012345678901"), resp: putResponse(3)},
{req: getRequest("key"), resp: getResponse("key", "123456789012345678901", 3, 3)},
{req: getRequest("key"), resp: getResponse("key", "012345678901234567890", 3, 3), expectFailure: true},
},
},
{
@ -196,12 +163,12 @@ var commonTestScenarios = []modelTestCase{
{
name: "Delete only increases revision on success",
operations: []testOperation{
{req: putRequest("key1", "11"), resp: putResponse(1)},
{req: putRequest("key2", "12"), resp: putResponse(2)},
{req: deleteRequest("key1"), resp: deleteResponse(1, 2), expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(1, 3)},
{req: deleteRequest("key1"), resp: deleteResponse(0, 4), expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(0, 3)},
{req: putRequest("key1", "11"), resp: putResponse(2)},
{req: putRequest("key2", "12"), resp: putResponse(3)},
{req: deleteRequest("key1"), resp: deleteResponse(1, 3), expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(1, 4)},
{req: deleteRequest("key1"), resp: deleteResponse(0, 5), expectFailure: true},
{req: deleteRequest("key1"), resp: deleteResponse(0, 4)},
},
},
{
@ -215,27 +182,27 @@ var commonTestScenarios = []modelTestCase{
{
name: "Delete clears value",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: deleteRequest("key"), resp: deleteResponse(1, 2)},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1), expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: deleteRequest("key"), resp: deleteResponse(1, 3)},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(2)},
{req: getRequest("key"), resp: getResponse("key", "1", 3, 3), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 3), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(3)},
},
},
{
name: "Txn executes onSuccess if revision matches expected",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 1), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 2), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 1), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 2)},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2), expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: compareRevisionAndPutResponse(true, 2), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: compareRevisionAndPutResponse(false, 3), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: compareRevisionAndPutResponse(false, 2), expectFailure: true},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: compareRevisionAndPutResponse(true, 3)},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 3), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 3, 3), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 3, 3)},
},
},
{
@ -252,13 +219,13 @@ var commonTestScenarios = []modelTestCase{
{
name: "Txn executes onFailure if revision doesn't match expected",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(false, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnPutResponse(true, 1), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 1), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 1)},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(false, 2)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(false, 3), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnEmptyResponse(false, 3), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 3), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnPutResponse(true, 2), expectFailure: true},
{req: txnRequestSingleOperation(compareRevision("key", 2), nil, putOperation("key", "2")), resp: txnEmptyResponse(true, 2)},
{req: txnRequestSingleOperation(compareRevision("key", 3), nil, putOperation("key", "2")), resp: txnPutResponse(false, 3)},
},
},
{

View File

@ -28,8 +28,7 @@ import (
// Failed requests fork the possible states, while successful requests merge and filter them.
var NonDeterministicModel = porcupine.Model{
Init: func() interface{} {
var states nonDeterministicState
data, err := json.Marshal(states)
data, err := json.Marshal(nonDeterministicState{freshEtcdState()})
if err != nil {
panic(err)
}
@ -56,12 +55,6 @@ var NonDeterministicModel = porcupine.Model{
type nonDeterministicState []etcdState
func (states nonDeterministicState) Step(request EtcdRequest, response MaybeEtcdResponse) (bool, nonDeterministicState) {
if len(states) == 0 {
if response.Err == nil && !response.PartialResponse {
return true, nonDeterministicState{initState(request, response.EtcdResponse)}
}
states = nonDeterministicState{emptyState()}
}
var newStates nonDeterministicState
switch {
case response.Err != nil:

View File

@ -46,12 +46,12 @@ func TestModelNonDeterministic(t *testing.T) {
{
name: "Put can fail and be lost before get",
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: putRequest("key", "1"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2)},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 3), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 3), expectFailure: true},
},
},
{
@ -83,24 +83,20 @@ func TestModelNonDeterministic(t *testing.T) {
{req: compareRevisionAndPutRequest("key", 2, "5"), resp: compareRevisionAndPutResponse(true, 3)},
},
},
{
name: "Put can fail and be lost before txn success",
operations: []testOperation{},
},
{
name: "Put can fail but be persisted and increase revision before get",
operations: []testOperation{
// One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: putRequest("key", "2"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "3", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "3", 1, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
{req: getRequest("key"), resp: getResponse("key", "3", 3, 3), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "3", 2, 3), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 3, 3)},
// Two failed request, two persisted.
{req: putRequest("key", "3"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "4"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4)},
{req: getRequest("key"), resp: getResponse("key", "4", 5, 5)},
},
},
{
@ -142,169 +138,169 @@ func TestModelNonDeterministic(t *testing.T) {
{
name: "Delete can fail and be lost before get",
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2)},
{req: getRequest("key"), resp: emptyGetResponse(3), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(3), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(2), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(2), expectFailure: true},
{req: getRequest("key"), resp: emptyGetResponse(1), expectFailure: true},
},
},
{
name: "Delete can fail and be lost before delete",
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 1), expectFailure: true},
{req: deleteRequest("key"), resp: deleteResponse(1, 2)},
{req: deleteRequest("key"), resp: deleteResponse(1, 2), expectFailure: true},
{req: deleteRequest("key"), resp: deleteResponse(1, 3)},
},
},
{
name: "Delete can fail and be lost before put",
operations: []testOperation{
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "1"), resp: putResponse(3)},
},
},
{
name: "Delete can fail but be persisted before get",
operations: []testOperation{
// One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: emptyGetResponse(2)},
{req: getRequest("key"), resp: emptyGetResponse(3)},
// Two failed request, one persisted.
{req: putRequest("key", "3"), resp: putResponse(3)},
{req: putRequest("key", "3"), resp: putResponse(4)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: emptyGetResponse(4)},
{req: getRequest("key"), resp: emptyGetResponse(5)},
},
},
{
name: "Delete can fail but be persisted before put",
operations: []testOperation{
// One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(3)},
{req: putRequest("key", "3"), resp: putResponse(4)},
// Two failed request, one persisted.
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "5"), resp: putResponse(5)},
{req: putRequest("key", "5"), resp: putResponse(6)},
},
},
{
name: "Delete can fail but be persisted before delete",
operations: []testOperation{
// One failed request, one persisted.
{req: putRequest("key", "1"), resp: putResponse(1)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(0, 2)},
{req: putRequest("key", "3"), resp: putResponse(3)},
{req: deleteRequest("key"), resp: deleteResponse(0, 3)},
{req: putRequest("key", "3"), resp: putResponse(4)},
// Two failed request, one persisted.
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(0, 4)},
{req: deleteRequest("key"), resp: deleteResponse(0, 5)},
},
},
{
name: "Delete can fail but be persisted before txn",
operations: []testOperation{
// Txn success
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 0, "3"), resp: compareRevisionAndPutResponse(true, 3)},
{req: compareRevisionAndPutRequest("key", 0, "3"), resp: compareRevisionAndPutResponse(true, 4)},
// Txn failure
{req: putRequest("key", "4"), resp: putResponse(4)},
{req: putRequest("key", "4"), resp: putResponse(5)},
{req: deleteRequest("key"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(false, 5)},
{req: compareRevisionAndPutRequest("key", 5, "5"), resp: compareRevisionAndPutResponse(false, 6)},
},
},
{
name: "Txn can fail and be lost before get",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2), expectFailure: true},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2)},
{req: getRequest("key"), resp: getResponse("key", "2", 3, 3), expectFailure: true},
},
},
{
name: "Txn can fail and be lost before delete",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 2)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 3)},
},
},
{
name: "Txn can fail and be lost before put",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(2)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(3)},
},
},
{
name: "Txn can fail but be persisted before get",
operations: []testOperation{
// One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2), expectFailure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 3, 3)},
// Two failed request, two persisted.
{req: putRequest("key", "3"), resp: putResponse(3)},
{req: compareRevisionAndPutRequest("key", 3, "4"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "5", 5, 5)},
{req: putRequest("key", "3"), resp: putResponse(4)},
{req: compareRevisionAndPutRequest("key", 4, "4"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 5, "5"), resp: failedResponse(errors.New("failed"))},
{req: getRequest("key"), resp: getResponse("key", "5", 6, 6)},
},
},
{
name: "Txn can fail but be persisted before put",
operations: []testOperation{
// One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(3)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "3"), resp: putResponse(4)},
// Two failed request, two persisted.
{req: putRequest("key", "4"), resp: putResponse(4)},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 5, "6"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "7"), resp: putResponse(7)},
{req: putRequest("key", "4"), resp: putResponse(5)},
{req: compareRevisionAndPutRequest("key", 5, "5"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 6, "6"), resp: failedResponse(errors.New("failed"))},
{req: putRequest("key", "7"), resp: putResponse(8)},
},
},
{
name: "Txn can fail but be persisted before delete",
operations: []testOperation{
// One failed request, one persisted.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 3)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 4)},
// Two failed request, two persisted.
{req: putRequest("key", "4"), resp: putResponse(4)},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 5, "6"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 7)},
{req: putRequest("key", "4"), resp: putResponse(5)},
{req: compareRevisionAndPutRequest("key", 5, "5"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 6, "6"), resp: failedResponse(errors.New("failed"))},
{req: deleteRequest("key"), resp: deleteResponse(1, 8)},
},
},
{
name: "Txn can fail but be persisted before txn",
operations: []testOperation{
// One failed request, one persisted with success.
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1)},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 2, "3"), resp: compareRevisionAndPutResponse(true, 3)},
{req: putRequest("key", "1"), resp: putResponse(2)},
{req: compareRevisionAndPutRequest("key", 2, "2"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 3, "3"), resp: compareRevisionAndPutResponse(true, 4)},
// Two failed request, two persisted with success.
{req: putRequest("key", "4"), resp: putResponse(4)},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 5, "6"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 6, "7"), resp: compareRevisionAndPutResponse(true, 7)},
{req: putRequest("key", "4"), resp: putResponse(5)},
{req: compareRevisionAndPutRequest("key", 5, "5"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 6, "6"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 7, "7"), resp: compareRevisionAndPutResponse(true, 8)},
// One failed request, one persisted with failure.
{req: putRequest("key", "8"), resp: putResponse(8)},
{req: compareRevisionAndPutRequest("key", 8, "9"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 8, "10"), resp: compareRevisionAndPutResponse(false, 9)},
{req: putRequest("key", "8"), resp: putResponse(9)},
{req: compareRevisionAndPutRequest("key", 9, "9"), resp: failedResponse(errors.New("failed"))},
{req: compareRevisionAndPutRequest("key", 9, "10"), resp: compareRevisionAndPutResponse(false, 10)},
},
},
{