2016-05-13 06:50:58 +03:00
|
|
|
// Copyright 2016 The etcd Authors
|
2016-01-30 03:11:42 +03:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package integration
|
|
|
|
|
|
|
|
import (
|
2017-09-07 00:57:25 +03:00
|
|
|
"context"
|
2016-01-30 03:11:42 +03:00
|
|
|
"fmt"
|
2016-06-08 03:03:48 +03:00
|
|
|
"math/rand"
|
2016-01-30 03:11:42 +03:00
|
|
|
"reflect"
|
|
|
|
"sort"
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
2019-05-29 00:59:02 +03:00
|
|
|
"go.etcd.io/etcd/clientv3"
|
|
|
|
"go.etcd.io/etcd/etcdserver/api/v3rpc"
|
|
|
|
"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
|
|
|
|
"go.etcd.io/etcd/integration"
|
|
|
|
mvccpb "go.etcd.io/etcd/mvcc/mvccpb"
|
|
|
|
"go.etcd.io/etcd/pkg/testutil"
|
2017-09-07 00:57:25 +03:00
|
|
|
|
2017-08-01 23:14:31 +03:00
|
|
|
"google.golang.org/grpc/metadata"
|
2016-01-30 03:11:42 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
type watcherTest func(*testing.T, *watchctx)
|
|
|
|
|
|
|
|
type watchctx struct {
|
2016-06-08 03:03:48 +03:00
|
|
|
clus *integration.ClusterV3
|
|
|
|
w clientv3.Watcher
|
|
|
|
kv clientv3.KV
|
|
|
|
wclientMember int
|
|
|
|
kvMember int
|
|
|
|
ch clientv3.WatchChan
|
2016-01-30 03:11:42 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
func runWatchTest(t *testing.T, f watcherTest) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
2016-06-08 03:03:48 +03:00
|
|
|
wclientMember := rand.Intn(3)
|
2017-02-07 02:22:48 +03:00
|
|
|
w := clus.Client(wclientMember).Watcher
|
2017-09-04 13:09:32 +03:00
|
|
|
// select a different client for KV operations so puts succeed if
|
|
|
|
// a test knocks out the watcher client.
|
2016-06-08 03:03:48 +03:00
|
|
|
kvMember := rand.Intn(3)
|
|
|
|
for kvMember == wclientMember {
|
|
|
|
kvMember = rand.Intn(3)
|
2016-01-30 03:11:42 +03:00
|
|
|
}
|
2017-02-07 02:22:48 +03:00
|
|
|
kv := clus.Client(kvMember).KV
|
2016-01-30 03:11:42 +03:00
|
|
|
|
2017-02-07 02:22:48 +03:00
|
|
|
wctx := &watchctx{clus, w, kv, wclientMember, kvMember, nil}
|
2016-01-30 03:11:42 +03:00
|
|
|
f(t, wctx)
|
|
|
|
}
|
|
|
|
|
|
|
|
// TestWatchMultiWatcher modifies multiple keys and observes the changes.
|
|
|
|
func TestWatchMultiWatcher(t *testing.T) {
|
|
|
|
runWatchTest(t, testWatchMultiWatcher)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchMultiWatcher(t *testing.T, wctx *watchctx) {
|
|
|
|
numKeyUpdates := 4
|
|
|
|
keys := []string{"foo", "bar", "baz"}
|
|
|
|
|
|
|
|
donec := make(chan struct{})
|
|
|
|
readyc := make(chan struct{})
|
|
|
|
for _, k := range keys {
|
|
|
|
// key watcher
|
|
|
|
go func(key string) {
|
2016-02-24 04:47:56 +03:00
|
|
|
ch := wctx.w.Watch(context.TODO(), key)
|
2016-01-30 03:11:42 +03:00
|
|
|
if ch == nil {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("expected watcher channel, got nil")
|
2016-01-30 03:11:42 +03:00
|
|
|
}
|
|
|
|
readyc <- struct{}{}
|
|
|
|
for i := 0; i < numKeyUpdates; i++ {
|
|
|
|
resp, ok := <-ch
|
|
|
|
if !ok {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("watcher unexpectedly closed")
|
2016-01-30 03:11:42 +03:00
|
|
|
}
|
|
|
|
v := fmt.Sprintf("%s-%d", key, i)
|
|
|
|
gotv := string(resp.Events[0].Kv.Value)
|
|
|
|
if gotv != v {
|
|
|
|
t.Errorf("#%d: got %s, wanted %s", i, gotv, v)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
donec <- struct{}{}
|
|
|
|
}(k)
|
|
|
|
}
|
|
|
|
// prefix watcher on "b" (bar and baz)
|
|
|
|
go func() {
|
2016-02-24 04:47:56 +03:00
|
|
|
prefixc := wctx.w.Watch(context.TODO(), "b", clientv3.WithPrefix())
|
2016-01-30 03:11:42 +03:00
|
|
|
if prefixc == nil {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("expected watcher channel, got nil")
|
2016-01-30 03:11:42 +03:00
|
|
|
}
|
|
|
|
readyc <- struct{}{}
|
2016-04-07 08:44:15 +03:00
|
|
|
evs := []*clientv3.Event{}
|
2016-01-30 03:11:42 +03:00
|
|
|
for i := 0; i < numKeyUpdates*2; i++ {
|
|
|
|
resp, ok := <-prefixc
|
|
|
|
if !ok {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("watcher unexpectedly closed")
|
2016-01-30 03:11:42 +03:00
|
|
|
}
|
|
|
|
evs = append(evs, resp.Events...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// check response
|
|
|
|
expected := []string{}
|
|
|
|
bkeys := []string{"bar", "baz"}
|
|
|
|
for _, k := range bkeys {
|
|
|
|
for i := 0; i < numKeyUpdates; i++ {
|
|
|
|
expected = append(expected, fmt.Sprintf("%s-%d", k, i))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
got := []string{}
|
|
|
|
for _, ev := range evs {
|
|
|
|
got = append(got, string(ev.Kv.Value))
|
|
|
|
}
|
|
|
|
sort.Strings(got)
|
2016-04-02 22:55:11 +03:00
|
|
|
if !reflect.DeepEqual(expected, got) {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Errorf("got %v, expected %v", got, expected)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ensure no extra data
|
|
|
|
select {
|
|
|
|
case resp, ok := <-prefixc:
|
|
|
|
if !ok {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("watcher unexpectedly closed")
|
2016-01-30 03:11:42 +03:00
|
|
|
}
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("unexpected event %+v", resp)
|
2016-01-30 03:11:42 +03:00
|
|
|
case <-time.After(time.Second):
|
|
|
|
}
|
|
|
|
donec <- struct{}{}
|
|
|
|
}()
|
|
|
|
|
|
|
|
// wait for watcher bring up
|
|
|
|
for i := 0; i < len(keys)+1; i++ {
|
|
|
|
<-readyc
|
|
|
|
}
|
|
|
|
// generate events
|
2016-02-10 04:33:30 +03:00
|
|
|
ctx := context.TODO()
|
2016-01-30 03:11:42 +03:00
|
|
|
for i := 0; i < numKeyUpdates; i++ {
|
|
|
|
for _, k := range keys {
|
|
|
|
v := fmt.Sprintf("%s-%d", k, i)
|
2016-02-11 01:44:35 +03:00
|
|
|
if _, err := wctx.kv.Put(ctx, k, v); err != nil {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// wait for watcher shutdown
|
|
|
|
for i := 0; i < len(keys)+1; i++ {
|
|
|
|
<-donec
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-02-29 10:11:17 +03:00
|
|
|
// TestWatchRange tests watcher creates ranges
|
|
|
|
func TestWatchRange(t *testing.T) {
|
2016-04-07 07:12:07 +03:00
|
|
|
runWatchTest(t, testWatchRange)
|
2016-02-29 10:11:17 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchRange(t *testing.T, wctx *watchctx) {
|
|
|
|
if wctx.ch = wctx.w.Watch(context.TODO(), "a", clientv3.WithRange("c")); wctx.ch == nil {
|
|
|
|
t.Fatalf("expected non-nil channel")
|
|
|
|
}
|
|
|
|
putAndWatch(t, wctx, "a", "a")
|
|
|
|
putAndWatch(t, wctx, "b", "b")
|
|
|
|
putAndWatch(t, wctx, "bar", "bar")
|
|
|
|
}
|
|
|
|
|
2016-02-12 00:32:58 +03:00
|
|
|
// TestWatchReconnRequest tests the send failure path when requesting a watcher.
|
|
|
|
func TestWatchReconnRequest(t *testing.T) {
|
|
|
|
runWatchTest(t, testWatchReconnRequest)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchReconnRequest(t *testing.T, wctx *watchctx) {
|
2016-02-19 01:28:52 +03:00
|
|
|
donec, stopc := make(chan struct{}), make(chan struct{}, 1)
|
2016-02-12 00:32:58 +03:00
|
|
|
go func() {
|
2016-02-19 01:28:52 +03:00
|
|
|
timer := time.After(2 * time.Second)
|
|
|
|
defer close(donec)
|
|
|
|
// take down watcher connection
|
2016-02-12 00:32:58 +03:00
|
|
|
for {
|
2016-06-08 03:03:48 +03:00
|
|
|
wctx.clus.Members[wctx.wclientMember].DropConnections()
|
2016-02-12 00:32:58 +03:00
|
|
|
select {
|
2016-02-19 01:28:52 +03:00
|
|
|
case <-timer:
|
|
|
|
// spinning on close may live lock reconnection
|
|
|
|
return
|
|
|
|
case <-stopc:
|
2016-02-12 00:32:58 +03:00
|
|
|
return
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
// should reconnect when requesting watch
|
2016-02-24 04:47:56 +03:00
|
|
|
if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil {
|
2016-02-12 00:32:58 +03:00
|
|
|
t.Fatalf("expected non-nil channel")
|
|
|
|
}
|
2016-02-19 01:28:52 +03:00
|
|
|
|
|
|
|
// wait for disconnections to stop
|
|
|
|
stopc <- struct{}{}
|
|
|
|
<-donec
|
|
|
|
|
2016-09-19 23:05:41 +03:00
|
|
|
// spinning on dropping connections may trigger a leader election
|
|
|
|
// due to resource starvation; l-read to ensure the cluster is stable
|
|
|
|
ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
|
|
|
|
if _, err := wctx.kv.Get(ctx, "_"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
cancel()
|
|
|
|
|
2016-02-12 00:32:58 +03:00
|
|
|
// ensure watcher works
|
|
|
|
putAndWatch(t, wctx, "a", "a")
|
|
|
|
}
|
|
|
|
|
2016-01-30 03:11:42 +03:00
|
|
|
// TestWatchReconnInit tests watcher resumes correctly if connection lost
|
|
|
|
// before any data was sent.
|
|
|
|
func TestWatchReconnInit(t *testing.T) {
|
|
|
|
runWatchTest(t, testWatchReconnInit)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchReconnInit(t *testing.T, wctx *watchctx) {
|
2016-02-24 04:47:56 +03:00
|
|
|
if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Fatalf("expected non-nil channel")
|
|
|
|
}
|
2016-06-08 03:03:48 +03:00
|
|
|
wctx.clus.Members[wctx.wclientMember].DropConnections()
|
2016-01-30 03:11:42 +03:00
|
|
|
// watcher should recover
|
|
|
|
putAndWatch(t, wctx, "a", "a")
|
|
|
|
}
|
|
|
|
|
|
|
|
// TestWatchReconnRunning tests watcher resumes correctly if connection lost
|
|
|
|
// after data was sent.
|
|
|
|
func TestWatchReconnRunning(t *testing.T) {
|
|
|
|
runWatchTest(t, testWatchReconnRunning)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchReconnRunning(t *testing.T, wctx *watchctx) {
|
2016-02-24 04:47:56 +03:00
|
|
|
if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Fatalf("expected non-nil channel")
|
|
|
|
}
|
|
|
|
putAndWatch(t, wctx, "a", "a")
|
|
|
|
// take down watcher connection
|
2016-06-08 03:03:48 +03:00
|
|
|
wctx.clus.Members[wctx.wclientMember].DropConnections()
|
2016-01-30 03:11:42 +03:00
|
|
|
// watcher should recover
|
|
|
|
putAndWatch(t, wctx, "a", "b")
|
|
|
|
}
|
|
|
|
|
2016-02-26 23:03:38 +03:00
|
|
|
// TestWatchCancelImmediate ensures a closed channel is returned
|
|
|
|
// if the context is cancelled.
|
|
|
|
func TestWatchCancelImmediate(t *testing.T) {
|
|
|
|
runWatchTest(t, testWatchCancelImmediate)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchCancelImmediate(t *testing.T, wctx *watchctx) {
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
cancel()
|
|
|
|
wch := wctx.w.Watch(ctx, "a")
|
|
|
|
select {
|
|
|
|
case wresp, ok := <-wch:
|
|
|
|
if ok {
|
|
|
|
t.Fatalf("read wch got %v; expected closed channel", wresp)
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
t.Fatalf("closed watcher channel should not block")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-01-30 03:11:42 +03:00
|
|
|
// TestWatchCancelInit tests watcher closes correctly after no events.
|
|
|
|
func TestWatchCancelInit(t *testing.T) {
|
|
|
|
runWatchTest(t, testWatchCancelInit)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchCancelInit(t *testing.T, wctx *watchctx) {
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
2016-02-24 04:47:56 +03:00
|
|
|
if wctx.ch = wctx.w.Watch(ctx, "a"); wctx.ch == nil {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Fatalf("expected non-nil watcher channel")
|
|
|
|
}
|
|
|
|
cancel()
|
|
|
|
select {
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
t.Fatalf("took too long to cancel")
|
|
|
|
case _, ok := <-wctx.ch:
|
|
|
|
if ok {
|
|
|
|
t.Fatalf("expected watcher channel to close")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// TestWatchCancelRunning tests watcher closes correctly after events.
|
|
|
|
func TestWatchCancelRunning(t *testing.T) {
|
|
|
|
runWatchTest(t, testWatchCancelRunning)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchCancelRunning(t *testing.T, wctx *watchctx) {
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
2016-02-24 04:47:56 +03:00
|
|
|
if wctx.ch = wctx.w.Watch(ctx, "a"); wctx.ch == nil {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Fatalf("expected non-nil watcher channel")
|
|
|
|
}
|
2016-02-11 01:44:35 +03:00
|
|
|
if _, err := wctx.kv.Put(ctx, "a", "a"); err != nil {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
cancel()
|
|
|
|
select {
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
t.Fatalf("took too long to cancel")
|
2017-10-03 12:14:22 +03:00
|
|
|
case _, ok := <-wctx.ch:
|
2016-01-30 03:11:42 +03:00
|
|
|
if !ok {
|
|
|
|
// closed before getting put; OK
|
|
|
|
break
|
|
|
|
}
|
|
|
|
// got the PUT; should close next
|
|
|
|
select {
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
t.Fatalf("took too long to close")
|
2017-10-03 12:14:22 +03:00
|
|
|
case v, ok2 := <-wctx.ch:
|
|
|
|
if ok2 {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Fatalf("expected watcher channel to close, got %v", v)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func putAndWatch(t *testing.T, wctx *watchctx, key, val string) {
|
2016-02-11 01:44:35 +03:00
|
|
|
if _, err := wctx.kv.Put(context.TODO(), key, val); err != nil {
|
2016-01-30 03:11:42 +03:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case <-time.After(5 * time.Second):
|
|
|
|
t.Fatalf("watch timed out")
|
|
|
|
case v, ok := <-wctx.ch:
|
|
|
|
if !ok {
|
|
|
|
t.Fatalf("unexpected watch close")
|
|
|
|
}
|
|
|
|
if string(v.Events[0].Kv.Value) != val {
|
|
|
|
t.Fatalf("bad value got %v, wanted %v", v.Events[0].Kv.Value, val)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-02-25 03:30:20 +03:00
|
|
|
|
2017-04-21 09:19:31 +03:00
|
|
|
func TestWatchResumeInitRev(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
|
|
|
cli := clus.Client(0)
|
|
|
|
if _, err := cli.Put(context.TODO(), "b", "2"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if _, err := cli.Put(context.TODO(), "a", "3"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
// if resume is broken, it'll pick up this key first instead of a=3
|
|
|
|
if _, err := cli.Put(context.TODO(), "a", "4"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
wch := clus.Client(0).Watch(context.Background(), "a", clientv3.WithRev(1), clientv3.WithCreatedNotify())
|
|
|
|
if resp, ok := <-wch; !ok || resp.Header.Revision != 4 {
|
|
|
|
t.Fatalf("got (%v, %v), expected create notification rev=4", resp, ok)
|
|
|
|
}
|
|
|
|
// pause wch
|
|
|
|
clus.Members[0].DropConnections()
|
|
|
|
clus.Members[0].PauseConnections()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case resp, ok := <-wch:
|
|
|
|
t.Skipf("wch should block, got (%+v, %v); drop not fast enough", resp, ok)
|
|
|
|
case <-time.After(100 * time.Millisecond):
|
|
|
|
}
|
|
|
|
|
|
|
|
// resume wch
|
|
|
|
clus.Members[0].UnpauseConnections()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case resp, ok := <-wch:
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("unexpected watch close")
|
|
|
|
}
|
|
|
|
if len(resp.Events) == 0 {
|
|
|
|
t.Fatal("expected event on watch")
|
|
|
|
}
|
|
|
|
if string(resp.Events[0].Kv.Value) != "3" {
|
|
|
|
t.Fatalf("expected value=3, got event %+v", resp.Events[0])
|
|
|
|
}
|
|
|
|
case <-time.After(5 * time.Second):
|
|
|
|
t.Fatal("watch timed out")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// TestWatchResumeCompacted checks that the watcher gracefully closes in case
|
2016-05-03 02:16:18 +03:00
|
|
|
// that it tries to resume to a revision that's been compacted out of the store.
|
2016-10-21 20:30:48 +03:00
|
|
|
// Since the watcher's server restarts with stale data, the watcher will receive
|
|
|
|
// either a compaction error or all keys by staying in sync before the compaction
|
|
|
|
// is finally applied.
|
2016-05-03 02:16:18 +03:00
|
|
|
func TestWatchResumeCompacted(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
|
|
|
// create a waiting watcher at rev 1
|
2017-02-07 02:22:48 +03:00
|
|
|
w := clus.Client(0)
|
2016-05-03 02:16:18 +03:00
|
|
|
wch := w.Watch(context.Background(), "foo", clientv3.WithRev(1))
|
|
|
|
select {
|
|
|
|
case w := <-wch:
|
|
|
|
t.Errorf("unexpected message from wch %v", w)
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
clus.Members[0].Stop(t)
|
|
|
|
|
|
|
|
ticker := time.After(time.Second * 10)
|
|
|
|
for clus.WaitLeader(t) <= 0 {
|
|
|
|
select {
|
|
|
|
case <-ticker:
|
|
|
|
t.Fatalf("failed to wait for new leader")
|
|
|
|
default:
|
|
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// put some data and compact away
|
2016-10-21 20:30:48 +03:00
|
|
|
numPuts := 5
|
2017-02-07 02:22:48 +03:00
|
|
|
kv := clus.Client(1)
|
2016-10-21 20:30:48 +03:00
|
|
|
for i := 0; i < numPuts; i++ {
|
2016-05-03 02:16:18 +03:00
|
|
|
if _, err := kv.Put(context.TODO(), "foo", "bar"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
2016-06-28 06:51:12 +03:00
|
|
|
if _, err := kv.Compact(context.TODO(), 3); err != nil {
|
2016-05-03 02:16:18 +03:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
clus.Members[0].Restart(t)
|
|
|
|
|
2016-10-21 20:30:48 +03:00
|
|
|
// since watch's server isn't guaranteed to be synced with the cluster when
|
|
|
|
// the watch resumes, there is a window where the watch can stay synced and
|
|
|
|
// read off all events; if the watcher misses the window, it will go out of
|
|
|
|
// sync and get a compaction error.
|
|
|
|
wRev := int64(2)
|
|
|
|
for int(wRev) <= numPuts+1 {
|
|
|
|
var wresp clientv3.WatchResponse
|
|
|
|
var ok bool
|
|
|
|
select {
|
|
|
|
case wresp, ok = <-wch:
|
|
|
|
if !ok {
|
|
|
|
t.Fatalf("expected wresp, but got closed channel")
|
|
|
|
}
|
|
|
|
case <-time.After(5 * time.Second):
|
|
|
|
t.Fatalf("compacted watch timed out")
|
|
|
|
}
|
|
|
|
for _, ev := range wresp.Events {
|
|
|
|
if ev.Kv.ModRevision != wRev {
|
|
|
|
t.Fatalf("expected modRev %v, got %+v", wRev, ev)
|
|
|
|
}
|
|
|
|
wRev++
|
|
|
|
}
|
|
|
|
if wresp.Err() == nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if wresp.Err() != rpctypes.ErrCompacted {
|
2017-04-07 04:26:42 +03:00
|
|
|
t.Fatalf("wresp.Err() expected %v, got %+v", rpctypes.ErrCompacted, wresp.Err())
|
2016-10-21 20:30:48 +03:00
|
|
|
}
|
|
|
|
break
|
2016-05-03 02:16:18 +03:00
|
|
|
}
|
2016-10-21 20:30:48 +03:00
|
|
|
if int(wRev) > numPuts+1 {
|
|
|
|
// got data faster than the compaction
|
|
|
|
return
|
2016-05-03 02:16:18 +03:00
|
|
|
}
|
2016-10-21 20:30:48 +03:00
|
|
|
// received compaction error; ensure the channel closes
|
|
|
|
select {
|
|
|
|
case wresp, ok := <-wch:
|
|
|
|
if ok {
|
|
|
|
t.Fatalf("expected closed channel, but got %v", wresp)
|
|
|
|
}
|
|
|
|
case <-time.After(5 * time.Second):
|
|
|
|
t.Fatalf("timed out waiting for channel close")
|
2016-05-03 02:16:18 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-04 00:13:10 +03:00
|
|
|
// TestWatchCompactRevision ensures the CompactRevision error is given on a
|
|
|
|
// compaction event ahead of a watcher.
|
|
|
|
func TestWatchCompactRevision(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
|
2016-07-19 19:31:44 +03:00
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
2016-03-04 00:13:10 +03:00
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
|
|
|
// set some keys
|
2017-02-07 02:22:48 +03:00
|
|
|
kv := clus.RandClient()
|
2016-03-04 00:13:10 +03:00
|
|
|
for i := 0; i < 5; i++ {
|
|
|
|
if _, err := kv.Put(context.TODO(), "foo", "bar"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-07 02:22:48 +03:00
|
|
|
w := clus.RandClient()
|
2016-03-04 00:13:10 +03:00
|
|
|
|
2016-06-28 06:51:12 +03:00
|
|
|
if _, err := kv.Compact(context.TODO(), 4); err != nil {
|
2016-03-04 00:13:10 +03:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
wch := w.Watch(context.Background(), "foo", clientv3.WithRev(2))
|
|
|
|
|
|
|
|
// get compacted error message
|
|
|
|
wresp, ok := <-wch
|
|
|
|
if !ok {
|
|
|
|
t.Fatalf("expected wresp, but got closed channel")
|
|
|
|
}
|
2016-03-17 20:08:16 +03:00
|
|
|
if wresp.Err() != rpctypes.ErrCompacted {
|
2016-04-29 21:03:37 +03:00
|
|
|
t.Fatalf("wresp.Err() expected %v, but got %v", rpctypes.ErrCompacted, wresp.Err())
|
2016-03-04 00:13:10 +03:00
|
|
|
}
|
2017-07-19 22:53:39 +03:00
|
|
|
if !wresp.Canceled {
|
|
|
|
t.Fatalf("wresp.Canceled expected true, got %+v", wresp)
|
|
|
|
}
|
2016-03-04 00:13:10 +03:00
|
|
|
|
|
|
|
// ensure the channel is closed
|
|
|
|
if wresp, ok = <-wch; ok {
|
|
|
|
t.Fatalf("expected closed channel, but got %v", wresp)
|
|
|
|
}
|
|
|
|
}
|
2016-03-04 20:47:13 +03:00
|
|
|
|
|
|
|
func TestWatchWithProgressNotify(t *testing.T) { testWatchWithProgressNotify(t, true) }
|
|
|
|
func TestWatchWithProgressNotifyNoEvent(t *testing.T) { testWatchWithProgressNotify(t, false) }
|
|
|
|
|
|
|
|
func testWatchWithProgressNotify(t *testing.T, watchOnPut bool) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
|
2016-03-27 12:39:46 +03:00
|
|
|
// accelerate report interval so test terminates quickly
|
2016-04-04 14:45:54 +03:00
|
|
|
oldpi := v3rpc.GetProgressReportInterval()
|
2016-03-27 12:39:46 +03:00
|
|
|
// using atomics to avoid race warnings
|
2016-04-04 14:45:54 +03:00
|
|
|
v3rpc.SetProgressReportInterval(3 * time.Second)
|
2016-03-27 12:39:46 +03:00
|
|
|
pi := 3 * time.Second
|
2016-04-04 14:45:54 +03:00
|
|
|
defer func() { v3rpc.SetProgressReportInterval(oldpi) }()
|
2016-03-27 12:39:46 +03:00
|
|
|
|
2016-03-04 20:47:13 +03:00
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
2017-02-07 02:22:48 +03:00
|
|
|
wc := clus.RandClient()
|
2016-03-04 20:47:13 +03:00
|
|
|
|
|
|
|
opts := []clientv3.OpOption{clientv3.WithProgressNotify()}
|
|
|
|
if watchOnPut {
|
|
|
|
opts = append(opts, clientv3.WithPrefix())
|
|
|
|
}
|
|
|
|
rch := wc.Watch(context.Background(), "foo", opts...)
|
|
|
|
|
|
|
|
select {
|
|
|
|
case resp := <-rch: // wait for notification
|
|
|
|
if len(resp.Events) != 0 {
|
|
|
|
t.Fatalf("resp.Events expected none, got %+v", resp.Events)
|
|
|
|
}
|
|
|
|
case <-time.After(2 * pi):
|
|
|
|
t.Fatalf("watch response expected in %v, but timed out", pi)
|
|
|
|
}
|
|
|
|
|
2017-02-07 02:22:48 +03:00
|
|
|
kvc := clus.RandClient()
|
2016-03-04 20:47:13 +03:00
|
|
|
if _, err := kvc.Put(context.TODO(), "foox", "bar"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case resp := <-rch:
|
|
|
|
if resp.Header.Revision != 2 {
|
|
|
|
t.Fatalf("resp.Header.Revision expected 2, got %d", resp.Header.Revision)
|
|
|
|
}
|
|
|
|
if watchOnPut { // wait for put if watch on the put key
|
2016-04-08 05:29:32 +03:00
|
|
|
ev := []*clientv3.Event{{Type: clientv3.EventTypePut,
|
2016-04-25 22:32:58 +03:00
|
|
|
Kv: &mvccpb.KeyValue{Key: []byte("foox"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}}}
|
2016-03-04 20:47:13 +03:00
|
|
|
if !reflect.DeepEqual(ev, resp.Events) {
|
|
|
|
t.Fatalf("expected %+v, got %+v", ev, resp.Events)
|
|
|
|
}
|
|
|
|
} else if len(resp.Events) != 0 { // wait for notification otherwise
|
|
|
|
t.Fatalf("expected no events, but got %+v", resp.Events)
|
|
|
|
}
|
2016-07-20 21:39:06 +03:00
|
|
|
case <-time.After(time.Duration(1.5 * float64(pi))):
|
2016-03-04 20:47:13 +03:00
|
|
|
t.Fatalf("watch response expected in %v, but timed out", pi)
|
|
|
|
}
|
|
|
|
}
|
2016-04-07 08:44:15 +03:00
|
|
|
|
2018-06-20 02:23:25 +03:00
|
|
|
func TestWatchRequestProgress(t *testing.T) {
|
|
|
|
testCases := []struct {
|
|
|
|
name string
|
|
|
|
watchers []string
|
|
|
|
}{
|
|
|
|
{"0-watcher", []string{}},
|
|
|
|
{"1-watcher", []string{"/"}},
|
|
|
|
{"2-watcher", []string{"/", "/"}},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, c := range testCases {
|
|
|
|
t.Run(c.name, func(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
|
|
|
|
watchTimeout := 3 * time.Second
|
|
|
|
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
|
|
|
wc := clus.RandClient()
|
|
|
|
|
|
|
|
var watchChans []clientv3.WatchChan
|
|
|
|
|
|
|
|
for _, prefix := range c.watchers {
|
|
|
|
watchChans = append(watchChans, wc.Watch(context.Background(), prefix, clientv3.WithPrefix()))
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err := wc.Put(context.Background(), "/a", "1")
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, rch := range watchChans {
|
|
|
|
select {
|
|
|
|
case resp := <-rch: // wait for notification
|
|
|
|
if len(resp.Events) != 1 {
|
|
|
|
t.Fatalf("resp.Events expected 1, got %d", len(resp.Events))
|
|
|
|
}
|
|
|
|
case <-time.After(watchTimeout):
|
|
|
|
t.Fatalf("watch response expected in %v, but timed out", watchTimeout)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// put a value not being watched to increment revision
|
|
|
|
_, err = wc.Put(context.Background(), "x", "1")
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = wc.RequestProgress(context.Background())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// verify all watch channels receive a progress notify
|
|
|
|
for _, rch := range watchChans {
|
|
|
|
select {
|
|
|
|
case resp := <-rch:
|
|
|
|
if !resp.IsProgressNotify() {
|
|
|
|
t.Fatalf("expected resp.IsProgressNotify() == true")
|
|
|
|
}
|
|
|
|
if resp.Header.Revision != 3 {
|
|
|
|
t.Fatalf("resp.Header.Revision expected 3, got %d", resp.Header.Revision)
|
|
|
|
}
|
|
|
|
case <-time.After(watchTimeout):
|
|
|
|
t.Fatalf("progress response expected in %v, but timed out", watchTimeout)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-07 08:44:15 +03:00
|
|
|
func TestWatchEventType(t *testing.T) {
|
|
|
|
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer cluster.Terminate(t)
|
|
|
|
|
|
|
|
client := cluster.RandClient()
|
|
|
|
ctx := context.Background()
|
|
|
|
watchChan := client.Watch(ctx, "/", clientv3.WithPrefix())
|
|
|
|
|
|
|
|
if _, err := client.Put(ctx, "/toDelete", "foo"); err != nil {
|
|
|
|
t.Fatalf("Put failed: %v", err)
|
|
|
|
}
|
|
|
|
if _, err := client.Put(ctx, "/toDelete", "bar"); err != nil {
|
|
|
|
t.Fatalf("Put failed: %v", err)
|
|
|
|
}
|
|
|
|
if _, err := client.Delete(ctx, "/toDelete"); err != nil {
|
|
|
|
t.Fatalf("Delete failed: %v", err)
|
|
|
|
}
|
2016-04-07 20:27:36 +03:00
|
|
|
lcr, err := client.Lease.Grant(ctx, 1)
|
2016-04-07 08:44:15 +03:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("lease create failed: %v", err)
|
|
|
|
}
|
2016-04-16 03:48:20 +03:00
|
|
|
if _, err := client.Put(ctx, "/toExpire", "foo", clientv3.WithLease(lcr.ID)); err != nil {
|
2016-04-07 08:44:15 +03:00
|
|
|
t.Fatalf("Put failed: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
tests := []struct {
|
2016-04-25 22:32:58 +03:00
|
|
|
et mvccpb.Event_EventType
|
2016-04-07 08:44:15 +03:00
|
|
|
isCreate bool
|
|
|
|
isModify bool
|
|
|
|
}{{
|
|
|
|
et: clientv3.EventTypePut,
|
|
|
|
isCreate: true,
|
|
|
|
}, {
|
|
|
|
et: clientv3.EventTypePut,
|
|
|
|
isModify: true,
|
|
|
|
}, {
|
|
|
|
et: clientv3.EventTypeDelete,
|
|
|
|
}, {
|
|
|
|
et: clientv3.EventTypePut,
|
|
|
|
isCreate: true,
|
|
|
|
}, {
|
|
|
|
et: clientv3.EventTypeDelete,
|
|
|
|
}}
|
|
|
|
|
|
|
|
var res []*clientv3.Event
|
|
|
|
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case wres := <-watchChan:
|
|
|
|
res = append(res, wres.Events...)
|
|
|
|
case <-time.After(10 * time.Second):
|
|
|
|
t.Fatalf("Should receive %d events and then break out loop", len(tests))
|
|
|
|
}
|
|
|
|
if len(res) == len(tests) {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, tt := range tests {
|
|
|
|
ev := res[i]
|
|
|
|
if tt.et != ev.Type {
|
|
|
|
t.Errorf("#%d: event type want=%s, get=%s", i, tt.et, ev.Type)
|
|
|
|
}
|
|
|
|
if tt.isCreate && !ev.IsCreate() {
|
|
|
|
t.Errorf("#%d: event should be CreateEvent", i)
|
|
|
|
}
|
|
|
|
if tt.isModify && !ev.IsModify() {
|
|
|
|
t.Errorf("#%d: event should be ModifyEvent", i)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-06-01 20:03:12 +03:00
|
|
|
|
|
|
|
func TestWatchErrConnClosed(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
|
|
|
cli := clus.Client(0)
|
|
|
|
|
|
|
|
donec := make(chan struct{})
|
|
|
|
go func() {
|
|
|
|
defer close(donec)
|
2017-02-07 02:22:48 +03:00
|
|
|
ch := cli.Watch(context.TODO(), "foo")
|
2018-05-02 03:07:37 +03:00
|
|
|
|
|
|
|
if wr := <-ch; !isCanceled(wr.Err()) {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("expected context canceled, got %v", wr.Err())
|
2016-06-01 20:03:12 +03:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
2016-11-01 21:13:33 +03:00
|
|
|
if err := cli.ActiveConnection().Close(); err != nil {
|
2016-06-01 20:03:12 +03:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
clus.TakeClient(0)
|
|
|
|
|
|
|
|
select {
|
2017-12-09 21:16:27 +03:00
|
|
|
case <-time.After(integration.RequestWaitTimeout):
|
2016-06-01 20:03:12 +03:00
|
|
|
t.Fatal("wc.Watch took too long")
|
|
|
|
case <-donec:
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestWatchAfterClose(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
|
|
|
cli := clus.Client(0)
|
|
|
|
clus.TakeClient(0)
|
|
|
|
if err := cli.Close(); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
donec := make(chan struct{})
|
|
|
|
go func() {
|
2017-02-07 02:22:48 +03:00
|
|
|
cli.Watch(context.TODO(), "foo")
|
2018-05-02 03:07:37 +03:00
|
|
|
if err := cli.Close(); err != nil && err != context.Canceled {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("expected %v, got %v", context.Canceled, err)
|
2016-06-01 20:03:12 +03:00
|
|
|
}
|
|
|
|
close(donec)
|
|
|
|
}()
|
|
|
|
select {
|
2017-12-09 21:16:27 +03:00
|
|
|
case <-time.After(integration.RequestWaitTimeout):
|
2016-06-01 20:03:12 +03:00
|
|
|
t.Fatal("wc.Watch took too long")
|
|
|
|
case <-donec:
|
|
|
|
}
|
|
|
|
}
|
2016-06-16 02:52:27 +03:00
|
|
|
|
|
|
|
// TestWatchWithRequireLeader checks the watch channel closes when no leader.
|
|
|
|
func TestWatchWithRequireLeader(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
2016-10-28 20:00:59 +03:00
|
|
|
// Put a key for the non-require leader watch to read as an event.
|
|
|
|
// The watchers will be on member[0]; put key through member[0] to
|
|
|
|
// ensure that it receives the update so watching after killing quorum
|
|
|
|
// is guaranteed to have the key.
|
|
|
|
liveClient := clus.Client(0)
|
|
|
|
if _, err := liveClient.Put(context.TODO(), "foo", "bar"); err != nil {
|
2016-06-16 02:52:27 +03:00
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
clus.Members[1].Stop(t)
|
|
|
|
clus.Members[2].Stop(t)
|
|
|
|
clus.Client(1).Close()
|
|
|
|
clus.Client(2).Close()
|
|
|
|
clus.TakeClient(1)
|
|
|
|
clus.TakeClient(2)
|
|
|
|
|
|
|
|
// wait for election timeout, then member[0] will not have a leader.
|
|
|
|
tickDuration := 10 * time.Millisecond
|
2017-02-11 02:46:48 +03:00
|
|
|
// existing streams need three elections before they're torn down; wait until 5 elections cycle
|
|
|
|
// so proxy tests receive a leader loss event on its existing watch before creating a new watch.
|
|
|
|
time.Sleep(time.Duration(5*clus.Members[0].ElectionTicks) * tickDuration)
|
2016-06-16 02:52:27 +03:00
|
|
|
|
2016-10-28 20:00:59 +03:00
|
|
|
chLeader := liveClient.Watch(clientv3.WithRequireLeader(context.TODO()), "foo", clientv3.WithRev(1))
|
|
|
|
chNoLeader := liveClient.Watch(context.TODO(), "foo", clientv3.WithRev(1))
|
2016-06-16 02:52:27 +03:00
|
|
|
|
|
|
|
select {
|
|
|
|
case resp, ok := <-chLeader:
|
|
|
|
if !ok {
|
|
|
|
t.Fatalf("expected %v watch channel, got closed channel", rpctypes.ErrNoLeader)
|
|
|
|
}
|
|
|
|
if resp.Err() != rpctypes.ErrNoLeader {
|
|
|
|
t.Fatalf("expected %v watch response error, got %+v", rpctypes.ErrNoLeader, resp)
|
|
|
|
}
|
2017-12-09 21:16:27 +03:00
|
|
|
case <-time.After(integration.RequestWaitTimeout):
|
2016-06-16 02:52:27 +03:00
|
|
|
t.Fatal("watch without leader took too long to close")
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case resp, ok := <-chLeader:
|
|
|
|
if ok {
|
|
|
|
t.Fatalf("expected closed channel, got response %v", resp)
|
|
|
|
}
|
2017-12-09 21:16:27 +03:00
|
|
|
case <-time.After(integration.RequestWaitTimeout):
|
2016-06-16 02:52:27 +03:00
|
|
|
t.Fatal("waited too long for channel to close")
|
|
|
|
}
|
|
|
|
|
|
|
|
if _, ok := <-chNoLeader; !ok {
|
|
|
|
t.Fatalf("expected response, got closed channel")
|
|
|
|
}
|
|
|
|
}
|
2016-07-28 07:25:06 +03:00
|
|
|
|
|
|
|
// TestWatchWithFilter checks that watch filtering works.
|
|
|
|
func TestWatchWithFilter(t *testing.T) {
|
|
|
|
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer cluster.Terminate(t)
|
|
|
|
|
|
|
|
client := cluster.RandClient()
|
|
|
|
ctx := context.Background()
|
|
|
|
|
|
|
|
wcNoPut := client.Watch(ctx, "a", clientv3.WithFilterPut())
|
|
|
|
wcNoDel := client.Watch(ctx, "a", clientv3.WithFilterDelete())
|
|
|
|
|
|
|
|
if _, err := client.Put(ctx, "a", "abc"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if _, err := client.Delete(ctx, "a"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
npResp := <-wcNoPut
|
|
|
|
if len(npResp.Events) != 1 || npResp.Events[0].Type != clientv3.EventTypeDelete {
|
|
|
|
t.Fatalf("expected delete event, got %+v", npResp.Events)
|
|
|
|
}
|
|
|
|
ndResp := <-wcNoDel
|
|
|
|
if len(ndResp.Events) != 1 || ndResp.Events[0].Type != clientv3.EventTypePut {
|
|
|
|
t.Fatalf("expected put event, got %+v", ndResp.Events)
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case resp := <-wcNoPut:
|
|
|
|
t.Fatalf("unexpected event on filtered put (%+v)", resp)
|
|
|
|
case resp := <-wcNoDel:
|
|
|
|
t.Fatalf("unexpected event on filtered delete (%+v)", resp)
|
|
|
|
case <-time.After(100 * time.Millisecond):
|
|
|
|
}
|
|
|
|
}
|
2016-08-03 00:56:58 +03:00
|
|
|
|
2017-09-04 13:09:32 +03:00
|
|
|
// TestWatchWithCreatedNotification checks that WithCreatedNotify returns a
|
|
|
|
// Created watch response.
|
2016-08-03 00:56:58 +03:00
|
|
|
func TestWatchWithCreatedNotification(t *testing.T) {
|
|
|
|
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer cluster.Terminate(t)
|
|
|
|
|
|
|
|
client := cluster.RandClient()
|
|
|
|
|
|
|
|
ctx := context.Background()
|
|
|
|
|
|
|
|
createC := client.Watch(ctx, "a", clientv3.WithCreatedNotify())
|
|
|
|
|
|
|
|
resp := <-createC
|
|
|
|
|
|
|
|
if !resp.Created {
|
|
|
|
t.Fatalf("expected created event, got %v", resp)
|
|
|
|
}
|
|
|
|
}
|
2016-08-09 09:47:16 +03:00
|
|
|
|
2016-10-27 21:11:16 +03:00
|
|
|
// TestWatchWithCreatedNotificationDropConn ensures that
|
|
|
|
// a watcher with created notify does not post duplicate
|
|
|
|
// created events from disconnect.
|
|
|
|
func TestWatchWithCreatedNotificationDropConn(t *testing.T) {
|
|
|
|
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer cluster.Terminate(t)
|
|
|
|
|
|
|
|
client := cluster.RandClient()
|
|
|
|
|
|
|
|
wch := client.Watch(context.Background(), "a", clientv3.WithCreatedNotify())
|
|
|
|
|
|
|
|
resp := <-wch
|
|
|
|
|
|
|
|
if !resp.Created {
|
|
|
|
t.Fatalf("expected created event, got %v", resp)
|
|
|
|
}
|
|
|
|
|
|
|
|
cluster.Members[0].DropConnections()
|
|
|
|
|
2017-09-04 13:09:32 +03:00
|
|
|
// check watch channel doesn't post another watch response.
|
2016-10-27 21:11:16 +03:00
|
|
|
select {
|
|
|
|
case wresp := <-wch:
|
|
|
|
t.Fatalf("got unexpected watch response: %+v\n", wresp)
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
// watcher may not reconnect by the time it hits the select,
|
|
|
|
// so it wouldn't have a chance to filter out the second create event
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-09 09:47:16 +03:00
|
|
|
// TestWatchCancelOnServer ensures client watcher cancels propagate back to the server.
|
|
|
|
func TestWatchCancelOnServer(t *testing.T) {
|
|
|
|
cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer cluster.Terminate(t)
|
|
|
|
|
|
|
|
client := cluster.RandClient()
|
2017-02-22 04:33:15 +03:00
|
|
|
numWatches := 10
|
2016-08-09 09:47:16 +03:00
|
|
|
|
2017-09-04 13:09:32 +03:00
|
|
|
// The grpc proxy starts watches to detect leadership after the proxy server
|
2017-06-23 01:02:41 +03:00
|
|
|
// returns as started; to avoid racing on the proxy's internal watches, wait
|
|
|
|
// until require leader watches get create responses to ensure the leadership
|
|
|
|
// watches have started.
|
|
|
|
for {
|
|
|
|
ctx, cancel := context.WithCancel(clientv3.WithRequireLeader(context.TODO()))
|
|
|
|
ww := client.Watch(ctx, "a", clientv3.WithCreatedNotify())
|
|
|
|
wresp := <-ww
|
|
|
|
cancel()
|
|
|
|
if wresp.Err() == nil {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-22 04:33:15 +03:00
|
|
|
cancels := make([]context.CancelFunc, numWatches)
|
|
|
|
for i := 0; i < numWatches; i++ {
|
2017-08-01 23:14:31 +03:00
|
|
|
// force separate streams in client
|
|
|
|
md := metadata.Pairs("some-key", fmt.Sprintf("%d", i))
|
|
|
|
mctx := metadata.NewOutgoingContext(context.Background(), md)
|
|
|
|
ctx, cancel := context.WithCancel(mctx)
|
2017-02-22 04:33:15 +03:00
|
|
|
cancels[i] = cancel
|
|
|
|
w := client.Watch(ctx, fmt.Sprintf("%d", i), clientv3.WithCreatedNotify())
|
|
|
|
<-w
|
|
|
|
}
|
|
|
|
|
|
|
|
// get max watches; proxy tests have leadership watches, so total may be >numWatches
|
|
|
|
maxWatches, _ := cluster.Members[0].Metric("etcd_debugging_mvcc_watcher_total")
|
|
|
|
|
|
|
|
// cancel all and wait for cancels to propagate to etcd server
|
|
|
|
for i := 0; i < numWatches; i++ {
|
|
|
|
cancels[i]()
|
2016-08-09 09:47:16 +03:00
|
|
|
}
|
|
|
|
time.Sleep(time.Second)
|
|
|
|
|
2017-02-22 04:33:15 +03:00
|
|
|
minWatches, err := cluster.Members[0].Metric("etcd_debugging_mvcc_watcher_total")
|
2016-08-09 09:47:16 +03:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2017-02-22 04:33:15 +03:00
|
|
|
|
|
|
|
maxWatchV, minWatchV := 0, 0
|
|
|
|
n, serr := fmt.Sscanf(maxWatches+" "+minWatches, "%d %d", &maxWatchV, &minWatchV)
|
|
|
|
if n != 2 || serr != nil {
|
|
|
|
t.Fatalf("expected n=2 and err=nil, got n=%d and err=%v", n, serr)
|
|
|
|
}
|
|
|
|
|
2017-06-23 01:02:41 +03:00
|
|
|
if maxWatchV-minWatchV < numWatches {
|
2017-02-22 04:33:15 +03:00
|
|
|
t.Fatalf("expected %d canceled watchers, got %d", numWatches, maxWatchV-minWatchV)
|
2016-08-09 09:47:16 +03:00
|
|
|
}
|
|
|
|
}
|
2016-09-20 01:11:11 +03:00
|
|
|
|
|
|
|
// TestWatchOverlapContextCancel stresses the watcher stream teardown path by
|
|
|
|
// creating/canceling watchers to ensure that new watchers are not taken down
|
|
|
|
// by a torn down watch stream. The sort of race that's being detected:
|
|
|
|
// 1. create w1 using a cancelable ctx with %v as "ctx"
|
|
|
|
// 2. cancel ctx
|
|
|
|
// 3. watcher client begins tearing down watcher grpc stream since no more watchers
|
|
|
|
// 3. start creating watcher w2 using a new "ctx" (not canceled), attaches to old grpc stream
|
|
|
|
// 4. watcher client finishes tearing down stream on "ctx"
|
|
|
|
// 5. w2 comes back canceled
|
|
|
|
func TestWatchOverlapContextCancel(t *testing.T) {
|
2016-09-24 02:40:29 +03:00
|
|
|
f := func(clus *integration.ClusterV3) {}
|
|
|
|
testWatchOverlapContextCancel(t, f)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestWatchOverlapDropConnContextCancel(t *testing.T) {
|
|
|
|
f := func(clus *integration.ClusterV3) {
|
|
|
|
clus.Members[0].DropConnections()
|
|
|
|
}
|
|
|
|
testWatchOverlapContextCancel(t, f)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchOverlapContextCancel(t *testing.T, f func(*integration.ClusterV3)) {
|
2016-09-20 01:11:11 +03:00
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
|
|
|
|
n := 100
|
|
|
|
ctxs, ctxc := make([]context.Context, 5), make([]chan struct{}, 5)
|
|
|
|
for i := range ctxs {
|
2017-08-01 23:14:31 +03:00
|
|
|
// make unique stream
|
|
|
|
md := metadata.Pairs("some-key", fmt.Sprintf("%d", i))
|
|
|
|
ctxs[i] = metadata.NewOutgoingContext(context.Background(), md)
|
2016-09-20 01:11:11 +03:00
|
|
|
// limits the maximum number of outstanding watchers per stream
|
|
|
|
ctxc[i] = make(chan struct{}, 2)
|
|
|
|
}
|
2016-09-24 02:40:29 +03:00
|
|
|
|
|
|
|
// issue concurrent watches on "abc" with cancel
|
|
|
|
cli := clus.RandClient()
|
|
|
|
if _, err := cli.Put(context.TODO(), "abc", "def"); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2016-09-20 01:11:11 +03:00
|
|
|
ch := make(chan struct{}, n)
|
|
|
|
for i := 0; i < n; i++ {
|
|
|
|
go func() {
|
|
|
|
defer func() { ch <- struct{}{} }()
|
|
|
|
idx := rand.Intn(len(ctxs))
|
|
|
|
ctx, cancel := context.WithCancel(ctxs[idx])
|
|
|
|
ctxc[idx] <- struct{}{}
|
2016-09-24 02:40:29 +03:00
|
|
|
wch := cli.Watch(ctx, "abc", clientv3.WithRev(1))
|
|
|
|
f(clus)
|
|
|
|
select {
|
|
|
|
case _, ok := <-wch:
|
|
|
|
if !ok {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("unexpected closed channel %p", wch)
|
2016-09-24 02:40:29 +03:00
|
|
|
}
|
|
|
|
// may take a second or two to reestablish a watcher because of
|
2017-09-04 13:09:32 +03:00
|
|
|
// grpc back off policies for disconnects
|
2016-09-24 02:40:29 +03:00
|
|
|
case <-time.After(5 * time.Second):
|
|
|
|
t.Errorf("timed out waiting for watch on %p", wch)
|
2016-09-20 01:11:11 +03:00
|
|
|
}
|
|
|
|
// randomize how cancel overlaps with watch creation
|
|
|
|
if rand.Intn(2) == 0 {
|
|
|
|
<-ctxc[idx]
|
|
|
|
cancel()
|
|
|
|
} else {
|
|
|
|
cancel()
|
|
|
|
<-ctxc[idx]
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
// join on watches
|
|
|
|
for i := 0; i < n; i++ {
|
|
|
|
select {
|
|
|
|
case <-ch:
|
2016-09-24 02:40:29 +03:00
|
|
|
case <-time.After(5 * time.Second):
|
2016-09-20 01:11:11 +03:00
|
|
|
t.Fatalf("timed out waiting for completed watch")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-10-05 00:47:40 +03:00
|
|
|
|
2017-09-04 13:09:32 +03:00
|
|
|
// TestWatchCancelAndCloseClient ensures that canceling a watcher then immediately
|
2016-10-05 00:47:40 +03:00
|
|
|
// closing the client does not return a client closing error.
|
|
|
|
func TestWatchCancelAndCloseClient(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
cli := clus.Client(0)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
wch := cli.Watch(ctx, "abc")
|
|
|
|
donec := make(chan struct{})
|
|
|
|
go func() {
|
|
|
|
defer close(donec)
|
|
|
|
select {
|
|
|
|
case wr, ok := <-wch:
|
|
|
|
if ok {
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Errorf("expected closed watch after cancel(), got resp=%+v err=%v", wr, wr.Err())
|
2016-10-05 00:47:40 +03:00
|
|
|
}
|
|
|
|
case <-time.After(5 * time.Second):
|
2019-02-01 22:17:58 +03:00
|
|
|
t.Error("timed out waiting for closed channel")
|
2016-10-05 00:47:40 +03:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
cancel()
|
|
|
|
if err := cli.Close(); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
<-donec
|
|
|
|
clus.TakeClient(0)
|
|
|
|
}
|
2016-10-13 02:42:27 +03:00
|
|
|
|
|
|
|
// TestWatchStressResumeClose establishes a bunch of watchers, disconnects
|
|
|
|
// to put them in resuming mode, cancels them so some resumes by cancel fail,
|
|
|
|
// then closes the watcher interface to ensure correct clean up.
|
|
|
|
func TestWatchStressResumeClose(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
cli := clus.Client(0)
|
|
|
|
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
// add more watches than can be resumed before the cancel
|
|
|
|
wchs := make([]clientv3.WatchChan, 2000)
|
|
|
|
for i := range wchs {
|
|
|
|
wchs[i] = cli.Watch(ctx, "abc")
|
|
|
|
}
|
|
|
|
clus.Members[0].DropConnections()
|
|
|
|
cancel()
|
|
|
|
if err := cli.Close(); err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
clus.TakeClient(0)
|
|
|
|
}
|
2016-11-07 23:04:41 +03:00
|
|
|
|
|
|
|
// TestWatchCancelDisconnected ensures canceling a watcher works when
|
|
|
|
// its grpc stream is disconnected / reconnecting.
|
|
|
|
func TestWatchCancelDisconnected(t *testing.T) {
|
|
|
|
defer testutil.AfterTest(t)
|
|
|
|
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
|
|
|
|
defer clus.Terminate(t)
|
|
|
|
cli := clus.Client(0)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
// add more watches than can be resumed before the cancel
|
|
|
|
wch := cli.Watch(ctx, "abc")
|
|
|
|
clus.Members[0].Stop(t)
|
|
|
|
cancel()
|
|
|
|
select {
|
|
|
|
case <-wch:
|
|
|
|
case <-time.After(time.Second):
|
|
|
|
t.Fatal("took too long to cancel disconnected watcher")
|
|
|
|
}
|
|
|
|
}
|
2019-01-20 19:39:28 +03:00
|
|
|
|
|
|
|
// TestWatchClose ensures that close does not return error
|
|
|
|
func TestWatchClose(t *testing.T) {
|
|
|
|
runWatchTest(t, testWatchClose)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testWatchClose(t *testing.T, wctx *watchctx) {
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
wch := wctx.w.Watch(ctx, "a")
|
|
|
|
cancel()
|
|
|
|
if wch == nil {
|
|
|
|
t.Fatalf("expected watcher channel, got nil")
|
|
|
|
}
|
|
|
|
if wctx.w.Close() != nil {
|
|
|
|
t.Fatalf("watch did not close successfully")
|
|
|
|
}
|
|
|
|
wresp, ok := <-wch
|
|
|
|
if ok {
|
|
|
|
t.Fatalf("read wch got %v; expected closed channel", wresp)
|
|
|
|
}
|
|
|
|
}
|