Merge pull request #7084 from heyitsanthony/watch-proxy-leak
integration: wait for watch proxy to finish on client closerelease-3.1
commit
24601ca24b
|
@ -104,7 +104,7 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
kvp := grpcproxy.NewKvProxy(client)
|
kvp := grpcproxy.NewKvProxy(client)
|
||||||
watchp := grpcproxy.NewWatchProxy(client)
|
watchp, _ := grpcproxy.NewWatchProxy(client)
|
||||||
clusterp := grpcproxy.NewClusterProxy(client)
|
clusterp := grpcproxy.NewClusterProxy(client)
|
||||||
leasep := grpcproxy.NewLeaseProxy(client)
|
leasep := grpcproxy.NewLeaseProxy(client)
|
||||||
mainp := grpcproxy.NewMaintenanceProxy(client)
|
mainp := grpcproxy.NewMaintenanceProxy(client)
|
||||||
|
|
|
@ -26,25 +26,43 @@ import (
|
||||||
|
|
||||||
var (
|
var (
|
||||||
pmu sync.Mutex
|
pmu sync.Mutex
|
||||||
proxies map[*clientv3.Client]grpcAPI = make(map[*clientv3.Client]grpcAPI)
|
proxies map[*clientv3.Client]grpcClientProxy = make(map[*clientv3.Client]grpcClientProxy)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type grpcClientProxy struct {
|
||||||
|
grpc grpcAPI
|
||||||
|
wdonec <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
func toGRPC(c *clientv3.Client) grpcAPI {
|
func toGRPC(c *clientv3.Client) grpcAPI {
|
||||||
pmu.Lock()
|
pmu.Lock()
|
||||||
defer pmu.Unlock()
|
defer pmu.Unlock()
|
||||||
|
|
||||||
if v, ok := proxies[c]; ok {
|
if v, ok := proxies[c]; ok {
|
||||||
return v
|
return v.grpc
|
||||||
}
|
}
|
||||||
api := grpcAPI{
|
|
||||||
|
wp, wpch := grpcproxy.NewWatchProxy(c)
|
||||||
|
grpc := grpcAPI{
|
||||||
pb.NewClusterClient(c.ActiveConnection()),
|
pb.NewClusterClient(c.ActiveConnection()),
|
||||||
grpcproxy.KvServerToKvClient(grpcproxy.NewKvProxy(c)),
|
grpcproxy.KvServerToKvClient(grpcproxy.NewKvProxy(c)),
|
||||||
pb.NewLeaseClient(c.ActiveConnection()),
|
pb.NewLeaseClient(c.ActiveConnection()),
|
||||||
grpcproxy.WatchServerToWatchClient(grpcproxy.NewWatchProxy(c)),
|
grpcproxy.WatchServerToWatchClient(wp),
|
||||||
pb.NewMaintenanceClient(c.ActiveConnection()),
|
pb.NewMaintenanceClient(c.ActiveConnection()),
|
||||||
}
|
}
|
||||||
proxies[c] = api
|
proxies[c] = grpcClientProxy{grpc: grpc, wdonec: wpch}
|
||||||
return api
|
return grpc
|
||||||
|
}
|
||||||
|
|
||||||
|
type watchCloser struct {
|
||||||
|
clientv3.Watcher
|
||||||
|
wdonec <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (wc *watchCloser) Close() error {
|
||||||
|
err := wc.Watcher.Close()
|
||||||
|
<-wc.wdonec
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func newClientV3(cfg clientv3.Config) (*clientv3.Client, error) {
|
func newClientV3(cfg clientv3.Config) (*clientv3.Client, error) {
|
||||||
|
@ -54,6 +72,11 @@ func newClientV3(cfg clientv3.Config) (*clientv3.Client, error) {
|
||||||
}
|
}
|
||||||
rpc := toGRPC(c)
|
rpc := toGRPC(c)
|
||||||
c.KV = clientv3.NewKVFromKVClient(rpc.KV)
|
c.KV = clientv3.NewKVFromKVClient(rpc.KV)
|
||||||
c.Watcher = clientv3.NewWatchFromWatchClient(rpc.Watch)
|
pmu.Lock()
|
||||||
|
c.Watcher = &watchCloser{
|
||||||
|
Watcher: clientv3.NewWatchFromWatchClient(rpc.Watch),
|
||||||
|
wdonec: proxies[c].wdonec,
|
||||||
|
}
|
||||||
|
pmu.Unlock()
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -407,6 +407,7 @@ func TestV3WatchCancelUnsynced(t *testing.T) {
|
||||||
|
|
||||||
func testV3WatchCancel(t *testing.T, startRev int64) {
|
func testV3WatchCancel(t *testing.T, startRev int64) {
|
||||||
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
|
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
|
||||||
|
defer clus.Terminate(t)
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
@ -455,8 +456,6 @@ func testV3WatchCancel(t *testing.T, startRev int64) {
|
||||||
if !rok {
|
if !rok {
|
||||||
t.Errorf("unexpected pb.WatchResponse is received %+v", nr)
|
t.Errorf("unexpected pb.WatchResponse is received %+v", nr)
|
||||||
}
|
}
|
||||||
|
|
||||||
clus.Terminate(t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestV3WatchCurrentPutOverlap ensures current watchers receive all events with
|
// TestV3WatchCurrentPutOverlap ensures current watchers receive all events with
|
||||||
|
@ -541,7 +540,10 @@ func TestV3WatchCurrentPutOverlap(t *testing.T) {
|
||||||
|
|
||||||
// TestV3WatchEmptyKey ensures synced watchers see empty key PUTs as PUT events
|
// TestV3WatchEmptyKey ensures synced watchers see empty key PUTs as PUT events
|
||||||
func TestV3WatchEmptyKey(t *testing.T) {
|
func TestV3WatchEmptyKey(t *testing.T) {
|
||||||
|
defer testutil.AfterTest(t)
|
||||||
|
|
||||||
clus := NewClusterV3(t, &ClusterConfig{Size: 1})
|
clus := NewClusterV3(t, &ClusterConfig{Size: 1})
|
||||||
|
defer clus.Terminate(t)
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
@ -581,8 +583,6 @@ func TestV3WatchEmptyKey(t *testing.T) {
|
||||||
if !reflect.DeepEqual(resp.Events, wevs) {
|
if !reflect.DeepEqual(resp.Events, wevs) {
|
||||||
t.Fatalf("got %v, expected %v", resp.Events, wevs)
|
t.Fatalf("got %v, expected %v", resp.Events, wevs)
|
||||||
}
|
}
|
||||||
|
|
||||||
clus.Terminate(t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestV3WatchMultipleWatchersSynced(t *testing.T) {
|
func TestV3WatchMultipleWatchersSynced(t *testing.T) {
|
||||||
|
@ -601,6 +601,8 @@ func TestV3WatchMultipleWatchersUnsynced(t *testing.T) {
|
||||||
// one watcher to test if it receives expected events.
|
// one watcher to test if it receives expected events.
|
||||||
func testV3WatchMultipleWatchers(t *testing.T, startRev int64) {
|
func testV3WatchMultipleWatchers(t *testing.T, startRev int64) {
|
||||||
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
|
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
|
||||||
|
defer clus.Terminate(t)
|
||||||
|
|
||||||
kvc := toGRPC(clus.RandClient()).KV
|
kvc := toGRPC(clus.RandClient()).KV
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
@ -686,8 +688,6 @@ func testV3WatchMultipleWatchers(t *testing.T, startRev int64) {
|
||||||
if !rok {
|
if !rok {
|
||||||
t.Errorf("unexpected pb.WatchResponse is received %+v", nr)
|
t.Errorf("unexpected pb.WatchResponse is received %+v", nr)
|
||||||
}
|
}
|
||||||
|
|
||||||
clus.Terminate(t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestV3WatchMultipleEventsTxnSynced(t *testing.T) {
|
func TestV3WatchMultipleEventsTxnSynced(t *testing.T) {
|
||||||
|
@ -703,6 +703,7 @@ func TestV3WatchMultipleEventsTxnUnsynced(t *testing.T) {
|
||||||
// testV3WatchMultipleEventsTxn tests Watch APIs when it receives multiple events.
|
// testV3WatchMultipleEventsTxn tests Watch APIs when it receives multiple events.
|
||||||
func testV3WatchMultipleEventsTxn(t *testing.T, startRev int64) {
|
func testV3WatchMultipleEventsTxn(t *testing.T, startRev int64) {
|
||||||
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
|
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
|
||||||
|
defer clus.Terminate(t)
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
@ -772,9 +773,6 @@ func testV3WatchMultipleEventsTxn(t *testing.T, startRev int64) {
|
||||||
if !rok {
|
if !rok {
|
||||||
t.Errorf("unexpected pb.WatchResponse is received %+v", nr)
|
t.Errorf("unexpected pb.WatchResponse is received %+v", nr)
|
||||||
}
|
}
|
||||||
|
|
||||||
// can't defer because tcp ports will be in use
|
|
||||||
clus.Terminate(t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type eventsSortByKey []*mvccpb.Event
|
type eventsSortByKey []*mvccpb.Event
|
||||||
|
@ -875,6 +873,8 @@ func TestV3WatchMultipleStreamsUnsynced(t *testing.T) {
|
||||||
// testV3WatchMultipleStreams tests multiple watchers on the same key on multiple streams.
|
// testV3WatchMultipleStreams tests multiple watchers on the same key on multiple streams.
|
||||||
func testV3WatchMultipleStreams(t *testing.T, startRev int64) {
|
func testV3WatchMultipleStreams(t *testing.T, startRev int64) {
|
||||||
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
|
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
|
||||||
|
defer clus.Terminate(t)
|
||||||
|
|
||||||
wAPI := toGRPC(clus.RandClient()).Watch
|
wAPI := toGRPC(clus.RandClient()).Watch
|
||||||
kvc := toGRPC(clus.RandClient()).KV
|
kvc := toGRPC(clus.RandClient()).KV
|
||||||
|
|
||||||
|
@ -939,8 +939,6 @@ func testV3WatchMultipleStreams(t *testing.T, startRev int64) {
|
||||||
}(i)
|
}(i)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
clus.Terminate(t)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitResponse waits on the given stream for given duration.
|
// waitResponse waits on the given stream for given duration.
|
||||||
|
|
|
@ -49,7 +49,7 @@ const (
|
||||||
retryPerSecond = 10
|
retryPerSecond = 10
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewWatchProxy(c *clientv3.Client) pb.WatchServer {
|
func NewWatchProxy(c *clientv3.Client) (pb.WatchServer, <-chan struct{}) {
|
||||||
wp := &watchProxy{
|
wp := &watchProxy{
|
||||||
cw: c.Watcher,
|
cw: c.Watcher,
|
||||||
ctx: clientv3.WithRequireLeader(c.Ctx()),
|
ctx: clientv3.WithRequireLeader(c.Ctx()),
|
||||||
|
@ -57,7 +57,9 @@ func NewWatchProxy(c *clientv3.Client) pb.WatchServer {
|
||||||
leaderc: make(chan struct{}),
|
leaderc: make(chan struct{}),
|
||||||
}
|
}
|
||||||
wp.ranges = newWatchRanges(wp)
|
wp.ranges = newWatchRanges(wp)
|
||||||
|
ch := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
|
defer close(ch)
|
||||||
// a new streams without opening any watchers won't catch
|
// a new streams without opening any watchers won't catch
|
||||||
// a lost leader event, so have a special watch to monitor it
|
// a lost leader event, so have a special watch to monitor it
|
||||||
rev := int64((uint64(1) << 63) - 2)
|
rev := int64((uint64(1) << 63) - 2)
|
||||||
|
@ -77,7 +79,7 @@ func NewWatchProxy(c *clientv3.Client) pb.WatchServer {
|
||||||
wp.wg.Wait()
|
wp.wg.Wait()
|
||||||
wp.ranges.stop()
|
wp.ranges.stop()
|
||||||
}()
|
}()
|
||||||
return wp
|
return wp, ch
|
||||||
}
|
}
|
||||||
|
|
||||||
func (wp *watchProxy) Watch(stream pb.Watch_WatchServer) (err error) {
|
func (wp *watchProxy) Watch(stream pb.Watch_WatchServer) (err error) {
|
||||||
|
|
Loading…
Reference in New Issue