2017-11-08 01:13:19 +03:00
// Copyright 2017 The etcd Authors
2016-10-15 00:05:34 +03:00
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clientv3
import (
2017-09-07 00:57:25 +03:00
"context"
2016-10-15 00:05:34 +03:00
"errors"
2017-03-09 00:39:11 +03:00
"net"
"sync"
2016-10-15 00:05:34 +03:00
"testing"
"time"
2017-03-09 00:39:11 +03:00
pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
"github.com/coreos/etcd/pkg/testutil"
2016-10-15 00:05:34 +03:00
"google.golang.org/grpc"
)
2017-11-08 01:13:19 +03:00
var endpoints = [ ] string { "localhost:2379" , "localhost:22379" , "localhost:32379" }
2016-10-15 00:05:34 +03:00
func TestBalancerGetUnblocking ( t * testing . T ) {
2017-11-08 01:13:19 +03:00
hb := newHealthBalancer ( endpoints , minHealthRetryDuration , func ( string ) ( bool , error ) { return true , nil } )
defer hb . Close ( )
if addrs := <- hb . Notify ( ) ; len ( addrs ) != len ( endpoints ) {
t . Errorf ( "Initialize newHealthBalancer should have triggered Notify() chan, but it didn't" )
2017-01-14 01:31:27 +03:00
}
2016-10-15 00:05:34 +03:00
unblockingOpts := grpc . BalancerGetOptions { BlockingWait : false }
2017-11-08 01:13:19 +03:00
_ , _ , err := hb . Get ( context . Background ( ) , unblockingOpts )
2016-10-15 00:05:34 +03:00
if err != ErrNoAddrAvilable {
t . Errorf ( "Get() with no up endpoints should return ErrNoAddrAvailable, got: %v" , err )
}
2017-11-08 01:13:19 +03:00
down1 := hb . Up ( grpc . Address { Addr : endpoints [ 1 ] } )
if addrs := <- hb . Notify ( ) ; len ( addrs ) != 1 {
2017-01-14 01:31:27 +03:00
t . Errorf ( "first Up() should have triggered balancer to send the first connected address via Notify chan so that other connections can be closed" )
}
2017-11-08 01:13:19 +03:00
down2 := hb . Up ( grpc . Address { Addr : endpoints [ 2 ] } )
addrFirst , putFun , err := hb . Get ( context . Background ( ) , unblockingOpts )
2016-10-15 00:05:34 +03:00
if err != nil {
2016-10-31 19:47:15 +03:00
t . Errorf ( "Get() with up endpoints should success, got %v" , err )
2016-10-15 00:05:34 +03:00
}
2017-01-13 00:07:44 +03:00
if addrFirst . Addr != endpoints [ 1 ] {
2016-10-15 00:05:34 +03:00
t . Errorf ( "Get() didn't return expected address, got %v" , addrFirst )
}
if putFun == nil {
t . Errorf ( "Get() returned unexpected nil put function" )
}
2017-11-08 01:13:19 +03:00
addrSecond , _ , _ := hb . Get ( context . Background ( ) , unblockingOpts )
2017-01-13 00:07:44 +03:00
if addrFirst . Addr != addrSecond . Addr {
2016-10-15 00:05:34 +03:00
t . Errorf ( "Get() didn't return the same address as previous call, got %v and %v" , addrFirst , addrSecond )
}
down1 ( errors . New ( "error" ) )
2017-11-11 00:44:45 +03:00
if addrs := <- hb . Notify ( ) ; len ( addrs ) != len ( endpoints ) - 1 { // we call down on one endpoint
t . Errorf ( "closing the only connection should triggered balancer to send the %d endpoints via Notify chan so that we can establish a connection" , len ( endpoints ) - 1 )
2017-01-14 01:31:27 +03:00
}
2016-10-15 00:05:34 +03:00
down2 ( errors . New ( "error" ) )
2017-11-08 01:13:19 +03:00
_ , _ , err = hb . Get ( context . Background ( ) , unblockingOpts )
2016-10-15 00:05:34 +03:00
if err != ErrNoAddrAvilable {
t . Errorf ( "Get() with no up endpoints should return ErrNoAddrAvailable, got: %v" , err )
}
}
func TestBalancerGetBlocking ( t * testing . T ) {
2017-11-08 01:13:19 +03:00
hb := newHealthBalancer ( endpoints , minHealthRetryDuration , func ( string ) ( bool , error ) { return true , nil } )
defer hb . Close ( )
if addrs := <- hb . Notify ( ) ; len ( addrs ) != len ( endpoints ) {
t . Errorf ( "Initialize newHealthBalancer should have triggered Notify() chan, but it didn't" )
2017-01-14 01:31:27 +03:00
}
2016-10-15 00:05:34 +03:00
blockingOpts := grpc . BalancerGetOptions { BlockingWait : true }
2017-09-07 01:58:07 +03:00
ctx , cancel := context . WithTimeout ( context . Background ( ) , time . Millisecond * 100 )
2017-11-08 01:13:19 +03:00
_ , _ , err := hb . Get ( ctx , blockingOpts )
2017-09-07 01:58:07 +03:00
cancel ( )
2016-10-15 00:05:34 +03:00
if err != context . DeadlineExceeded {
t . Errorf ( "Get() with no up endpoints should timeout, got %v" , err )
}
downC := make ( chan func ( error ) , 1 )
go func ( ) {
2017-11-08 01:13:19 +03:00
// ensure hb.Up() will be called after hb.Get() to see if Up() releases blocking Get()
2016-10-15 00:05:34 +03:00
time . Sleep ( time . Millisecond * 100 )
2017-11-08 01:13:19 +03:00
f := hb . Up ( grpc . Address { Addr : endpoints [ 1 ] } )
if addrs := <- hb . Notify ( ) ; len ( addrs ) != 1 {
2017-01-14 01:31:27 +03:00
t . Errorf ( "first Up() should have triggered balancer to send the first connected address via Notify chan so that other connections can be closed" )
}
2017-03-09 23:13:42 +03:00
downC <- f
2016-10-15 00:05:34 +03:00
} ( )
2017-11-08 01:13:19 +03:00
addrFirst , putFun , err := hb . Get ( context . Background ( ) , blockingOpts )
2016-10-15 00:05:34 +03:00
if err != nil {
2016-10-31 19:47:15 +03:00
t . Errorf ( "Get() with up endpoints should success, got %v" , err )
2016-10-15 00:05:34 +03:00
}
if addrFirst . Addr != endpoints [ 1 ] {
t . Errorf ( "Get() didn't return expected address, got %v" , addrFirst )
}
if putFun == nil {
t . Errorf ( "Get() returned unexpected nil put function" )
}
down1 := <- downC
2017-11-08 01:13:19 +03:00
down2 := hb . Up ( grpc . Address { Addr : endpoints [ 2 ] } )
addrSecond , _ , _ := hb . Get ( context . Background ( ) , blockingOpts )
2017-01-13 00:07:44 +03:00
if addrFirst . Addr != addrSecond . Addr {
2016-10-15 00:05:34 +03:00
t . Errorf ( "Get() didn't return the same address as previous call, got %v and %v" , addrFirst , addrSecond )
}
down1 ( errors . New ( "error" ) )
2017-11-11 00:44:45 +03:00
if addrs := <- hb . Notify ( ) ; len ( addrs ) != len ( endpoints ) - 1 { // we call down on one endpoint
t . Errorf ( "closing the only connection should triggered balancer to send the %d endpoints via Notify chan so that we can establish a connection" , len ( endpoints ) - 1 )
2017-01-14 01:31:27 +03:00
}
2016-10-15 00:05:34 +03:00
down2 ( errors . New ( "error" ) )
2017-09-07 01:58:07 +03:00
ctx , cancel = context . WithTimeout ( context . Background ( ) , time . Millisecond * 100 )
2017-11-08 01:13:19 +03:00
_ , _ , err = hb . Get ( ctx , blockingOpts )
2017-09-07 01:58:07 +03:00
cancel ( )
2016-10-15 00:05:34 +03:00
if err != context . DeadlineExceeded {
t . Errorf ( "Get() with no up endpoints should timeout, got %v" , err )
}
}
2017-03-09 00:39:11 +03:00
2017-09-12 12:13:47 +03:00
// TestHealthBalancerGraylist checks one endpoint is tried after the other
// due to gray listing.
func TestHealthBalancerGraylist ( t * testing . T ) {
var wg sync . WaitGroup
// Use 3 endpoints so gray list doesn't fallback to all connections
// after failing on 2 endpoints.
lns , eps := make ( [ ] net . Listener , 3 ) , make ( [ ] string , 3 )
wg . Add ( 3 )
connc := make ( chan string , 2 )
for i := range eps {
ln , err := net . Listen ( "tcp" , ":0" )
testutil . AssertNil ( t , err )
lns [ i ] , eps [ i ] = ln , ln . Addr ( ) . String ( )
go func ( ) {
defer wg . Done ( )
for {
conn , err := ln . Accept ( )
if err != nil {
return
}
_ , err = conn . Read ( make ( [ ] byte , 512 ) )
conn . Close ( )
if err == nil {
select {
case connc <- ln . Addr ( ) . String ( ) :
// sleep some so balancer catches up
// before attempted next reconnect.
time . Sleep ( 50 * time . Millisecond )
default :
}
}
}
} ( )
}
tf := func ( s string ) ( bool , error ) { return false , nil }
2017-11-08 01:13:19 +03:00
hb := newHealthBalancer ( eps , 5 * time . Second , tf )
2017-09-12 12:13:47 +03:00
conn , err := grpc . Dial ( "" , grpc . WithInsecure ( ) , grpc . WithBalancer ( hb ) )
testutil . AssertNil ( t , err )
defer conn . Close ( )
kvc := pb . NewKVClient ( conn )
<- hb . ready ( )
kvc . Range ( context . TODO ( ) , & pb . RangeRequest { } )
ep1 := <- connc
kvc . Range ( context . TODO ( ) , & pb . RangeRequest { } )
ep2 := <- connc
for _ , ln := range lns {
ln . Close ( )
}
wg . Wait ( )
if ep1 == ep2 {
t . Fatalf ( "expected %q != %q" , ep1 , ep2 )
}
}
2017-03-09 00:39:11 +03:00
// TestBalancerDoNotBlockOnClose ensures that balancer and grpc don't deadlock each other
// due to rapid open/close conn. The deadlock causes balancer.Close() to block forever.
// See issue: https://github.com/coreos/etcd/issues/7283 for more detail.
func TestBalancerDoNotBlockOnClose ( t * testing . T ) {
defer testutil . AfterTest ( t )
kcl := newKillConnListener ( t , 3 )
defer kcl . close ( )
for i := 0 ; i < 5 ; i ++ {
2017-11-08 01:13:19 +03:00
hb := newHealthBalancer ( kcl . endpoints ( ) , minHealthRetryDuration , func ( string ) ( bool , error ) { return true , nil } )
conn , err := grpc . Dial ( "" , grpc . WithInsecure ( ) , grpc . WithBalancer ( hb ) )
2017-03-09 00:39:11 +03:00
if err != nil {
t . Fatal ( err )
}
kvc := pb . NewKVClient ( conn )
2017-11-08 01:13:19 +03:00
<- hb . readyc
2017-03-15 00:03:32 +03:00
var wg sync . WaitGroup
wg . Add ( 100 )
cctx , cancel := context . WithCancel ( context . TODO ( ) )
2017-03-09 00:39:11 +03:00
for j := 0 ; j < 100 ; j ++ {
2017-03-15 00:03:32 +03:00
go func ( ) {
defer wg . Done ( )
kvc . Range ( cctx , & pb . RangeRequest { } , grpc . FailFast ( false ) )
} ( )
2017-03-09 00:39:11 +03:00
}
// balancer.Close() might block
// if balancer and grpc deadlock each other.
2017-03-15 00:03:32 +03:00
bclosec , cclosec := make ( chan struct { } ) , make ( chan struct { } )
2017-03-09 00:39:11 +03:00
go func ( ) {
2017-03-15 00:03:32 +03:00
defer close ( bclosec )
2017-11-08 01:13:19 +03:00
hb . Close ( )
2017-03-09 00:39:11 +03:00
} ( )
2017-03-15 00:03:32 +03:00
go func ( ) {
defer close ( cclosec )
conn . Close ( )
} ( )
2017-03-09 00:39:11 +03:00
select {
2017-03-15 00:03:32 +03:00
case <- bclosec :
2017-03-09 00:39:11 +03:00
case <- time . After ( 3 * time . Second ) :
testutil . FatalStack ( t , "balancer close timeout" )
}
2017-03-15 00:03:32 +03:00
select {
case <- cclosec :
case <- time . After ( 3 * time . Second ) :
t . Fatal ( "grpc conn close timeout" )
}
cancel ( )
wg . Wait ( )
2017-03-09 00:39:11 +03:00
}
}
// killConnListener listens incoming conn and kills it immediately.
type killConnListener struct {
wg sync . WaitGroup
eps [ ] string
stopc chan struct { }
t * testing . T
}
func newKillConnListener ( t * testing . T , size int ) * killConnListener {
kcl := & killConnListener { stopc : make ( chan struct { } ) , t : t }
for i := 0 ; i < size ; i ++ {
ln , err := net . Listen ( "tcp" , ":0" )
if err != nil {
t . Fatal ( err )
}
kcl . eps = append ( kcl . eps , ln . Addr ( ) . String ( ) )
kcl . wg . Add ( 1 )
go kcl . listen ( ln )
}
return kcl
}
func ( kcl * killConnListener ) endpoints ( ) [ ] string {
return kcl . eps
}
func ( kcl * killConnListener ) listen ( l net . Listener ) {
go func ( ) {
defer kcl . wg . Done ( )
for {
conn , err := l . Accept ( )
select {
case <- kcl . stopc :
return
default :
}
if err != nil {
kcl . t . Fatal ( err )
}
time . Sleep ( 1 * time . Millisecond )
conn . Close ( )
}
} ( )
<- kcl . stopc
l . Close ( )
}
func ( kcl * killConnListener ) close ( ) {
close ( kcl . stopc )
kcl . wg . Wait ( )
}