2016-06-08 02:28:08 +03:00
|
|
|
// Copyright 2016 The etcd Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package clientv3
|
|
|
|
|
|
|
|
import (
|
|
|
|
"net/url"
|
|
|
|
"strings"
|
2016-07-01 21:29:37 +03:00
|
|
|
"sync"
|
2016-06-08 02:28:08 +03:00
|
|
|
|
|
|
|
"golang.org/x/net/context"
|
|
|
|
"google.golang.org/grpc"
|
2016-10-23 07:40:10 +03:00
|
|
|
"google.golang.org/grpc/codes"
|
2016-06-08 02:28:08 +03:00
|
|
|
)
|
|
|
|
|
2016-10-15 00:05:34 +03:00
|
|
|
// ErrNoAddrAvilable is returned by Get() when the balancer does not have
|
|
|
|
// any active connection to endpoints at the time.
|
|
|
|
// This error is returned only when opts.BlockingWait is true.
|
2016-10-23 07:40:10 +03:00
|
|
|
var ErrNoAddrAvilable = grpc.Errorf(codes.Unavailable, "there is no address available")
|
2016-10-15 00:05:34 +03:00
|
|
|
|
2016-06-08 02:28:08 +03:00
|
|
|
// simpleBalancer does the bare minimum to expose multiple eps
|
|
|
|
// to the grpc reconnection code path
|
|
|
|
type simpleBalancer struct {
|
2016-07-01 21:29:37 +03:00
|
|
|
// addrs are the client's endpoints for grpc
|
|
|
|
addrs []grpc.Address
|
|
|
|
// notifyCh notifies grpc of the set of addresses for connecting
|
|
|
|
notifyCh chan []grpc.Address
|
|
|
|
|
|
|
|
// readyc closes once the first connection is up
|
|
|
|
readyc chan struct{}
|
|
|
|
readyOnce sync.Once
|
|
|
|
|
|
|
|
// mu protects upEps, pinAddr, and connectingAddr
|
|
|
|
mu sync.RWMutex
|
2017-01-14 01:31:27 +03:00
|
|
|
|
2016-07-01 21:29:37 +03:00
|
|
|
// upc closes when upEps transitions from empty to non-zero or the balancer closes.
|
|
|
|
upc chan struct{}
|
|
|
|
|
2017-03-09 00:39:11 +03:00
|
|
|
// downc closes when grpc calls down() on pinAddr
|
|
|
|
downc chan struct{}
|
|
|
|
|
|
|
|
// stopc is closed to signal updateNotifyLoop should stop.
|
|
|
|
stopc chan struct{}
|
|
|
|
|
|
|
|
// donec closes when all goroutines are exited
|
|
|
|
donec chan struct{}
|
|
|
|
|
2017-03-15 01:42:56 +03:00
|
|
|
// updateAddrsC notifies updateNotifyLoop to update addrs.
|
|
|
|
updateAddrsC chan struct{}
|
|
|
|
|
2016-09-19 22:36:01 +03:00
|
|
|
// grpc issues TLS cert checks using the string passed into dial so
|
|
|
|
// that string must be the host. To recover the full scheme://host URL,
|
|
|
|
// have a map from hosts to the original endpoint.
|
|
|
|
host2ep map[string]string
|
|
|
|
|
2016-07-01 21:29:37 +03:00
|
|
|
// pinAddr is the currently pinned address; set to the empty string on
|
|
|
|
// intialization and shutdown.
|
|
|
|
pinAddr string
|
2016-09-14 08:52:16 +03:00
|
|
|
|
|
|
|
closed bool
|
2016-06-08 02:28:08 +03:00
|
|
|
}
|
|
|
|
|
2016-07-01 21:29:37 +03:00
|
|
|
func newSimpleBalancer(eps []string) *simpleBalancer {
|
|
|
|
notifyCh := make(chan []grpc.Address, 1)
|
2016-06-08 02:28:08 +03:00
|
|
|
addrs := make([]grpc.Address, len(eps))
|
|
|
|
for i := range eps {
|
|
|
|
addrs[i].Addr = getHost(eps[i])
|
|
|
|
}
|
2016-07-01 21:29:37 +03:00
|
|
|
sb := &simpleBalancer{
|
2017-03-15 01:42:56 +03:00
|
|
|
addrs: addrs,
|
|
|
|
notifyCh: notifyCh,
|
|
|
|
readyc: make(chan struct{}),
|
|
|
|
upc: make(chan struct{}),
|
|
|
|
stopc: make(chan struct{}),
|
|
|
|
downc: make(chan struct{}),
|
|
|
|
donec: make(chan struct{}),
|
|
|
|
updateAddrsC: make(chan struct{}, 1),
|
|
|
|
host2ep: getHost2ep(eps),
|
2016-07-01 21:29:37 +03:00
|
|
|
}
|
2017-04-29 02:57:22 +03:00
|
|
|
close(sb.downc)
|
2017-03-09 00:39:11 +03:00
|
|
|
go sb.updateNotifyLoop()
|
2016-07-01 21:29:37 +03:00
|
|
|
return sb
|
|
|
|
}
|
|
|
|
|
2016-10-10 21:07:55 +03:00
|
|
|
func (b *simpleBalancer) Start(target string, config grpc.BalancerConfig) error { return nil }
|
2016-07-01 21:29:37 +03:00
|
|
|
|
2016-08-13 07:16:09 +03:00
|
|
|
func (b *simpleBalancer) ConnectNotify() <-chan struct{} {
|
|
|
|
b.mu.Lock()
|
|
|
|
defer b.mu.Unlock()
|
|
|
|
return b.upc
|
|
|
|
}
|
|
|
|
|
2016-09-19 22:36:01 +03:00
|
|
|
func (b *simpleBalancer) getEndpoint(host string) string {
|
|
|
|
b.mu.Lock()
|
|
|
|
defer b.mu.Unlock()
|
|
|
|
return b.host2ep[host]
|
|
|
|
}
|
|
|
|
|
|
|
|
func getHost2ep(eps []string) map[string]string {
|
|
|
|
hm := make(map[string]string, len(eps))
|
|
|
|
for i := range eps {
|
|
|
|
_, host, _ := parseEndpoint(eps[i])
|
|
|
|
hm[host] = eps[i]
|
|
|
|
}
|
|
|
|
return hm
|
|
|
|
}
|
|
|
|
|
|
|
|
func (b *simpleBalancer) updateAddrs(eps []string) {
|
2016-09-21 19:10:25 +03:00
|
|
|
np := getHost2ep(eps)
|
|
|
|
|
2016-09-19 22:36:01 +03:00
|
|
|
b.mu.Lock()
|
|
|
|
|
2016-09-21 19:10:25 +03:00
|
|
|
match := len(np) == len(b.host2ep)
|
|
|
|
for k, v := range np {
|
|
|
|
if b.host2ep[k] != v {
|
|
|
|
match = false
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if match {
|
|
|
|
// same endpoints, so no need to update address
|
2017-03-15 01:42:56 +03:00
|
|
|
b.mu.Unlock()
|
2016-09-21 19:10:25 +03:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
b.host2ep = np
|
2016-09-19 22:36:01 +03:00
|
|
|
|
|
|
|
addrs := make([]grpc.Address, 0, len(eps))
|
|
|
|
for i := range eps {
|
|
|
|
addrs = append(addrs, grpc.Address{Addr: getHost(eps[i])})
|
|
|
|
}
|
|
|
|
b.addrs = addrs
|
2017-03-15 01:42:56 +03:00
|
|
|
|
2017-01-26 23:12:08 +03:00
|
|
|
// updating notifyCh can trigger new connections,
|
2017-03-15 01:42:56 +03:00
|
|
|
// only update addrs if all connections are down
|
|
|
|
// or addrs does not include pinAddr.
|
|
|
|
update := !hasAddr(addrs, b.pinAddr)
|
|
|
|
b.mu.Unlock()
|
|
|
|
|
|
|
|
if update {
|
|
|
|
select {
|
|
|
|
case b.updateAddrsC <- struct{}{}:
|
|
|
|
case <-b.stopc:
|
|
|
|
}
|
2017-01-26 23:12:08 +03:00
|
|
|
}
|
2016-09-19 22:36:01 +03:00
|
|
|
}
|
|
|
|
|
2017-03-15 01:42:56 +03:00
|
|
|
func hasAddr(addrs []grpc.Address, targetAddr string) bool {
|
|
|
|
for _, addr := range addrs {
|
|
|
|
if targetAddr == addr.Addr {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2017-03-09 00:39:11 +03:00
|
|
|
func (b *simpleBalancer) updateNotifyLoop() {
|
|
|
|
defer close(b.donec)
|
|
|
|
|
|
|
|
for {
|
|
|
|
b.mu.RLock()
|
2017-04-29 02:57:22 +03:00
|
|
|
upc, downc, addr := b.upc, b.downc, b.pinAddr
|
2017-03-09 00:39:11 +03:00
|
|
|
b.mu.RUnlock()
|
2017-04-29 02:57:22 +03:00
|
|
|
// downc or upc should be closed
|
|
|
|
select {
|
|
|
|
case <-downc:
|
|
|
|
downc = nil
|
|
|
|
default:
|
|
|
|
}
|
2017-03-09 00:39:11 +03:00
|
|
|
select {
|
|
|
|
case <-upc:
|
2017-04-29 02:57:22 +03:00
|
|
|
upc = nil
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
switch {
|
|
|
|
case downc == nil && upc == nil:
|
|
|
|
// stale
|
|
|
|
select {
|
|
|
|
case <-b.stopc:
|
|
|
|
return
|
|
|
|
default:
|
2017-03-09 00:39:11 +03:00
|
|
|
}
|
2017-04-29 02:57:22 +03:00
|
|
|
case downc == nil:
|
|
|
|
b.notifyAddrs()
|
2017-03-09 00:39:11 +03:00
|
|
|
select {
|
2017-04-29 02:57:22 +03:00
|
|
|
case <-upc:
|
|
|
|
case <-b.updateAddrsC:
|
|
|
|
b.notifyAddrs()
|
|
|
|
case <-b.stopc:
|
|
|
|
return
|
|
|
|
}
|
|
|
|
case upc == nil:
|
|
|
|
select {
|
|
|
|
// close connections that are not the pinned address
|
2017-03-09 00:39:11 +03:00
|
|
|
case b.notifyCh <- []grpc.Address{{Addr: addr}}:
|
2017-04-29 02:57:22 +03:00
|
|
|
case <-downc:
|
|
|
|
case <-b.stopc:
|
|
|
|
return
|
|
|
|
}
|
|
|
|
select {
|
|
|
|
case <-downc:
|
|
|
|
case <-b.updateAddrsC:
|
2017-03-09 00:39:11 +03:00
|
|
|
case <-b.stopc:
|
|
|
|
return
|
|
|
|
}
|
2017-03-15 01:42:56 +03:00
|
|
|
b.notifyAddrs()
|
2017-03-09 00:39:11 +03:00
|
|
|
}
|
2017-03-15 01:42:56 +03:00
|
|
|
}
|
|
|
|
}
|
2017-03-09 00:39:11 +03:00
|
|
|
|
2017-03-15 01:42:56 +03:00
|
|
|
func (b *simpleBalancer) notifyAddrs() {
|
|
|
|
b.mu.RLock()
|
|
|
|
addrs := b.addrs
|
|
|
|
b.mu.RUnlock()
|
|
|
|
select {
|
|
|
|
case b.notifyCh <- addrs:
|
|
|
|
case <-b.stopc:
|
2017-03-09 00:39:11 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-07-01 21:29:37 +03:00
|
|
|
func (b *simpleBalancer) Up(addr grpc.Address) func(error) {
|
|
|
|
b.mu.Lock()
|
2016-09-14 08:52:16 +03:00
|
|
|
defer b.mu.Unlock()
|
|
|
|
|
|
|
|
// gRPC might call Up after it called Close. We add this check
|
|
|
|
// to "fix" it up at application layer. Or our simplerBalancer
|
|
|
|
// might panic since b.upc is closed.
|
|
|
|
if b.closed {
|
|
|
|
return func(err error) {}
|
|
|
|
}
|
2017-03-15 01:42:56 +03:00
|
|
|
// gRPC might call Up on a stale address.
|
|
|
|
// Prevent updating pinAddr with a stale address.
|
|
|
|
if !hasAddr(b.addrs, addr.Addr) {
|
|
|
|
return func(err error) {}
|
|
|
|
}
|
2017-04-29 02:57:22 +03:00
|
|
|
if b.pinAddr != "" {
|
|
|
|
return func(err error) {}
|
2016-07-01 21:29:37 +03:00
|
|
|
}
|
2017-04-29 02:57:22 +03:00
|
|
|
// notify waiting Get()s and pin first connected address
|
|
|
|
close(b.upc)
|
|
|
|
b.downc = make(chan struct{})
|
|
|
|
b.pinAddr = addr.Addr
|
|
|
|
// notify client that a connection is up
|
|
|
|
b.readyOnce.Do(func() { close(b.readyc) })
|
2016-07-01 21:29:37 +03:00
|
|
|
return func(err error) {
|
|
|
|
b.mu.Lock()
|
2017-04-29 02:57:22 +03:00
|
|
|
b.upc = make(chan struct{})
|
|
|
|
close(b.downc)
|
|
|
|
b.pinAddr = ""
|
2016-07-01 21:29:37 +03:00
|
|
|
b.mu.Unlock()
|
|
|
|
}
|
2016-06-08 02:28:08 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
func (b *simpleBalancer) Get(ctx context.Context, opts grpc.BalancerGetOptions) (grpc.Address, func(), error) {
|
2017-01-14 01:31:27 +03:00
|
|
|
var (
|
|
|
|
addr string
|
|
|
|
closed bool
|
|
|
|
)
|
2016-10-15 00:05:34 +03:00
|
|
|
|
|
|
|
// If opts.BlockingWait is false (for fail-fast RPCs), it should return
|
|
|
|
// an address it has notified via Notify immediately instead of blocking.
|
|
|
|
if !opts.BlockingWait {
|
|
|
|
b.mu.RLock()
|
2017-01-14 01:31:27 +03:00
|
|
|
closed = b.closed
|
2016-10-15 00:05:34 +03:00
|
|
|
addr = b.pinAddr
|
|
|
|
b.mu.RUnlock()
|
2016-10-23 07:40:10 +03:00
|
|
|
if closed {
|
|
|
|
return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
|
|
|
|
}
|
2017-01-14 01:31:27 +03:00
|
|
|
if addr == "" {
|
2016-10-15 00:05:34 +03:00
|
|
|
return grpc.Address{Addr: ""}, nil, ErrNoAddrAvilable
|
|
|
|
}
|
|
|
|
return grpc.Address{Addr: addr}, func() {}, nil
|
|
|
|
}
|
|
|
|
|
2016-07-01 21:29:37 +03:00
|
|
|
for {
|
|
|
|
b.mu.RLock()
|
|
|
|
ch := b.upc
|
|
|
|
b.mu.RUnlock()
|
|
|
|
select {
|
|
|
|
case <-ch:
|
2017-04-29 02:57:22 +03:00
|
|
|
case <-b.donec:
|
|
|
|
return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
|
2016-07-01 21:29:37 +03:00
|
|
|
case <-ctx.Done():
|
|
|
|
return grpc.Address{Addr: ""}, nil, ctx.Err()
|
|
|
|
}
|
|
|
|
b.mu.RLock()
|
2017-01-14 01:31:27 +03:00
|
|
|
closed = b.closed
|
2016-07-01 21:29:37 +03:00
|
|
|
addr = b.pinAddr
|
|
|
|
b.mu.RUnlock()
|
2017-01-14 01:31:27 +03:00
|
|
|
// Close() which sets b.closed = true can be called before Get(), Get() must exit if balancer is closed.
|
|
|
|
if closed {
|
2016-07-01 21:29:37 +03:00
|
|
|
return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
|
|
|
|
}
|
2017-01-14 01:31:27 +03:00
|
|
|
if addr != "" {
|
2016-07-01 21:29:37 +03:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return grpc.Address{Addr: addr}, func() {}, nil
|
2016-06-08 02:28:08 +03:00
|
|
|
}
|
2016-07-01 21:29:37 +03:00
|
|
|
|
|
|
|
func (b *simpleBalancer) Notify() <-chan []grpc.Address { return b.notifyCh }
|
|
|
|
|
2016-06-08 02:28:08 +03:00
|
|
|
func (b *simpleBalancer) Close() error {
|
2016-07-01 21:29:37 +03:00
|
|
|
b.mu.Lock()
|
2016-09-14 08:52:16 +03:00
|
|
|
// In case gRPC calls close twice. TODO: remove the checking
|
|
|
|
// when we are sure that gRPC wont call close twice.
|
|
|
|
if b.closed {
|
2017-03-09 00:39:11 +03:00
|
|
|
b.mu.Unlock()
|
|
|
|
<-b.donec
|
2016-09-14 08:52:16 +03:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
b.closed = true
|
2017-03-09 00:39:11 +03:00
|
|
|
close(b.stopc)
|
2016-07-01 21:29:37 +03:00
|
|
|
b.pinAddr = ""
|
2017-01-14 01:31:27 +03:00
|
|
|
|
2017-01-26 23:12:08 +03:00
|
|
|
// In the case of following scenario:
|
2017-01-14 01:31:27 +03:00
|
|
|
// 1. upc is not closed; no pinned address
|
|
|
|
// 2. client issues an rpc, calling invoke(), which calls Get(), enters for loop, blocks
|
|
|
|
// 3. clientconn.Close() calls balancer.Close(); closed = true
|
|
|
|
// 4. for loop in Get() never exits since ctx is the context passed in by the client and may not be canceled
|
|
|
|
// we must close upc so Get() exits from blocking on upc
|
|
|
|
select {
|
|
|
|
case <-b.upc:
|
|
|
|
default:
|
|
|
|
// terminate all waiting Get()s
|
2016-07-01 21:29:37 +03:00
|
|
|
close(b.upc)
|
|
|
|
}
|
2017-03-09 00:39:11 +03:00
|
|
|
|
|
|
|
b.mu.Unlock()
|
|
|
|
|
|
|
|
// wait for updateNotifyLoop to finish
|
|
|
|
<-b.donec
|
|
|
|
close(b.notifyCh)
|
|
|
|
|
2016-06-08 02:28:08 +03:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func getHost(ep string) string {
|
|
|
|
url, uerr := url.Parse(ep)
|
|
|
|
if uerr != nil || !strings.Contains(ep, "://") {
|
|
|
|
return ep
|
|
|
|
}
|
|
|
|
return url.Host
|
|
|
|
}
|