version: 3.3.26

Signed-off-by: Sam Batschelet <sbatsche@redhat.com>

.travis.yml

@@ -6,7 +6,7 @@ sudo: required
 services: docker
 
 go:
-- 1.12.9
+- 1.12.12
 
 env:
 - GO111MODULE=on
@@ -28,7 +28,7 @@ env:
 matrix:
   fast_finish: true
   allow_failures:
-  - go: 1.12.9
+  - go: 1.12.12
     env: TARGET=linux-386-unit
 
 install:

.words

@@ -25,6 +25,8 @@ healthcheck
 iff
 inflight
 keepalive
+hasleader
+racey
 keepalives
 keyspace
 linearization
@@ -41,4 +43,4 @@ too_many_pings
 uncontended
 unprefixed
 unlisting
-
+WithDialer

Documentation/op-guide/authentication.md

@@ -174,3 +174,5 @@ As of version v3.2 if an etcd server is launched with the option `--client-cert-
 As of version v3.3 if an etcd server is launched with the option `--peer-cert-allowed-cn` filtering of CN inter-peer connections is enabled.  Nodes can only join the etcd cluster if their CN match the allowed one.
 See [etcd security page](https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/security.md) for more details.
 
+## Notes on password strength
+`etcdctl` command line interface and etcd API don't check a strength (length, coexistence of numbers and alphabets, etc) of the password during creating a new user or updating password of an existing user. An administrator needs to care about a requirement of password strength by themselves.

Documentation/op-guide/configuration.md

@@ -328,6 +328,11 @@ The security flags help to [build a secure etcd cluster][security].
 + default: ""
 + env variable: ETCD_CIPHER_SUITES
 
+### --experimental-peer-skip-client-san-verification 
++ Skip verification of SAN field in client certificate for peer connections.
++ default: false
++ env variable: ETCD_EXPERIMENTAL_PEER_SKIP_CLIENT_SAN_VERIFICATION
+
 ## Logging flags
 
 ### --logger

Documentation/op-guide/gateway.md

@@ -4,7 +4,7 @@ title: etcd gateway
 
 ## What is etcd gateway
 
-etcd gateway is a simple TCP proxy that forwards network data to the etcd cluster. The gateway is stateless and transparent; it neither inspects client requests nor interferes with cluster responses.
+etcd gateway is a simple TCP proxy that forwards network data to the etcd cluster. The gateway is stateless and transparent; it neither inspects client requests nor interferes with cluster responses. It does not terminate TLS connections, do TLS handshakes on behalf of its clients, or verify if the connection is secured.
 
 The gateway supports multiple etcd server endpoints and works on a simple round-robin policy. It only routes to available endpoints and hides failures from its clients. Other retry policies, such as weighted round-robin, may be supported in the future.
 
@@ -74,7 +74,7 @@ $ etcd gateway start --discovery-srv=example.com
 
  * Comma-separated list of etcd server targets for forwarding client connections.
  * Default: `127.0.0.1:2379`
- * Invalid example: `https://127.0.0.1:2379` (gateway does not terminate TLS)
+ * Invalid example: `https://127.0.0.1:2379` (gateway does not terminate TLS). Note that the gateway does not verify the HTTP schema or inspect the requests, it only forwards requests to the given endpoints.
 
 #### --discovery-srv
 
@@ -103,5 +103,5 @@ $ etcd gateway start --discovery-srv=example.com
 
 #### --trusted-ca-file
 
- * Path to the client TLS CA file for the etcd cluster. Used to authenticate endpoints.
+ * Path to the client TLS CA file for the etcd cluster to verify the endpoints returned from SRV discovery. Note that it is ONLY used for authenticating the discovered endpoints rather than creating connections for data transferring. The gateway never terminates TLS connections or create TLS connections on behalf of its clients.
  * Default: (not set)

Documentation/op-guide/security.md

@@ -2,7 +2,7 @@
 title: Transport security model
 ---
 
-etcd supports automatic TLS as well as authentication through client certificates for both clients to server as well as peer (server to server / cluster) communication.
+etcd supports automatic TLS as well as authentication through client certificates for both clients to server as well as peer (server to server / cluster) communication. **Note that etcd doesn't enable [RBAC based authentication][auth] or the authentication feature in the transport layer by default to reduce friction for users getting started with the database. Further, changing this default would be a breaking change for the project which was established since 2013. An etcd cluster which doesn't enable security features can expose its data to any clients.**
 
 To get up and running, first have a CA certificate and a signed key pair for one member. It is recommended to create and sign a new key pair for every member in a cluster.
 
@@ -426,8 +426,17 @@ Make sure to sign the certificates with a Subject Name the member's public IP ad
 
 The certificate needs to be signed for the member's FQDN in its Subject Name, use Subject Alternative Names (short IP SANs) to add the IP address. The `etcd-ca` tool provides `--domain=` option for its `new-cert` command, and openssl can make [it][alt-name] too.
 
+### Does etcd encrypt data stored on disk drives?
+No. etcd doesn't encrypt key/value data stored on disk drives. If a user need to encrypt data stored on etcd, there are some options:
+* Let client applications encrypt and decrypt the data
+* Use a feature of underlying storage systems for encrypting stored data like [dm-crypt]
+
+### I’m seeing a log warning that "directory X exist without recommended permission -rwx------"
+When etcd create certain new directories it sets file permission to 700 to prevent unprivileged access as possible. However, if user has already created a directory with own preference, etcd uses the existing directory and logs a warning message if the permission is different than 700.
+
 [cfssl]: https://github.com/cloudflare/cfssl
 [tls-setup]: ../../hack/tls-setup
 [tls-guide]: https://github.com/coreos/docs/blob/master/os/generate-self-signed-certificates.md
 [alt-name]: http://wiki.cacert.org/FAQ/subjectAltName
 [auth]: authentication.md
+[dm-crypt]: https://en.wikipedia.org/wiki/Dm-crypt

auth/metrics.go

@@ -0,0 +1,42 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sync"
+)
+
+var (
+	currentAuthRevision = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "etcd_debugging",
+		Subsystem: "auth",
+		Name:      "revision",
+		Help:      "The current revision of auth store.",
+	},
+		func() float64 {
+			reportCurrentAuthRevMu.RLock()
+			defer reportCurrentAuthRevMu.RUnlock()
+			return reportCurrentAuthRev()
+		},
+	)
+	// overridden by auth store initialization
+	reportCurrentAuthRevMu sync.RWMutex
+	reportCurrentAuthRev   = func() float64 { return 0 }
+)
+
+func init() {
+	prometheus.MustRegister(currentAuthRevision)
+}

auth/simple_token.go

@@ -35,7 +35,7 @@ const (
 
 // var for testing purposes
 var (
-	simpleTokenTTL           = 5 * time.Minute
+	simpleTokenTTLDefault    = 300 * time.Second
 	simpleTokenTTLResolution = 1 * time.Second
 )
 
@@ -45,6 +45,7 @@ type simpleTokenTTLKeeper struct {
 	stopc           chan struct{}
 	deleteTokenFunc func(string)
 	mu              *sync.Mutex
+	simpleTokenTTL  time.Duration
 }
 
 func (tm *simpleTokenTTLKeeper) stop() {
@@ -56,12 +57,12 @@ func (tm *simpleTokenTTLKeeper) stop() {
 }
 
 func (tm *simpleTokenTTLKeeper) addSimpleToken(token string) {
-	tm.tokens[token] = time.Now().Add(simpleTokenTTL)
+	tm.tokens[token] = time.Now().Add(tm.simpleTokenTTL)
 }
 
 func (tm *simpleTokenTTLKeeper) resetSimpleToken(token string) {
 	if _, ok := tm.tokens[token]; ok {
-		tm.tokens[token] = time.Now().Add(simpleTokenTTL)
+		tm.tokens[token] = time.Now().Add(tm.simpleTokenTTL)
 	}
 }
 
@@ -98,6 +99,7 @@ type tokenSimple struct {
 	simpleTokenKeeper *simpleTokenTTLKeeper
 	simpleTokensMu    sync.Mutex
 	simpleTokens      map[string]string // token -> username
+	simpleTokenTTL    time.Duration
 }
 
 func (t *tokenSimple) genTokenPrefix() (string, error) {
@@ -146,6 +148,10 @@ func (t *tokenSimple) invalidateUser(username string) {
 }
 
 func (t *tokenSimple) enable() {
+	if t.simpleTokenTTL <= 0 {
+		t.simpleTokenTTL = simpleTokenTTLDefault
+	}
+
 	delf := func(tk string) {
 		if username, ok := t.simpleTokens[tk]; ok {
 			plog.Infof("deleting token %s for user %s", tk, username)
@@ -158,6 +164,7 @@ func (t *tokenSimple) enable() {
 		stopc:           make(chan struct{}),
 		deleteTokenFunc: delf,
 		mu:              &t.simpleTokensMu,
+		simpleTokenTTL:  t.simpleTokenTTL,
 	}
 	go t.simpleTokenKeeper.run()
 }
@@ -215,9 +222,10 @@ func (t *tokenSimple) isValidSimpleToken(ctx context.Context, token string) bool
 	return false
 }
 
-func newTokenProviderSimple(indexWaiter func(uint64) <-chan struct{}) *tokenSimple {
+func newTokenProviderSimple(indexWaiter func(uint64) <-chan struct{}, TokenTTL time.Duration) *tokenSimple {
 	return &tokenSimple{
-		simpleTokens: make(map[string]string),
-		indexWaiter:  indexWaiter,
+		simpleTokens:   make(map[string]string),
+		indexWaiter:    indexWaiter,
+		simpleTokenTTL: TokenTTL,
 	}
 }

auth/simple_token_test.go

@@ -22,9 +22,9 @@ import (
 // TestSimpleTokenDisabled ensures that TokenProviderSimple behaves correctly when
 // disabled.
 func TestSimpleTokenDisabled(t *testing.T) {
-	initialState := newTokenProviderSimple(dummyIndexWaiter)
+	initialState := newTokenProviderSimple(dummyIndexWaiter, simpleTokenTTLDefault)
 
-	explicitlyDisabled := newTokenProviderSimple(dummyIndexWaiter)
+	explicitlyDisabled := newTokenProviderSimple(dummyIndexWaiter, simpleTokenTTLDefault)
 	explicitlyDisabled.enable()
 	explicitlyDisabled.disable()
 
@@ -46,7 +46,7 @@ func TestSimpleTokenDisabled(t *testing.T) {
 // TestSimpleTokenAssign ensures that TokenProviderSimple can correctly assign a
 // token, look it up with info, and invalidate it by user.
 func TestSimpleTokenAssign(t *testing.T) {
-	tp := newTokenProviderSimple(dummyIndexWaiter)
+	tp := newTokenProviderSimple(dummyIndexWaiter, simpleTokenTTLDefault)
 	tp.enable()
 	ctx := context.WithValue(context.WithValue(context.TODO(), AuthenticateParamIndex{}, uint64(1)), AuthenticateParamSimpleTokenPrefix{}, "dummy")
 	token, err := tp.assign(ctx, "user1", 0)

auth/store.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/coreos/etcd/auth/authpb"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
@@ -90,6 +91,10 @@ type AuthenticateParamIndex struct{}
 // AuthenticateParamSimpleTokenPrefix is used for a key of context in the parameters of Authenticate()
 type AuthenticateParamSimpleTokenPrefix struct{}
 
+// saveConsistentIndexFunc is used to sync consistentIndex to backend, now reusing store.saveIndex
+type saveConsistentIndexFunc func(tx backend.BatchTx)
+
+// AuthStore defines auth storage interface.
 type AuthStore interface {
 	// AuthEnable turns on the authentication feature
 	AuthEnable() error
@@ -178,6 +183,9 @@ type AuthStore interface {
 
 	// HasRole checks that user has role
 	HasRole(user, role string) bool
+
+	// SetConsistentIndexSyncer sets consistentIndex syncer
+	SetConsistentIndexSyncer(syncer saveConsistentIndexFunc)
 }
 
 type TokenProvider interface {
@@ -200,9 +208,13 @@ type authStore struct {
 
 	rangePermCache map[string]*unifiedRangePermissions // username -> unifiedRangePermissions
 
-	tokenProvider TokenProvider
+	tokenProvider       TokenProvider
+	syncConsistentIndex saveConsistentIndexFunc
 }
 
+func (as *authStore) SetConsistentIndexSyncer(syncer saveConsistentIndexFunc) {
+	as.syncConsistentIndex = syncer
+}
 func (as *authStore) AuthEnable() error {
 	as.enabledMu.Lock()
 	defer as.enabledMu.Unlock()
@@ -252,6 +264,7 @@ func (as *authStore) AuthDisable() {
 	tx.Lock()
 	tx.UnsafePut(authBucketName, enableFlagKey, authDisabled)
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 	tx.Unlock()
 	b.ForceCommit()
 
@@ -368,6 +381,7 @@ func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse,
 	putUser(tx, newUser)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("added a new user: %s", r.Name)
 
@@ -392,6 +406,7 @@ func (as *authStore) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDelete
 	delUser(tx, r.Name)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	as.invalidateCachedPerm(r.Name)
 	as.tokenProvider.invalidateUser(r.Name)
@@ -428,6 +443,7 @@ func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*p
 	putUser(tx, updatedUser)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	as.invalidateCachedPerm(r.Name)
 	as.tokenProvider.invalidateUser(r.Name)
@@ -468,6 +484,7 @@ func (as *authStore) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUser
 	as.invalidateCachedPerm(r.User)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("granted role %s to user %s", r.Role, r.User)
 	return &pb.AuthUserGrantRoleResponse{}, nil
@@ -536,6 +553,7 @@ func (as *authStore) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUs
 	as.invalidateCachedPerm(r.Name)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("revoked role %s from user %s", r.Role, r.Name)
 	return &pb.AuthUserRevokeRoleResponse{}, nil
@@ -600,6 +618,7 @@ func (as *authStore) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest)
 	as.clearCachedPerm()
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("revoked key %s from role %s", r.Key, r.Role)
 	return &pb.AuthRoleRevokePermissionResponse{}, nil
@@ -645,6 +664,7 @@ func (as *authStore) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDelete
 	}
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("deleted role %s", r.Role)
 	return &pb.AuthRoleDeleteResponse{}, nil
@@ -667,6 +687,7 @@ func (as *authStore) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse,
 	putRole(tx, newRole)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("Role %s is created", r.Name)
 
@@ -727,6 +748,7 @@ func (as *authStore) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (
 	as.clearCachedPerm()
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("role %s's permission of key %s is updated as %s", r.Name, r.Perm.Key, authpb.Permission_Type_name[int32(r.Perm.PermType)])
 
@@ -743,8 +765,13 @@ func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeE
 	if revision == 0 {
 		return ErrUserEmpty
 	}
-
-	if revision < as.Revision() {
+	rev := as.Revision()
+	if revision < rev {
+		plog.Warningf("request auth revision is less than current node auth revision,"+
+			"current node auth revision is %d,"+
+			"request auth revision is %d,"+
+			"request key is %s, "+
+			"err is %v", rev, revision, key, ErrAuthOldRevision)
 		return ErrAuthOldRevision
 	}
 
@@ -786,7 +813,7 @@ func (as *authStore) IsAdminPermitted(authInfo *AuthInfo) error {
 	if !as.isAuthEnabled() {
 		return nil
 	}
-	if authInfo == nil {
+	if authInfo == nil || authInfo.Username == "" {
 		return ErrUserEmpty
 	}
 
@@ -933,6 +960,8 @@ func NewAuthStore(be backend.Backend, tp TokenProvider) *authStore {
 		as.commitRevision(tx)
 	}
 
+	as.setupMetricsReporter()
+
 	tx.Unlock()
 	be.ForceCommit()
 
@@ -1059,7 +1088,11 @@ func decomposeOpts(optstr string) (string, map[string]string, error) {
 
 }
 
-func NewTokenProvider(tokenOpts string, indexWaiter func(uint64) <-chan struct{}) (TokenProvider, error) {
+// NewTokenProvider creates a new token provider.
+func NewTokenProvider(
+	tokenOpts string,
+	indexWaiter func(uint64) <-chan struct{},
+	TokenTTL time.Duration) (TokenProvider, error) {
 	tokenType, typeSpecificOpts, err := decomposeOpts(tokenOpts)
 	if err != nil {
 		return nil, ErrInvalidAuthOpts
@@ -1068,7 +1101,7 @@ func NewTokenProvider(tokenOpts string, indexWaiter func(uint64) <-chan struct{}
 	switch tokenType {
 	case tokenTypeSimple:
 		plog.Warningf("simple token is not cryptographically signed")
-		return newTokenProviderSimple(indexWaiter), nil
+		return newTokenProviderSimple(indexWaiter, TokenTTL), nil
 
 	case tokenTypeJWT:
 		return newTokenProviderJWT(typeSpecificOpts)
@@ -1134,3 +1167,19 @@ func (as *authStore) HasRole(user, role string) bool {
 
 	return false
 }
+
+func (as *authStore) saveConsistentIndex(tx backend.BatchTx) {
+	if as.syncConsistentIndex != nil {
+		as.syncConsistentIndex(tx)
+	} else {
+		plog.Errorf("failed to save consistentIndex,syncConsistentIndex is nil")
+	}
+}
+
+func (as *authStore) setupMetricsReporter() {
+	reportCurrentAuthRevMu.Lock()
+	reportCurrentAuthRev = func() float64 {
+		return float64(as.Revision())
+	}
+	reportCurrentAuthRevMu.Unlock()
+}

auth/store_test.go

@@ -48,7 +48,7 @@ func TestNewAuthStoreRevision(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func TestNewAuthStoreRevision(t *testing.T) {
 func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testing.T)) {
 	b, tPath := backend.NewDefaultTmpBackend()
 
-	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -513,7 +513,7 @@ func TestAuthInfoFromCtxRace(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -545,6 +545,12 @@ func TestIsAdminPermitted(t *testing.T) {
 		t.Errorf("expected %v, got %v", ErrUserNotFound, err)
 	}
 
+	// empty user
+	err = as.IsAdminPermitted(&AuthInfo{Username: "", Revision: 1})
+	if err != ErrUserEmpty {
+		t.Errorf("expected %v, got %v", ErrUserEmpty, err)
+	}
+
 	// non-admin user
 	err = as.IsAdminPermitted(&AuthInfo{Username: "foo", Revision: 1})
 	if err != ErrPermissionDenied {
@@ -579,7 +585,7 @@ func TestRecoverFromSnapshot(t *testing.T) {
 
 	as.Close()
 
-	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -612,13 +618,13 @@ func contains(array []string, str string) bool {
 
 func TestHammerSimpleAuthenticate(t *testing.T) {
 	// set TTL values low to try to trigger races
-	oldTTL, oldTTLRes := simpleTokenTTL, simpleTokenTTLResolution
+	oldTTL, oldTTLRes := simpleTokenTTLDefault, simpleTokenTTLResolution
 	defer func() {
-		simpleTokenTTL = oldTTL
+		simpleTokenTTLDefault = oldTTL
 		simpleTokenTTLResolution = oldTTLRes
 	}()
-	simpleTokenTTL = 10 * time.Millisecond
-	simpleTokenTTLResolution = simpleTokenTTL
+	simpleTokenTTLDefault = 10 * time.Millisecond
+	simpleTokenTTLResolution = simpleTokenTTLDefault
 	users := make(map[string]struct{})
 
 	as, tearDown := setupAuthStore(t)
@@ -661,7 +667,7 @@ func TestRolesOrder(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter)
+	tp, err := NewTokenProvider(tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -716,7 +722,7 @@ func testAuthInfoFromCtxWithRoot(t *testing.T, opts string) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer os.Remove(tPath)
 
-	tp, err := NewTokenProvider(opts, dummyIndexWaiter)
+	tp, err := NewTokenProvider(opts, dummyIndexWaiter, simpleTokenTTLDefault)
 	if err != nil {
 		t.Fatal(err)
 	}

bill-of-materials.json

@@ -103,7 +103,7 @@
 		"licenses": [
 			{
 				"type": "BSD 3-clause \"New\" or \"Revised\" License",
-				"confidence": 0.9090909090909091
+				"confidence": 0.9163346613545816
 			}
 		]
 	},
@@ -121,7 +121,7 @@
 		"licenses": [
 			{
 				"type": "BSD 3-clause \"New\" or \"Revised\" License",
-				"confidence": 0.92
+				"confidence": 0.9663865546218487
 			}
 		]
 	},
@@ -134,6 +134,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/google/uuid",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
+	},
 	{
 		"project": "github.com/gorilla/websocket",
 		"licenses": [
@@ -143,6 +152,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/grpc-ecosystem/go-grpc-middleware",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
 	{
 		"project": "github.com/grpc-ecosystem/go-grpc-prometheus",
 		"licenses": [
@@ -164,10 +182,6 @@
 	{
 		"project": "github.com/inconshreveable/mousetrap",
 		"licenses": [
-			{
-				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
-				"confidence": 1
-			},
 			{
 				"type": "Apache License 2.0",
 				"confidence": 1
@@ -418,7 +432,7 @@
 		]
 	},
 	{
-		"project": "google.golang.org/genproto/googleapis/rpc/status",
+		"project": "google.golang.org/genproto/googleapis",
 		"licenses": [
 			{
 				"type": "Apache License 2.0",
@@ -448,8 +462,8 @@
 		"project": "gopkg.in/yaml.v2",
 		"licenses": [
 			{
-				"type": "The Unlicense",
-				"confidence": 0.35294117647058826
+				"type": "Apache License 2.0",
+				"confidence": 1
 			},
 			{
 				"type": "MIT License",
@@ -460,6 +474,10 @@
 	{
 		"project": "sigs.k8s.io/yaml",
 		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			},
 			{
 				"type": "BSD 3-clause \"New\" or \"Revised\" License",
 				"confidence": 1

clientv3/balancer/resolver/endpoint/endpoint.go

@@ -16,7 +16,9 @@
 package endpoint
 
 import (
+	"context"
 	"fmt"
+	"net"
 	"net/url"
 	"strings"
 	"sync"
@@ -228,13 +230,18 @@ func ParseTarget(target string) (string, string, error) {
 	return parts[0], parts[1], nil
 }
 
-// ParseHostPort splits a "<host>:<port>" string into the host and port parts.
-// The port part is optional.
-func ParseHostPort(hostPort string) (host string, port string) {
-	parts := strings.SplitN(hostPort, ":", 2)
-	host = parts[0]
-	if len(parts) > 1 {
-		port = parts[1]
+// Dialer dials a endpoint using net.Dialer.
+// Context cancelation and timeout are supported.
+func Dialer(ctx context.Context, dialEp string) (net.Conn, error) {
+	proto, host, _ := ParseEndpoint(dialEp)
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
 	}
-	return host, port
+	dialer := &net.Dialer{}
+	if deadline, ok := ctx.Deadline(); ok {
+		dialer.Deadline = deadline
+	}
+	return dialer.DialContext(ctx, proto, host)
 }

clientv3/client.go

@@ -25,19 +25,19 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/coreos/etcd/clientv3/balancer"
 	"github.com/coreos/etcd/clientv3/balancer/picker"
 	"github.com/coreos/etcd/clientv3/balancer/resolver/endpoint"
 	"github.com/coreos/etcd/clientv3/credentials"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
 	"github.com/coreos/etcd/pkg/logutil"
+	"github.com/coreos/pkg/capnslog"
+	"github.com/google/uuid"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	grpccredentials "google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/keepalive"
-	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 )
 
@@ -48,6 +48,10 @@ var (
 	roundRobinBalancerName = fmt.Sprintf("etcd-%s", picker.RoundrobinBalanced.String())
 )
 
+var (
+	plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "clientv3")
+)
+
 func init() {
 	lg := zap.NewNop()
 	if os.Getenv("ETCD_CLIENT_DEBUG") != "" {
@@ -226,24 +230,17 @@ func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts
 	}
 	opts = append(opts, dopts...)
 
-	// Provide a net dialer that supports cancelation and timeout.
-	f := func(dialEp string, t time.Duration) (net.Conn, error) {
-		proto, host, _ := endpoint.ParseEndpoint(dialEp)
-		select {
-		case <-c.ctx.Done():
-			return nil, c.ctx.Err()
-		default:
-		}
-		dialer := &net.Dialer{Timeout: t}
-		return dialer.DialContext(c.ctx, proto, host)
-	}
-	opts = append(opts, grpc.WithDialer(f))
-
+	dialer := endpoint.Dialer
 	if creds != nil {
 		opts = append(opts, grpc.WithTransportCredentials(creds))
+		// gRPC load balancer workaround. See credentials.transportCredential for details.
+		if credsDialer, ok := creds.(TransportCredentialsWithDialer); ok {
+			dialer = credsDialer.Dialer
+		}
 	} else {
 		opts = append(opts, grpc.WithInsecure())
 	}
+	opts = append(opts, grpc.WithContextDialer(dialer))
 
 	// Interceptor retry and backoff.
 	// TODO: Replace all of clientv3/retry.go with interceptor based retry, or with
@@ -262,7 +259,10 @@ func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts
 
 // Dial connects to a single endpoint using the client's config.
 func (c *Client) Dial(ep string) (*grpc.ClientConn, error) {
-	creds := c.directDialCreds(ep)
+	creds, err := c.directDialCreds(ep)
+	if err != nil {
+		return nil, err
+	}
 	// Use the grpc passthrough resolver to directly dial a single endpoint.
 	// This resolver passes through the 'unix' and 'unixs' endpoints schemes used
 	// by etcd without modification, allowing us to directly dial endpoints and
@@ -365,8 +365,8 @@ func (c *Client) dial(target string, creds grpccredentials.TransportCredentials,
 	return conn, nil
 }
 
-func (c *Client) directDialCreds(ep string) grpccredentials.TransportCredentials {
-	_, hostPort, scheme := endpoint.ParseEndpoint(ep)
+func (c *Client) directDialCreds(ep string) (grpccredentials.TransportCredentials, error) {
+	_, host, scheme := endpoint.ParseEndpoint(ep)
 	creds := c.creds
 	if len(scheme) != 0 {
 		creds = c.processCreds(scheme)
@@ -375,12 +375,17 @@ func (c *Client) directDialCreds(ep string) grpccredentials.TransportCredentials
 			// Set the server name must to the endpoint hostname without port since grpc
 			// otherwise attempts to check if x509 cert is valid for the full endpoint
 			// including the scheme and port, which fails.
-			host, _ := endpoint.ParseHostPort(hostPort)
-			clone.OverrideServerName(host)
+			overrideServerName, _, err := net.SplitHostPort(host)
+			if err != nil {
+				// Either the host didn't have a port or the host could not be parsed. Either way, continue with the
+				// original host string.
+				overrideServerName = host
+			}
+			clone.OverrideServerName(overrideServerName)
 			creds = clone
 		}
 	}
-	return creds
+	return creds, nil
 }
 
 func (c *Client) dialWithBalancerCreds(ep string) grpccredentials.TransportCredentials {
@@ -392,13 +397,6 @@ func (c *Client) dialWithBalancerCreds(ep string) grpccredentials.TransportCrede
 	return creds
 }
 
-// WithRequireLeader requires client requests to only succeed
-// when the cluster has a leader.
-func WithRequireLeader(ctx context.Context) context.Context {
-	md := metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader)
-	return metadata.NewOutgoingContext(ctx, md)
-}
-
 func newClient(cfg *Config) (*Client, error) {
 	if cfg == nil {
 		cfg = &Config{}
@@ -659,3 +657,9 @@ func IsConnCanceled(err error) bool {
 	// <= gRPC v1.7.x returns 'errors.New("grpc: the client connection is closing")'
 	return strings.Contains(err.Error(), "grpc: the client connection is closing")
 }
+
+// TransportCredentialsWithDialer is for a gRPC load balancer workaround. See credentials.transportCredential for details.
+type TransportCredentialsWithDialer interface {
+	grpccredentials.TransportCredentials
+	Dialer(ctx context.Context, dialEp string) (net.Conn, error)
+}

clientv3/concurrency/election_test.go

@@ -17,9 +17,9 @@ package concurrency_test
 import (
 	"context"
 	"log"
+	"strings"
 	"testing"
 	"time"
-	"strings"
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/clientv3/concurrency"

clientv3/credentials/credentials.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"sync"
 
+	"github.com/coreos/etcd/clientv3/balancer/resolver/endpoint"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
 	grpccredentials "google.golang.org/grpc/credentials"
 )
@@ -65,38 +66,37 @@ func (b *bundle) NewWithMode(mode string) (grpccredentials.Bundle, error) {
 }
 
 // transportCredential implements "grpccredentials.TransportCredentials" interface.
+// transportCredential wraps TransportCredentials to track which
+// addresses are dialed for which endpoints, and then sets the authority when checking the endpoint's cert to the
+// hostname or IP of the dialed endpoint.
+// This is a workaround of a gRPC load balancer issue. gRPC uses the dialed target's service name as the authority when
+// checking all endpoint certs, which does not work for etcd servers using their hostname or IP as the Subject Alternative Name
+// in their TLS certs.
+// To enable, include both WithTransportCredentials(creds) and WithContextDialer(creds.Dialer)
+// when dialing.
 type transportCredential struct {
 	gtc grpccredentials.TransportCredentials
+	mu  sync.Mutex
+	// addrToEndpoint maps from the connection addresses that are dialed to the hostname or IP of the
+	// endpoint provided to the dialer when dialing
+	addrToEndpoint map[string]string
 }
 
 func newTransportCredential(cfg *tls.Config) *transportCredential {
 	return &transportCredential{
-		gtc: grpccredentials.NewTLS(cfg),
+		gtc:            grpccredentials.NewTLS(cfg),
+		addrToEndpoint: map[string]string{},
 	}
 }
 
 func (tc *transportCredential) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, grpccredentials.AuthInfo, error) {
-	// Only overwrite when authority is an IP address!
-	// Let's say, a server runs SRV records on "etcd.local" that resolves
-	// to "m1.etcd.local", and its SAN field also includes "m1.etcd.local".
-	// But what if SAN does not include its resolved IP address (e.g. 127.0.0.1)?
-	// Then, the server should only authenticate using its DNS hostname "m1.etcd.local",
-	// instead of overwriting it with its IP address.
-	// And we do not overwrite "localhost" either. Only overwrite IP addresses!
-	if isIP(authority) {
-		target := rawConn.RemoteAddr().String()
-		if authority != target {
-			// When user dials with "grpc.WithDialer", "grpc.DialContext" "cc.parsedTarget"
-			// update only happens once. This is problematic, because when TLS is enabled,
-			// retries happen through "grpc.WithDialer" with static "cc.parsedTarget" from
-			// the initial dial call.
-			// If the server authenticates by IP addresses, we want to set a new endpoint as
-			// a new authority. Otherwise
-			// "transport: authentication handshake failed: x509: certificate is valid for 127.0.0.1, 192.168.121.180, not 192.168.223.156"
-			// when the new dial target is "192.168.121.180" whose certificate host name is also "192.168.121.180"
-			// but client tries to authenticate with previously set "cc.parsedTarget" field "192.168.223.156"
-			authority = target
-		}
+	// Set the authority when checking the endpoint's cert to the hostname or IP of the dialed endpoint
+	tc.mu.Lock()
+	dialEp, ok := tc.addrToEndpoint[rawConn.RemoteAddr().String()]
+	tc.mu.Unlock()
+	if ok {
+		_, host, _ := endpoint.ParseEndpoint(dialEp)
+		authority = host
 	}
 	return tc.gtc.ClientHandshake(ctx, authority, rawConn)
 }
@@ -115,8 +115,15 @@ func (tc *transportCredential) Info() grpccredentials.ProtocolInfo {
 }
 
 func (tc *transportCredential) Clone() grpccredentials.TransportCredentials {
+	copy := map[string]string{}
+	tc.mu.Lock()
+	for k, v := range tc.addrToEndpoint {
+		copy[k] = v
+	}
+	tc.mu.Unlock()
 	return &transportCredential{
-		gtc: tc.gtc.Clone(),
+		gtc:            tc.gtc.Clone(),
+		addrToEndpoint: copy,
 	}
 }
 
@@ -124,6 +131,17 @@ func (tc *transportCredential) OverrideServerName(serverNameOverride string) err
 	return tc.gtc.OverrideServerName(serverNameOverride)
 }
 
+func (tc *transportCredential) Dialer(ctx context.Context, dialEp string) (net.Conn, error) {
+	// Keep track of which addresses are dialed for which endpoints
+	conn, err := endpoint.Dialer(ctx, dialEp)
+	if conn != nil {
+		tc.mu.Lock()
+		tc.addrToEndpoint[conn.RemoteAddr().String()] = dialEp
+		tc.mu.Unlock()
+	}
+	return conn, err
+}
+
 // perRPCCredential implements "grpccredentials.PerRPCCredentials" interface.
 type perRPCCredential struct {
 	authToken   string

clientv3/ctx.go

@@ -0,0 +1,64 @@
+// Copyright 2020 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	"context"
+	"strings"
+
+	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	"github.com/coreos/etcd/version"
+	"google.golang.org/grpc/metadata"
+)
+
+// WithRequireLeader requires client requests to only succeed
+// when the cluster has a leader.
+func WithRequireLeader(ctx context.Context) context.Context {
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if !ok { // no outgoing metadata ctx key, create one
+		md = metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader)
+		return metadata.NewOutgoingContext(ctx, md)
+	}
+	copied := md.Copy() // avoid racey updates
+	// overwrite/add 'hasleader' key/value
+	metadataSet(copied, rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader)
+	return metadata.NewOutgoingContext(ctx, copied)
+}
+
+// embeds client version
+func withVersion(ctx context.Context) context.Context {
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if !ok { // no outgoing metadata ctx key, create one
+		md = metadata.Pairs(rpctypes.MetadataClientAPIVersionKey, version.APIVersion)
+		return metadata.NewOutgoingContext(ctx, md)
+	}
+	copied := md.Copy() // avoid racey updates
+	// overwrite/add version key/value
+	metadataSet(copied, rpctypes.MetadataClientAPIVersionKey, version.APIVersion)
+	return metadata.NewOutgoingContext(ctx, copied)
+}
+
+func metadataGet(md metadata.MD, k string) []string {
+	k = strings.ToLower(k)
+	return md[k]
+}
+
+func metadataSet(md metadata.MD, k string, vals ...string) {
+	if len(vals) == 0 {
+		return
+	}
+	k = strings.ToLower(k)
+	md[k] = vals
+}

clientv3/ctx_test.go

@@ -0,0 +1,67 @@
+// Copyright 2020 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	"github.com/coreos/etcd/version"
+	"google.golang.org/grpc/metadata"
+)
+
+func TestMetadataWithRequireLeader(t *testing.T) {
+	ctx := context.TODO()
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if ok {
+		t.Fatal("expected no outgoing metadata ctx key")
+	}
+
+	// add a conflicting key with some other value
+	md = metadata.Pairs(rpctypes.MetadataRequireLeaderKey, "invalid")
+	// add a key, and expect not be overwritten
+	metadataSet(md, "hello", "1", "2")
+	ctx = metadata.NewOutgoingContext(ctx, md)
+
+	// expect overwrites but still keep other keys
+	ctx = WithRequireLeader(ctx)
+	md, ok = metadata.FromOutgoingContext(ctx)
+	if !ok {
+		t.Fatal("expected outgoing metadata ctx key")
+	}
+	if ss := metadataGet(md, rpctypes.MetadataRequireLeaderKey); !reflect.DeepEqual(ss, []string{rpctypes.MetadataHasLeader}) {
+		t.Fatalf("unexpected metadata for %q %v", rpctypes.MetadataRequireLeaderKey, ss)
+	}
+	if ss := metadataGet(md, "hello"); !reflect.DeepEqual(ss, []string{"1", "2"}) {
+		t.Fatalf("unexpected metadata for 'hello' %v", ss)
+	}
+}
+
+func TestMetadataWithClientAPIVersion(t *testing.T) {
+	ctx := withVersion(WithRequireLeader(context.TODO()))
+
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if !ok {
+		t.Fatal("expected outgoing metadata ctx key")
+	}
+	if ss := metadataGet(md, rpctypes.MetadataRequireLeaderKey); !reflect.DeepEqual(ss, []string{rpctypes.MetadataHasLeader}) {
+		t.Fatalf("unexpected metadata for %q %v", rpctypes.MetadataRequireLeaderKey, ss)
+	}
+	if ss := metadataGet(md, rpctypes.MetadataClientAPIVersionKey); !reflect.DeepEqual(ss, []string{version.APIVersion}) {
+		t.Fatalf("unexpected metadata for %q %v", rpctypes.MetadataClientAPIVersionKey, ss)
+	}
+}

clientv3/integration/user_test.go

@@ -63,8 +63,8 @@ func TestUserErrorAuth(t *testing.T) {
 	authSetupRoot(t, authapi.Auth)
 
 	// unauthenticated client
-	if _, err := authapi.UserAdd(context.TODO(), "foo", "bar"); err != rpctypes.ErrUserNotFound {
-		t.Fatalf("expected %v, got %v", rpctypes.ErrUserNotFound, err)
+	if _, err := authapi.UserAdd(context.TODO(), "foo", "bar"); err != rpctypes.ErrUserEmpty {
+		t.Fatalf("expected %v, got %v", rpctypes.ErrUserEmpty, err)
 	}
 
 	// wrong id or password

clientv3/maintenance.go

@@ -193,23 +193,32 @@ func (m *maintenance) Snapshot(ctx context.Context) (io.ReadCloser, error) {
 		return nil, toErr(ctx, err)
 	}
 
+	plog.Info("opened snapshot stream; downloading")
 	pr, pw := io.Pipe()
 	go func() {
 		for {
 			resp, err := ss.Recv()
 			if err != nil {
+				switch err {
+				case io.EOF:
+					plog.Info("completed snapshot read; closing")
+				default:
+					plog.Warningf("failed to receive from snapshot stream; closing (%v)", err)
+				}
 				pw.CloseWithError(err)
 				return
 			}
-			if resp == nil && err == nil {
-				break
-			}
+
+			// can "resp == nil && err == nil"
+			// before we receive snapshot SHA digest?
+			// No, server sends EOF with an empty response
+			// after it sends SHA digest at the end
+
 			if _, werr := pw.Write(resp.Blob); werr != nil {
 				pw.CloseWithError(werr)
 				return
 			}
 		}
-		pw.Close()
 	}()
 	return &snapshotReadCloser{ctx: ctx, ReadCloser: pr}, nil
 }

clientv3/retry_interceptor.go

@@ -38,6 +38,7 @@ import (
 func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.UnaryClientInterceptor {
 	intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs)
 	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		ctx = withVersion(ctx)
 		grpcOpts, retryOpts := filterCallOptions(opts)
 		callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts)
 		// short circuit for simplicity, and avoiding allocations.
@@ -103,6 +104,17 @@ func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOpt
 func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.StreamClientInterceptor {
 	intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs)
 	return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+		ctx = withVersion(ctx)
+		// getToken automatically
+		// TODO(cfc4n): keep this code block, remove codes about getToken in client.go after pr #12165 merged.
+		if c.authTokenBundle != nil {
+			// equal to c.Username != "" && c.Password != ""
+			err := c.getToken(ctx)
+			if err != nil && rpctypes.Error(err) != rpctypes.ErrAuthNotEnabled {
+				logger.Error("clientv3/retry_interceptor: getToken failed", zap.Error(err))
+				return nil, err
+			}
+		}
 		grpcOpts, retryOpts := filterCallOptions(opts)
 		callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts)
 		// short circuit for simplicity, and avoiding allocations.
@@ -113,10 +125,9 @@ func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOp
 			return nil, status.Errorf(codes.Unimplemented, "clientv3/retry_interceptor: cannot retry on ClientStreams, set Disable()")
 		}
 		newStreamer, err := streamer(ctx, desc, cc, method, grpcOpts...)
-		logger.Warn("retry stream intercept", zap.Error(err))
 		if err != nil {
-			// TODO(mwitkow): Maybe dial and transport errors should be retriable?
-			return nil, err
+			logger.Error("streamer failed to create ClientStream", zap.Error(err))
+			return nil, err // TODO(mwitkow): Maybe dial and transport errors should be retriable?
 		}
 		retryingStreamer := &serverStreamingRetryingStream{
 			client:       c,
@@ -185,6 +196,7 @@ func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error {
 	if !attemptRetry {
 		return lastErr // success or hard failure
 	}
+
 	// We start off from attempt 1, because zeroth was already made on normal SendMsg().
 	for attempt := uint(1); attempt < s.callOpts.max; attempt++ {
 		if err := waitRetryBackoff(s.ctx, attempt, s.callOpts); err != nil {
@@ -192,12 +204,13 @@ func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error {
 		}
 		newStream, err := s.reestablishStreamAndResendBuffer(s.ctx)
 		if err != nil {
-			// TODO(mwitkow): Maybe dial and transport errors should be retriable?
-			return err
+			s.client.lg.Error("failed reestablishStreamAndResendBuffer", zap.Error(err))
+			return err // TODO(mwitkow): Maybe dial and transport errors should be retriable?
 		}
 		s.setStream(newStream)
+
+		s.client.lg.Warn("retrying RecvMsg", zap.Error(lastErr))
 		attemptRetry, lastErr = s.receiveMsgAndIndicateRetry(m)
-		//fmt.Printf("Received message and indicate: %v  %v\n", attemptRetry, lastErr)
 		if !attemptRetry {
 			return lastErr
 		}

clientv3/snapshot/v3_snapshot.go

@@ -44,6 +44,7 @@ import (
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/etcd/wal"
 	"github.com/coreos/etcd/wal/walpb"
+	"github.com/dustin/go-humanize"
 	"go.uber.org/zap"
 )
 
@@ -87,6 +88,14 @@ type v3Manager struct {
 	skipHashCheck bool
 }
 
+// hasChecksum returns "true" if the file size "n"
+// has appended sha256 hash digest.
+func hasChecksum(n int64) bool {
+	// 512 is chosen because it's a minimum disk sector size
+	// smaller than (and multiplies to) OS page size in most systems
+	return (n % 512) == sha256.Size
+}
+
 // Save fetches snapshot from remote etcd server and saves data to target path.
 func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string) error {
 	if len(cfg.Endpoints) != 1 {
@@ -106,10 +115,7 @@ func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string
 	if err != nil {
 		return fmt.Errorf("could not open %s (%v)", partpath, err)
 	}
-	s.lg.Info(
-		"created temporary db file",
-		zap.String("path", partpath),
-	)
+	s.lg.Info("created temporary db file", zap.String("path", partpath))
 
 	now := time.Now()
 	var rd io.ReadCloser
@@ -117,13 +123,15 @@ func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string
 	if err != nil {
 		return err
 	}
-	s.lg.Info(
-		"fetching snapshot",
-		zap.String("endpoint", cfg.Endpoints[0]),
-	)
-	if _, err = io.Copy(f, rd); err != nil {
+	s.lg.Info("fetching snapshot", zap.String("endpoint", cfg.Endpoints[0]))
+	var size int64
+	size, err = io.Copy(f, rd)
+	if err != nil {
 		return err
 	}
+	if !hasChecksum(size) {
+		return fmt.Errorf("sha256 checksum not found [bytes: %d]", size)
+	}
 	if err = fileutil.Fsync(f); err != nil {
 		return err
 	}
@@ -133,6 +141,7 @@ func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string
 	s.lg.Info(
 		"fetched snapshot",
 		zap.String("endpoint", cfg.Endpoints[0]),
+		zap.String("size", humanize.Bytes(uint64(size))),
 		zap.Duration("took", time.Since(now)),
 	)
 
@@ -344,7 +353,7 @@ func (s *v3Manager) saveDB() error {
 	if serr != nil {
 		return serr
 	}
-	hasHash := (off % 512) == sha256.Size
+	hasHash := hasChecksum(off)
 	if hasHash {
 		if err := db.Truncate(off - sha256.Size); err != nil {
 			return err

discovery/discovery.go

@@ -211,7 +211,7 @@ func (d *discovery) createSelf(contents string) error {
 	return err
 }
 
-func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) {
+func (d *discovery) checkCluster() ([]*client.Node, uint64, uint64, error) {
 	configKey := path.Join("/", d.cluster, "_config")
 	ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
 	// find cluster size
@@ -230,7 +230,7 @@ func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) {
 		}
 		return nil, 0, 0, err
 	}
-	size, err := strconv.Atoi(resp.Node.Value)
+	size, err := strconv.ParseUint(resp.Node.Value, 10, 0)
 	if err != nil {
 		return nil, 0, 0, ErrBadSizeKey
 	}
@@ -261,7 +261,7 @@ func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) {
 		if path.Base(nodes[i].Key) == path.Base(d.selfKey()) {
 			break
 		}
-		if i >= size-1 {
+		if uint64(i) >= size-1 {
 			return nodes[:size], size, resp.Index, ErrFullCluster
 		}
 	}
@@ -280,7 +280,7 @@ func (d *discovery) logAndBackoffForRetry(step string) {
 	d.clock.Sleep(retryTimeInSecond)
 }
 
-func (d *discovery) checkClusterRetry() ([]*client.Node, int, uint64, error) {
+func (d *discovery) checkClusterRetry() ([]*client.Node, uint64, uint64, error) {
 	if d.retries < nRetries {
 		d.logAndBackoffForRetry("cluster status check")
 		return d.checkCluster()
@@ -300,8 +300,8 @@ func (d *discovery) waitNodesRetry() ([]*client.Node, error) {
 	return nil, ErrTooManyRetries
 }
 
-func (d *discovery) waitNodes(nodes []*client.Node, size int, index uint64) ([]*client.Node, error) {
-	if len(nodes) > size {
+func (d *discovery) waitNodes(nodes []*client.Node, size uint64, index uint64) ([]*client.Node, error) {
+	if uint64(len(nodes)) > size {
 		nodes = nodes[:size]
 	}
 	// watch from the next index
@@ -317,8 +317,8 @@ func (d *discovery) waitNodes(nodes []*client.Node, size int, index uint64) ([]*
 	}
 
 	// wait for others
-	for len(all) < size {
-		plog.Noticef("found %d peer(s), waiting for %d more", len(all), size-len(all))
+	for uint64(len(all)) < size {
+		plog.Noticef("found %d peer(s), waiting for %d more", len(all), int(size-uint64(len(all))))
 		resp, err := w.Next(context.Background())
 		if err != nil {
 			if ce, ok := err.(*client.ClusterError); ok {
@@ -338,7 +338,7 @@ func (d *discovery) selfKey() string {
 	return path.Join("/", d.cluster, d.id.String())
 }
 
-func nodesToCluster(ns []*client.Node, size int) (string, error) {
+func nodesToCluster(ns []*client.Node, size uint64) (string, error) {
 	s := make([]string, len(ns))
 	for i, n := range ns {
 		s[i] = n.Value
@@ -348,7 +348,7 @@ func nodesToCluster(ns []*client.Node, size int) (string, error) {
 	if err != nil {
 		return us, ErrInvalidURL
 	}
-	if m.Len() != size {
+	if uint64(m.Len()) != size {
 		return us, ErrDuplicateName
 	}
 	return us, nil

discovery/discovery_test.go

@@ -215,7 +215,7 @@ func TestCheckCluster(t *testing.T) {
 			if reflect.DeepEqual(ns, tt.nodes) {
 				t.Errorf("#%d: nodes = %v, want %v", i, ns, tt.nodes)
 			}
-			if size != tt.wsize {
+			if size != uint64(tt.wsize) {
 				t.Errorf("#%d: size = %v, want %d", i, size, tt.wsize)
 			}
 			if index != tt.index {
@@ -299,7 +299,7 @@ func TestWaitNodes(t *testing.T) {
 					fc.Advance(time.Second * (0x1 << i))
 				}
 			}()
-			g, err := d.waitNodes(tt.nodes, 3, 0) // we do not care about index in this test
+			g, err := d.waitNodes(tt.nodes, uint64(3), 0) // we do not care about index in this test
 			if err != nil {
 				t.Errorf("#%d: err = %v, want %v", i, err, nil)
 			}
@@ -346,7 +346,7 @@ func TestCreateSelf(t *testing.T) {
 func TestNodesToCluster(t *testing.T) {
 	tests := []struct {
 		nodes    []*client.Node
-		size     int
+		size     uint64
 		wcluster string
 		werr     error
 	}{

embed/config.go

@@ -222,6 +222,9 @@ type Config struct {
 
 	// Experimental flags
 
+	//The AuthTokenTTL in seconds of the simple token
+	AuthTokenTTL uint `json:"auth-token-ttl"`
+
 	ExperimentalInitialCorruptCheck bool          `json:"experimental-initial-corrupt-check"`
 	ExperimentalCorruptCheckTime    time.Duration `json:"experimental-corrupt-check-time"`
 	ExperimentalEnableV2V3          string        `json:"experimental-enable-v2v3"`
@@ -273,17 +276,18 @@ func NewConfig() *Config {
 		TickMs:                     100,
 		ElectionMs:                 1000,
 		InitialElectionTickAdvance: true,
-		LPUrls:              []url.URL{*lpurl},
-		LCUrls:              []url.URL{*lcurl},
-		APUrls:              []url.URL{*apurl},
-		ACUrls:              []url.URL{*acurl},
-		ClusterState:        ClusterStateFlagNew,
-		InitialClusterToken: "etcd-cluster",
-		StrictReconfigCheck: DefaultStrictReconfigCheck,
-		LogOutput:           DefaultLogOutput,
-		Metrics:             "basic",
-		EnableV2:            DefaultEnableV2,
-		AuthToken:           "simple",
+		LPUrls:                     []url.URL{*lpurl},
+		LCUrls:                     []url.URL{*lcurl},
+		APUrls:                     []url.URL{*apurl},
+		ACUrls:                     []url.URL{*acurl},
+		ClusterState:               ClusterStateFlagNew,
+		InitialClusterToken:        "etcd-cluster",
+		StrictReconfigCheck:        DefaultStrictReconfigCheck,
+		LogOutput:                  DefaultLogOutput,
+		Metrics:                    "basic",
+		EnableV2:                   DefaultEnableV2,
+		AuthToken:                  "simple",
+		AuthTokenTTL: 300,
 	}
 	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
 	return cfg

embed/etcd.go

@@ -171,6 +171,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		StrictReconfigCheck:        cfg.StrictReconfigCheck,
 		ClientCertAuthEnabled:      cfg.ClientTLSInfo.ClientCertAuth,
 		AuthToken:                  cfg.AuthToken,
+		TokenTTL:                   cfg.AuthTokenTTL,
 		InitialCorruptCheck:        cfg.ExperimentalInitialCorruptCheck,
 		CorruptCheckTime:           cfg.ExperimentalCorruptCheckTime,
 		Debug:                      cfg.Debug,
@@ -564,7 +565,7 @@ func (e *Etcd) errHandler(err error) {
 
 func parseCompactionRetention(mode, retention string) (ret time.Duration, err error) {
 	h, err := strconv.Atoi(retention)
-	if err == nil {
+	if err == nil && h >= 0 {
 		switch mode {
 		case compactor.ModeRevision:
 			ret = time.Duration(int64(h))

etcdctl/ctlv3/command/member_command.go

@@ -118,28 +118,8 @@ func memberAddCommandFunc(cmd *cobra.Command, args []string) {
 	display.MemberAdd(*resp)
 
 	if _, ok := (display).(*simplePrinter); ok {
-		ctx, cancel = commandCtx(cmd)
-		listResp, err := cli.MemberList(ctx)
-		// get latest member list; if there's failover new member might have outdated list
-		for {
-			if err != nil {
-				ExitWithError(ExitError, err)
-			}
-			if listResp.Header.MemberId == resp.Header.MemberId {
-				break
-			}
-			// quorum get to sync cluster list
-			gresp, gerr := cli.Get(ctx, "_")
-			if gerr != nil {
-				ExitWithError(ExitError, err)
-			}
-			resp.Header.MemberId = gresp.Header.MemberId
-			listResp, err = cli.MemberList(ctx)
-		}
-		cancel()
-
 		conf := []string{}
-		for _, memb := range listResp.Members {
+		for _, memb := range resp.Members {
 			for _, u := range memb.PeerURLs {
 				n := memb.Name
 				if memb.ID == newID {

etcdctl/ctlv3/ctl.go

@@ -59,7 +59,7 @@ func init() {
 	// TODO: secure by default when etcd enables secure gRPC by default.
 	rootCmd.PersistentFlags().BoolVar(&globalFlags.Insecure, "insecure-transport", true, "disable transport security for client connections")
 	rootCmd.PersistentFlags().BoolVar(&globalFlags.InsecureDiscovery, "insecure-discovery", true, "accept insecure SRV records describing cluster endpoints")
-	rootCmd.PersistentFlags().BoolVar(&globalFlags.InsecureSkipVerify, "insecure-skip-tls-verify", false, "skip server certificate verification")
+	rootCmd.PersistentFlags().BoolVar(&globalFlags.InsecureSkipVerify, "insecure-skip-tls-verify", false, "skip server certificate verification (CAUTION: this option should be enabled only for testing purposes)")
 	rootCmd.PersistentFlags().StringVar(&globalFlags.TLS.CertFile, "cert", "", "identify secure client using this TLS certificate file")
 	rootCmd.PersistentFlags().StringVar(&globalFlags.TLS.KeyFile, "key", "", "identify secure client using this TLS key file")
 	rootCmd.PersistentFlags().StringVar(&globalFlags.TLS.CAFile, "cacert", "", "verify certificates of TLS-enabled secure servers using this CA bundle")

etcdmain/config.go

@@ -189,6 +189,7 @@ func newConfig() *config {
 	fs.BoolVar(&cfg.ec.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.AllowedCN, "peer-cert-allowed-cn", "", "Allowed CN for inter peer authentication.")
+	fs.BoolVar(&cfg.ec.PeerTLSInfo.SkipClientSANVerify, "experimental-peer-skip-client-san-verification", false, "Skip verification of SAN field in client certificate for peer connections.")
 
 	fs.Var(flags.NewStringsValueV2(""), "cipher-suites", "Comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).")
 
@@ -214,6 +215,7 @@ func newConfig() *config {
 
 	// auth
 	fs.StringVar(&cfg.ec.AuthToken, "auth-token", cfg.ec.AuthToken, "Specify auth token specific options.")
+	fs.UintVar(&cfg.ec.AuthTokenTTL, "auth-token-ttl", cfg.ec.AuthTokenTTL, "The lifetime in seconds of the auth token.")
 
 	// experimental
 	fs.BoolVar(&cfg.ec.ExperimentalInitialCorruptCheck, "experimental-initial-corrupt-check", cfg.ec.ExperimentalInitialCorruptCheck, "Enable to check data corruption before serving any client/peer traffic.")

etcdmain/etcd.go

@@ -218,7 +218,7 @@ func startProxy(cfg *config) error {
 	}
 
 	cfg.ec.Dir = filepath.Join(cfg.ec.Dir, "proxy")
-	err = os.MkdirAll(cfg.ec.Dir, fileutil.PrivateDirMode)
+	err = fileutil.TouchDirAll(cfg.ec.Dir)
 	if err != nil {
 		return err
 	}

etcdmain/gateway.go

@@ -68,7 +68,7 @@ func newGatewayStartCommand() *cobra.Command {
 	cmd.Flags().StringVar(&gatewayListenAddr, "listen-addr", "127.0.0.1:23790", "listen address")
 	cmd.Flags().StringVar(&gatewayDNSCluster, "discovery-srv", "", "DNS domain used to bootstrap initial cluster")
 	cmd.Flags().BoolVar(&gatewayInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records")
-	cmd.Flags().StringVar(&gatewayCA, "trusted-ca-file", "", "path to the client server TLS CA file.")
+	cmd.Flags().StringVar(&gatewayCA, "trusted-ca-file", "", "path to the client server TLS CA file for verifying the discovered endpoints when discovery-srv is provided.")
 
 	cmd.Flags().StringSliceVar(&gatewayEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints")
 
@@ -112,6 +112,40 @@ func startGateway(cmd *cobra.Command, args []string) {
 		}
 	}
 
+	lhost, lport, err := net.SplitHostPort(gatewayListenAddr)
+	if err != nil {
+		fmt.Println("failed to validate listen address:", gatewayListenAddr)
+		os.Exit(1)
+	}
+
+	laddrs, err := net.LookupHost(lhost)
+	if err != nil {
+		fmt.Println("failed to resolve listen host:", lhost)
+		os.Exit(1)
+	}
+	laddrsMap := make(map[string]bool)
+	for _, addr := range laddrs {
+		laddrsMap[addr] = true
+	}
+
+	for _, srv := range srvs.SRVs {
+		eaddrs, err := net.LookupHost(srv.Target)
+		if err != nil {
+			fmt.Println("failed to resolve endpoint host:", srv.Target)
+			os.Exit(1)
+		}
+		if fmt.Sprintf("%d", srv.Port) != lport {
+			continue
+		}
+
+		for _, ea := range eaddrs {
+			if laddrsMap[ea] {
+				fmt.Printf("SRV or endpoint (%s:%d->%s:%d) should not resolve to the gateway listen addr (%s)\n", srv.Target, srv.Port, ea, srv.Port, gatewayListenAddr)
+				os.Exit(1)
+			}
+		}
+	}
+
 	if len(srvs.Endpoints) == 0 {
 		plog.Fatalf("no endpoints found")
 	}

etcdmain/grpc_proxy.go

@@ -127,7 +127,7 @@ func newGRPCProxyStartCommand() *cobra.Command {
 	cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file")
 	cmd.Flags().StringVar(&grpcProxyKey, "key", "", "identify secure connections with etcd servers using this TLS key file")
 	cmd.Flags().StringVar(&grpcProxyCA, "cacert", "", "verify certificates of TLS-enabled secure etcd servers using this CA bundle")
-	cmd.Flags().BoolVar(&grpcProxyInsecureSkipTLSVerify, "insecure-skip-tls-verify", false, "skip authentication of etcd server TLS certificates")
+	cmd.Flags().BoolVar(&grpcProxyInsecureSkipTLSVerify, "insecure-skip-tls-verify", false, "skip authentication of etcd server TLS certificates (CAUTION: this option should be enabled only for testing purposes)")
 
 	// client TLS for connecting to proxy
 	cmd.Flags().StringVar(&grpcProxyListenCert, "cert-file", "", "identify secure connections to the proxy using this TLS certificate file")
@@ -267,6 +267,9 @@ func newClientCfg(eps []string) (*clientv3.Config, error) {
 			return nil, err
 		}
 		clientTLS.InsecureSkipVerify = grpcProxyInsecureSkipTLSVerify
+		if clientTLS.InsecureSkipVerify {
+			plog.Warningf("--insecure-skip-tls-verify was given, this grpc proxy process skips authentication of etcd server TLS certificates. This option should be enabled only for testing purposes.")
+		}
 		cfg.TLS = clientTLS
 		plog.Infof("ClientTLS: %s", tls)
 	}

etcdmain/help.go

@@ -162,6 +162,8 @@ security flags:
 		path to the peer certificate revocation list file.
 	--cipher-suites ''
 		comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).
+	--experimental-peer-skip-client-san-verification 'false'
+		Skip verification of SAN field in client certificate for peer connections.
 
 logging flags
 
@@ -191,6 +193,8 @@ profiling flags:
 auth flags:
 	--auth-token 'simple'
 		Specify a v3 authentication token type and its options ('simple' or 'jwt').
+	--auth-token-ttl 300
+    Time (in seconds) of the auth-token-ttl.
 
 experimental flags:
 	--experimental-initial-corrupt-check 'false'

etcdserver/api/etcdhttp/metrics.go

@@ -50,6 +50,7 @@ func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
 		if r.Method != http.MethodGet {
 			w.Header().Set("Allow", http.MethodGet)
 			http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+			plog.Warningf("/health error (status code %d)", http.StatusMethodNotAllowed)
 			return
 		}
 		h := hfunc()
@@ -97,11 +98,15 @@ func checkHealth(srv etcdserver.ServerV2) Health {
 	as := srv.Alarms()
 	if len(as) > 0 {
 		h.Health = "false"
+		for _, v := range as {
+			plog.Warningf("/health error due to an alarm %s", v.String())
+		}
 	}
 
 	if h.Health == "true" {
 		if uint64(srv.Leader()) == raft.None {
 			h.Health = "false"
+			plog.Warningf("/health error; no leader (status code %d)", http.StatusServiceUnavailable)
 		}
 	}
 
@@ -111,11 +116,13 @@ func checkHealth(srv etcdserver.ServerV2) Health {
 		cancel()
 		if err != nil {
 			h.Health = "false"
+			plog.Warningf("/health error; QGET failed %v (status code %d)", err, http.StatusServiceUnavailable)
 		}
 	}
 
 	if h.Health == "true" {
 		healthSuccess.Inc()
+		plog.Infof("/health OK (status code %d)", http.StatusOK)
 	} else {
 		healthFailed.Inc()
 	}

etcdserver/api/v2http/client.go

@@ -73,11 +73,11 @@ func handleV2(mux *http.ServeMux, server etcdserver.ServerV2, timeout time.Durat
 	}
 
 	mh := &membersHandler{
-		sec:     sec,
-		server:  server,
-		cluster: server.Cluster(),
-		timeout: timeout,
-		clock:   clockwork.NewRealClock(),
+		sec:                   sec,
+		server:                server,
+		cluster:               server.Cluster(),
+		timeout:               timeout,
+		clock:                 clockwork.NewRealClock(),
 		clientCertAuthEnabled: server.ClientCertAuthEnabled(),
 	}
 

etcdserver/api/v3rpc/interceptor.go

@@ -16,16 +16,16 @@ package v3rpc
 
 import (
 	"context"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/coreos/etcd/etcdserver"
 	"github.com/coreos/etcd/etcdserver/api"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft"
-
-	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
@@ -49,6 +49,12 @@ func newUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor {
 
 		md, ok := metadata.FromIncomingContext(ctx)
 		if ok {
+			ver, vs := "unknown", metadataGet(md, rpctypes.MetadataClientAPIVersionKey)
+			if len(vs) > 0 {
+				ver = vs[0]
+			}
+			clientRequests.WithLabelValues("unary", ver).Inc()
+
 			if ks := md[rpctypes.MetadataRequireLeaderKey]; len(ks) > 0 && ks[0] == rpctypes.MetadataHasLeader {
 				if s.Leader() == types.ID(raft.None) {
 					return nil, rpctypes.ErrGRPCNoLeader
@@ -187,6 +193,12 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor
 
 		md, ok := metadata.FromIncomingContext(ss.Context())
 		if ok {
+			ver, vs := "unknown", metadataGet(md, rpctypes.MetadataClientAPIVersionKey)
+			if len(vs) > 0 {
+				ver = vs[0]
+			}
+			clientRequests.WithLabelValues("stream", ver).Inc()
+
 			if ks := md[rpctypes.MetadataRequireLeaderKey]; len(ks) > 0 && ks[0] == rpctypes.MetadataHasLeader {
 				if s.Leader() == types.ID(raft.None) {
 					return rpctypes.ErrGRPCNoLeader
@@ -205,7 +217,6 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor
 					smap.mu.Unlock()
 					cancel()
 				}()
-
 			}
 		}
 
@@ -261,3 +272,8 @@ func monitorLeader(s *etcdserver.EtcdServer) *streamsMap {
 
 	return smap
 }
+
+func metadataGet(md metadata.MD, k string) []string {
+	k = strings.ToLower(k)
+	return md[k]
+}

etcdserver/api/v3rpc/maintenance.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"io"
+	"time"
 
 	"github.com/coreos/etcd/auth"
 	"github.com/coreos/etcd/etcdserver"
@@ -27,6 +28,7 @@ import (
 	"github.com/coreos/etcd/mvcc/backend"
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/version"
+	"github.com/dustin/go-humanize"
 )
 
 type KVGetter interface {
@@ -81,6 +83,9 @@ func (ms *maintenanceServer) Defragment(ctx context.Context, sr *pb.DefragmentRe
 	return &pb.DefragmentResponse{}, nil
 }
 
+// big enough size to hold >1 OS pages in the buffer
+const snapshotSendBufferSize = 32 * 1024
+
 func (ms *maintenanceServer) Snapshot(sr *pb.SnapshotRequest, srv pb.Maintenance_SnapshotServer) error {
 	snap := ms.bg.Backend().Snapshot()
 	pr, pw := io.Pipe()
@@ -95,19 +100,39 @@ func (ms *maintenanceServer) Snapshot(sr *pb.SnapshotRequest, srv pb.Maintenance
 		pw.Close()
 	}()
 
-	// send file data
+	// record SHA digest of snapshot data
+	// used for integrity checks during snapshot restore operation
 	h := sha256.New()
-	br := int64(0)
-	buf := make([]byte, 32*1024)
-	sz := snap.Size()
-	for br < sz {
+
+	// buffer just holds read bytes from stream
+	// response size is multiple of OS page size, fetched in boltdb
+	// e.g. 4*1024
+	buf := make([]byte, snapshotSendBufferSize)
+
+	sent := int64(0)
+	total := snap.Size()
+	size := humanize.Bytes(uint64(total))
+
+	start := time.Now()
+	plog.Infof("sending database snapshot to client %s [%d bytes]", size, total)
+	for total-sent > 0 {
 		n, err := io.ReadFull(pr, buf)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			return togRPCError(err)
 		}
-		br += int64(n)
+		sent += int64(n)
+
+		// if total is x * snapshotSendBufferSize. it is possible that
+		// resp.RemainingBytes == 0
+		// resp.Blob == zero byte but not nil
+		// does this make server response sent to client nil in proto
+		// and client stops receiving from snapshot stream before
+		// server sends snapshot SHA?
+		// No, the client will still receive non-nil response
+		// until server closes the stream with EOF
+
 		resp := &pb.SnapshotResponse{
-			RemainingBytes: uint64(sz - br),
+			RemainingBytes: uint64(total - sent),
 			Blob:           buf[:n],
 		}
 		if err = srv.Send(resp); err != nil {
@@ -116,13 +141,17 @@ func (ms *maintenanceServer) Snapshot(sr *pb.SnapshotRequest, srv pb.Maintenance
 		h.Write(buf[:n])
 	}
 
-	// send sha
+	// send SHA digest for integrity checks
+	// during snapshot restore operation
 	sha := h.Sum(nil)
+
+	plog.Infof("sending database sha256 checksum to client [%d bytes]", len(sha))
 	hresp := &pb.SnapshotResponse{RemainingBytes: 0, Blob: sha}
 	if err := srv.Send(hresp); err != nil {
 		return togRPCError(err)
 	}
 
+	plog.Infof("successfully sent database snapshot to client %s [%d bytes, took %s]", size, total, humanize.Time(start))
 	return nil
 }
 

etcdserver/api/v3rpc/metrics.go

@@ -30,9 +30,19 @@ var (
 		Name:      "client_grpc_received_bytes_total",
 		Help:      "The total number of bytes received from grpc clients.",
 	})
+
+	clientRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "etcd",
+		Subsystem: "server",
+		Name:      "client_requests_total",
+		Help:      "The total number of client requests per client version.",
+	},
+		[]string{"type", "client_api_version"},
+	)
 )
 
 func init() {
 	prometheus.MustRegister(sentBytes)
 	prometheus.MustRegister(receivedBytes)
+	prometheus.MustRegister(clientRequests)
 }

etcdserver/api/v3rpc/rpctypes/md.go

@@ -17,4 +17,6 @@ package rpctypes
 var (
 	MetadataRequireLeaderKey = "hasleader"
 	MetadataHasLeader        = "true"
+
+	MetadataClientAPIVersionKey = "client-api-version"
 )

etcdserver/api/v3rpc/watch.go

@@ -230,9 +230,10 @@ func (sws *serverWatchStream) recvLoop() error {
 
 				select {
 				case sws.ctrlStream <- wr:
+					continue
 				case <-sws.closec:
+					return nil
 				}
-				return nil
 			}
 
 			filters := FiltersFromRequest(creq)

etcdserver/api/v3rpc/watch_test.go

@@ -92,4 +92,4 @@ func createResponse(dataSize, events int) (resp *pb.WatchResponse) {
 		}
 	}
 	return resp
-}
+}

etcdserver/apply.go

@@ -110,6 +110,9 @@ func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
 	ar := &applyResult{}
 	defer func(start time.Time) {
 		warnOfExpensiveRequest(start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
+		if ar.err != nil {
+			warnOfFailedRequest(start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
+		}
 	}(time.Now())
 
 	// call into a.s.applyV3.F instead of a.F so upper appliers can check individual calls

etcdserver/backend.go

@@ -71,7 +71,7 @@ func openBackend(cfg ServerConfig) backend.Backend {
 // case, replace the db with the snapshot db sent by the leader.
 func recoverSnapshotBackend(cfg ServerConfig, oldbe backend.Backend, snapshot raftpb.Snapshot) (backend.Backend, error) {
 	var cIndex consistentIndex
-	kv := mvcc.New(oldbe, &lease.FakeLessor{}, &cIndex)
+	kv := mvcc.New(oldbe, &lease.FakeLessor{}, nil, &cIndex)
 	defer kv.Close()
 	if snapshot.Metadata.Index <= kv.ConsistentIndex() {
 		return oldbe, nil

etcdserver/config.go

@@ -95,6 +95,7 @@ type ServerConfig struct {
 	ClientCertAuthEnabled bool
 
 	AuthToken string
+	TokenTTL   uint
 
 	// InitialCorruptCheck is true to check data corruption on boot
 	// before serving any peer/client traffic.

etcdserver/config_test.go

@@ -189,8 +189,8 @@ func TestWALDir(t *testing.T) {
 
 func TestShouldDiscover(t *testing.T) {
 	tests := map[string]bool{
-		"":    false,
-		"foo": true,
+		"":                              false,
+		"foo":                           true,
 		"http://discovery.etcd.io/asdf": true,
 	}
 	for durl, w := range tests {

etcdserver/etcdserverpb/raft_internal_stringer.go

@@ -137,7 +137,7 @@ type loggableValueCompare struct {
 	Result    Compare_CompareResult `protobuf:"varint,1,opt,name=result,proto3,enum=etcdserverpb.Compare_CompareResult"`
 	Target    Compare_CompareTarget `protobuf:"varint,2,opt,name=target,proto3,enum=etcdserverpb.Compare_CompareTarget"`
 	Key       []byte                `protobuf:"bytes,3,opt,name=key,proto3"`
-	ValueSize int                   `protobuf:"bytes,7,opt,name=value_size,proto3"`
+	ValueSize int64                 `protobuf:"varint,7,opt,name=value_size,proto3"`
 	RangeEnd  []byte                `protobuf:"bytes,64,opt,name=range_end,proto3"`
 }
 
@@ -146,7 +146,7 @@ func newLoggableValueCompare(c *Compare, cv *Compare_Value) *loggableValueCompar
 		c.Result,
 		c.Target,
 		c.Key,
-		len(cv.Value),
+		int64(len(cv.Value)),
 		c.RangeEnd,
 	}
 }
@@ -160,7 +160,7 @@ func (*loggableValueCompare) ProtoMessage()    {}
 // To preserve proto encoding of the key bytes, a faked out proto type is used here.
 type loggablePutRequest struct {
 	Key         []byte `protobuf:"bytes,1,opt,name=key,proto3"`
-	ValueSize   int    `protobuf:"varint,2,opt,name=value_size,proto3"`
+	ValueSize   int64  `protobuf:"varint,2,opt,name=value_size,proto3"`
 	Lease       int64  `protobuf:"varint,3,opt,name=lease,proto3"`
 	PrevKv      bool   `protobuf:"varint,4,opt,name=prev_kv,proto3"`
 	IgnoreValue bool   `protobuf:"varint,5,opt,name=ignore_value,proto3"`
@@ -170,7 +170,7 @@ type loggablePutRequest struct {
 func NewLoggablePutRequest(request *PutRequest) *loggablePutRequest {
 	return &loggablePutRequest{
 		request.Key,
-		len(request.Value),
+		int64(len(request.Value)),
 		request.Lease,
 		request.PrevKv,
 		request.IgnoreValue,

etcdserver/membership/cluster.go

@@ -36,6 +36,7 @@ import (
 	"github.com/coreos/etcd/version"
 
 	"github.com/coreos/go-semver/semver"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // RaftCluster is a list of Members that belong to the same raft cluster
@@ -368,6 +369,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version
 	} else {
 		plog.Noticef("set the initial cluster version to %v", version.Cluster(ver.String()))
 	}
+	oldVer := c.version
 	c.version = ver
 	mustDetectDowngrade(c.version)
 	if c.store != nil {
@@ -376,6 +378,10 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version
 	if c.be != nil {
 		mustSaveClusterVersionToBackend(c.be, ver)
 	}
+	if oldVer != nil {
+		ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(oldVer.String())}).Set(0)
+	}
+	ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(ver.String())}).Set(1)
 	onSet(ver)
 }
 

etcdserver/membership/metrics.go

@@ -0,0 +1,31 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package membership
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	ClusterVersionMetrics = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "etcd",
+		Subsystem: "cluster",
+		Name:      "version",
+		Help:      "Which version is running. 1 for 'cluster_version' label with current cluster version",
+	},
+		[]string{"cluster_version"})
+)
+
+func init() {
+	prometheus.MustRegister(ClusterVersionMetrics)
+}

etcdserver/metrics.go

@@ -129,6 +129,19 @@ var (
 		Help:      "Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID.",
 	},
 		[]string{"server_id"})
+
+	fdUsed = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "os",
+		Subsystem: "fd",
+		Name:      "used",
+		Help:      "The number of used file descriptors.",
+	})
+	fdLimit = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "os",
+		Subsystem: "fd",
+		Name:      "limit",
+		Help:      "The file descriptor limit.",
+	})
 )
 
 func init() {
@@ -149,6 +162,8 @@ func init() {
 	prometheus.MustRegister(currentVersion)
 	prometheus.MustRegister(currentGoVersion)
 	prometheus.MustRegister(serverID)
+	prometheus.MustRegister(fdUsed)
+	prometheus.MustRegister(fdLimit)
 
 	currentVersion.With(prometheus.Labels{
 		"server_version": version.Version,
@@ -159,7 +174,12 @@ func init() {
 }
 
 func monitorFileDescriptor(done <-chan struct{}) {
-	ticker := time.NewTicker(5 * time.Second)
+	// This ticker will check File Descriptor Requirements ,and count all fds in used.
+	// And recorded some logs when in used >= limit/5*4. Just recorded message.
+	// If fds was more than 10K,It's low performance due to FDUsage() works.
+	// So need to increase it.
+	// See https://github.com/etcd-io/etcd/issues/11969 for more detail.
+	ticker := time.NewTicker(10 * time.Minute)
 	defer ticker.Stop()
 	for {
 		used, err := runtime.FDUsage()
@@ -167,11 +187,13 @@ func monitorFileDescriptor(done <-chan struct{}) {
 			plog.Errorf("cannot monitor file descriptor usage (%v)", err)
 			return
 		}
+		fdUsed.Set(float64(used))
 		limit, err := runtime.FDLimit()
 		if err != nil {
 			plog.Errorf("cannot monitor file descriptor usage (%v)", err)
 			return
 		}
+		fdLimit.Set(float64(limit))
 		if used >= limit/5*4 {
 			plog.Warningf("80%% of the file descriptor limit is used [used = %d, limit = %d]", used, limit)
 		}

etcdserver/raft.go

@@ -228,9 +228,19 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					r.transport.Send(r.processMessages(rd.Messages))
 				}
 
+				// Must save the snapshot file and WAL snapshot entry before saving any other entries or hardstate to
+				// ensure that recovery after a snapshot restore is possible.
+				if !raft.IsEmptySnap(rd.Snapshot) {
+					// gofail: var raftBeforeSaveSnap struct{}
+					if err := r.storage.SaveSnap(rd.Snapshot); err != nil {
+						plog.Fatalf("failed to save Raft snapshot %v", err)
+					}
+					// gofail: var raftAfterSaveSnap struct{}
+				}
+
 				// gofail: var raftBeforeSave struct{}
 				if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
-					plog.Fatalf("raft save state and entries error: %v", err)
+					plog.Fatalf("failed to raft save state and entries %v", err)
 				}
 				if !raft.IsEmptyHardState(rd.HardState) {
 					proposalsCommitted.Set(float64(rd.HardState.Commit))
@@ -238,10 +248,14 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 				// gofail: var raftAfterSave struct{}
 
 				if !raft.IsEmptySnap(rd.Snapshot) {
-					// gofail: var raftBeforeSaveSnap struct{}
-					if err := r.storage.SaveSnap(rd.Snapshot); err != nil {
-						plog.Fatalf("raft save snapshot error: %v", err)
+					// Force WAL to fsync its hard state before Release() releases
+					// old data from the WAL. Otherwise could get an error like:
+					// panic: tocommit(107) is out of range [lastIndex(84)]. Was the raft log corrupted, truncated, or lost?
+					// See https://github.com/etcd-io/etcd/issues/10219 for more details.
+					if err := r.storage.Sync(); err != nil {
+						plog.Fatalf("failed to sync Raft snapshot %v", err)
 					}
+
 					// etcdserver now claim the snapshot has been persisted onto the disk
 					notifyc <- struct{}{}
 
@@ -249,6 +263,10 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					r.raftStorage.ApplySnapshot(rd.Snapshot)
 					plog.Infof("raft applied incoming snapshot at index %d", rd.Snapshot.Metadata.Index)
 					// gofail: var raftAfterApplySnap struct{}
+
+					if err := r.storage.Release(rd.Snapshot); err != nil {
+						plog.Fatalf("failed to release Raft wal %v", err)
+					}
 				}
 
 				r.raftStorage.Append(rd.Entries)

etcdserver/server.go

@@ -375,7 +375,15 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 		if cfg.ShouldDiscover() {
 			plog.Warningf("discovery token ignored since a cluster has already been initialized. Valid log found at %q", cfg.WALDir())
 		}
-		snapshot, err = ss.Load()
+		
+		// Find a snapshot to start/restart a raft node
+		walSnaps, serr := wal.ValidSnapshotEntries(cfg.WALDir())
+		if serr != nil {
+			return nil, serr
+		}
+		// snapshot files can be orphaned if etcd crashes after writing them but before writing the corresponding
+		// wal log entries
+		snapshot, err = ss.LoadNewestAvailable(walSnaps)
 		if err != nil && err != snap.ErrNoSnapshot {
 			return nil, err
 		}
@@ -448,7 +456,20 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 	// always recover lessor before kv. When we recover the mvcc.KV it will reattach keys to its leases.
 	// If we recover mvcc.KV first, it will attach the keys to the wrong lessor before it recovers.
 	srv.lessor = lease.NewLessor(srv.be, int64(math.Ceil(minTTL.Seconds())))
-	srv.kv = mvcc.New(srv.be, srv.lessor, &srv.consistIndex)
+
+	tp, err := auth.NewTokenProvider(cfg.AuthToken,
+		func(index uint64) <-chan struct{} {
+			return srv.applyWait.Wait(index)
+		},
+		time.Duration(cfg.TokenTTL)*time.Second,
+	)
+	if err != nil {
+		plog.Warningf("failed to create token provider,err is %v", err)
+		return nil, err
+	}
+	srv.authStore = auth.NewAuthStore(srv.be, tp)
+
+	srv.kv = mvcc.New(srv.be, srv.lessor, srv.authStore, &srv.consistIndex)
 	if beExist {
 		kvindex := srv.kv.ConsistentIndex()
 		// TODO: remove kvindex != 0 checking when we do not expect users to upgrade
@@ -470,16 +491,6 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 	}()
 
 	srv.consistIndex.setConsistentIndex(srv.kv.ConsistentIndex())
-	tp, err := auth.NewTokenProvider(cfg.AuthToken,
-		func(index uint64) <-chan struct{} {
-			return srv.applyWait.Wait(index)
-		},
-	)
-	if err != nil {
-		plog.Errorf("failed to create token provider: %s", err)
-		return nil, err
-	}
-	srv.authStore = auth.NewAuthStore(srv.be, tp)
 	if num := cfg.AutoCompactionRetention; num != 0 {
 		srv.compactor, err = compactor.New(cfg.AutoCompactionMode, num, srv.kv, srv)
 		if err != nil {
@@ -602,6 +613,7 @@ func (s *EtcdServer) start() {
 	s.leaderChanged = make(chan struct{})
 	if s.ClusterVersion() != nil {
 		plog.Infof("starting server... [version: %v, cluster version: %v]", version.Version, version.Cluster(s.ClusterVersion().String()))
+		membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(s.ClusterVersion().String())}).Set(1)
 	} else {
 		plog.Infof("starting server... [version: %v, cluster version: to_be_decided]", version.Version)
 	}
@@ -612,12 +624,13 @@ func (s *EtcdServer) start() {
 
 func (s *EtcdServer) purgeFile() {
 	var dberrc, serrc, werrc <-chan error
+	var dbdonec, sdonec, wdonec <-chan struct{}
 	if s.Cfg.MaxSnapFiles > 0 {
-		dberrc = fileutil.PurgeFile(s.Cfg.SnapDir(), "snap.db", s.Cfg.MaxSnapFiles, purgeFileInterval, s.done)
-		serrc = fileutil.PurgeFile(s.Cfg.SnapDir(), "snap", s.Cfg.MaxSnapFiles, purgeFileInterval, s.done)
+		dbdonec, dberrc = fileutil.PurgeFileWithDoneNotify(s.Cfg.SnapDir(), "snap.db", s.Cfg.MaxSnapFiles, purgeFileInterval, s.stopping)
+		sdonec, serrc = fileutil.PurgeFileWithDoneNotify(s.Cfg.SnapDir(), "snap", s.Cfg.MaxSnapFiles, purgeFileInterval, s.stopping)
 	}
 	if s.Cfg.MaxWALFiles > 0 {
-		werrc = fileutil.PurgeFile(s.Cfg.WALDir(), "wal", s.Cfg.MaxWALFiles, purgeFileInterval, s.done)
+		wdonec, werrc = fileutil.PurgeFileWithDoneNotify(s.Cfg.WALDir(), "wal", s.Cfg.MaxWALFiles, purgeFileInterval, s.stopping)
 	}
 	select {
 	case e := <-dberrc:
@@ -627,6 +640,15 @@ func (s *EtcdServer) purgeFile() {
 	case e := <-werrc:
 		plog.Fatalf("failed to purge wal file %v", e)
 	case <-s.stopping:
+		if dbdonec != nil {
+			<-dbdonec
+		}
+		if sdonec != nil {
+			<-sdonec
+		}
+		if wdonec != nil {
+			<-wdonec
+		}
 		return
 	}
 }
@@ -1543,6 +1565,10 @@ func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) {
 		}
 		plog.Infof("saved snapshot at index %d", snap.Metadata.Index)
 
+		if err = s.r.storage.Release(snap); err != nil {
+				plog.Panicf("failed to release wal %v", err)
+		}
+
 		// When sending a snapshot, etcd will pause compaction.
 		// After receives a snapshot, the slow follower needs to get all the entries right after
 		// the snapshot sent to catch up. If we do not pause compaction, the log entries right after

etcdserver/server_test.go

@@ -916,17 +916,17 @@ func TestSnapshot(t *testing.T) {
 		r:     *r,
 		store: st,
 	}
-	srv.kv = mvcc.New(be, &lease.FakeLessor{}, &srv.consistIndex)
+	srv.kv = mvcc.New(be, &lease.FakeLessor{}, nil, &srv.consistIndex)
 	srv.be = be
 
 	ch := make(chan struct{}, 2)
 
 	go func() {
-		gaction, _ := p.Wait(1)
+		gaction, _ := p.Wait(2)
 		defer func() { ch <- struct{}{} }()
 
-		if len(gaction) != 1 {
-			t.Fatalf("len(action) = %d, want 1", len(gaction))
+		if len(gaction) != 2 {
+			t.Fatalf("len(action) = %d, want 2", len(gaction))
 		}
 		if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "SaveSnap"}) {
 			t.Errorf("action = %s, want SaveSnap", gaction[0])
@@ -994,7 +994,7 @@ func TestSnapshotOrdering(t *testing.T) {
 
 	be, tmpPath := backend.NewDefaultTmpBackend()
 	defer os.RemoveAll(tmpPath)
-	s.kv = mvcc.New(be, &lease.FakeLessor{}, &s.consistIndex)
+	s.kv = mvcc.New(be, &lease.FakeLessor{}, nil, &s.consistIndex)
 	s.be = be
 
 	s.start()
@@ -1013,6 +1013,9 @@ func TestSnapshotOrdering(t *testing.T) {
 	if ac := <-p.Chan(); ac.Name != "Save" {
 		t.Fatalf("expected Save, got %+v", ac)
 	}
+	if ac := <-p.Chan(); ac.Name != "SaveSnap" {
+		t.Fatalf("expected Save, got %+v", ac)
+	}
 	if ac := <-p.Chan(); ac.Name != "Save" {
 		t.Fatalf("expected Save, got %+v", ac)
 	}
@@ -1022,7 +1025,10 @@ func TestSnapshotOrdering(t *testing.T) {
 		t.Fatalf("expected file %q, got missing", snapPath)
 	}
 	// unblock SaveSnapshot, etcdserver now permitted to move snapshot file
-	if ac := <-p.Chan(); ac.Name != "SaveSnap" {
+	if ac := <-p.Chan(); ac.Name != "Sync" {
+		t.Fatalf("expected SaveSnap, got %+v", ac)
+	}
+	if ac := <-p.Chan(); ac.Name != "Release" {
 		t.Fatalf("expected SaveSnap, got %+v", ac)
 	}
 }
@@ -1052,23 +1058,27 @@ func TestTriggerSnap(t *testing.T) {
 	}
 	srv.applyV2 = &applierV2store{store: srv.store, cluster: srv.cluster}
 
-	srv.kv = mvcc.New(be, &lease.FakeLessor{}, &srv.consistIndex)
+	srv.kv = mvcc.New(be, &lease.FakeLessor{}, nil, &srv.consistIndex)
 	srv.be = be
 
 	srv.start()
 
 	donec := make(chan struct{})
 	go func() {
-		wcnt := 2 + snapc
+		wcnt := 3 + snapc
 		gaction, _ := p.Wait(wcnt)
 
 		// each operation is recorded as a Save
-		// (SnapCount+1) * Puts + SaveSnap = (SnapCount+1) * Save + SaveSnap
+		// (SnapCount+1) * Puts + SaveSnap = (SnapCount+1) * Save + SaveSnap + Release
 		if len(gaction) != wcnt {
+			t.Logf("gaction: %v", gaction)
 			t.Fatalf("len(action) = %d, want %d", len(gaction), wcnt)
 		}
-		if !reflect.DeepEqual(gaction[wcnt-1], testutil.Action{Name: "SaveSnap"}) {
-			t.Errorf("action = %s, want SaveSnap", gaction[wcnt-1])
+		if !reflect.DeepEqual(gaction[wcnt-2], testutil.Action{Name: "SaveSnap"}) {
+			t.Errorf("action = %s, want SaveSnap", gaction[wcnt-2])
+		}
+		if !reflect.DeepEqual(gaction[wcnt-1], testutil.Action{Name: "Release"}) {
+			t.Errorf("action = %s, want Release", gaction[wcnt-1])
 		}
 		close(donec)
 	}()
@@ -1121,7 +1131,7 @@ func TestConcurrentApplyAndSnapshotV3(t *testing.T) {
 	defer func() {
 		os.RemoveAll(tmpPath)
 	}()
-	s.kv = mvcc.New(be, &lease.FakeLessor{}, &s.consistIndex)
+	s.kv = mvcc.New(be, &lease.FakeLessor{}, nil, &s.consistIndex)
 	s.be = be
 
 	s.start()

etcdserver/storage.go

@@ -34,6 +34,10 @@ type Storage interface {
 	SaveSnap(snap raftpb.Snapshot) error
 	// Close closes the Storage and performs finalization.
 	Close() error
+	// Release releases the locked wal files older than the provided snapshot.
+	Release(snap raftpb.Snapshot) error
+	// Sync WAL
+	Sync() error
 }
 
 type storage struct {
@@ -45,22 +49,32 @@ func NewStorage(w *wal.WAL, s *snap.Snapshotter) Storage {
 	return &storage{w, s}
 }
 
-// SaveSnap saves the snapshot to disk and release the locked
-// wal files since they will not be used.
+// SaveSnap saves the snapshot file to disk and writes the WAL snapshot entry.
 func (st *storage) SaveSnap(snap raftpb.Snapshot) error {
 	walsnap := walpb.Snapshot{
 		Index: snap.Metadata.Index,
 		Term:  snap.Metadata.Term,
 	}
-	err := st.WAL.SaveSnapshot(walsnap)
+
+	// save the snapshot file before writing the snapshot to the wal.
+	// This makes it possible for the snapshot file to become orphaned, but prevents
+	// a WAL snapshot entry from having no corresponding snapshot file.
+	err := st.Snapshotter.SaveSnap(snap)
 	if err != nil {
 		return err
 	}
-	err = st.Snapshotter.SaveSnap(snap)
-	if err != nil {
+
+	return st.WAL.SaveSnapshot(walsnap)
+}
+
+// Release releases resources older than the given snap and are no longer needed:
+// - releases the locks to the wal files that are older than the provided wal for the given snap.
+// - deletes any .snap.db files that are older than the given snap.
+func (st *storage) Release(snap raftpb.Snapshot) error {
+	if err := st.WAL.ReleaseLockTo(snap.Metadata.Index); err != nil {
 		return err
 	}
-	return st.WAL.ReleaseLockTo(snap.Metadata.Index)
+	return st.Snapshotter.ReleaseSnapDBs(snap)
 }
 
 func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) {

etcdserver/util.go

@@ -109,6 +109,15 @@ func warnOfExpensiveRequest(now time.Time, reqStringer fmt.Stringer, respMsg pro
 	warnOfExpensiveGenericRequest(now, reqStringer, "", resp, err)
 }
 
+func warnOfFailedRequest(now time.Time, reqStringer fmt.Stringer, respMsg proto.Message, err error) {
+	var resp string
+	if !isNil(respMsg) {
+		resp = fmt.Sprintf("size:%d", proto.Size(respMsg))
+	}
+	d := time.Since(now)
+	plog.Warningf("failed to apply request,took %v,request %s,resp %s,err is %v", d, reqStringer.String(), resp, err)
+}
+
 func warnOfExpensiveReadOnlyTxnRequest(now time.Time, r *pb.TxnRequest, txnResponse *pb.TxnResponse, err error) {
 	reqStringer := pb.NewLoggableTxnRequest(r)
 	var resp string

etcdserver/v3_server.go

@@ -378,9 +378,10 @@ func (s *EtcdServer) Authenticate(ctx context.Context, r *pb.AuthenticateRequest
 			return nil, err
 		}
 
+		// internalReq doesn't need to have Password because the above s.AuthStore().CheckPassword() already did it.
+		// In addition, it will let a WAL entry not record password as a plain text.
 		internalReq := &pb.InternalAuthenticateRequest{
 			Name:        r.Name,
-			Password:    r.Password,
 			SimpleToken: st,
 		}
 
@@ -513,39 +514,30 @@ func (s *EtcdServer) raftRequestOnce(ctx context.Context, r pb.InternalRaftReque
 }
 
 func (s *EtcdServer) raftRequest(ctx context.Context, r pb.InternalRaftRequest) (proto.Message, error) {
-	for {
-		resp, err := s.raftRequestOnce(ctx, r)
-		if err != auth.ErrAuthOldRevision {
-			return resp, err
-		}
-	}
+	return s.raftRequestOnce(ctx, r)
 }
 
 // doSerialize handles the auth logic, with permissions checked by "chk", for a serialized request "get". Returns a non-nil error on authentication failure.
 func (s *EtcdServer) doSerialize(ctx context.Context, chk func(*auth.AuthInfo) error, get func()) error {
-	for {
-		ai, err := s.AuthInfoFromCtx(ctx)
-		if err != nil {
-			return err
-		}
-		if ai == nil {
-			// chk expects non-nil AuthInfo; use empty credentials
-			ai = &auth.AuthInfo{}
-		}
-		if err = chk(ai); err != nil {
-			if err == auth.ErrAuthOldRevision {
-				continue
-			}
-			return err
-		}
-		// fetch response for serialized request
-		get()
-		//  empty credentials or current auth info means no need to retry
-		if ai.Revision == 0 || ai.Revision == s.authStore.Revision() {
-			return nil
-		}
-		// avoid TOCTOU error, retry of the request is required.
+	ai, err := s.AuthInfoFromCtx(ctx)
+	if err != nil {
+		return err
 	}
+	if ai == nil {
+		// chk expects non-nil AuthInfo; use empty credentials
+		ai = &auth.AuthInfo{}
+	}
+	if err = chk(ai); err != nil {
+		return err
+	}
+	// fetch response for serialized request
+	get()
+	// check for stale token revision in case the auth store was updated while
+	// the request has been handled.
+	if ai.Revision != 0 && ai.Revision != s.authStore.Revision() {
+		return auth.ErrAuthOldRevision
+	}
+	return nil
 }
 
 func (s *EtcdServer) processInternalRaftRequestOnce(ctx context.Context, r pb.InternalRaftRequest) (*applyResult, error) {

functional/agent/server.go

@@ -61,10 +61,10 @@ func NewServer(
 	address string,
 ) *Server {
 	return &Server{
-		lg:      lg,
-		network: network,
-		address: address,
-		last:    rpcpb.Operation_NOT_STARTED,
+		lg:                         lg,
+		network:                    network,
+		address:                    address,
+		last:                       rpcpb.Operation_NOT_STARTED,
 		advertiseClientPortToProxy: make(map[int]proxy.Server),
 		advertisePeerPortToProxy:   make(map[int]proxy.Server),
 	}

glide.lock

@@ -0,0 +1,230 @@
+hash: 87224cbe021ea625798ac508c8963f49ff4ceb852d99da50db9d6b7fe95282f4
+updated: 2020-05-04T21:27:57.570766+08:00
+imports:
+- name: github.com/beorn7/perks
+  version: 37c8de3658fcb183f997c4e13e8337516ab753e6
+  subpackages:
+  - quantile
+- name: github.com/bgentry/speakeasy
+  version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd
+- name: github.com/coreos/bbolt
+  version: a0458a2b35708eef59eb5f620ceb3cd1c01a824d
+- name: github.com/coreos/go-semver
+  version: 8ab6407b697782a06568d4b7f1db25550ec2e4c6
+  subpackages:
+  - semver
+- name: github.com/coreos/go-systemd
+  version: e64a0ec8b42a61e2a9801dc1d0abe539dea79197
+  subpackages:
+  - daemon
+  - journal
+  - util
+- name: github.com/coreos/pkg
+  version: 97fdf19511ea361ae1c100dd393cc47f8dcfa1e1
+  subpackages:
+  - capnslog
+  - dlopen
+- name: github.com/cpuguy83/go-md2man
+  version: 23709d0847197db6021a51fdb193e66e9222d4e7
+  subpackages:
+  - md2man
+- name: github.com/dgrijalva/jwt-go
+  version: d2709f9f1f31ebcda9651b03077758c1f3a0018c
+- name: github.com/dustin/go-humanize
+  version: 9f541cc9db5d55bce703bd99987c9d5cb8eea45e
+- name: github.com/ghodss/yaml
+  version: 0ca9ea5df5451ffdf184b4428c902747c2c11cd7
+- name: github.com/gogo/protobuf
+  version: ba06b47c162d49f2af050fb4c75bcbc86a159d5c
+  subpackages:
+  - gogoproto
+  - proto
+  - protoc-gen-gogo/descriptor
+- name: github.com/golang/groupcache
+  version: 869f871628b6baa9cfbc11732cdf6546b17c1298
+  subpackages:
+  - lru
+- name: github.com/golang/protobuf
+  version: 6c65a5562fc06764971b7c5d05c76c75e84bdbf7
+  subpackages:
+  - jsonpb
+  - proto
+  - protoc-gen-go/descriptor
+  - ptypes
+  - ptypes/any
+  - ptypes/duration
+  - ptypes/struct
+  - ptypes/timestamp
+- name: github.com/google/btree
+  version: 4030bb1f1f0c35b30ca7009e9ebd06849dd45306
+- name: github.com/google/uuid
+  version: d460ce9f8df2e77fb1ba55ca87fafed96c607494
+- name: github.com/gorilla/websocket
+  version: 4201258b820c74ac8e6922fc9e6b52f71fe46f8d
+- name: github.com/grpc-ecosystem/go-grpc-middleware
+  version: c250d6563d4d4c20252cd865923440e829844f4e
+- name: github.com/grpc-ecosystem/go-grpc-prometheus
+  version: 0dafe0d496ea71181bf2dd039e7e3f44b6bd11a7
+- name: github.com/grpc-ecosystem/grpc-gateway
+  version: 07f5e79768022f9a3265235f0db4ac8c3f675fec
+  subpackages:
+  - runtime
+  - runtime/internal
+  - utilities
+- name: github.com/inconshreveable/mousetrap
+  version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
+- name: github.com/jonboulle/clockwork
+  version: 2eee05ed794112d45db504eb05aa693efd2b8b09
+- name: github.com/json-iterator/go
+  version: 27518f6661eba504be5a7a9a9f6d9460d892ade3
+- name: github.com/kr/pty
+  version: 2c10821df3c3cf905230d078702dfbe9404c9b23
+- name: github.com/mattn/go-runewidth
+  version: 9e777a8366cce605130a531d2cd6363d07ad7317
+  subpackages:
+  - runewidth.go
+- name: github.com/matttproud/golang_protobuf_extensions
+  version: c182affec369e30f25d3eb8cd8a478dee585ae7d
+  subpackages:
+  - pbutil
+- name: github.com/modern-go/concurrent
+  version: bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94
+- name: github.com/modern-go/reflect2
+  version: 94122c33edd36123c84d5368cfb2b69df93a0ec8
+- name: github.com/olekukonko/tablewriter
+  version: a0225b3f23b5ce0cbec6d7a66a968f8a59eca9c4
+- name: github.com/prometheus/client_golang
+  version: 5cec1d0429b02e4323e042eb04dafdb079ddf568
+  subpackages:
+  - prometheus
+  - prometheus/promhttp
+- name: github.com/prometheus/client_model
+  version: 6f3806018612930941127f2a7c6c453ba2c527d2
+  subpackages:
+  - go
+- name: github.com/prometheus/common
+  version: e3fb1a1acd7605367a2b378bc2e2f893c05174b7
+  subpackages:
+  - expfmt
+  - internal/bitbucket.org/ww/goautoneg
+  - model
+- name: github.com/prometheus/procfs
+  version: a6e9df898b1336106c743392c48ee0b71f5c4efa
+  subpackages:
+  - xfs
+- name: github.com/russross/blackfriday
+  version: 4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c
+- name: github.com/sirupsen/logrus
+  version: f006c2ac4710855cf0f916dd6b77acf6b048dc6e
+- name: github.com/soheilhy/cmux
+  version: e09e9389d85d8492d313d73d1469c029e710623f
+- name: github.com/spf13/cobra
+  version: 1c44ec8d3f1552cac48999f9306da23c4d8a288b
+- name: github.com/spf13/pflag
+  version: e57e3eeb33f795204c1ca35f56c44f83227c6e66
+- name: github.com/tmc/grpc-websocket-proxy
+  version: 89b8d40f7ca833297db804fcb3be53a76d01c238
+  subpackages:
+  - wsproxy
+- name: github.com/ugorji/go
+  version: bdcc60b419d136a85cdf2e7cbcac34b3f1cd6e57
+  subpackages:
+  - codec
+- name: github.com/urfave/cli
+  version: 1efa31f08b9333f1bd4882d61f9d668a70cd902e
+- name: github.com/xiang90/probing
+  version: 07dd2e8dfe18522e9c447ba95f2fe95262f63bb2
+- name: go.uber.org/atomic
+  version: 845920076a298bdb984fb0f1b86052e4ca0a281c
+- name: go.uber.org/multierr
+  version: b587143a48b62b01d337824eab43700af6ffe222
+- name: go.uber.org/zap
+  version: 27376062155ad36be76b0f12cf1572a221d3a48c
+  subpackages:
+  - buffer
+  - internal/bufferpool
+  - internal/color
+  - internal/exit
+  - zapcore
+- name: golang.org/x/crypto
+  version: c2843e01d9a2bc60bb26ad24e09734fdc2d9ec58
+  subpackages:
+  - bcrypt
+  - blowfish
+  - ssh/terminal
+- name: golang.org/x/net
+  version: 74dc4d7220e7acc4e100824340f3e66577424772
+  subpackages:
+  - context
+  - http/httpguts
+  - http2
+  - http2/hpack
+  - idna
+  - internal/timeseries
+  - trace
+- name: golang.org/x/sys
+  version: fde4db37ae7ad8191b03d30d27f258b5291ae4e3
+  subpackages:
+  - unix
+  - windows
+- name: golang.org/x/text
+  version: 342b2e1fbaa52c93f31447ad2c6abc048c63e475
+  subpackages:
+  - secure/bidirule
+  - transform
+  - unicode/bidi
+  - unicode/norm
+- name: golang.org/x/time
+  version: c06e80d9300e4443158a03817b8a8cb37d230320
+  subpackages:
+  - rate
+- name: google.golang.org/genproto
+  version: 09f6ed296fc66555a25fe4ce95173148778dfa85
+  subpackages:
+  - googleapis/api/annotations
+  - googleapis/rpc/status
+- name: google.golang.org/grpc
+  version: 6eaf6f47437a6b4e2153a190160ef39a92c7eceb
+  subpackages:
+  - balancer
+  - balancer/base
+  - balancer/roundrobin
+  - binarylog/grpc_binarylog_v1
+  - codes
+  - connectivity
+  - credentials
+  - credentials/internal
+  - encoding
+  - encoding/proto
+  - grpclog
+  - health
+  - health/grpc_health_v1
+  - internal
+  - internal/backoff
+  - internal/balancerload
+  - internal/binarylog
+  - internal/channelz
+  - internal/envconfig
+  - internal/grpcrand
+  - internal/grpcsync
+  - internal/syscall
+  - internal/transport
+  - keepalive
+  - metadata
+  - naming
+  - peer
+  - resolver
+  - resolver/dns
+  - resolver/passthrough
+  - serviceconfig
+  - stats
+  - status
+  - tap
+  - transport
+- name: gopkg.in/cheggaaa/pb.v1
+  version: 226d21d43a305fac52b3a104ef83e721b15275e0
+- name: gopkg.in/yaml.v2
+  version: 51d6538a90f86fe93ac480b35f37b2be17fef232
+- name: sigs.k8s.io/yaml
+  version: fd68e9863619f6ec2fdd8625fe1f02e7c877e480
+testImports: []

glide.yaml

@@ -0,0 +1,168 @@
+package: github.com/coreos/etcd
+ignore:
+- google.golang.org/appengine
+import:
+- package: github.com/bgentry/speakeasy
+  version: v0.1.0
+- package: github.com/coreos/bbolt
+  version: v1.3.3
+- package: github.com/coreos/go-semver
+  version: v0.2.0
+  subpackages:
+  - semver
+- package: github.com/coreos/go-systemd
+  version: v20
+  subpackages:
+  - daemon
+  - journal
+  - util
+- package: go.uber.org/zap
+  version: v1.10.0
+- package: github.com/coreos/pkg
+  version: v4
+  subpackages:
+  - capnslog
+- package: github.com/cpuguy83/go-md2man
+  version: 23709d0847197db6021a51fdb193e66e9222d4e7
+- package: github.com/dustin/go-humanize
+  version: v1.0.0
+- package: github.com/ghodss/yaml
+  version: v1.0.0
+- package: github.com/gogo/protobuf
+  version: v1.2.1
+  subpackages:
+  - proto
+  - gogoproto
+- package: github.com/gorilla/websocket
+  version: 4201258b820c74ac8e6922fc9e6b52f71fe46f8d
+- package: github.com/golang/groupcache
+  version: 869f871628b6baa9cfbc11732cdf6546b17c1298
+  subpackages:
+  - lru
+- package: github.com/golang/protobuf
+  version: v1.3.2
+  subpackages:
+  - jsonpb
+  - proto
+- package: github.com/google/btree
+  version: v1.0.0
+- package: github.com/google/uuid
+  version: v1.0.0
+- package: github.com/grpc-ecosystem/grpc-gateway
+  version: v1.3.1
+- package: github.com/jonboulle/clockwork
+  version: v0.1.0
+- package: github.com/kr/pty
+  version: v1.0.0
+- package: github.com/olekukonko/tablewriter
+  version: a0225b3f23b5ce0cbec6d7a66a968f8a59eca9c4
+- package: github.com/mattn/go-runewidth
+  version: v0.0.2
+  subpackages:
+  - runewidth.go
+- package: github.com/prometheus/client_golang
+  version: 5cec1d0429b02e4323e042eb04dafdb079ddf568
+  subpackages:
+  - prometheus
+  - prometheus/promhttp
+- package: github.com/prometheus/client_model
+  version: 6f3806018612930941127f2a7c6c453ba2c527d2
+  subpackages:
+  - go
+- package: github.com/prometheus/common
+  version: e3fb1a1acd7605367a2b378bc2e2f893c05174b7
+- package: github.com/prometheus/procfs
+  version: a6e9df898b1336106c743392c48ee0b71f5c4efa
+  subpackages:
+  - xfs
+- package: github.com/grpc-ecosystem/go-grpc-middleware
+  version: v1.0.0
+- package: github.com/grpc-ecosystem/go-grpc-prometheus
+  version: 0dafe0d496ea71181bf2dd039e7e3f44b6bd11a7
+- package: github.com/spf13/cobra
+  version: 1c44ec8d3f1552cac48999f9306da23c4d8a288b
+- package: github.com/spf13/pflag
+  version: v1.0.0
+- package: github.com/ugorji/go
+  version: bdcc60b419d136a85cdf2e7cbcac34b3f1cd6e57
+  subpackages:
+  - codec
+- package: github.com/json-iterator/go
+  version: v1.1.7
+- package: github.com/modern-go/reflect2
+  version: v1.0.1
+- package: github.com/urfave/cli
+  version: v1.18.0
+- package: github.com/xiang90/probing
+  version: 0.0.1
+- package: golang.org/x/crypto
+  version: c2843e01d9a2bc60bb26ad24e09734fdc2d9ec58
+  subpackages:
+  - bcrypt
+  - blowfish
+- package: golang.org/x/net
+  version: 74dc4d7220e7acc4e100824340f3e66577424772
+  subpackages:
+  - context
+  - http2
+  - http2/hpack
+  - internal/timeseries
+  - trace
+- package: golang.org/x/sys
+  version: fde4db37ae7ad8191b03d30d27f258b5291ae4e3
+  subpackages:
+  - unix
+  - windows
+- package: golang.org/x/time
+  version: c06e80d9300e4443158a03817b8a8cb37d230320
+  subpackages:
+  - rate
+- package: google.golang.org/grpc
+  version: v1.23.0
+  subpackages:
+  - balancer
+  - codes
+  - connectivity
+  - credentials
+  - grpclog
+  - internal
+  - keepalive
+  - status
+  - metadata
+  - naming
+  - peer
+  - transport
+  - health
+  - health/grpc_health_v1
+  - resolver
+  - resolver/dns
+  - resolver/passthrough
+- package: gopkg.in/cheggaaa/pb.v1
+  version: v1.0.2
+- package: gopkg.in/yaml.v2
+  version: v2.2.2
+- package: sigs.k8s.io/yaml
+  version: v1.1.0
+- package: github.com/dgrijalva/jwt-go
+  version: v3.0.0
+- package: google.golang.org/genproto
+  version: 09f6ed296fc66555a25fe4ce95173148778dfa85
+  subpackages:
+  - googleapis/rpc/status
+- package: golang.org/x/text
+  version: 342b2e1fbaa52c93f31447ad2c6abc048c63e475
+  subpackages:
+  - secure/bidirule
+  - transform
+  - unicode/bidi
+  - unicode/norm
+- package: github.com/russross/blackfriday
+  version: 4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c
+- package: github.com/sirupsen/logrus
+  version: v1.0.3
+- package: github.com/soheilhy/cmux
+  version: v0.1.4
+- package: github.com/tmc/grpc-websocket-proxy
+  version: 89b8d40f7ca833297db804fcb3be53a76d01c238
+  subpackages:
+  - wsproxy

go.mod

@@ -1,62 +0,0 @@
-module github.com/coreos/etcd
-
-go 1.12
-
-require (
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
-	github.com/bgentry/speakeasy v0.1.0
-	github.com/coreos/bbolt v1.3.1-coreos.6
-	github.com/coreos/go-semver v0.2.0
-	github.com/coreos/go-systemd v0.0.0-20170731111925-d21964639418
-	github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf
-	github.com/cpuguy83/go-md2man v0.0.0-20170603125239-23709d084719 // indirect
-	github.com/dgrijalva/jwt-go v3.0.0+incompatible
-	github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4
-	github.com/fatih/color v1.7.0 // indirect
-	github.com/gogo/protobuf v1.2.1
-	github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903
-	github.com/golang/protobuf v1.3.2
-	github.com/google/btree v1.0.0
-	github.com/google/uuid v1.0.0
-	github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
-	github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170826090648-0dafe0d496ea
-	github.com/grpc-ecosystem/grpc-gateway v1.3.0
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jonboulle/clockwork v0.1.0
-	github.com/json-iterator/go v1.1.7
-	github.com/kr/pty v1.0.0
-	github.com/mattn/go-colorable v0.1.2 // indirect
-	github.com/mattn/go-runewidth v0.0.2 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/modern-go/reflect2 v1.0.1
-	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
-	github.com/pkg/errors v0.8.1 // indirect
-	github.com/prometheus/client_golang v0.0.0-20171005112915-5cec1d0429b0
-	github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612
-	github.com/prometheus/common v0.0.0-20171104095907-e3fb1a1acd76 // indirect
-	github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13 // indirect
-	github.com/russross/blackfriday v0.0.0-20170728175326-4048872b16cc // indirect
-	github.com/sirupsen/logrus v1.0.3 // indirect
-	github.com/soheilhy/cmux v0.1.4
-	github.com/spf13/cobra v0.0.0-20151124153217-1c44ec8d3f15
-	github.com/spf13/pflag v1.0.0
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8
-	github.com/urfave/cli v1.18.0
-	github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18
-	go.uber.org/atomic v1.3.1 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
-	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
-	golang.org/x/net v0.0.0-20190813000000-74dc4d7220e7
-	golang.org/x/time v0.0.0-20170420181420-c06e80d9300e
-	google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8
-	google.golang.org/grpc v1.23.0
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/cheggaaa/pb.v1 v1.0.2
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
-	gopkg.in/yaml.v2 v2.2.2
-	sigs.k8s.io/yaml v1.1.0
-)

go.sum

@@ -1,169 +0,0 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A=
-github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20170731111925-d21964639418 h1:0QH6fTJVDpblGjjozilaO++YRbnQNnTYh3yuFJHH0o8=
-github.com/coreos/go-systemd v0.0.0-20170731111925-d21964639418/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man v0.0.0-20170603125239-23709d084719 h1:5igivQzpoMP+VGgMjEsCmitTIislsbKlA5V1u968mX0=
-github.com/cpuguy83/go-md2man v0.0.0-20170603125239-23709d084719/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.0.0+incompatible h1:nfVqwkkhaRUethVJaQf5TUFdFr3YUF4lJBTf/F2XwVI=
-github.com/dgrijalva/jwt-go v3.0.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170826090648-0dafe0d496ea h1:Bzd/0fcg24qAEJyr7pTtDOn806SRBtzyloCuLTEvSOo=
-github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170826090648-0dafe0d496ea/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.3.0 h1:HJtP6RRwj2EpPCD/mhAWzSvLL/dFTdPm1UrWwanoFos=
-github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/pty v1.0.0 h1:jR04h3bskdxb8xt+5B6MoxPwDhMCe0oEgxug4Ca1YSA=
-github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.0.0-20171005112915-5cec1d0429b0 h1:uEiENdm9N5Nj3ezfwdvwBGc2EHLiUgD3hUTOaMfBn5E=
-github.com/prometheus/client_golang v0.0.0-20171005112915-5cec1d0429b0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612 h1:13pIdM2tpaDi4OVe24fgoIS7ZTqMt0QI+bwQsX5hq+g=
-github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20171104095907-e3fb1a1acd76 h1:g2v6dZgmqj2wYGPgHYX5WVaQ9IwV1ylsSiD+f8RvS1Y=
-github.com/prometheus/common v0.0.0-20171104095907-e3fb1a1acd76/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13 h1:leRfx9kcgnSDkqAFhaaUcRqpAZgnFdwZkZcdRcea1h0=
-github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/russross/blackfriday v0.0.0-20170728175326-4048872b16cc h1:Ng688TEbTGosxh0B0IQ7NqUMYZiERtWbGGGEvRLKjh4=
-github.com/russross/blackfriday v0.0.0-20170728175326-4048872b16cc/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/sirupsen/logrus v1.0.3 h1:B5C/igNWoiULof20pKfY4VntcIPqKuwEmoLZrabbUrc=
-github.com/sirupsen/logrus v1.0.3/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spf13/cobra v0.0.0-20151124153217-1c44ec8d3f15 h1:x2QTuHA5vXuhN7ZzzJNnGNItiOsvFZrq23mWI5qLjAE=
-github.com/spf13/cobra v0.0.0-20151124153217-1c44ec8d3f15/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/pflag v1.0.0 h1:oaPbdDe/x0UncahuwiPxW1GYJyilRAdsPnq3e1yaPcI=
-github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/urfave/cli v1.18.0 h1:m9MfmZWX7bwr9kUcs/Asr95j0IVXzGNNc+/5ku2m26Q=
-github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs=
-github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-go.uber.org/atomic v1.3.1 h1:U8WaWEmp56LGz7PReduqHRVF6zzs9GbMC2NEZ42dxSQ=
-go.uber.org/atomic v1.3.1/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190813000000-74dc4d7220e7 h1:HOTjhHFecQCpFnEhQ4MAH6Gf7yGklZnqaHvtezTKqnQ=
-golang.org/x/net v0.0.0-20190813000000-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/time v0.0.0-20170420181420-c06e80d9300e h1:J4S0GBcCoJ2pYYVewfgT7HU8SvahFdrkNK7FRuGCZdo=
-golang.org/x/time v0.0.0-20170420181420-c06e80d9300e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
-gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/cheggaaa/pb.v1 v1.0.2 h1:TBVEWjGePKAUdeFNbT3qP6Tt0I8GkybpMWmynFpeLd4=
-gopkg.in/cheggaaa/pb.v1 v1.0.2/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

integration/fixtures/ca-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvdN4pteH5d6f+xcxBg7rGKK6fwuuAIV79Q4ejh5A/VG6ErA+
+Z5aw30nOXDSVTmScHYCc/vdwLJ/cu3tUuOgrZE2J6X8AUW2wow78Ck4icrpwsQLR
+DN4/dU0T8Ff4d0AYucMkRdmgE0tOQX/8YvBAZr42sEwAmRVxW2V19znszZNOHJQj
+CXh6As6YBGqRvQSPr/LitmLQ8cQ77FFQ/6oC3/a5Biwdr2Pd14q1vU+nZVVkCfW5
+fbNDPSYNDlG6jU8abQhM5WslIqyIcy5kM6+rCnj+Ks2CyS0LqwZONTgQLfTfQJGc
+dwrztsTTm4+Ut4dPqouM6jQru3tC1hlNX3VFGwIDAQABAoIBADpvslGyQfyqtxcm
+AYp65zazSbQ4lT1F2m4LBa78c0dIdH6yUNO02Qg0AVnzOg9i+4g9gpce9yJVqC7y
+/Zbaqhj7obwGw8NNgDYCM+a8PPCSooRreI18kY57xuqTOkMDsVjmUPFL2HJ4GUQi
+sUCH81ttrQpCq2B08GuRZWpRI6v46DR9fUt+EQNIOgCyGUtDZWuSAtq+XwviqqLm
+nkfHpC3Zm1b69qsqPxDz0rplYc+lz/V/kcIa1uw9x/4bRBdUWgiCTY4HNLuP0ePm
+pFzOLRZGAo3uOidl/mR2ArFTKlUhjSEpRmFR1Gx7fH2cc2mszmUv0yvpA9JspYGT
+e/tH+qkCgYEA2j/t4Wv+KQ5KcCHUczPSzqS/9iwmFHWQm2jGNpvbfGNYsJk7MBku
+Lfe/OTGdxNuzA7UbQXN5e35eue2vGCiL0w8UA6bftA5hFDVFUM0YKlLj1Vy5lIJV
+HeV/DI3CNca6i6FYtD0mVLRrStj7geNlmStxPgd5dQZ//s8+OUQHOw8CgYEA3qjy
+HRv09h+A4ODsQIzqiO78YaMN0+sensFsm/OwH4Jxu7nYWp8X07FRCr+KZOqbsbbu
+tF4RHJuox4ZcB8ibUQzrD3ecKzd8p/py+ecmTkxSUOpjxMneL+aXN5znqjTuI4WP
+YCeeRtedSy3jyQ2pJDFXimM6az11SKSKr9RFRTUCgYBnLBODHe8fb28HBScOcrA0
+GbPZZtN1loIOxX/2LsWaTsiNa1KMkUrAVj7iha8Ecat1lDbXQQRrubiEAnVkYT6A
+Pr+CXm+gCbAgwnILGXlUAK1NnrDoIJimMmhWAemOTGzBNzvcsI+fOU8DKgHzTBEq
+UFwPK69h//mf9k5++cbFRwKBgB2fXm6u/H0OHehNJEFGPiGvodYfikRqYG2AkEGD
+PyhCA32VMQqFZfcc/QowB2p7p/ERxFostZwXvXGmF3JVpww2asNpB1bcj/INKOTE
+ct0x3DW1qUZSEQRQakfU0SFc313MdBG305/bKasJ1Oc3sQwGoH3hy7DewU7DzUut
+MvYNAoGBAMHxoqdTi/9ewLhKxKdwvhsyRa9Ll9d9Y85+VoI/eAGAEY2iFq9aPzBA
+pdFBkPLoApLqoPPH3m0/RHO3t/yLsyjDuF1xPlBRAfnT9GF4kTvw8sMMPQoWDLoO
+qLq31y4yzdlKQTT9SauXb7KRfBj0ClG863dM6ot9YWqVBPKuV5Yk
+-----END RSA PRIVATE KEY-----

integration/fixtures/ca.crt

@@ -1,23 +1,22 @@
 -----BEGIN CERTIFICATE-----
-MIID0jCCArqgAwIBAgIUGMTka1d/PO3J5ui12qLiCKjR1rMwDQYJKoZIhvcNAQEL
+MIIDrjCCApagAwIBAgIUeR3i5w/rlikvZJQTuUqZqOnoygswDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzEwMDUyMTU5MDBaFw0yNzEwMDMyMTU5
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMjM4MDBaFw0yOTEwMDUyMjM4
 MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
-AoIBAQC/oSHQqlc05uZYMHHxkkF/eTM4uh7Z/6TSjChMfnNk+5rZJzwSkLeL47Am
-kmcGnB9xo4tKkiDjq6MbHwgK2YpgdF5rRSC89BYBWPVvSMxMWV4Pvx/q3mdUjh7N
-tb9udJxG6c3rgI+2t2zcx1+qFGeKw0JSHKpcI6UKNm7E/xY0lqJ0ptHruBgFQaqy
-VdqVa5QkyF2imnFEdgakO2EuQR3vVAr5sM552LLnngNsfsxLStxXWDJZ/+34k2ON
-jJy5KEQ1KgYmVwi8843yQRDkvAjZ4Z1LzbcQQ83/6oT3kAfsOjy8bdsxMZBWSA53
-W9LuKUOk6WLRY0S3vUjzdw8To15RAgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIBBjAS
-BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBSt4lqZZ20BpzYG8GSym10Intbr
-EzAfBgNVHSMEGDAWgBSt4lqZZ20BpzYG8GSym10IntbrEzANBgkqhkiG9w0BAQsF
-AAOCAQEAA6TqqLkPdI6zuda71LL68myXbN2qdxzs7HK8jkGPM2cU6Ii0G66TvesM
-gf1k3VddbvF8mPCyhdLYRArQeDLxKDLq/efosOj/NiLtepXea6Ib7XEo2AKCe21S
-SpBAZ2Szx8mGa7IY3ISfxkY0PpGhe2G0Rf0kOX5SYhQ/4TdAoe2pr8jFX6b7Kbdl
-yBxq9nDYjSCA7fC5Yr3Pup7Uu9fh1TJr+62DMBeeQN1XFZ0hEdu5sk4jkNq3ijC7
-/vpDyzRduUGpbp8Jy4HzoyWrCmp+KEznEWNmXb/HoPHKBlAz0ovjzU+jnFYi9tVN
-WODbdeGuXqBBmdAGaWBEowVP8ZhdQg==
+AoIBAQC903im14fl3p/7FzEGDusYorp/C64AhXv1Dh6OHkD9UboSsD5nlrDfSc5c
+NJVOZJwdgJz+93Asn9y7e1S46CtkTYnpfwBRbbCjDvwKTiJyunCxAtEM3j91TRPw
+V/h3QBi5wyRF2aATS05Bf/xi8EBmvjawTACZFXFbZXX3OezNk04clCMJeHoCzpgE
+apG9BI+v8uK2YtDxxDvsUVD/qgLf9rkGLB2vY93XirW9T6dlVWQJ9bl9s0M9Jg0O
+UbqNTxptCEzlayUirIhzLmQzr6sKeP4qzYLJLQurBk41OBAt9N9AkZx3CvO2xNOb
+j5S3h0+qi4zqNCu7e0LWGU1fdUUbAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
+BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBT+Ax2zZdq5BEBtmtBecH/QP8ruZjAN
+BgkqhkiG9w0BAQsFAAOCAQEAtiJqHvQH3iqdXwThZjzaj+4evDY5QJVcHsGmZiYl
+g8gKsyWfE4tCuBHvHvoggnYm8BdToXVJ9cvUG+u6FWxdVXDlU17kVYf3SXM39/es
+Kf2HMZ+wc95xUol77f0Nmq6XXqIMpX5TSWHVucDWFHROTYjMDW9xau1fEV810b2r
+4CjieTyNtd6FUKFgdFNPTvse5fFp0W+DZmZIDeYv1wHqQsQObLsk/MOz7ybNqRBp
+JgNJKUr3Y9YaqH3jq4eokgPpzqnZ4t+teY/gAISwtQwx4Zspa8ZJ8ybyUvAM86hN
+rYa5usa2Obd13qwgPSYYuUkAwpYy/s/cQLgAQu68ZHxsKg==
 -----END CERTIFICATE-----

integration/fixtures/ca.csr

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIC1jCCAb4CAQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlh
+MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQL
+Ew1ldGNkIFNlY3VyaXR5MQswCQYDVQQDEwJjYTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAL3TeKbXh+Xen/sXMQYO6xiiun8LrgCFe/UOHo4eQP1RuhKw
+PmeWsN9Jzlw0lU5knB2AnP73cCyf3Lt7VLjoK2RNiel/AFFtsKMO/ApOInK6cLEC
+0QzeP3VNE/BX+HdAGLnDJEXZoBNLTkF//GLwQGa+NrBMAJkVcVtldfc57M2TThyU
+Iwl4egLOmARqkb0Ej6/y4rZi0PHEO+xRUP+qAt/2uQYsHa9j3deKtb1Pp2VVZAn1
+uX2zQz0mDQ5Ruo1PGm0ITOVrJSKsiHMuZDOvqwp4/irNgsktC6sGTjU4EC3030CR
+nHcK87bE05uPlLeHT6qLjOo0K7t7QtYZTV91RRsCAwEAAaAiMCAGCSqGSIb3DQEJ
+DjETMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAmUD/50fV
+uzw/NWXztsSDULzgCTxesWz33PaW237U6WuRfNAzz1NUtHkmo0rXm1r987ajhaBQ
+jD9aEpLUvmaqohRfnZLHsGrKYj+5DZ8j5UiVF34RjSw3FyPzimSzkHjfq+zl7kh0
+FbYTfTHsxhcICm62hBIWN7bgCSTPSTINa9VCQJTOorcFAUt0xTjfIFDm6EpSKwzh
+1jO/Q1NR6CvXYwd0uxIxtVvxYtqsTdSO+pI8sSsFcxBXu+5LbgSemy+y60tfXwZR
+hF7x4a95XVn0H6UyDISc3ZmlmPNB+K3wOJ/eTAu8lMda6izzoPR2E/1kkkfF8Zol
+LUSG/f8OEQ/AHg==
+-----END CERTIFICATE REQUEST-----

integration/fixtures/gencerts.sh

@@ -23,6 +23,15 @@ cfssl gencert \
 mv server.pem server.crt
 mv server-key.pem server.key.insecure
 
+# generate IPv6: [::1], CN: example.com certificates
+cfssl gencert \
+  --ca ./ca.crt \
+  --ca-key ./ca-key.pem \
+  --config ./gencert.json \
+  ./server-ca-csr-ipv6.json | cfssljson --bare ./server-ip
+mv server-ip.pem server-ipv6.crt
+mv server-ip-key.pem server-ipv6.key.insecure
+
 # generate DNS: localhost, IP: 127.0.0.1, CN: example2.com certificates
 cfssl gencert \
     --ca ./ca.crt \

integration/fixtures/server-ca-csr-ipv6.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "example.com",
+  "hosts": [
+    "::1"
+  ]
+}

integration/fixtures/server-ecdsa.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDRzCCAi+gAwIBAgIUKgQJ/CMaFxc4JcwwGyiT/7KpedIwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjO
+PQMBBwNCAARXbc8naiFZ3Y2LujrnDCScVNRks/TR+aXPmnuPGjDxbuHxSSbC8Q2z
+iTvCkgsIcsifmUIEQcI4v3Kbkj3qMF1so4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAd
+BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNV
+HQ4EFgQU3z1DifT82BfoU5DfMe08meeYmSUwHwYDVR0jBBgwFoAUZQe+r42mchS6
+xFBpug0H8o5jej4wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3
+DQEBCwUAA4IBAQAE3bhZcJuGrnMGMgebCFMuAXvoF9twYIHXpxNOg6u0HTIWOsMB
+njEJW/rfZFE/RAJ6JdOMNE2bq2LbJ8dUA25PX3uz6V4omm9B3EvEG9Hh3J+C77XQ
+P+ofiUd+j06SdewoxrmmQmjZZdotpFUQG3EEncs+v94jsamwGNLdq4yWDjFdmyuC
+hqzSkD48aGqP2Q93wfv8uIiCEmJS1vITTm2LxssCLfiYGortpCx32/DWme8nUlni
+1U/pRTx8Brx00dMeruTGjCCpwb8k453oNV6u0D1LsQ9y5DuyEwmZtBEHBN1kVPro
+yYW3/b1jcmZk8W9GXqcXy16LbWmpvJmTHPsj
+-----END CERTIFICATE-----

integration/fixtures/server-ecdsa.key.insecure

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIK3K2gimOw2P0pZ4soFAopriuORuqpRptllFXNRhCRV0oAoGCCqGSM49
+AwEHoUQDQgAEV23PJ2ohWd2Ni7o65wwknFTUZLP00fmlz5p7jxow8W7h8UkmwvEN
+s4k7wpILCHLIn5lCBEHCOL9ym5I96jBdbA==
+-----END EC PRIVATE KEY-----

integration/fixtures/server-ip.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEBzCCAu+gAwIBAgIUSvxuG1lgImYpnaK4sPaCiMAd0lgwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQC7mJOiyqWfmNM5ptQZ22plotVfgoBf9fHTzMw/ap2Vl0/0
+4V3GEyYCdPt6V87GWzjBSO9GAmlISBQQybMieZTaTm8KKW2066iJDKseBCv9m4nS
+mHv0oDqp3SHsZQ2xHis4lbi7ws2thdqpmjw4Dv96SUiCJUjhcBX4kBMRcOGgk1RF
+ENIOInTSKlAiwNF1NSnhj8wMNw7mjw90jpAGAuPuuiQ7+AYHJBJqtT9mRikR8ppw
+isjEE6kslCCg2RC45AiF4LXNp7A7Xwm6P34XJ6T9PJUh/r3pa0xHRuI2zQLaW8Z/
+b6NYkUGMbHR7AY/+2JzOfnnnQcSB8EYC9bHadvHnAgMBAAGjgZEwgY4wDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBSPaFA2Jh7s/IJN/Yw/QFFR4pO3nDAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3
+DQEBCwUAA4IBAQAO2EnUXDlZAzOJLmkzQQF/d88PjvzspFtBfj/jCGzK6bpjeZwq
+oM1fQOkjuFeNvVLA3WHVT0XEpZEM8lwAr/YwnBWMFlNd3Vb2Cho5VaQq0nVfhYoB
+tpzoWcf0Qx4cALesQZ3y2EnXePpzky1R4MfHqulYrmZKSBQsERob/7YgSBk+ucV9
+OHLzYxm4OvYvDoR54REq+vgZ3ohoDmBrNNv9OmUHLIrUi+nBpBgnww85Dc7cKB27
+EEKxqIfCNTeHSemvzfK/1M6manQX6eyGe48nOwQMV/ocfY6SeA7RABT0l/UsbeMp
+g/b2RU+liZ3e8FziW4/1VTt1pmFAN/2hnb0v
+-----END CERTIFICATE-----

integration/fixtures/server-ip.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAu5iTosqln5jTOabUGdtqZaLVX4KAX/Xx08zMP2qdlZdP9OFd
+xhMmAnT7elfOxls4wUjvRgJpSEgUEMmzInmU2k5vCilttOuoiQyrHgQr/ZuJ0ph7
+9KA6qd0h7GUNsR4rOJW4u8LNrYXaqZo8OA7/eklIgiVI4XAV+JATEXDhoJNURRDS
+DiJ00ipQIsDRdTUp4Y/MDDcO5o8PdI6QBgLj7rokO/gGByQSarU/ZkYpEfKacIrI
+xBOpLJQgoNkQuOQIheC1zaewO18Juj9+Fyek/TyVIf696WtMR0biNs0C2lvGf2+j
+WJFBjGx0ewGP/ticzn5550HEgfBGAvWx2nbx5wIDAQABAoIBAB0jBpM7TFwsfWov
+6jOV68Gbd+6cs1m0NnpCDdsvsQgh904+jrUMFlQ9XS3UY45Vbsw+isNh7n5Gi69L
+1KHfJmp90itO4fY+v++BYzaHSVnbhZ2LB32oQVROv00bKPRAjk/8mTO4fv+bkanU
+BdRjJ/UTWsq0BczV/uObZQrJcJHi6+sAMYw4b/kxzTALd+UuvmOP7Z/NoWW6x8Mm
+ahHgqaMwA0O1f4DsdKYnSUVMF9DNGsxKCUYSYR6RH93Bq/Eo0q1U2egmLIMcTVW9
+7QSWsJoZuXlzkq7Hb7mxGdppa6kSzA/VM26qPNE9Cjg4tCMu1RJSfgkcnv27Y8vZ
+fZSq3zkCgYEA68VjIqG6sj43SZSvD+Z+Dfuzc+lO4YBSI0Yru8B4ZZq0vfTVQdM/
+uf0Bpk/nMbqec/kfcPMHP8zznLe8rcmfZXNQFIaajOb6rzWhCRSgbX98MeGnUe/y
+9sG+zFSRrAPDaVRJZwSYILs6o6Hz4o6DBCvr8iKFfm26SLB7hIjwx8UCgYEAy7EL
+dIMdsGDzfmxAYqad3oy/N1KVp96zfdnHEiIC0oiXz3YfI7YLFj54yXxx5rHR2/AK
+wOo7b90Rc8R0PgtKedKrz5p/E0Bz723ToTxHjsqgVRZqYaEKUOp8wR2t2DJOF9b9
+0C/qp6iUy0IOTBYyu3BCMV0aB5kRW62jXJIsQbsCgYB6uO7mOurUFsBug38wNpjM
+rIR3RCz0Afg/NipTe1bwBDwqWEOdFNmp9QEj0ZmU7//EfBsajtXqJsNzgswqZbWb
+eA9p77qItz4rby3YbS0oceByknOmmdCNEsI+15JPyFGyBNaEUgbhmrNmM0mgVu/p
+fvc8vS1hZro9VeelUCaMxQKBgFDgnXHH1fQAqu4ZwX7qNWj2bb5jtjSPgqmH3Tlf
+88rwnYasmjStxb0xVPh7xyYYmQFBUKPE3ZDPMGzNJnK0PQAeHEY0TByyzNXWv98X
+djpGTl86pUbakKQMVzi+thZP8x4YKXOOcxfbIimKsu6XKdGvAzlihEFcD75dNa4+
+BACdAoGBAJevnrC7M/KyDDGW3ci4sFcn7MxRGqLBulwGoCuM+zecbG7NBvDynoaH
+NRGpASiboRJyCEoIQivvkZf+K7L/oB4bL/ThF2ZpJUe471tq0444xnXdHRDLG0Dw
+OnBl27e3iAiUctqR51ufXKOUaNEf4gcsS9duELMPBxM70GE2Q/2r
+-----END RSA PRIVATE KEY-----

integration/fixtures/server-ipv6.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEEzCCAvugAwIBAgIUYTkp3oUkde9wFRkJA1LlvwFrZ3MwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDcRWZxskwCNXhprj8XCtkxj9GP4z9hVgUxgquSBync1hic
+or6qNgrUztv6nlALdQdf+TbPKyGEwCgAlKU/hnJK6lAG3+riyShnyM74/ulV1wYS
+F3Rkeh0nNCo95TPNq4GLB+sMfzwoSsT0srPX7KzCqpGy+G7sB0JBNwkTZLkCuMZf
+dkkmcZJ3zqIiOzJPlcQa4iBa0L1nV3Uuv49kLZLMCLMslg//IZxC09fnmjn+XLcV
+4+RpOKIn7AMN1kqPqmaB6gk2aCbYTZZ8aS9+cOJmTERbynyX4y4sRV18ED3dRNvs
+HCedgPOp53nqDneSOqOhhg+Mb95tnMQq1on0+TRDAgMBAAGjgZ0wgZowDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBTFoXLQVq+Yg2AlRIirXj5ho0PMrjAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAbBgNVHREEFDAShxAAAAAAAAAAAAAAAAAA
+AAABMA0GCSqGSIb3DQEBCwUAA4IBAQB4bl4f8TI7k+nlHe4MhJuHP1BKHB5O5SeG
+wrgI2+qV38UrKvTag2Z3OVKw12ANGN1vcOUrDS7cCtIZ8Aar7JpBgWrYvVlhAtc5
+3syj74Iapg1Prc0PFRmMQTZ4mahRHEqUTm3rdzkwMjNDekBs9yyBsKa08Qrm9+Cz
+Z84k/cQTBc3Bg6Xw3vUiL4EmeRQudBQAvh/vdxj6X+fwKmvLbPpgogXuQS/lHhFQ
+/rZ+s22RHLlqzAMuordjxS4Nw91dqYFwdYVvEmsK89ZnSWqwLvFCJ4uNnAe8siS7
+53YTpGbpLdNkQKAQJdMQSyvcDbQoQ7FI19a1EtSwpg5qSMOTpQ/C
+-----END CERTIFICATE-----

integration/fixtures/server-ipv6.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA3EVmcbJMAjV4aa4/FwrZMY/Rj+M/YVYFMYKrkgcp3NYYnKK+
+qjYK1M7b+p5QC3UHX/k2zyshhMAoAJSlP4ZySupQBt/q4skoZ8jO+P7pVdcGEhd0
+ZHodJzQqPeUzzauBiwfrDH88KErE9LKz1+yswqqRsvhu7AdCQTcJE2S5ArjGX3ZJ
+JnGSd86iIjsyT5XEGuIgWtC9Z1d1Lr+PZC2SzAizLJYP/yGcQtPX55o5/ly3FePk
+aTiiJ+wDDdZKj6pmgeoJNmgm2E2WfGkvfnDiZkxEW8p8l+MuLEVdfBA93UTb7Bwn
+nYDzqed56g53kjqjoYYPjG/ebZzEKtaJ9Pk0QwIDAQABAoIBABBdY5gM3BLJ8DFB
+zdQjbTF+ct5SztGnd2lPQPnvaE/M5DU27h1tOG7JE5TSEDZZsnuR412O4cWgFRi9
+8mz+yxz/vYRVPHku4r6bL61WGvXSrNPJRE92txXDjWPd1HRySoSOyQq7pTeFHo7j
+e/MN1WP9EigOxwboHycDNLxpHkmyV1DIlAgNkCZV56//liU/b+4vAVIJrgWfwfGH
+NkFd9nkm93oCFOroJ2f30E1wLPlC+ZhIn4ysau+zlWDLYeils0xHwS2GD7gjp/if
+i/ibVPgMVW/WPb67olm3nMUsan6CLmKWTiG+yklJT2djoam/iCZWE8/SAZj3qsxy
+6W9rafkCgYEA+D8tPM8h0oHlKriFDQZx37EH1dfGJRqxr+SgQiJ03d9pGYEsT+jC
+yr/l5ntzTwEEJjp/biIRwCwSWPYQtN4dNqn+11ICQzjhQbfWTfeT6vhSoBNxkeTT
+R8tUM0fmoUNrXhPbGZ9XdIxDFgD1pJL96KtyaQGjIRAhyG+khIT7oIUCgYEA4yaM
+Mw65KDonnKSVfMiOuG0QNYf70UcIiLSH8USnhbQhzT/c2LG7tNmru9UtQhZtmrpc
+vezuOYTkfcAIUjwqm12Ra8Px8WMzwHwKx3C2SrFCLFgjNFyoQ+VIGjtAL1lNKvEx
+MObSX7kVIf5+gaO9+KRBEdu55R16yQpW/UVAwCcCgYEA8XdqRkLoED2/Ln3LFW9W
+ZpJpH61BlCfR/FhzNcEUUhiUv3UxKA0tJE/ijP05nPhNE+5Es1i6UWXM9vFqMLP4
+UIqsUr73anGyUd1CvBX8sEqY/BHNn26nwKbboQHoKKZOknTX4qVmSPyB6K5IQaul
+BKN3pwIrreZmJfPKYAiGRY0CgYAYgEbtFvB321X8enA5ZnSmhfUSoRlTaIMOI9Lp
+/krHjDd9KR9MLFef2T7B4uufzkWCRAnO3qiPgbsXqUf8fsrluUD/S8JkFBw37elH
+u+udwOLvX45kjn4D3M5bLfrtYIeHUz7IFI2qj48s/INuvle2Yxk1sOqrQPPGjZv2
+c6rZTwKBgQCHSa+ToxicPJBZ5E7ezgue0LyRGWIMsr2OR16PBL2lPPiCWPH8Ez+l
+mTClHll4KVZyqc0VOZDbjMjZBnTiAq/1lb8ZvwsXLi0ue1obkkEYfXLWcxYD3Yne
+iBCGhjkqaUA4rESb22j7yqB8WGT83qV0kB9JwElzE9SxnyR9iw2FmA==
+-----END RSA PRIVATE KEY-----

integration/fixtures/server-revoked.crt

@@ -1,4 +1,5 @@
 -----BEGIN CERTIFICATE-----
+<<<<<<< HEAD
 MIIEEjCCAvqgAwIBAgIUZf8MqK2zoEIlXqd8LqfVPpuEtLwwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
@@ -21,4 +22,28 @@ zI8J0/RtYCibCAcVKmNwmfhoUqMTERhSL4dcloU9n/45anZgQXqNCHXJk8+I6nAY
 ZLEJ2aGFhvNypPTYrr4BvHx+LnrUzPWcd7JwXGLXGJtDEF45HIMLgduof+azDp/X
 HJHVra4ChMbyJHiiC9nCJruGAtF2aJuwqrGG7KnPifDLPBsplE3zvDA6dtEPvGui
 l/IE15sZ++GqTgf4fn2CNJ0PK/xYCtcBejodus88SJviaEftEB0=
+=======
+MIIEEjCCAvqgAwIBAgIUBwoN2+J27JtT6IaqV9sWhsHii2IwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDHS/zscOjq013InbTlwsBVwasv8e5+ukZNGDQx5RNaXYxI
+NVUM/5Bai4R3CS+DTbr+jBDylKi55gPQ/UIDKlU/NQH+x6UJB050G+aLDWAuRmxi
+w8dq7kRw2QJvuMxI+quiZhWk2HYjtvZRZLCUGl//QTL/VCT1smXwXRBU19S2uOfy
+g9KgZL/DCkJ9VBUh3+bFVKXBDnIphY4N/0+B/sW71cvRj8zvW3iD0R5T1J+QVEFz
+sFRT99/OhV2kUEwMaAYOFv/mMIEO6qc7vf6pB91qdUfEP8AbsOlmiSuOOLuR6X/2
+FHUjc8JrFfMuOVHnedRR5quxXbP8o83ilat0tXeVAgMBAAGjgZwwgZkwDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBS7gBJSFrjAHryiQpe38OMTzCKH1TAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAE2tsLHtgF1T3d/anKf543q9uh61tr46HHf0
+RrGZF+RJuJY5XIAiCN514Z/I7i2wG/x1TDTKwZUebajbk4GvaI4nEnCXs05jwm/n
+wpdyRE1EUy5PkVFfXKCNQd096mpZu6EYXBGnQ2fQjg5zFvZSDnYaIf0vBF1WxE4W
+0a4a9na3N77OSamPEljM1RJ1Sk+Zg5yI+nwyKcWWk3OlD0j668Vp6/m5VZKyQEkx
+crfSj7kgRJWZRhMeh6li3xa9vDmzdF6ojGkgRN3Qljrs36JnmsTono2ETF8GIc+g
+eNByAQNppLJjMn+zsaG9J5pr0gDLubFA7oa8aAJgYgJMM/GecAg=
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END CERTIFICATE-----

integration/fixtures/server-revoked.key.insecure

@@ -1,4 +1,5 @@
 -----BEGIN RSA PRIVATE KEY-----
+<<<<<<< HEAD
 MIIEpQIBAAKCAQEA5BJlqSA8qtDvbgwdNfkTxRpXspQgqHJ2PyJuol2Z5Abhcw3L
 ZSCb361IM4pdrQejrJoIcliwqUVVsHCvTCwJEZ8nIYzyA3UmCRhRKprlD97mRgXB
 UZZvCRpCSCDvbxgUKVubvdNBc2ME+C5oyt/Ffs0eqhU0JJwjcDsdD0wv/8P99LO8
@@ -24,4 +25,31 @@ rzTd0hqEvwI8XDhYlDfOte+gYXgZeL6fqJXyUzoB/LCeysk+de8fQSmBk/qJ4dtI
 se7BWZUCgYEAvRn3UqhEVq6gZgJ48LKtCPAVdDH9I2gXf0ywa8ezRcuKSsCexKOH
 gM8/MuL6KeKZMj9X01fySx9KFGIAN7GQ6bm4kZLIAQCLVhBkG8YV6t0i44oVQbSz
 qTapBzKVPyuJPVE79adX+pOgQjIfnljFlrO7JCQ+XCfKGuU9MhJfuMU=
+=======
+MIIEpAIBAAKCAQEAx0v87HDo6tNdyJ205cLAVcGrL/HufrpGTRg0MeUTWl2MSDVV
+DP+QWouEdwkvg026/owQ8pSoueYD0P1CAypVPzUB/selCQdOdBvmiw1gLkZsYsPH
+au5EcNkCb7jMSPqromYVpNh2I7b2UWSwlBpf/0Ey/1Qk9bJl8F0QVNfUtrjn8oPS
+oGS/wwpCfVQVId/mxVSlwQ5yKYWODf9Pgf7Fu9XL0Y/M71t4g9EeU9SfkFRBc7BU
+U/ffzoVdpFBMDGgGDhb/5jCBDuqnO73+qQfdanVHxD/AG7DpZokrjji7kel/9hR1
+I3PCaxXzLjlR53nUUearsV2z/KPN4pWrdLV3lQIDAQABAoIBAQC2y+TVvY51bJ81
+lilJIIMnZTauCDqXdCVtKwkcxp8koG89/+Tdwj7WPeenAv7YcWBVf4U/6siDkgzo
+EJMOsjJ0ghstZFLkYBY+eyTPX9pbN27MfAQZ+Sc/VlxcuuRs/7aTgwzRIVXi1jtB
+Vph7j2GDj3rGJJit3w6PE90Z5MkPOhXwbPD+T2OCIhO0OQCv9YNrdHmQzFZJ8vn/
+FuKUjZuoKKnwgXvBVBKsUPvvSdPTWpavNYdA7WQtjpVYVjVHgEHZWtxUwQ43JHzb
+pABWqYp/XJNiGhZ+cEXsw5dBBWp/BPxbu1P2iagZTmNr/8EfGCq04fEkKhv22x0y
+FbQa+2e1AoGBAOCztIuf1Magca9mFD+3YZHgBv2TA2XSujwYBr/664dLjL/9NQIK
+00IBykiNykiWZ0ixcaJI1j+af7fWr5OuSzBVwdXMUZraKUEwrKI3hh764yX8aUYt
+JsqpAFhyro7smp3LaUyMCW2ZFVxayp60h8fQXcNepFwmK5o5BnsTsFHHAoGBAOMO
+ZooI0Yz/fzBKOEMM1Vdf3PpUqYnjCyJSXag8OeZn/OPgiYkwXWL0idfC49B0ArVZ
+/j2zMXJduIrwa3UIfd6tjPf8O24YOiO2SenkVkcsUwJgsB1nM1QlOamGw8BB+nbT
+O5V44r7vy3HldHHQgbPvjs0z5de3b3eBBTZC/2vDAoGAVNroSnYAV0YNyIwHB4zL
+9tegLDBRbylmFP2JxwQN39ji/Tm0w+Gsp9efOUj6Y/EQbf48iGlzJy/EHXugcGe4
+kzc/bOqswoqyW6DzAItxRc++6gBpDQxOAuhRbhVY4DZvqTlAuZyEjvPpgifzLn3E
+bOu+DOJ3tSjg/Guei+oCgs8CgYAUfwxKkZk4/SdiGJETnGj1xjWQc2wKgnBS3NSP
+h0BCyEhP2ckQlUkY0bJPw8wE2TQVYtZMg4yHImayRBmvKuER5ODA0ggbXByDdMUf
+U/ll215y7H95aAN+KQ4Xe47YIByX9WF/kLYHPmZDFc95JrVOpOVjKLgqzOhHBWKP
+D2U3OQKBgQDbmwsNr0mopOYiAp60KsKJmICUQO27RyL87UfdFysDrTZ+K1Pc6X1e
+HOFtma4zNftDym9Xjzz2eOXT6flHeJNu1qZwvurNV1g0JZdXnY83q4C130bAFJLt
+I0+I3vDpJt9wznYnC3jDI24gCbEJ2D//8dpeDNUPKk94rjsEjXxDFg==
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END RSA PRIVATE KEY-----

integration/fixtures/server-wildcard.crt

@@ -1,4 +1,5 @@
 -----BEGIN CERTIFICATE-----
+<<<<<<< HEAD
 MIIELDCCAxSgAwIBAgIUGtkVdLvghfSjGwEVSothEo9W2mcwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
@@ -22,4 +23,29 @@ vADeofmoiAA6a9HUJbDfcJYz0mUoPZPcN4emCMv9PNOOybxjRqDL2HeZWwHedPth
 kfJkOHM0NXwb+XyRY3uZHdRC5VkBBmI7H/Jo0kGYB3T7YlREfGAkPd9Iop9pfitY
 FfYVu2hcxQCmGYtLNC4csP8C/nL/0o/2pIz4ldFNsYqH1swRnZQ7A+xwo7oFhvfK
 RchIgJR6qcPHkR7oloYDxA==
+=======
+MIIELDCCAxSgAwIBAgIUEQuXXKtjueOgUpZjzr1ORDG/zHwwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDPAGzCcM5JSlitl/iHLYJ6eGO8MJ3S6R1qzAvmdB9+KsGD
+F99gWVTrJRzz/hJyo9Lt8GvNj9Ll3iT2QnXyyaSOX1+uT4cxBM2MFBBfERDh0WUY
+43uLQKY45H4zrS4tJTOuSGKM/LlK2ZMj++pQBqHsONrNG+nOhqe3qLqPDV3yBfmD
+PXfjASNvHINgxb9AwQWJydgjfGDiAwWHnKbnVScYBFgWfMG0Gm1wa8EfRfWD0NPd
+L61XwQwgb5VsYAs7XH7bxVbPm6E+/oQTOJXQHMzQYve9DFPy62KFSIkfvNwVRctL
+NE+k3HnyviDzbs387ys56ZjPG1/XpbFmeQuDRndJAgMBAAGjgbYwgbMwDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBRyAdJmiDFhC3CYXPLW8kufz4zp6DAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjA0BgNVHREELTArggwqLmV0Y2QubG9jYWyC
+CmV0Y2QubG9jYWyCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA
+sc0JnS1udx4HFStcPZXY+kyVwYPuRDv3GisO+TTrxzdupQQl4gPmuGa/cik4tKxQ
+o//XFgCBQCapO2cY+JerjMSOLHtt4YmdyYDSXeMjsRG0sP36njH3nHIYsFAoEyvg
+nJQX7iDWj/9bzMT4dU2ac3t9RgCtyABRoT4G/MNhWMlJt9XkwVTN2Pqf4TMV0GlV
+54GOScsWAIwoIDPOCoO8Q40jtFSSnehrlrW7x6B37gY/EbbYpZNrIDNckNfLJMvl
+dak9P/ovtjLk8GM11gE4s2ANWA3o5bIm17b1x7Fw122sB4Rxptbc/BDpv2GIY5Zm
+tqDculro2C7Ib5GyEBEl3Q==
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END CERTIFICATE-----

integration/fixtures/server-wildcard.key.insecure

@@ -1,4 +1,5 @@
 -----BEGIN RSA PRIVATE KEY-----
+<<<<<<< HEAD
 MIIEpQIBAAKCAQEAszEM04KJY6EdfcthECFGYVQQbXo+ndP5WB3PsiNBJlu+ShjH
 Lmw3RTmP4OnHoObfjCZFqbySeYdpe0lVd1Ewx7cScskHCXCexQUPUKsK6yvuF1NZ
 tlrAeqqs3ecf8DUap3KvUJej5zILXptRWmWEeSuR4nr09/tuL21hQFrWzmdSKCJ2
@@ -24,4 +25,31 @@ mJM4d4hgFhfCqsIjVpIs+z2/e91zadnbQx0BSO3KFk3L72evUzpQjHpfhBA/p/51
 dJ3NoMUCgYEAr1menJ2CogbPAy57h+0LST/w9tBQaYeu5krf3HTWrhzUaXVobkil
 3aVpO9Ia8Oo5SkeTSODJoa4U/oeuThJzrBJgGRxo8mXeELmpFCKjHOyj4h+8dgcK
 8KAamUqmT9WVDP+8RqTKbt/jA7HulC4ew76PMPty49Ln9t/o8BXBGVY=
+=======
+MIIEpAIBAAKCAQEAzwBswnDOSUpYrZf4hy2CenhjvDCd0ukdaswL5nQffirBgxff
+YFlU6yUc8/4ScqPS7fBrzY/S5d4k9kJ18smkjl9frk+HMQTNjBQQXxEQ4dFlGON7
+i0CmOOR+M60uLSUzrkhijPy5StmTI/vqUAah7DjazRvpzoant6i6jw1d8gX5gz13
+4wEjbxyDYMW/QMEFicnYI3xg4gMFh5ym51UnGARYFnzBtBptcGvBH0X1g9DT3S+t
+V8EMIG+VbGALO1x+28VWz5uhPv6EEziV0BzM0GL3vQxT8utihUiJH7zcFUXLSzRP
+pNx58r4g827N/O8rOemYzxtf16WxZnkLg0Z3SQIDAQABAoIBAQCd5u0PxY0WSygq
+A2sJcqW9Vmh9/XfmkvxloxDQ0nPTgjnrDiLPFFW6qazUUlMwL9eOuX8CZ1uxDSuU
+zk26ziZAlHAgP3oY4lkJKaTzX8lI+Lntqllrd/1UGLhMIya+OUqa/4xtj7qoZh/f
+qyKpuOV7lEMTgt9vMzhs2MC2rrOjEZxcpuwpnZLKvpuwBMcxD1ccRdCA1zHvKdQ9
+ukPTRVjz9WUEOgANRkndHTZKWMz2p4QC9Id35HlksZi0/M6oboz2Eg1mtZEpbgUX
+loMv1CPtWP1uj9PFWiOmnBC2/v/2MVGg2fJ1Lf4c72ZVFEIU4l3YNiV4IqFb38F+
+GJVcmiGhAoGBAOvrTjYYl5vodK57gRRT0UsaU6x64/IK2i0vbBGTATywuijJ313X
+vwZBU9I2rLZqr7FZ27g5ANorw8dUKn92otr/TVS/c/VZOSw/+gTM9Rl4ZGji+qKt
+4zY/dA38jlDJNWmFwK/9KNOfXNS+WLsA2QJlONgUfkFPb3yXJUGLsU8nAoGBAOCf
+AUcyDHjGwtYsLc/4aiKtQUIdeX0v6jCWtl9EI9cZ/o414iapE05sOGb724itSFN/
+EI4biQGw8CaMcaqMaRJ8+xVQQJ7qkXItzZEFVGqz0PKwiwYAwFp6raXuio9y+cTw
+savJIM8IDijph9ezRCalef4Qj6I0zFI8H7PmiwwPAoGABWvY1kFmanzDAadw5eiv
+LIykU5hXWJ6LOPKYBydbpethu8I30c49Y4VoybHb8i0tcGPiOq+Ep37N9uymNVui
+jmnDeykTHxY3zB6EPkv/beBoXkio/cgFKp/2qMOe+ZhGE/Cw5tpob8R/u5vMKi/w
+zK9KyRxfclzC8RgAESuGnY0CgYEAlHC/+Xrbvx0rOTps9Blomo4AqF6uIMr/ayjO
+UNrJDKfDD9wQHhhyB8uA4p3ikMpjF7rLB/6uZg22RuNdYqXz8iHiFE26xsqhX+Fh
+DkuFZBZ9KUT+OvNYKvMTuqqPqwkCguHFqI78PZVHNkZOXX+8tAV7PylWoo1d0aKm
+GM9saIUCgYBXO9TtUTiaoxIVTe+r+Abt5iwasAwxai/RvymdykwYFwq2NOEl02oi
+fU3gbqDV3oirHAsAKJipnrASc70hTn4SM9hUKTQrD3fNIABch811ZDd8vaHEzLZG
+pp9yKam09sPvQo6O4E7TJPccrddV286jZq+qO9YNIsRlYJiCnjzihw==
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END RSA PRIVATE KEY-----

integration/fixtures/server.crt

@@ -1,24 +1,24 @@
 -----BEGIN CERTIFICATE-----
-MIIEEjCCAvqgAwIBAgIURbT0TUoUtitOg0ell5xYd3mAiDUwDQYJKoZIhvcNAQEL
+MIIEEjCCAvqgAwIBAgIUemioNeSN8XEGGLLqfS4uF733NiMwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzEwMDUyMTU5MDBaFw0yNzEwMDMyMTU5
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMjM4MDBaFw0yOTEwMDUyMjM4
 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
-A4IBDwAwggEKAoIBAQC+k2hb+zgEk2WxfXtypI03y3HUo1muX7FZdHX1yKfm/TCl
-5OCR1Id8tZv3Rn3+hZEVvjNlOi/Ct5ic0SVa4OpeQTo7u5ku5/RiXE7c55I+wpmw
-o1/IUlgeq3nyDKX4RlBJfymSUD+lWHsmhXkpKdU5wcERwL5FnrkdbTQwo+4nBMcc
-UhLgWC9awpT7sAXW7OmTlg/szTIOzyJp2YRBoXp6mGsHF0rujVElQyCOBnAi/+zP
-hGGycCOoa1eVjZebpFgcisyIwZRO6KugfYIZrQ47Swcqsy4lfkbzXxZ6UMh+FtUz
-iXYSG2c8rzdpnHlFQcpfkNrbAu5Q2ObmQh21E18BAgMBAAGjgZwwgZkwDgYDVR0P
+A4IBDwAwggEKAoIBAQC58XVTmNuwAt6LstiaSQQro3IEjVaG3fiqxh0rjcWnWKEc
+6kZFxTZyRepfQtLSb52SDXTGf7kPJQX4kst/NR9qndrdT6gwwNcAXdEqVJIx9i92
+eMaYuAS0nIoslq/6ULPviK6TocTAIKJp2mrDU2ylU7PqvmRUuNxxieVCSIMAevNa
+ZFhucW/uGtbNd4sSzo2Xwav391DDWDIZYYvucGFitjY1mOHDyQb54ECVM4G57ITn
+t3fvPPjGYTSVDktgKIdcAZxd4pBDHJB8pB5IuvQPqvjtbbB4aubKEJVXaCFK22zJ
+wBTseo1S5hctdtswcqENKzgbqrtQTpHfB0XZXYKvAgMBAAGjgZwwgZkwDgYDVR0P
 AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
-Af8EAjAAMB0GA1UdDgQWBBTATssOHl8+T6AfYWGXqKHyCtUqMTAfBgNVHSMEGDAW
-gBSt4lqZZ20BpzYG8GSym10IntbrEzAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
-AAEwDQYJKoZIhvcNAQELBQADggEBAAzU+ZRqgGaVTGCl21QnfUY4Hj9aqt6uMhVv
-b+BG24gNUJgwkCIZODOsig7RWXjPAMkgeGmgyw1QbvV2AZo1NFJT111YhxjdqWLg
-ganDb7K2Jm5vm8mVvBVTe2y7ZBdmxJwfo9UYZkEXNQtlbvYcvYnzr0nr5QEc9v8X
-bkrbxG1DVB9wU+7hy6s4v9946xQavGUqOOC70wHUMj8gKGnGkd1mOTYaabx3VzPU
-uD82AsCQnxOIzk0qze3jCVVoTdzKt9iWpgLdFHY2pa0fdirTN1s80sLpXhUOihSP
-+gu2NGP3+C0I6SW6Wu35vpYI+uMWrggu6OCxomAC872d6CVdtcs=
+Af8EAjAAMB0GA1UdDgQWBBSj5OsDxzEYEqO3SBCEkegbA1MJ3TAfBgNVHSMEGDAW
+gBT+Ax2zZdq5BEBtmtBecH/QP8ruZjAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAKIhy98tSD8SgW2y6DA/CyennGzSYs5aj6r5
+BMCDAuaa7OH9MM6zk4EQPH6TU06/wrQ0Jhwk7BKTslEH+OFA5X0WYNfMQ+462v2Y
+QYGO68128domd2XSMNOsk2we1eN4J0RkAqua9/k1R5NwFpUki5f2G1zNOKFXklLg
+HleRsqGk5vTauICa79XPz3940K4f2oobSXHZzBOVyIFRTaUD7t8pF2wLF1tgwCDZ
+87UXkSpsR7nLTkEXEm/pk5gNPbyHMmPsXvxZ0lheAiZh2RgRMRSdOcvJcE4Vcbfw
+Vw+aQbo96GHHMU323DgSwbVLs4umBucvW1Ny2nZkrc7V56Qqdc8=
 -----END CERTIFICATE-----

integration/fixtures/server.csr

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIC6jCCAdICAQAweDEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlh
+MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQL
+Ew1ldGNkIFNlY3VyaXR5MRQwEgYDVQQDEwtleGFtcGxlLmNvbTCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBALnxdVOY27AC3ouy2JpJBCujcgSNVobd+KrG
+HSuNxadYoRzqRkXFNnJF6l9C0tJvnZINdMZ/uQ8lBfiSy381H2qd2t1PqDDA1wBd
+0SpUkjH2L3Z4xpi4BLSciiyWr/pQs++IrpOhxMAgomnaasNTbKVTs+q+ZFS43HGJ
+5UJIgwB681pkWG5xb+4a1s13ixLOjZfBq/f3UMNYMhlhi+5wYWK2NjWY4cPJBvng
+QJUzgbnshOe3d+88+MZhNJUOS2Aoh1wBnF3ikEMckHykHki69A+q+O1tsHhq5soQ
+lVdoIUrbbMnAFOx6jVLmFy122zByoQ0rOBuqu1BOkd8HRdldgq8CAwEAAaAtMCsG
+CSqGSIb3DQEJDjEeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqG
+SIb3DQEBCwUAA4IBAQCwf7Q0Mkc6uMPFCa5W3cvMmRHSvfGGi7IcXDC2sdrEK+K3
+cOm4h7jCsWdKvE6+uxoPnDro0CgUeieHFDBpgZ613l9MCiYqBJwFHZeMoNqhItge
+GIx5TMF57Qt/TQ+5Mm/Oh8hbXEOwZUwSOW0sl1rPi+UWv8PXtmPb/di2dpbmRChq
+s/7yZQCNvQwwcdg21aWDB/rsYHlsD9InHy9NHdCzcA/QAiTTaTh15ej2s9/yo/8o
+lHOmytUAIhw44qXCg/ZLYX1Lb2LLKMSMlwFAnB9SV+EMvvrsAo07p6prjJk4pYp8
+PYDXZpnmR9zmtnKFN+2F2AWhknsdOmqaguLW/Z71
+-----END CERTIFICATE REQUEST-----

integration/fixtures/server.key.insecure

@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpQIBAAKCAQEAvpNoW/s4BJNlsX17cqSNN8tx1KNZrl+xWXR19cin5v0wpeTg
-kdSHfLWb90Z9/oWRFb4zZTovwreYnNElWuDqXkE6O7uZLuf0YlxO3OeSPsKZsKNf
-yFJYHqt58gyl+EZQSX8pklA/pVh7JoV5KSnVOcHBEcC+RZ65HW00MKPuJwTHHFIS
-4FgvWsKU+7AF1uzpk5YP7M0yDs8iadmEQaF6ephrBxdK7o1RJUMgjgZwIv/sz4Rh
-snAjqGtXlY2Xm6RYHIrMiMGUTuiroH2CGa0OO0sHKrMuJX5G818WelDIfhbVM4l2
-EhtnPK83aZx5RUHKX5Da2wLuUNjm5kIdtRNfAQIDAQABAoIBAQCDTcjfZw1Hic7N
-JWnCqUFrKc759Lo7fE8TFTyY5XFZoyS7iCB6GXZoJDCbhIQWsywtUOjUW+zAOgL6
-ONeF7+VKn6JhuXVnbgVhJ7xmU17dwvJlU4sQ2DtCll7kuHY5wyhaGzUnTAcuAvKG
-rfu2ss3oh2hgtO3jxeJBNhZ5VNknI+EycvW1JsMXG7qwySqLwtuUhptHmxyMTcXt
-LZi2zwmBKX7s44fCQrLq2CAo+GMJ7OFoUtZezu57ySki32VnL6uwM8ErKnvxUbHu
-H64erRRJ7Pw/qDQyYyo9pX/pKmcjGYIpt/ywYqbvszBY6ad3cdjtBHcS+9CeLeaN
-LrxAqNcpAoGBANr3dX2TMiCRiT6sTR+6ainzj/FHFA6xSU4ipe6uc/65zxh2xGaN
-+FrkDMmlvEEQLboEFdkiQikZK3xNHB/GEqAEmMsDgzYsg36JjsxLYLR9XQR3TDnu
-leNtjk/P05jiJchrL9m5xW1WM4QxwD7rf7TSRia/BrVyvQvhI3p8BKGbAoGBAN7O
-t/UajS8xjJmx1+u92FPXHnFi+tLuEfdd6ooKdw9rXUARVgeBsGgrLLrMKoCzdfWr
-txw7j5DjOlcx8ZXpyrcVyGJWuMboV3uf1IEiYZsMd7Le6yfnz4qgUPFmFy3JUQb1
-cbzc9dBuhCLQ2H7JU4EhlhtxyyY585kZtDaThmmTAoGBAIw9bVxuB+7gB1zCkeq+
-Q/x2aDyJ34jBd0e53TiPNu9wJflvJ77fMq9T2/TSV038hKzcrPmSfXlBC57i7B5V
-h9xA1XNA3qq1u8oxY+noZRl0KT0RAxsfeZRduIXZf5YtUTGZpN33o0CxsvD5xD0I
-K5SuEAwE0NEpmXagTU7HW1f9AoGAb+z0aEJQTjbb5JF8YEZcF7Hm7xrD2ZYSnGsn
-WPTs3mgWzgpnZxn1Hj8iFyxc5Y5BYYpDUAFzm1sqgYbrT13Eobhlk1DxPaqV19pw
-i/ZThen7b3WgN8mxbngecUXRuwR4mcBOxItTSMNbyYmUWAyW0DWpDFxbqvZNsslA
-yHHPgdUCgYEAuIRHMNanm5eZz7iXUMShAIUgaqcEIFOQ4W43zPQ1/F2beJBx+VoN
-u1Bvs7K9GBRDvJHcsjRxhYnwBwGu06M1NRG3QBW5VNuezpKzvchCgWR96ulzNOIe
-5C+j3zQmut4sOx2IY0zsJCfXnLJSoYwwtM1eVzY06uHwx+F4SMv1z8w=
+MIIEogIBAAKCAQEAufF1U5jbsALei7LYmkkEK6NyBI1Wht34qsYdK43Fp1ihHOpG
+RcU2ckXqX0LS0m+dkg10xn+5DyUF+JLLfzUfap3a3U+oMMDXAF3RKlSSMfYvdnjG
+mLgEtJyKLJav+lCz74iuk6HEwCCiadpqw1NspVOz6r5kVLjccYnlQkiDAHrzWmRY
+bnFv7hrWzXeLEs6Nl8Gr9/dQw1gyGWGL7nBhYrY2NZjhw8kG+eBAlTOBueyE57d3
+7zz4xmE0lQ5LYCiHXAGcXeKQQxyQfKQeSLr0D6r47W2weGrmyhCVV2ghSttsycAU
+7HqNUuYXLXbbMHKhDSs4G6q7UE6R3wdF2V2CrwIDAQABAoIBABdY9zdw+RzHr3Px
+F/t/cZOdKULjEFtM4d1nlOAUJB5YJZrA2+QE2EpO9xfRBzG/LBTbta76+JyrNG26
+2Ox3GiPEAGhLiT3d6OE15n6fMeAuHlWM4rroWEbpZPwmS6Bto7pO+kS1SkQplNsf
+vsoZ3ol/0haGo4hz0dI701qYSkKz9kldTYUp6iP1Lt9icSbIom95DP6wtKEOty9i
+j7WwZRRWWjmVQZEcL6CkB9B/XlWlOP6STqLwnN2pGmqHxxUfpWJYLZkuf9fYwNaf
+5xcUvztlMYHXHaCAD3MEZ0gh/YxlzgqdnCBQSO5w4niWIjCwi6Q/UANm3qzLk/7n
+HiUdwkkCgYEAwRjlrot5BCVwyF5f6qMz5SFKhE8xYUDfy2MkPb0tb7UoCj9/d9Ee
+Vu4ysPsyqALwG8gvDUhCBtEgqrs2QmlyTVlvZDOnN8JDU8DALeTitn0U+wUqHG8C
+fd9FqicsnUug9tKz92+npNvI6S/RmiEVid6OEUUxLxaIJVDjdL3xajUCgYEA9oP1
+Gd3C9yiInnQHZuhQzBxjar1IndrHiUV5t5VmUp+a2M7DDJupQ8o7903fabheWsHK
+ZLPiCFE2dOUBlAa15lNoQST4pW0jQaspWmljInjIbGBD1/3oNY2P5QsMVRpYR/jc
+zkEMu6UhccODobkDDKR1QuBXE3gUXB1NYKuONdMCgYA5che6gqHA+wZ/hZwRaPYi
+X3IUxJ6TKUKq9lasy1/+EK3Vxqg8VAkroXeRMVoTo7Qc/8QHtox2DLckM8fjoA0y
+N74s2DUSIIf5HanOvX44/iNEvneVt+zM16SZB0h9jydW5r5FiYEdSYmO3fwEV31Z
+48zuFGCaeqCMQXE5pNxyVQKBgHXQIN5ozpnCpHBd8X+r/Zle4+CKb98JRR2Et6QW
+YZ2RK7b8MdmftyhvyXLqo6Bp/aYQQcMY+SiSb500KSnufxaBvCyOF8svuo7S12Of
+REKgSOEKrMHYVhbp/eAwBIfTnGdhN13XVteDFtXA4/LxcdSj8GqyvsrOM6TGWEmg
+E8oDAoGAQqDV89Ed6r+IdyA2QY3IYDN+x7PJEt3xySVcfPGGFE2QNLEY5aNtwa5r
+y7uj33hg6Bkzacrw/9mzcm71wBv0RNX5C51hIKsdruzftOVDJFnK6qtoS5lQXtxd
+nngvlDI4CJRSj3O6d6uLT1cXjCQnT0hLjrpVIr72448w7M870mE=
 -----END RSA PRIVATE KEY-----

integration/fixtures/server2.crt

@@ -1,4 +1,5 @@
 -----BEGIN CERTIFICATE-----
+<<<<<<< HEAD
 MIIEEzCCAvugAwIBAgIUev7+NZl9RzdnsOGshKbMEHIxtD8wDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
@@ -21,4 +22,28 @@ R12Tnph2+RVsUaCcSmKNNcJxKjYXlU1bH3vyZ8/EmtGzsYnUlGtHfsHSHBdL/eHN
 g958qOHHYLtUWQsCf0az51mXO0yeaD+9pzTcnUX6tk2Er3OVKF51AdrdQVjQ9uub
 ST8onCbuICF6nRzXiF+sxv8h78ilIkdr3iCJw3TnIOLgXs8uh9PK3Du7Qh/2UD/5
 EucAVCeNgJQQ7Bvtcw+VIPLWwXnq71qu8p9Datir5dghK9FGmYeu
+=======
+MIIEEzCCAvugAwIBAgIUCHfFkPZDhLT2oK0fhu2TP4xZtcwwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHkxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEVMBMGA1UEAxMMZXhhbXBsZTIuY29tMIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAv6FwE/OeAwAmZutb+DTP+y7FKpONm0+aHtESlrmTRmDo
+iLGuoEXjYcFOCHr6gQJTYKEAivBK3TIN0dRirNBqLQxssDlsbU4ZXG++OciH/OoR
+8E+VmMsbqof/E0nhVFYDumnuoS+waX8elzuDjDX4u7F+d1/gIb8aYU1VDjtZxF2b
+vqaPyroOn5HvBs4MW+BpAB4guHfDXpK/oAnJDsq9JTUZqoG1xOZfHNhA/iVBSAJv
+hO6aBxDCjzwO1gT5kTbNELrBCJ0V2NXlHFcBOPhxyl4+DIPwa0Q8oExUEOzxnHFu
+U3GPoklbLp0RNNiqHHKfD4yaLI1rf8dH5AZG/QjK4wIDAQABo4GcMIGZMA4GA1Ud
+DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0T
+AQH/BAIwADAdBgNVHQ4EFgQUS6HUgF/GO55DtrxLN2dcyGACgnQwHwYDVR0jBBgw
+FoAUZQe+r42mchS6xFBpug0H8o5jej4wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/
+AAABMA0GCSqGSIb3DQEBCwUAA4IBAQC2pcpQZVkvZO4TC1zluGhG6Yhz+fzZuNlp
+jHa0U+WIdTPerqZZ98iU1hCm3F6voNjAQ0eHRV3z5q/rDph8AR7cVl9cT2rLO/zA
+F6M4QDMetddj7EIUq2/B1CWzHkOvwcJFgc0OfBWzJYAShFv/B7Ir1WpdtixOvyOH
+kWsWoy1WmatQvBQ2jDrvdGRWhqsPmg2uGbJrUjABeYtc5whQr0zscy+jEIrDpqPT
+VVuUu19/ALvdv2kOC+ayhH+vTAvEA38P6wlavDlgsv/M902ORWahdLQ/H0XX+gVP
+QDe3MyrBR6QkjAfKJvnMv/5x8mj+AFWfnALnRb9j3/q1UOwITWN2
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END CERTIFICATE-----

integration/fixtures/server2.key.insecure

@@ -1,4 +1,5 @@
 -----BEGIN RSA PRIVATE KEY-----
+<<<<<<< HEAD
 MIIEpAIBAAKCAQEA3VZyO4MoZjGpPU9hbb6SDV3ESNjUiS+pojigIVJi7C4bW1eJ
 vPDpodetKQg22Htq+Vj8mpCmnqn7/8pegbxV9Fc3Fyy0kxiEfHnaOpzo8GWhpReu
 bHJ2+kj2I6gY7HnKPJ3KP3Z9txKECNm9CtHuoOxjRvHR8SThcVYZqxz6cjkgZB/A
@@ -24,4 +25,31 @@ B3axr3feZ12lpqFGSEJAXqtN+UKgrx7oE4jibIDdPE78jW3YgIhagc0d4MhE6FPW
 Y+dqLQKBgQCUQEMiTwGadmDy0VmzAYVmKQV0qqb+hHswPt2Nyr5tNs7ZGJXaiW1+
 t2AtWeFFvpvdCaN9q7G5sZedIeB/zAByNTGF8ztBB3dSyv7IiMhthAWaC1UZefD6
 zw7N2UCuvjwvh+T1fi8BpMvK4YizmSyw04tEvoZMnhK1fQ2r/PWsjw==
+=======
+MIIEpAIBAAKCAQEAv6FwE/OeAwAmZutb+DTP+y7FKpONm0+aHtESlrmTRmDoiLGu
+oEXjYcFOCHr6gQJTYKEAivBK3TIN0dRirNBqLQxssDlsbU4ZXG++OciH/OoR8E+V
+mMsbqof/E0nhVFYDumnuoS+waX8elzuDjDX4u7F+d1/gIb8aYU1VDjtZxF2bvqaP
+yroOn5HvBs4MW+BpAB4guHfDXpK/oAnJDsq9JTUZqoG1xOZfHNhA/iVBSAJvhO6a
+BxDCjzwO1gT5kTbNELrBCJ0V2NXlHFcBOPhxyl4+DIPwa0Q8oExUEOzxnHFuU3GP
+oklbLp0RNNiqHHKfD4yaLI1rf8dH5AZG/QjK4wIDAQABAoIBAE5QTHxq4BV71zXS
+U7ig5KpTV9JpkMJ7CpIzgTRFzNFDQ2SxsJrhVOabWCeREpTsfWSNB6rAPugcz5cE
+A/t6BRo57KUsIoqdEzI6nHQC5shOZFxgOdPClaDgiTa5x7Nun4FsT1BiK+dBQyAs
++zqux+L0y6k/blp8Peyr7OmvCaV8osB4/JLLH/WHt2wWgqFWisyIT7/D/gQlQn81
+Hvv84BAL9y8iyCmCzWhQL0YisLPyaFkGkb7DK4wznWxfQn3jRAkZDQMH675o/OHj
+8nL0NSdCA/MGLEtPAXegM7kMPCf68JwZV3gPZDyEK0JES1oT1z+op7JHuatlhgdL
+WTA10fkCgYEA1QXRRpOeZvHzGVMzrXrrgS5GeaR+XgjUalBGgu0w8nSET195oXu6
+Y8dVco4FlEZ0Wq7evA4M9XVJyKQkkEGR7Nkv922p8RhG+U73ajODANAQURIwqOPJ
+01IrfMIK2mkXDZwzkAxPaOnny2OMZtUznmZnNdJ/vLd7U0sScNPVQp8CgYEA5krD
+ImQ8U9/S4VOK78i3FMWMoutffXpW71lEc9tsz1YWUPf170raujjF5mqtBBXup9ko
+37CmVk6mOO2TdXLg1feMaVBsoblL6iPoBZot/fLdzgmICccpimst2yrUZEHwJpdk
+9k95xEZUQhN73eY/Ih6b5HZZ/ygxfAVvhDFzNT0CgYEAru5aDvUGbU9e7HsQwvNg
+FfMkWJwmUZ46oRtO7BFP0qqwRGYJAf0S8QEuQCY0mrDIt/dGXXPEXIV2k9eHVxch
+eDhaVXuuxJfFINIiBwpKGA7Ed27Smr6EbI7bu1W1h+oozjppdW9GfscmXDVhhMir
+3PYG54H298hM8/eAKzsps80CgYEAzk5Vp76ySNVv3spv4kYmtaYQSnef8RIjRYLs
+DvqY7NmLXnf0y618a22m5LfWTZ20Uov50QM40ILe6Ir1GjeS8jw1frc8yljsiFIo
+brRj1We4ivcA9vmD3mwMBZbF9RcZJAlmuj4SsOHsY9F+mxjEoDVZpP7duvbv9dIM
+yBlgw2UCgYB27Zwd3ta3x9mx2wRHaFMlcnwh+ERRGO6yAKndsr3L80Gkb2HxHlR3
+cA3dIg/YAGocjROOIO8MO5rclZm30qM5rlv0SH6gQ4nW16PZgHSuakgeX5fVHb4v
+nT8TKt0PY9r5e5cdHzpwlrDD1Q8GiAHvn0oH6oioGxlIV190YAHvGQ==
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END RSA PRIVATE KEY-----

integration/fixtures/server3.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID/DCCAuSgAwIBAgIUMgfWSv3FrD0dRcHmPNxLr7p9BXcwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMGIxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKKf9zhwcgu
+S53Vd92PLqHtDNpTbl4nM/5qYy7nHV5FHW7uolQHFA0BjJwxz7/LC39G950PpSIs
+K7Y99P7aMPVqVH5FGL+uoDbHMzt/gIUGgWj51J38+x6zN/9vIvAVIBClBhzEuB2k
+WJU6KyB1V5G+1wnnKRXLB9QC0f/7vqd9f21O7sJmWeVhGJuEUwwAp1p5WDGM2Tn8
+Fjy57O9f1nT4WVqWhB5EbvYGDF2Z5DWyKz90EWOwVw30ThcQHF57X6WJdlNiQOrY
+KIWO475QKPwbUSpRkvw1jwvllU8s4l6pB1hKTAcUK3UsWrzpf2+m8v8ou35uFD/c
+pd2bBx5MVlkCAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYI
+KwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFAcMdmvX
+8NC4V4Z67Jb/pNUwai7cMB8GA1UdIwQYMBaAFGUHvq+NpnIUusRQaboNB/KOY3o+
+MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA
+btQjJEWHD/0gYsNLFg3tDxZ64U/HfNlh4USGOK02VL2LteMcV8AoYlZ3jwmp4+33
+D3HlqLclJNABax2pOvTHVnQlf25TSNwJRtmzOvcg+6xYbPdgRoeEVsWbmgbpX7Vi
+P8FYelYCiYTPezjqZgPG1gmq0Uf/drlTrjwsG2njEcuK7hip+LdJnIrtpIrabpIk
+lZRa7Y/JBM3gP/rR1fu9lhzJ97s3NabuHzPwyouSTTknaaiGwSV8F5frh9NGcFhd
+G7giCLZLKklQB4IUTOFcVFSZmeAGy6KBqyT10N2kkBrsrcWhyMKIU9X0+6hh3Tlc
+JEla9as6qFvt1dFGp+qeww==
+-----END CERTIFICATE-----

integration/fixtures/server3.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAsop/3OHByC5LndV33Y8uoe0M2lNuXicz/mpjLucdXkUdbu6i
+VAcUDQGMnDHPv8sLf0b3nQ+lIiwrtj30/tow9WpUfkUYv66gNsczO3+AhQaBaPnU
+nfz7HrM3/28i8BUgEKUGHMS4HaRYlTorIHVXkb7XCecpFcsH1ALR//u+p31/bU7u
+wmZZ5WEYm4RTDACnWnlYMYzZOfwWPLns71/WdPhZWpaEHkRu9gYMXZnkNbIrP3QR
+Y7BXDfROFxAcXntfpYl2U2JA6tgohY7jvlAo/BtRKlGS/DWPC+WVTyziXqkHWEpM
+BxQrdSxavOl/b6by/yi7fm4UP9yl3ZsHHkxWWQIDAQABAoIBAQCVyfrCDqlsT+Li
+1UBOIp0l/uIEnXCAD3XgodL6e6249FVgR1brFlEtJDqapHO+XhQUQS7ml0ScqeA2
+cj6EPfxLOV0P3tqHnnMN4gvKhAsID9AsiUVnEuJ//C4j4FK4h5CyRjEdm7E4NTSY
+ZgfeoHPKdAinZ0eh4Ad+SKt0jvmCPDD1L+6bxpJ6E258xPDxB71rHTCgnwZZmXrN
+rHDg07tVVU6lYtXEsZAsyIBIxXV/RaCt4xSijM1C7kuSsUE3+CCw+pzQUvHDxkuk
+FPxE5hONzkUaCSBKTv4L6gVaiYa30Jo9THuTRWvzDJmcnNwlYgRLeIR8PJY0eHqv
+FYJbLdQBAoGBAOeiW1aHpNO+K4BqVMzm7fvz0a3DBkml/wIKSvw3dbTYnyIXFvDB
+Be7OZLhiWPwaE/58aQWh+/OGCr88yyCLAt2mOQ5aMalKpdSCaY5swEnzt923pY9z
+jt3DnhX8aXggPbqM6eJjaxJY7jIMSDNKUQZKJeSesr/EzFo0CRkkepyBAoGBAMVS
+Z2nIY+G2+P1VSUEbI3dbaZ3ciMEDLc+rZQs0fx8+xQCD71eCU9ggxf+O0R1/0smm
+9so65KrmKl3yOt8OQW3YgUpQqIQdJPHnfuTnsU7y/+zRl6k119gV89LLDlCM7nfW
+5/ey/iJLXQxfC7OoFF/hQM0odmorA8jBuqKDWy3ZAoGAZL6gq0njzpRvpzKYH2Zx
+K5woHkMsgOvJtcF0S65za2ysCc+xEpVhVzQ9alScD0noWE8T/nctdgVetz5huo27
+eVvKhQuFffQRnBP8hQ2XtJJj7fLp9zJzeNCT+UwHM1ASiQiw0N4cu6YiM3JUFLrF
+8s5dHMpJRE778l+fdWgATAECgYEAmJ3osE+2uUCtCjvpwbp8zvdcFCYbe7W6vBGj
+wGvlGsSQ2JozB2sc8GBA5C2RHhDcdu11meq9LFWDVVBiKl27S3uWXGVQQYbNKXDU
+m7V8VUTrnz5o4A5uGIq6IEK/mpu2YehNWC8QEnRZzpTA1z7cK2Bsn4F5PRpx/deh
+Q8r3PdkCgYBd2rT+S4/51C5AnIhgMF/PYl0+DYMFD8HAfsx7VTaIUmFQ4devHMOz
+J6lbqRyEITZtXgna1n35LkyBDcPwsEtntjJOP+xneCtKzozdzhXyoAw2xQR3gqvV
+7YtV3miYQiOTqjOefViMhR/XiOV2zng3OId1AQObfOUZODJPfSN26g==
+-----END RSA PRIVATE KEY-----

integration/v3_auth_test.go

@@ -347,6 +347,7 @@ func TestV3AuthNonAuthorizedRPCs(t *testing.T) {
 }
 
 func TestV3AuthOldRevConcurrent(t *testing.T) {
+	t.Skip() // TODO(jingyih): re-enable the test when #10408 is fixed.
 	defer testutil.AfterTest(t)
 	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
 	defer clus.Terminate(t)

integration/v3_watch_test.go

@@ -781,9 +781,11 @@ func testV3WatchMultipleEventsTxn(t *testing.T, startRev int64) {
 
 type eventsSortByKey []*mvccpb.Event
 
-func (evs eventsSortByKey) Len() int           { return len(evs) }
-func (evs eventsSortByKey) Swap(i, j int)      { evs[i], evs[j] = evs[j], evs[i] }
-func (evs eventsSortByKey) Less(i, j int) bool { return bytes.Compare(evs[i].Kv.Key, evs[j].Kv.Key) < 0 }
+func (evs eventsSortByKey) Len() int      { return len(evs) }
+func (evs eventsSortByKey) Swap(i, j int) { evs[i], evs[j] = evs[j], evs[i] }
+func (evs eventsSortByKey) Less(i, j int) bool {
+	return bytes.Compare(evs[i].Kv.Key, evs[j].Kv.Key) < 0
+}
 
 func TestV3WatchMultipleEventsPutUnsynced(t *testing.T) {
 	defer testutil.AfterTest(t)

mvcc/backend/backend.go

@@ -292,7 +292,7 @@ func (b *backend) Defrag() error {
 
 func (b *backend) defrag() error {
 	now := time.Now()
-	
+
 	// TODO: make this non-blocking?
 	// lock batchTx to ensure nobody is using previous tx, and then
 	// close previous ongoing tx.
@@ -310,21 +310,38 @@ func (b *backend) defrag() error {
 	b.batchTx.unsafeCommit(true)
 	b.batchTx.tx = nil
 
-	tmpdb, err := bolt.Open(b.db.Path()+".tmp", 0600, boltOpenOptions)
+	// Create a temporary file to ensure we start with a clean slate.
+	// Snapshotter.cleanupSnapdir cleans up any of these that are found during startup.
+	dir := filepath.Dir(b.db.Path())
+	temp, err := ioutil.TempFile(dir, "db.tmp.*")
+	if err != nil {
+		return err
+	}
+	options := bolt.Options{}
+	if boltOpenOptions != nil {
+		options = *boltOpenOptions
+	}
+	options.OpenFile = func(path string, i int, mode os.FileMode) (file *os.File, err error) {
+		return temp, nil
+	}
+	tdbp := temp.Name()
+	tmpdb, err := bolt.Open(tdbp, 0600, &options)
 	if err != nil {
 		return err
 	}
 
+	// gofail: var defragBeforeCopy struct{}
 	err = defragdb(b.db, tmpdb, defragLimit)
 
 	if err != nil {
 		tmpdb.Close()
-		os.RemoveAll(tmpdb.Path())
+		if rmErr := os.RemoveAll(tmpdb.Path()); rmErr != nil {
+			plog.Fatalf("failed to remove db.tmp after defragmentation completed: %v", rmErr)
+		}
 		return err
 	}
 
 	dbp := b.db.Path()
-	tdbp := tmpdb.Path()
 
 	err = b.db.Close()
 	if err != nil {
@@ -334,6 +351,7 @@ func (b *backend) defrag() error {
 	if err != nil {
 		plog.Fatalf("cannot close database (%s)", err)
 	}
+	// gofail: var defragBeforeRename struct{}
 	err = os.Rename(tdbp, dbp)
 	if err != nil {
 		plog.Fatalf("cannot rename database (%s)", err)

mvcc/kv_test.go

@@ -710,7 +710,7 @@ func TestKVSnapshot(t *testing.T) {
 
 func TestWatchableKVWatch(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()

mvcc/kvstore.go

@@ -142,14 +142,18 @@ func NewStore(b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) *sto
 
 func (s *store) compactBarrier(ctx context.Context, ch chan struct{}) {
 	if ctx == nil || ctx.Err() != nil {
-		s.mu.Lock()
 		select {
 		case <-s.stopc:
 		default:
+			// fix deadlock in mvcc,for more information, please refer to pr 11817.
+			// s.stopc is only updated in restore operation, which is called by apply
+			// snapshot call, compaction and apply snapshot requests are serialized by
+			// raft, and do not happen at the same time.
+			s.mu.Lock()
 			f := func(ctx context.Context) { s.compactBarrier(ctx, ch) }
 			s.fifoSched.Schedule(f)
+			s.mu.Unlock()
 		}
-		s.mu.Unlock()
 		return
 	}
 	close(ch)
@@ -309,14 +313,7 @@ func (s *store) Restore(b backend.Backend) error {
 }
 
 func (s *store) restore() error {
-	b := s.b
-
-	reportDbTotalSizeInBytesMu.Lock()
-	reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) }
-	reportDbTotalSizeInBytesMu.Unlock()
-	reportDbTotalSizeInUseInBytesMu.Lock()
-	reportDbTotalSizeInUseInBytes = func() float64 { return float64(b.SizeInUse()) }
-	reportDbTotalSizeInUseInBytesMu.Unlock()
+	s.setupMetricsReporter()
 
 	min, max := newRevBytes(), newRevBytes()
 	revToBytes(revision{main: 1}, min)
@@ -496,6 +493,30 @@ func (s *store) ConsistentIndex() uint64 {
 	return v
 }
 
+func (s *store) setupMetricsReporter() {
+	b := s.b
+	reportDbTotalSizeInBytesMu.Lock()
+	reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) }
+	reportDbTotalSizeInBytesMu.Unlock()
+	reportDbTotalSizeInUseInBytesMu.Lock()
+	reportDbTotalSizeInUseInBytes = func() float64 { return float64(b.SizeInUse()) }
+	reportDbTotalSizeInUseInBytesMu.Unlock()
+	reportCurrentRevMu.Lock()
+	reportCurrentRev = func() float64 {
+		s.revMu.RLock()
+		defer s.revMu.RUnlock()
+		return float64(s.currentRev)
+	}
+	reportCurrentRevMu.Unlock()
+	reportCompactRevMu.Lock()
+	reportCompactRev = func() float64 {
+		s.revMu.RLock()
+		defer s.revMu.RUnlock()
+		return float64(s.compactMainRev)
+	}
+	reportCompactRevMu.Unlock()
+}
+
 // appendMarkTombstone appends tombstone mark to normal revision bytes.
 func appendMarkTombstone(b []byte) []byte {
 	if len(b) != revBytesLen {

mvcc/metrics.go

@@ -206,6 +206,46 @@ var (
 		// highest bucket start of 0.01 sec * 2^14 == 163.84 sec
 		Buckets: prometheus.ExponentialBuckets(.01, 2, 15),
 	})
+
+	currentRev = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "etcd_debugging",
+		Subsystem: "mvcc",
+		Name:      "current_revision",
+		Help:      "The current revision of store.",
+	},
+		func() float64 {
+			reportCurrentRevMu.RLock()
+			defer reportCurrentRevMu.RUnlock()
+			return reportCurrentRev()
+		},
+	)
+	// overridden by mvcc initialization
+	reportCurrentRevMu sync.RWMutex
+	reportCurrentRev   = func() float64 { return 0 }
+
+	compactRev = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "etcd_debugging",
+		Subsystem: "mvcc",
+		Name:      "compact_revision",
+		Help:      "The revision of the last compaction in store.",
+	},
+		func() float64 {
+			reportCompactRevMu.RLock()
+			defer reportCompactRevMu.RUnlock()
+			return reportCompactRev()
+		},
+	)
+	// overridden by mvcc initialization
+	reportCompactRevMu sync.RWMutex
+	reportCompactRev   = func() float64 { return 0 }
+
+	totalPutSizeGauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "etcd_debugging",
+			Subsystem: "mvcc",
+			Name:      "total_put_size_in_bytes",
+			Help:      "The total size of put kv pairs seen by this member.",
+		})
 )
 
 func init() {
@@ -228,6 +268,9 @@ func init() {
 	prometheus.MustRegister(dbTotalSizeInUse)
 	prometheus.MustRegister(hashDurations)
 	prometheus.MustRegister(hashRevDurations)
+	prometheus.MustRegister(currentRev)
+	prometheus.MustRegister(compactRev)
+	prometheus.MustRegister(totalPutSizeGauge)
 }
 
 // ReportEventReceived reports that an event is received.

mvcc/metrics_txn.go

@@ -23,14 +23,15 @@ type metricsTxnWrite struct {
 	ranges  uint
 	puts    uint
 	deletes uint
+	putSize int64
 }
 
 func newMetricsTxnRead(tr TxnRead) TxnRead {
-	return &metricsTxnWrite{&txnReadWrite{tr}, 0, 0, 0}
+	return &metricsTxnWrite{&txnReadWrite{tr}, 0, 0, 0, 0}
 }
 
 func newMetricsTxnWrite(tw TxnWrite) TxnWrite {
-	return &metricsTxnWrite{tw, 0, 0, 0}
+	return &metricsTxnWrite{tw, 0, 0, 0, 0}
 }
 
 func (tw *metricsTxnWrite) Range(key, end []byte, ro RangeOptions) (*RangeResult, error) {
@@ -45,6 +46,8 @@ func (tw *metricsTxnWrite) DeleteRange(key, end []byte) (n, rev int64) {
 
 func (tw *metricsTxnWrite) Put(key, value []byte, lease lease.LeaseID) (rev int64) {
 	tw.puts++
+	size := int64(len(key) + len(value))
+	tw.putSize += size
 	return tw.TxnWrite.Put(key, value, lease)
 }
 
@@ -55,5 +58,6 @@ func (tw *metricsTxnWrite) End() {
 	}
 	rangeCounter.Add(float64(tw.ranges))
 	putCounter.Add(float64(tw.puts))
+	totalPutSizeGauge.Add(float64(tw.putSize))
 	deleteCounter.Add(float64(tw.deletes))
 }

mvcc/watchable_store.go

@@ -18,6 +18,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/coreos/etcd/auth"
 	"github.com/coreos/etcd/lease"
 	"github.com/coreos/etcd/mvcc/backend"
 	"github.com/coreos/etcd/mvcc/mvccpb"
@@ -27,9 +28,8 @@ import (
 var (
 	// chanBufLen is the length of the buffered chan
 	// for sending out watched events.
-	// TODO: find a good buf value. 1024 is just a random one that
-	// seems to be reasonable.
-	chanBufLen = 1024
+	// See https://github.com/etcd-io/etcd/issues/11906 for more detail.
+	chanBufLen = 128
 
 	// maxWatchersPerSync is the number of watchers to sync in a single batch
 	maxWatchersPerSync = 512
@@ -67,11 +67,11 @@ type watchableStore struct {
 // cancel operations.
 type cancelFunc func()
 
-func New(b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) ConsistentWatchableKV {
-	return newWatchableStore(b, le, ig)
+func New(b backend.Backend, le lease.Lessor, as auth.AuthStore, ig ConsistentIndexGetter) ConsistentWatchableKV {
+	return newWatchableStore(b, le, as, ig)
 }
 
-func newWatchableStore(b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) *watchableStore {
+func newWatchableStore(b backend.Backend, le lease.Lessor, as auth.AuthStore, ig ConsistentIndexGetter) *watchableStore {
 	s := &watchableStore{
 		store:    NewStore(b, le, ig),
 		victimc:  make(chan struct{}, 1),
@@ -85,6 +85,10 @@ func newWatchableStore(b backend.Backend, le lease.Lessor, ig ConsistentIndexGet
 		// use this store as the deleter so revokes trigger watch events
 		s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write() })
 	}
+	if as != nil {
+		// TODO: encapsulating consistentindex into a separate package
+		as.SetConsistentIndexSyncer(s.store.saveIndex)
+	}
 	s.wg.Add(2)
 	go s.syncWatchersLoop()
 	go s.syncVictimsLoop()

mvcc/watchable_store_bench_test.go

@@ -25,7 +25,7 @@ import (
 
 func BenchmarkWatchableStorePut(b *testing.B) {
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	s := New(be, &lease.FakeLessor{}, nil)
+	s := New(be, &lease.FakeLessor{}, nil, nil)
 	defer cleanup(s, be, tmpPath)
 
 	// arbitrary number of bytes
@@ -46,7 +46,7 @@ func BenchmarkWatchableStorePut(b *testing.B) {
 func BenchmarkWatchableStoreTxnPut(b *testing.B) {
 	var i fakeConsistentIndex
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	s := New(be, &lease.FakeLessor{}, &i)
+	s := New(be, &lease.FakeLessor{}, nil, &i)
 	defer cleanup(s, be, tmpPath)
 
 	// arbitrary number of bytes
@@ -67,7 +67,7 @@ func BenchmarkWatchableStoreTxnPut(b *testing.B) {
 // many synced watchers receiving a Put notification.
 func BenchmarkWatchableStoreWatchSyncPut(b *testing.B) {
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(be, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(be, &lease.FakeLessor{}, nil, nil)
 	defer cleanup(s, be, tmpPath)
 
 	k := []byte("testkey")
@@ -162,7 +162,7 @@ func BenchmarkWatchableStoreUnsyncedCancel(b *testing.B) {
 
 func BenchmarkWatchableStoreSyncedCancel(b *testing.B) {
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(be, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(be, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()

mvcc/watchable_store_test.go

@@ -30,7 +30,7 @@ import (
 
 func TestWatch(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -52,7 +52,7 @@ func TestWatch(t *testing.T) {
 
 func TestNewWatcherCancel(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -222,7 +222,7 @@ func TestSyncWatchers(t *testing.T) {
 // TestWatchCompacted tests a watcher that watches on a compacted revision.
 func TestWatchCompacted(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -259,7 +259,7 @@ func TestWatchCompacted(t *testing.T) {
 
 func TestWatchFutureRev(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -300,7 +300,7 @@ func TestWatchRestore(t *testing.T) {
 	test := func(delay time.Duration) func(t *testing.T) {
 		return func(t *testing.T) {
 			b, tmpPath := backend.NewDefaultTmpBackend()
-			s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+			s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 			defer cleanup(s, b, tmpPath)
 
 			testKey := []byte("foo")
@@ -308,7 +308,7 @@ func TestWatchRestore(t *testing.T) {
 			rev := s.Put(testKey, testValue, lease.NoLease)
 
 			newBackend, newPath := backend.NewDefaultTmpBackend()
-			newStore := newWatchableStore(newBackend, &lease.FakeLessor{}, nil)
+			newStore := newWatchableStore(newBackend, &lease.FakeLessor{}, nil, nil)
 			defer cleanup(newStore, newBackend, newPath)
 
 			w := newStore.NewWatchStream()
@@ -346,11 +346,11 @@ func TestWatchRestore(t *testing.T) {
 //   5. choose the watcher from step 1, without panic
 func TestWatchRestoreSyncedWatcher(t *testing.T) {
 	b1, b1Path := backend.NewDefaultTmpBackend()
-	s1 := newWatchableStore(b1, &lease.FakeLessor{}, nil)
+	s1 := newWatchableStore(b1, &lease.FakeLessor{}, nil, nil)
 	defer cleanup(s1, b1, b1Path)
 
 	b2, b2Path := backend.NewDefaultTmpBackend()
-	s2 := newWatchableStore(b2, &lease.FakeLessor{}, nil)
+	s2 := newWatchableStore(b2, &lease.FakeLessor{}, nil, nil)
 	defer cleanup(s2, b2, b2Path)
 
 	testKey, testValue := []byte("foo"), []byte("bar")
@@ -397,7 +397,7 @@ func TestWatchRestoreSyncedWatcher(t *testing.T) {
 // TestWatchBatchUnsynced tests batching on unsynced watchers
 func TestWatchBatchUnsynced(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	oldMaxRevs := watchBatchMaxRevs
 	defer func() {
@@ -531,7 +531,7 @@ func TestWatchVictims(t *testing.T) {
 	oldChanBufLen, oldMaxWatchersPerSync := chanBufLen, maxWatchersPerSync
 
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -609,7 +609,7 @@ func TestWatchVictims(t *testing.T) {
 // canceling its watches.
 func TestStressWatchCancelClose(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()

mvcc/watcher_bench_test.go

@@ -24,7 +24,7 @@ import (
 
 func BenchmarkKVWatcherMemoryUsage(b *testing.B) {
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	watchable := newWatchableStore(be, &lease.FakeLessor{}, nil)
+	watchable := newWatchableStore(be, &lease.FakeLessor{}, nil, nil)
 
 	defer cleanup(watchable, be, tmpPath)
 

mvcc/watcher_test.go

@@ -31,7 +31,7 @@ import (
 // and the watched event attaches the correct watchID.
 func TestWatcherWatchID(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()
@@ -83,7 +83,7 @@ func TestWatcherWatchID(t *testing.T) {
 // and returns events with matching prefixes.
 func TestWatcherWatchPrefix(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()
@@ -157,7 +157,7 @@ func TestWatcherWatchPrefix(t *testing.T) {
 // does not create watcher, which panics when canceling in range tree.
 func TestWatcherWatchWrongRange(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()
@@ -177,7 +177,7 @@ func TestWatcherWatchWrongRange(t *testing.T) {
 
 func TestWatchDeleteRange(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -216,7 +216,7 @@ func TestWatchDeleteRange(t *testing.T) {
 // with given id inside watchStream.
 func TestWatchStreamCancelWatcherByID(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()
@@ -308,7 +308,7 @@ func TestWatcherRequestProgress(t *testing.T) {
 
 func TestWatcherWatchWithFilter(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()

pkg/fileutil/dir_unix.go

@@ -18,5 +18,10 @@ package fileutil
 
 import "os"
 
+const (
+	// PrivateDirMode grants owner to make/remove files inside the directory.
+	PrivateDirMode = 0700
+)
+
 // OpenDir opens a directory for syncing.
 func OpenDir(path string) (*os.File, error) { return os.Open(path) }

pkg/fileutil/dir_windows.go

@@ -21,6 +21,11 @@ import (
 	"syscall"
 )
 
+const (
+	// PrivateDirMode grants owner to make/remove files inside the directory.
+	PrivateDirMode = 0777
+)
+
 // OpenDir opens a directory in windows with write access for syncing.
 func OpenDir(path string) (*os.File, error) {
 	fd, err := openDir(path)

pkg/fileutil/fileutil.go

@@ -29,8 +29,6 @@ import (
 const (
 	// PrivateFileMode grants owner to read/write a file.
 	PrivateFileMode = 0600
-	// PrivateDirMode grants owner to make/remove files inside the directory.
-	PrivateDirMode = 0700
 )
 
 var (
@@ -65,14 +63,22 @@ func ReadDir(dirpath string) ([]string, error) {
 // TouchDirAll is similar to os.MkdirAll. It creates directories with 0700 permission if any directory
 // does not exists. TouchDirAll also ensures the given directory is writable.
 func TouchDirAll(dir string) error {
-	// If path is already a directory, MkdirAll does nothing
-	// and returns nil.
-	err := os.MkdirAll(dir, PrivateDirMode)
-	if err != nil {
-		// if mkdirAll("a/text") and "text" is not
-		// a directory, this will return syscall.ENOTDIR
-		return err
+	// If path is already a directory, MkdirAll does nothing and returns nil, so,
+	// first check if dir exist with an expected permission mode.
+	if Exist(dir) {
+		err := CheckDirPermission(dir, PrivateDirMode)
+		if err != nil {
+			plog.Warningf("check file permission: %v", err)
+		}
+	} else {
+		err := os.MkdirAll(dir, PrivateDirMode)
+		if err != nil {
+			// if mkdirAll("a/text") and "text" is not
+			// a directory, this will return syscall.ENOTDIR
+			return err
+		}
 	}
+
 	return IsDirWriteable(dir)
 }
 
@@ -120,3 +126,22 @@ func ZeroToEnd(f *os.File) error {
 	_, err = f.Seek(off, io.SeekStart)
 	return err
 }
+
+// CheckDirPermission checks permission on an existing dir.
+// Returns error if dir is empty or exist with a different permission than specified.
+func CheckDirPermission(dir string, perm os.FileMode) error {
+	if !Exist(dir) {
+		return fmt.Errorf("directory %q empty, cannot check permission.", dir)
+	}
+	//check the existing permission on the directory
+	dirInfo, err := os.Stat(dir)
+	if err != nil {
+		return err
+	}
+	dirMode := dirInfo.Mode().Perm()
+	if dirMode != perm {
+		err = fmt.Errorf("directory %q exist, but the permission is %q. The recommended permission is %q to prevent possible unprivileged access to the data.", dir, dirInfo.Mode(), os.FileMode(PrivateDirMode))
+		return err
+	}
+	return nil
+}

pkg/fileutil/fileutil_test.go

@@ -163,3 +163,21 @@ func TestZeroToEnd(t *testing.T) {
 		}
 	}
 }
+
+func TestDirPermission(t *testing.T) {
+	tmpdir, err := ioutil.TempDir(os.TempDir(), "foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	tmpdir2 := filepath.Join(tmpdir, "testpermission")
+	// create a new dir with 0700
+	if err = CreateDirAll(tmpdir2); err != nil {
+		t.Fatal(err)
+	}
+	// check dir permission with mode different than created dir
+	if err = CheckDirPermission(tmpdir2, 0600); err == nil {
+		t.Errorf("expected error, got nil")
+	}
+}

pkg/fileutil/lock_linux.go

@@ -29,7 +29,7 @@ import (
 //
 // constants from /usr/include/bits/fcntl-linux.h
 const (
-	F_OFD_GETLK  = 37
+	F_OFD_GETLK  = 36
 	F_OFD_SETLK  = 37
 	F_OFD_SETLKW = 38
 )

pkg/fileutil/purge.go

@@ -23,13 +23,23 @@ import (
 )
 
 func PurgeFile(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) <-chan error {
-	return purgeFile(dirname, suffix, max, interval, stop, nil)
+	return purgeFile(dirname, suffix, max, interval, stop, nil, nil)
+}
+
+func PurgeFileWithDoneNotify(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) (<-chan struct{}, <-chan error) {
+	doneC := make(chan struct{})
+	errC := purgeFile(dirname, suffix, max, interval, stop, nil, doneC)
+	return doneC, errC
 }
 
 // purgeFile is the internal implementation for PurgeFile which can post purged files to purgec if non-nil.
-func purgeFile(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string) <-chan error {
+// if donec is non-nil, the function closes it to notify its exit.
+func purgeFile(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string, donec chan<- struct{}) <-chan error {
 	errC := make(chan error, 1)
 	go func() {
+		if donec != nil {
+			defer close(donec)
+		}
 		for {
 			fnames, err := ReadDir(dirname)
 			if err != nil {