version: 3.3.22

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>

.travis.yml

@@ -6,7 +6,7 @@ sudo: required
 services: docker
 
 go:
-- 1.12.9
+- 1.12.12
 
 env:
 - GO111MODULE=on
@@ -28,7 +28,7 @@ env:
 matrix:
   fast_finish: true
   allow_failures:
-  - go: 1.12.9
+  - go: 1.12.12
     env: TARGET=linux-386-unit
 
 install:

.words

@@ -25,6 +25,8 @@ healthcheck
 iff
 inflight
 keepalive
+hasleader
+racey
 keepalives
 keyspace
 linearization
@@ -41,4 +43,4 @@ too_many_pings
 uncontended
 unprefixed
 unlisting
-
+WithDialer

Documentation/op-guide/configuration.md

@@ -328,6 +328,11 @@ The security flags help to [build a secure etcd cluster][security].
 + default: ""
 + env variable: ETCD_CIPHER_SUITES
 
+### --experimental-peer-skip-client-san-verification 
++ Skip verification of SAN field in client certificate for peer connections.
++ default: false
++ env variable: ETCD_EXPERIMENTAL_PEER_SKIP_CLIENT_SAN_VERIFICATION
+
 ## Logging flags
 
 ### --logger

auth/metrics.go

@@ -0,0 +1,42 @@
+// Copyright 2015 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sync"
+)
+
+var (
+	currentAuthRevision = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "etcd_debugging",
+		Subsystem: "auth",
+		Name:      "revision",
+		Help:      "The current revision of auth store.",
+	},
+		func() float64 {
+			reportCurrentAuthRevMu.RLock()
+			defer reportCurrentAuthRevMu.RUnlock()
+			return reportCurrentAuthRev()
+		},
+	)
+	// overridden by auth store initialization
+	reportCurrentAuthRevMu sync.RWMutex
+	reportCurrentAuthRev   = func() float64 { return 0 }
+)
+
+func init() {
+	prometheus.MustRegister(currentAuthRevision)
+}

auth/store.go

@@ -90,6 +90,10 @@ type AuthenticateParamIndex struct{}
 // AuthenticateParamSimpleTokenPrefix is used for a key of context in the parameters of Authenticate()
 type AuthenticateParamSimpleTokenPrefix struct{}
 
+// saveConsistentIndexFunc is used to sync consistentIndex to backend, now reusing store.saveIndex
+type saveConsistentIndexFunc func(tx backend.BatchTx)
+
+// AuthStore defines auth storage interface.
 type AuthStore interface {
 	// AuthEnable turns on the authentication feature
 	AuthEnable() error
@@ -178,6 +182,9 @@ type AuthStore interface {
 
 	// HasRole checks that user has role
 	HasRole(user, role string) bool
+
+	// SetConsistentIndexSyncer sets consistentIndex syncer
+	SetConsistentIndexSyncer(syncer saveConsistentIndexFunc)
 }
 
 type TokenProvider interface {
@@ -200,9 +207,13 @@ type authStore struct {
 
 	rangePermCache map[string]*unifiedRangePermissions // username -> unifiedRangePermissions
 
-	tokenProvider TokenProvider
+	tokenProvider       TokenProvider
+	syncConsistentIndex saveConsistentIndexFunc
 }
 
+func (as *authStore) SetConsistentIndexSyncer(syncer saveConsistentIndexFunc) {
+	as.syncConsistentIndex = syncer
+}
 func (as *authStore) AuthEnable() error {
 	as.enabledMu.Lock()
 	defer as.enabledMu.Unlock()
@@ -252,6 +263,7 @@ func (as *authStore) AuthDisable() {
 	tx.Lock()
 	tx.UnsafePut(authBucketName, enableFlagKey, authDisabled)
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 	tx.Unlock()
 	b.ForceCommit()
 
@@ -368,6 +380,7 @@ func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse,
 	putUser(tx, newUser)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("added a new user: %s", r.Name)
 
@@ -392,6 +405,7 @@ func (as *authStore) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDelete
 	delUser(tx, r.Name)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	as.invalidateCachedPerm(r.Name)
 	as.tokenProvider.invalidateUser(r.Name)
@@ -428,6 +442,7 @@ func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*p
 	putUser(tx, updatedUser)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	as.invalidateCachedPerm(r.Name)
 	as.tokenProvider.invalidateUser(r.Name)
@@ -468,6 +483,7 @@ func (as *authStore) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUser
 	as.invalidateCachedPerm(r.User)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("granted role %s to user %s", r.Role, r.User)
 	return &pb.AuthUserGrantRoleResponse{}, nil
@@ -536,6 +552,7 @@ func (as *authStore) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUs
 	as.invalidateCachedPerm(r.Name)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("revoked role %s from user %s", r.Role, r.Name)
 	return &pb.AuthUserRevokeRoleResponse{}, nil
@@ -600,6 +617,7 @@ func (as *authStore) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest)
 	as.clearCachedPerm()
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("revoked key %s from role %s", r.Key, r.Role)
 	return &pb.AuthRoleRevokePermissionResponse{}, nil
@@ -645,6 +663,7 @@ func (as *authStore) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDelete
 	}
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("deleted role %s", r.Role)
 	return &pb.AuthRoleDeleteResponse{}, nil
@@ -667,6 +686,7 @@ func (as *authStore) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse,
 	putRole(tx, newRole)
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("Role %s is created", r.Name)
 
@@ -727,6 +747,7 @@ func (as *authStore) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (
 	as.clearCachedPerm()
 
 	as.commitRevision(tx)
+	as.saveConsistentIndex(tx)
 
 	plog.Noticef("role %s's permission of key %s is updated as %s", r.Name, r.Perm.Key, authpb.Permission_Type_name[int32(r.Perm.PermType)])
 
@@ -743,8 +764,13 @@ func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeE
 	if revision == 0 {
 		return ErrUserEmpty
 	}
-
-	if revision < as.Revision() {
+	rev := as.Revision()
+	if revision < rev {
+		plog.Warningf("request auth revision is less than current node auth revision,"+
+			"current node auth revision is %d,"+
+			"request auth revision is %d,"+
+			"request key is %s, "+
+			"err is %v", rev, revision, key, ErrAuthOldRevision)
 		return ErrAuthOldRevision
 	}
 
@@ -933,6 +959,8 @@ func NewAuthStore(be backend.Backend, tp TokenProvider) *authStore {
 		as.commitRevision(tx)
 	}
 
+	as.setupMetricsReporter()
+
 	tx.Unlock()
 	be.ForceCommit()
 
@@ -1134,3 +1162,19 @@ func (as *authStore) HasRole(user, role string) bool {
 
 	return false
 }
+
+func (as *authStore) saveConsistentIndex(tx backend.BatchTx) {
+	if as.syncConsistentIndex != nil {
+		as.syncConsistentIndex(tx)
+	} else {
+		plog.Errorf("failed to save consistentIndex,syncConsistentIndex is nil")
+	}
+}
+
+func (as *authStore) setupMetricsReporter() {
+	reportCurrentAuthRevMu.Lock()
+	reportCurrentAuthRev = func() float64 {
+		return float64(as.Revision())
+	}
+	reportCurrentAuthRevMu.Unlock()
+}

bill-of-materials.json

@@ -103,7 +103,7 @@
 		"licenses": [
 			{
 				"type": "BSD 3-clause \"New\" or \"Revised\" License",
-				"confidence": 0.9090909090909091
+				"confidence": 0.9163346613545816
 			}
 		]
 	},
@@ -121,7 +121,7 @@
 		"licenses": [
 			{
 				"type": "BSD 3-clause \"New\" or \"Revised\" License",
-				"confidence": 0.92
+				"confidence": 0.9663865546218487
 			}
 		]
 	},
@@ -134,6 +134,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/google/uuid",
+		"licenses": [
+			{
+				"type": "BSD 3-clause \"New\" or \"Revised\" License",
+				"confidence": 0.9663865546218487
+			}
+		]
+	},
 	{
 		"project": "github.com/gorilla/websocket",
 		"licenses": [
@@ -143,6 +152,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/grpc-ecosystem/go-grpc-middleware",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
 	{
 		"project": "github.com/grpc-ecosystem/go-grpc-prometheus",
 		"licenses": [
@@ -164,10 +182,6 @@
 	{
 		"project": "github.com/inconshreveable/mousetrap",
 		"licenses": [
-			{
-				"type": "MIT License and BSD 3-clause \"New\" or \"Revised\" License",
-				"confidence": 1
-			},
 			{
 				"type": "Apache License 2.0",
 				"confidence": 1
@@ -418,7 +432,7 @@
 		]
 	},
 	{
-		"project": "google.golang.org/genproto/googleapis/rpc/status",
+		"project": "google.golang.org/genproto/googleapis",
 		"licenses": [
 			{
 				"type": "Apache License 2.0",
@@ -448,8 +462,8 @@
 		"project": "gopkg.in/yaml.v2",
 		"licenses": [
 			{
-				"type": "The Unlicense",
-				"confidence": 0.35294117647058826
+				"type": "Apache License 2.0",
+				"confidence": 1
 			},
 			{
 				"type": "MIT License",
@@ -460,6 +474,10 @@
 	{
 		"project": "sigs.k8s.io/yaml",
 		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			},
 			{
 				"type": "BSD 3-clause \"New\" or \"Revised\" License",
 				"confidence": 1

clientv3/balancer/resolver/endpoint/endpoint.go

@@ -16,7 +16,9 @@
 package endpoint
 
 import (
+	"context"
 	"fmt"
+	"net"
 	"net/url"
 	"strings"
 	"sync"
@@ -228,13 +230,18 @@ func ParseTarget(target string) (string, string, error) {
 	return parts[0], parts[1], nil
 }
 
-// ParseHostPort splits a "<host>:<port>" string into the host and port parts.
-// The port part is optional.
-func ParseHostPort(hostPort string) (host string, port string) {
-	parts := strings.SplitN(hostPort, ":", 2)
-	host = parts[0]
-	if len(parts) > 1 {
-		port = parts[1]
+// Dialer dials a endpoint using net.Dialer.
+// Context cancelation and timeout are supported.
+func Dialer(ctx context.Context, dialEp string) (net.Conn, error) {
+	proto, host, _ := ParseEndpoint(dialEp)
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
 	}
-	return host, port
+	dialer := &net.Dialer{}
+	if deadline, ok := ctx.Deadline(); ok {
+		dialer.Deadline = deadline
+	}
+	return dialer.DialContext(ctx, proto, host)
 }

clientv3/client.go

@@ -25,19 +25,19 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/coreos/etcd/clientv3/balancer"
 	"github.com/coreos/etcd/clientv3/balancer/picker"
 	"github.com/coreos/etcd/clientv3/balancer/resolver/endpoint"
 	"github.com/coreos/etcd/clientv3/credentials"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
 	"github.com/coreos/etcd/pkg/logutil"
+	"github.com/coreos/pkg/capnslog"
+	"github.com/google/uuid"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	grpccredentials "google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/keepalive"
-	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 )
 
@@ -48,6 +48,10 @@ var (
 	roundRobinBalancerName = fmt.Sprintf("etcd-%s", picker.RoundrobinBalanced.String())
 )
 
+var (
+	plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "clientv3")
+)
+
 func init() {
 	lg := zap.NewNop()
 	if os.Getenv("ETCD_CLIENT_DEBUG") != "" {
@@ -226,24 +230,17 @@ func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts
 	}
 	opts = append(opts, dopts...)
 
-	// Provide a net dialer that supports cancelation and timeout.
-	f := func(dialEp string, t time.Duration) (net.Conn, error) {
-		proto, host, _ := endpoint.ParseEndpoint(dialEp)
-		select {
-		case <-c.ctx.Done():
-			return nil, c.ctx.Err()
-		default:
-		}
-		dialer := &net.Dialer{Timeout: t}
-		return dialer.DialContext(c.ctx, proto, host)
-	}
-	opts = append(opts, grpc.WithDialer(f))
-
+	dialer := endpoint.Dialer
 	if creds != nil {
 		opts = append(opts, grpc.WithTransportCredentials(creds))
+		// gRPC load balancer workaround. See credentials.transportCredential for details.
+		if credsDialer, ok := creds.(TransportCredentialsWithDialer); ok {
+			dialer = credsDialer.Dialer
+		}
 	} else {
 		opts = append(opts, grpc.WithInsecure())
 	}
+	opts = append(opts, grpc.WithContextDialer(dialer))
 
 	// Interceptor retry and backoff.
 	// TODO: Replace all of clientv3/retry.go with interceptor based retry, or with
@@ -262,7 +259,10 @@ func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts
 
 // Dial connects to a single endpoint using the client's config.
 func (c *Client) Dial(ep string) (*grpc.ClientConn, error) {
-	creds := c.directDialCreds(ep)
+	creds, err := c.directDialCreds(ep)
+	if err != nil {
+		return nil, err
+	}
 	// Use the grpc passthrough resolver to directly dial a single endpoint.
 	// This resolver passes through the 'unix' and 'unixs' endpoints schemes used
 	// by etcd without modification, allowing us to directly dial endpoints and
@@ -365,8 +365,8 @@ func (c *Client) dial(target string, creds grpccredentials.TransportCredentials,
 	return conn, nil
 }
 
-func (c *Client) directDialCreds(ep string) grpccredentials.TransportCredentials {
-	_, hostPort, scheme := endpoint.ParseEndpoint(ep)
+func (c *Client) directDialCreds(ep string) (grpccredentials.TransportCredentials, error) {
+	_, host, scheme := endpoint.ParseEndpoint(ep)
 	creds := c.creds
 	if len(scheme) != 0 {
 		creds = c.processCreds(scheme)
@@ -375,12 +375,17 @@ func (c *Client) directDialCreds(ep string) grpccredentials.TransportCredentials
 			// Set the server name must to the endpoint hostname without port since grpc
 			// otherwise attempts to check if x509 cert is valid for the full endpoint
 			// including the scheme and port, which fails.
-			host, _ := endpoint.ParseHostPort(hostPort)
-			clone.OverrideServerName(host)
+			overrideServerName, _, err := net.SplitHostPort(host)
+			if err != nil {
+				// Either the host didn't have a port or the host could not be parsed. Either way, continue with the
+				// original host string.
+				overrideServerName = host
+			}
+			clone.OverrideServerName(overrideServerName)
 			creds = clone
 		}
 	}
-	return creds
+	return creds, nil
 }
 
 func (c *Client) dialWithBalancerCreds(ep string) grpccredentials.TransportCredentials {
@@ -392,13 +397,6 @@ func (c *Client) dialWithBalancerCreds(ep string) grpccredentials.TransportCrede
 	return creds
 }
 
-// WithRequireLeader requires client requests to only succeed
-// when the cluster has a leader.
-func WithRequireLeader(ctx context.Context) context.Context {
-	md := metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader)
-	return metadata.NewOutgoingContext(ctx, md)
-}
-
 func newClient(cfg *Config) (*Client, error) {
 	if cfg == nil {
 		cfg = &Config{}
@@ -659,3 +657,9 @@ func IsConnCanceled(err error) bool {
 	// <= gRPC v1.7.x returns 'errors.New("grpc: the client connection is closing")'
 	return strings.Contains(err.Error(), "grpc: the client connection is closing")
 }
+
+// TransportCredentialsWithDialer is for a gRPC load balancer workaround. See credentials.transportCredential for details.
+type TransportCredentialsWithDialer interface {
+	grpccredentials.TransportCredentials
+	Dialer(ctx context.Context, dialEp string) (net.Conn, error)
+}

clientv3/concurrency/election_test.go

@@ -17,9 +17,9 @@ package concurrency_test
 import (
 	"context"
 	"log"
+	"strings"
 	"testing"
 	"time"
-	"strings"
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/clientv3/concurrency"

clientv3/credentials/credentials.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"sync"
 
+	"github.com/coreos/etcd/clientv3/balancer/resolver/endpoint"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
 	grpccredentials "google.golang.org/grpc/credentials"
 )
@@ -65,38 +66,37 @@ func (b *bundle) NewWithMode(mode string) (grpccredentials.Bundle, error) {
 }
 
 // transportCredential implements "grpccredentials.TransportCredentials" interface.
+// transportCredential wraps TransportCredentials to track which
+// addresses are dialed for which endpoints, and then sets the authority when checking the endpoint's cert to the
+// hostname or IP of the dialed endpoint.
+// This is a workaround of a gRPC load balancer issue. gRPC uses the dialed target's service name as the authority when
+// checking all endpoint certs, which does not work for etcd servers using their hostname or IP as the Subject Alternative Name
+// in their TLS certs.
+// To enable, include both WithTransportCredentials(creds) and WithContextDialer(creds.Dialer)
+// when dialing.
 type transportCredential struct {
 	gtc grpccredentials.TransportCredentials
+	mu  sync.Mutex
+	// addrToEndpoint maps from the connection addresses that are dialed to the hostname or IP of the
+	// endpoint provided to the dialer when dialing
+	addrToEndpoint map[string]string
 }
 
 func newTransportCredential(cfg *tls.Config) *transportCredential {
 	return &transportCredential{
-		gtc: grpccredentials.NewTLS(cfg),
+		gtc:            grpccredentials.NewTLS(cfg),
+		addrToEndpoint: map[string]string{},
 	}
 }
 
 func (tc *transportCredential) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, grpccredentials.AuthInfo, error) {
-	// Only overwrite when authority is an IP address!
-	// Let's say, a server runs SRV records on "etcd.local" that resolves
-	// to "m1.etcd.local", and its SAN field also includes "m1.etcd.local".
-	// But what if SAN does not include its resolved IP address (e.g. 127.0.0.1)?
-	// Then, the server should only authenticate using its DNS hostname "m1.etcd.local",
-	// instead of overwriting it with its IP address.
-	// And we do not overwrite "localhost" either. Only overwrite IP addresses!
-	if isIP(authority) {
-		target := rawConn.RemoteAddr().String()
-		if authority != target {
-			// When user dials with "grpc.WithDialer", "grpc.DialContext" "cc.parsedTarget"
-			// update only happens once. This is problematic, because when TLS is enabled,
-			// retries happen through "grpc.WithDialer" with static "cc.parsedTarget" from
-			// the initial dial call.
-			// If the server authenticates by IP addresses, we want to set a new endpoint as
-			// a new authority. Otherwise
-			// "transport: authentication handshake failed: x509: certificate is valid for 127.0.0.1, 192.168.121.180, not 192.168.223.156"
-			// when the new dial target is "192.168.121.180" whose certificate host name is also "192.168.121.180"
-			// but client tries to authenticate with previously set "cc.parsedTarget" field "192.168.223.156"
-			authority = target
-		}
+	// Set the authority when checking the endpoint's cert to the hostname or IP of the dialed endpoint
+	tc.mu.Lock()
+	dialEp, ok := tc.addrToEndpoint[rawConn.RemoteAddr().String()]
+	tc.mu.Unlock()
+	if ok {
+		_, host, _ := endpoint.ParseEndpoint(dialEp)
+		authority = host
 	}
 	return tc.gtc.ClientHandshake(ctx, authority, rawConn)
 }
@@ -115,8 +115,15 @@ func (tc *transportCredential) Info() grpccredentials.ProtocolInfo {
 }
 
 func (tc *transportCredential) Clone() grpccredentials.TransportCredentials {
+	copy := map[string]string{}
+	tc.mu.Lock()
+	for k, v := range tc.addrToEndpoint {
+		copy[k] = v
+	}
+	tc.mu.Unlock()
 	return &transportCredential{
-		gtc: tc.gtc.Clone(),
+		gtc:            tc.gtc.Clone(),
+		addrToEndpoint: copy,
 	}
 }
 
@@ -124,6 +131,17 @@ func (tc *transportCredential) OverrideServerName(serverNameOverride string) err
 	return tc.gtc.OverrideServerName(serverNameOverride)
 }
 
+func (tc *transportCredential) Dialer(ctx context.Context, dialEp string) (net.Conn, error) {
+	// Keep track of which addresses are dialed for which endpoints
+	conn, err := endpoint.Dialer(ctx, dialEp)
+	if conn != nil {
+		tc.mu.Lock()
+		tc.addrToEndpoint[conn.RemoteAddr().String()] = dialEp
+		tc.mu.Unlock()
+	}
+	return conn, err
+}
+
 // perRPCCredential implements "grpccredentials.PerRPCCredentials" interface.
 type perRPCCredential struct {
 	authToken   string

clientv3/ctx.go

@@ -0,0 +1,64 @@
+// Copyright 2020 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	"context"
+	"strings"
+
+	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	"github.com/coreos/etcd/version"
+	"google.golang.org/grpc/metadata"
+)
+
+// WithRequireLeader requires client requests to only succeed
+// when the cluster has a leader.
+func WithRequireLeader(ctx context.Context) context.Context {
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if !ok { // no outgoing metadata ctx key, create one
+		md = metadata.Pairs(rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader)
+		return metadata.NewOutgoingContext(ctx, md)
+	}
+	copied := md.Copy() // avoid racey updates
+	// overwrite/add 'hasleader' key/value
+	metadataSet(copied, rpctypes.MetadataRequireLeaderKey, rpctypes.MetadataHasLeader)
+	return metadata.NewOutgoingContext(ctx, copied)
+}
+
+// embeds client version
+func withVersion(ctx context.Context) context.Context {
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if !ok { // no outgoing metadata ctx key, create one
+		md = metadata.Pairs(rpctypes.MetadataClientAPIVersionKey, version.APIVersion)
+		return metadata.NewOutgoingContext(ctx, md)
+	}
+	copied := md.Copy() // avoid racey updates
+	// overwrite/add version key/value
+	metadataSet(copied, rpctypes.MetadataClientAPIVersionKey, version.APIVersion)
+	return metadata.NewOutgoingContext(ctx, copied)
+}
+
+func metadataGet(md metadata.MD, k string) []string {
+	k = strings.ToLower(k)
+	return md[k]
+}
+
+func metadataSet(md metadata.MD, k string, vals ...string) {
+	if len(vals) == 0 {
+		return
+	}
+	k = strings.ToLower(k)
+	md[k] = vals
+}

clientv3/ctx_test.go

@@ -0,0 +1,67 @@
+// Copyright 2020 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	"github.com/coreos/etcd/version"
+	"google.golang.org/grpc/metadata"
+)
+
+func TestMetadataWithRequireLeader(t *testing.T) {
+	ctx := context.TODO()
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if ok {
+		t.Fatal("expected no outgoing metadata ctx key")
+	}
+
+	// add a conflicting key with some other value
+	md = metadata.Pairs(rpctypes.MetadataRequireLeaderKey, "invalid")
+	// add a key, and expect not be overwritten
+	metadataSet(md, "hello", "1", "2")
+	ctx = metadata.NewOutgoingContext(ctx, md)
+
+	// expect overwrites but still keep other keys
+	ctx = WithRequireLeader(ctx)
+	md, ok = metadata.FromOutgoingContext(ctx)
+	if !ok {
+		t.Fatal("expected outgoing metadata ctx key")
+	}
+	if ss := metadataGet(md, rpctypes.MetadataRequireLeaderKey); !reflect.DeepEqual(ss, []string{rpctypes.MetadataHasLeader}) {
+		t.Fatalf("unexpected metadata for %q %v", rpctypes.MetadataRequireLeaderKey, ss)
+	}
+	if ss := metadataGet(md, "hello"); !reflect.DeepEqual(ss, []string{"1", "2"}) {
+		t.Fatalf("unexpected metadata for 'hello' %v", ss)
+	}
+}
+
+func TestMetadataWithClientAPIVersion(t *testing.T) {
+	ctx := withVersion(WithRequireLeader(context.TODO()))
+
+	md, ok := metadata.FromOutgoingContext(ctx)
+	if !ok {
+		t.Fatal("expected outgoing metadata ctx key")
+	}
+	if ss := metadataGet(md, rpctypes.MetadataRequireLeaderKey); !reflect.DeepEqual(ss, []string{rpctypes.MetadataHasLeader}) {
+		t.Fatalf("unexpected metadata for %q %v", rpctypes.MetadataRequireLeaderKey, ss)
+	}
+	if ss := metadataGet(md, rpctypes.MetadataClientAPIVersionKey); !reflect.DeepEqual(ss, []string{version.APIVersion}) {
+		t.Fatalf("unexpected metadata for %q %v", rpctypes.MetadataClientAPIVersionKey, ss)
+	}
+}

clientv3/maintenance.go

@@ -193,23 +193,32 @@ func (m *maintenance) Snapshot(ctx context.Context) (io.ReadCloser, error) {
 		return nil, toErr(ctx, err)
 	}
 
+	plog.Info("opened snapshot stream; downloading")
 	pr, pw := io.Pipe()
 	go func() {
 		for {
 			resp, err := ss.Recv()
 			if err != nil {
+				switch err {
+				case io.EOF:
+					plog.Info("completed snapshot read; closing")
+				default:
+					plog.Warningf("failed to receive from snapshot stream; closing (%v)", err)
+				}
 				pw.CloseWithError(err)
 				return
 			}
-			if resp == nil && err == nil {
-				break
-			}
+
+			// can "resp == nil && err == nil"
+			// before we receive snapshot SHA digest?
+			// No, server sends EOF with an empty response
+			// after it sends SHA digest at the end
+
 			if _, werr := pw.Write(resp.Blob); werr != nil {
 				pw.CloseWithError(werr)
 				return
 			}
 		}
-		pw.Close()
 	}()
 	return &snapshotReadCloser{ctx: ctx, ReadCloser: pr}, nil
 }

clientv3/retry_interceptor.go

@@ -38,6 +38,7 @@ import (
 func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.UnaryClientInterceptor {
 	intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs)
 	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		ctx = withVersion(ctx)
 		grpcOpts, retryOpts := filterCallOptions(opts)
 		callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts)
 		// short circuit for simplicity, and avoiding allocations.
@@ -103,6 +104,7 @@ func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOpt
 func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.StreamClientInterceptor {
 	intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs)
 	return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+		ctx = withVersion(ctx)
 		grpcOpts, retryOpts := filterCallOptions(opts)
 		callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts)
 		// short circuit for simplicity, and avoiding allocations.
@@ -113,10 +115,9 @@ func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOp
 			return nil, status.Errorf(codes.Unimplemented, "clientv3/retry_interceptor: cannot retry on ClientStreams, set Disable()")
 		}
 		newStreamer, err := streamer(ctx, desc, cc, method, grpcOpts...)
-		logger.Warn("retry stream intercept", zap.Error(err))
 		if err != nil {
-			// TODO(mwitkow): Maybe dial and transport errors should be retriable?
-			return nil, err
+			logger.Error("streamer failed to create ClientStream", zap.Error(err))
+			return nil, err // TODO(mwitkow): Maybe dial and transport errors should be retriable?
 		}
 		retryingStreamer := &serverStreamingRetryingStream{
 			client:       c,
@@ -185,6 +186,7 @@ func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error {
 	if !attemptRetry {
 		return lastErr // success or hard failure
 	}
+
 	// We start off from attempt 1, because zeroth was already made on normal SendMsg().
 	for attempt := uint(1); attempt < s.callOpts.max; attempt++ {
 		if err := waitRetryBackoff(s.ctx, attempt, s.callOpts); err != nil {
@@ -192,12 +194,13 @@ func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error {
 		}
 		newStream, err := s.reestablishStreamAndResendBuffer(s.ctx)
 		if err != nil {
-			// TODO(mwitkow): Maybe dial and transport errors should be retriable?
-			return err
+			s.client.lg.Error("failed reestablishStreamAndResendBuffer", zap.Error(err))
+			return err // TODO(mwitkow): Maybe dial and transport errors should be retriable?
 		}
 		s.setStream(newStream)
+
+		s.client.lg.Warn("retrying RecvMsg", zap.Error(lastErr))
 		attemptRetry, lastErr = s.receiveMsgAndIndicateRetry(m)
-		//fmt.Printf("Received message and indicate: %v  %v\n", attemptRetry, lastErr)
 		if !attemptRetry {
 			return lastErr
 		}

clientv3/snapshot/v3_snapshot.go

@@ -44,6 +44,7 @@ import (
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/etcd/wal"
 	"github.com/coreos/etcd/wal/walpb"
+	"github.com/dustin/go-humanize"
 	"go.uber.org/zap"
 )
 
@@ -87,6 +88,14 @@ type v3Manager struct {
 	skipHashCheck bool
 }
 
+// hasChecksum returns "true" if the file size "n"
+// has appended sha256 hash digest.
+func hasChecksum(n int64) bool {
+	// 512 is chosen because it's a minimum disk sector size
+	// smaller than (and multiplies to) OS page size in most systems
+	return (n % 512) == sha256.Size
+}
+
 // Save fetches snapshot from remote etcd server and saves data to target path.
 func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string) error {
 	if len(cfg.Endpoints) != 1 {
@@ -106,10 +115,7 @@ func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string
 	if err != nil {
 		return fmt.Errorf("could not open %s (%v)", partpath, err)
 	}
-	s.lg.Info(
-		"created temporary db file",
-		zap.String("path", partpath),
-	)
+	s.lg.Info("created temporary db file", zap.String("path", partpath))
 
 	now := time.Now()
 	var rd io.ReadCloser
@@ -117,13 +123,15 @@ func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string
 	if err != nil {
 		return err
 	}
-	s.lg.Info(
-		"fetching snapshot",
-		zap.String("endpoint", cfg.Endpoints[0]),
-	)
-	if _, err = io.Copy(f, rd); err != nil {
+	s.lg.Info("fetching snapshot", zap.String("endpoint", cfg.Endpoints[0]))
+	var size int64
+	size, err = io.Copy(f, rd)
+	if err != nil {
 		return err
 	}
+	if !hasChecksum(size) {
+		return fmt.Errorf("sha256 checksum not found [bytes: %d]", size)
+	}
 	if err = fileutil.Fsync(f); err != nil {
 		return err
 	}
@@ -133,6 +141,7 @@ func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string
 	s.lg.Info(
 		"fetched snapshot",
 		zap.String("endpoint", cfg.Endpoints[0]),
+		zap.String("size", humanize.Bytes(uint64(size))),
 		zap.Duration("took", time.Since(now)),
 	)
 
@@ -344,7 +353,7 @@ func (s *v3Manager) saveDB() error {
 	if serr != nil {
 		return serr
 	}
-	hasHash := (off % 512) == sha256.Size
+	hasHash := hasChecksum(off)
 	if hasHash {
 		if err := db.Truncate(off - sha256.Size); err != nil {
 			return err

embed/config.go

@@ -273,17 +273,17 @@ func NewConfig() *Config {
 		TickMs:                     100,
 		ElectionMs:                 1000,
 		InitialElectionTickAdvance: true,
-		LPUrls:              []url.URL{*lpurl},
-		LCUrls:              []url.URL{*lcurl},
-		APUrls:              []url.URL{*apurl},
-		ACUrls:              []url.URL{*acurl},
-		ClusterState:        ClusterStateFlagNew,
-		InitialClusterToken: "etcd-cluster",
-		StrictReconfigCheck: DefaultStrictReconfigCheck,
-		LogOutput:           DefaultLogOutput,
-		Metrics:             "basic",
-		EnableV2:            DefaultEnableV2,
-		AuthToken:           "simple",
+		LPUrls:                     []url.URL{*lpurl},
+		LCUrls:                     []url.URL{*lcurl},
+		APUrls:                     []url.URL{*apurl},
+		ACUrls:                     []url.URL{*acurl},
+		ClusterState:               ClusterStateFlagNew,
+		InitialClusterToken:        "etcd-cluster",
+		StrictReconfigCheck:        DefaultStrictReconfigCheck,
+		LogOutput:                  DefaultLogOutput,
+		Metrics:                    "basic",
+		EnableV2:                   DefaultEnableV2,
+		AuthToken:                  "simple",
 	}
 	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
 	return cfg

etcdctl/ctlv3/command/member_command.go

@@ -118,28 +118,8 @@ func memberAddCommandFunc(cmd *cobra.Command, args []string) {
 	display.MemberAdd(*resp)
 
 	if _, ok := (display).(*simplePrinter); ok {
-		ctx, cancel = commandCtx(cmd)
-		listResp, err := cli.MemberList(ctx)
-		// get latest member list; if there's failover new member might have outdated list
-		for {
-			if err != nil {
-				ExitWithError(ExitError, err)
-			}
-			if listResp.Header.MemberId == resp.Header.MemberId {
-				break
-			}
-			// quorum get to sync cluster list
-			gresp, gerr := cli.Get(ctx, "_")
-			if gerr != nil {
-				ExitWithError(ExitError, err)
-			}
-			resp.Header.MemberId = gresp.Header.MemberId
-			listResp, err = cli.MemberList(ctx)
-		}
-		cancel()
-
 		conf := []string{}
-		for _, memb := range listResp.Members {
+		for _, memb := range resp.Members {
 			for _, u := range memb.PeerURLs {
 				n := memb.Name
 				if memb.ID == newID {

etcdmain/config.go

@@ -189,6 +189,7 @@ func newConfig() *config {
 	fs.BoolVar(&cfg.ec.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.")
 	fs.StringVar(&cfg.ec.PeerTLSInfo.AllowedCN, "peer-cert-allowed-cn", "", "Allowed CN for inter peer authentication.")
+	fs.BoolVar(&cfg.ec.PeerTLSInfo.SkipClientSANVerify, "experimental-peer-skip-client-san-verification", false, "Skip verification of SAN field in client certificate for peer connections.")
 
 	fs.Var(flags.NewStringsValueV2(""), "cipher-suites", "Comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).")
 

etcdmain/help.go

@@ -162,6 +162,8 @@ security flags:
 		path to the peer certificate revocation list file.
 	--cipher-suites ''
 		comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).
+	--experimental-peer-skip-client-san-verification 'false'
+		Skip verification of SAN field in client certificate for peer connections.
 
 logging flags
 

etcdserver/api/etcdhttp/metrics.go

@@ -50,6 +50,7 @@ func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
 		if r.Method != http.MethodGet {
 			w.Header().Set("Allow", http.MethodGet)
 			http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+			plog.Warningf("/health error (status code %d)", http.StatusMethodNotAllowed)
 			return
 		}
 		h := hfunc()
@@ -97,11 +98,15 @@ func checkHealth(srv etcdserver.ServerV2) Health {
 	as := srv.Alarms()
 	if len(as) > 0 {
 		h.Health = "false"
+		for _, v := range as {
+			plog.Warningf("/health error due to an alarm %s", v.String())
+		}
 	}
 
 	if h.Health == "true" {
 		if uint64(srv.Leader()) == raft.None {
 			h.Health = "false"
+			plog.Warningf("/health error; no leader (status code %d)", http.StatusServiceUnavailable)
 		}
 	}
 
@@ -111,11 +116,13 @@ func checkHealth(srv etcdserver.ServerV2) Health {
 		cancel()
 		if err != nil {
 			h.Health = "false"
+			plog.Warningf("/health error; QGET failed %v (status code %d)", err, http.StatusServiceUnavailable)
 		}
 	}
 
 	if h.Health == "true" {
 		healthSuccess.Inc()
+		plog.Infof("/health OK (status code %d)", http.StatusOK)
 	} else {
 		healthFailed.Inc()
 	}

etcdserver/api/v2http/client.go

@@ -73,11 +73,11 @@ func handleV2(mux *http.ServeMux, server etcdserver.ServerV2, timeout time.Durat
 	}
 
 	mh := &membersHandler{
-		sec:     sec,
-		server:  server,
-		cluster: server.Cluster(),
-		timeout: timeout,
-		clock:   clockwork.NewRealClock(),
+		sec:                   sec,
+		server:                server,
+		cluster:               server.Cluster(),
+		timeout:               timeout,
+		clock:                 clockwork.NewRealClock(),
 		clientCertAuthEnabled: server.ClientCertAuthEnabled(),
 	}
 

etcdserver/api/v3rpc/interceptor.go

@@ -16,16 +16,16 @@ package v3rpc
 
 import (
 	"context"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/coreos/etcd/etcdserver"
 	"github.com/coreos/etcd/etcdserver/api"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft"
-
-	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
@@ -49,6 +49,12 @@ func newUnaryInterceptor(s *etcdserver.EtcdServer) grpc.UnaryServerInterceptor {
 
 		md, ok := metadata.FromIncomingContext(ctx)
 		if ok {
+			ver, vs := "unknown", metadataGet(md, rpctypes.MetadataClientAPIVersionKey)
+			if len(vs) > 0 {
+				ver = vs[0]
+			}
+			clientRequests.WithLabelValues("unary", ver).Inc()
+
 			if ks := md[rpctypes.MetadataRequireLeaderKey]; len(ks) > 0 && ks[0] == rpctypes.MetadataHasLeader {
 				if s.Leader() == types.ID(raft.None) {
 					return nil, rpctypes.ErrGRPCNoLeader
@@ -187,6 +193,12 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor
 
 		md, ok := metadata.FromIncomingContext(ss.Context())
 		if ok {
+			ver, vs := "unknown", metadataGet(md, rpctypes.MetadataClientAPIVersionKey)
+			if len(vs) > 0 {
+				ver = vs[0]
+			}
+			clientRequests.WithLabelValues("stream", ver).Inc()
+
 			if ks := md[rpctypes.MetadataRequireLeaderKey]; len(ks) > 0 && ks[0] == rpctypes.MetadataHasLeader {
 				if s.Leader() == types.ID(raft.None) {
 					return rpctypes.ErrGRPCNoLeader
@@ -205,7 +217,6 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor
 					smap.mu.Unlock()
 					cancel()
 				}()
-
 			}
 		}
 
@@ -261,3 +272,8 @@ func monitorLeader(s *etcdserver.EtcdServer) *streamsMap {
 
 	return smap
 }
+
+func metadataGet(md metadata.MD, k string) []string {
+	k = strings.ToLower(k)
+	return md[k]
+}

etcdserver/api/v3rpc/maintenance.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"io"
+	"time"
 
 	"github.com/coreos/etcd/auth"
 	"github.com/coreos/etcd/etcdserver"
@@ -27,6 +28,7 @@ import (
 	"github.com/coreos/etcd/mvcc/backend"
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/version"
+	"github.com/dustin/go-humanize"
 )
 
 type KVGetter interface {
@@ -81,6 +83,9 @@ func (ms *maintenanceServer) Defragment(ctx context.Context, sr *pb.DefragmentRe
 	return &pb.DefragmentResponse{}, nil
 }
 
+// big enough size to hold >1 OS pages in the buffer
+const snapshotSendBufferSize = 32 * 1024
+
 func (ms *maintenanceServer) Snapshot(sr *pb.SnapshotRequest, srv pb.Maintenance_SnapshotServer) error {
 	snap := ms.bg.Backend().Snapshot()
 	pr, pw := io.Pipe()
@@ -95,19 +100,39 @@ func (ms *maintenanceServer) Snapshot(sr *pb.SnapshotRequest, srv pb.Maintenance
 		pw.Close()
 	}()
 
-	// send file data
+	// record SHA digest of snapshot data
+	// used for integrity checks during snapshot restore operation
 	h := sha256.New()
-	br := int64(0)
-	buf := make([]byte, 32*1024)
-	sz := snap.Size()
-	for br < sz {
+
+	// buffer just holds read bytes from stream
+	// response size is multiple of OS page size, fetched in boltdb
+	// e.g. 4*1024
+	buf := make([]byte, snapshotSendBufferSize)
+
+	sent := int64(0)
+	total := snap.Size()
+	size := humanize.Bytes(uint64(total))
+
+	start := time.Now()
+	plog.Infof("sending database snapshot to client %s [%d bytes]", size, total)
+	for total-sent > 0 {
 		n, err := io.ReadFull(pr, buf)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			return togRPCError(err)
 		}
-		br += int64(n)
+		sent += int64(n)
+
+		// if total is x * snapshotSendBufferSize. it is possible that
+		// resp.RemainingBytes == 0
+		// resp.Blob == zero byte but not nil
+		// does this make server response sent to client nil in proto
+		// and client stops receiving from snapshot stream before
+		// server sends snapshot SHA?
+		// No, the client will still receive non-nil response
+		// until server closes the stream with EOF
+
 		resp := &pb.SnapshotResponse{
-			RemainingBytes: uint64(sz - br),
+			RemainingBytes: uint64(total - sent),
 			Blob:           buf[:n],
 		}
 		if err = srv.Send(resp); err != nil {
@@ -116,13 +141,17 @@ func (ms *maintenanceServer) Snapshot(sr *pb.SnapshotRequest, srv pb.Maintenance
 		h.Write(buf[:n])
 	}
 
-	// send sha
+	// send SHA digest for integrity checks
+	// during snapshot restore operation
 	sha := h.Sum(nil)
+
+	plog.Infof("sending database sha256 checksum to client [%d bytes]", len(sha))
 	hresp := &pb.SnapshotResponse{RemainingBytes: 0, Blob: sha}
 	if err := srv.Send(hresp); err != nil {
 		return togRPCError(err)
 	}
 
+	plog.Infof("successfully sent database snapshot to client %s [%d bytes, took %s]", size, total, humanize.Time(start))
 	return nil
 }
 

etcdserver/api/v3rpc/metrics.go

@@ -30,9 +30,19 @@ var (
 		Name:      "client_grpc_received_bytes_total",
 		Help:      "The total number of bytes received from grpc clients.",
 	})
+
+	clientRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "etcd",
+		Subsystem: "server",
+		Name:      "client_requests_total",
+		Help:      "The total number of client requests per client version.",
+	},
+		[]string{"type", "client_api_version"},
+	)
 )
 
 func init() {
 	prometheus.MustRegister(sentBytes)
 	prometheus.MustRegister(receivedBytes)
+	prometheus.MustRegister(clientRequests)
 }

etcdserver/api/v3rpc/rpctypes/md.go

@@ -17,4 +17,6 @@ package rpctypes
 var (
 	MetadataRequireLeaderKey = "hasleader"
 	MetadataHasLeader        = "true"
+
+	MetadataClientAPIVersionKey = "client-api-version"
 )

etcdserver/api/v3rpc/watch_test.go

@@ -92,4 +92,4 @@ func createResponse(dataSize, events int) (resp *pb.WatchResponse) {
 		}
 	}
 	return resp
-}
+}

etcdserver/apply.go

@@ -110,6 +110,9 @@ func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
 	ar := &applyResult{}
 	defer func(start time.Time) {
 		warnOfExpensiveRequest(start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
+		if ar.err != nil {
+			warnOfFailedRequest(start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
+		}
 	}(time.Now())
 
 	// call into a.s.applyV3.F instead of a.F so upper appliers can check individual calls

etcdserver/backend.go

@@ -71,7 +71,7 @@ func openBackend(cfg ServerConfig) backend.Backend {
 // case, replace the db with the snapshot db sent by the leader.
 func recoverSnapshotBackend(cfg ServerConfig, oldbe backend.Backend, snapshot raftpb.Snapshot) (backend.Backend, error) {
 	var cIndex consistentIndex
-	kv := mvcc.New(oldbe, &lease.FakeLessor{}, &cIndex)
+	kv := mvcc.New(oldbe, &lease.FakeLessor{}, nil, &cIndex)
 	defer kv.Close()
 	if snapshot.Metadata.Index <= kv.ConsistentIndex() {
 		return oldbe, nil

etcdserver/config_test.go

@@ -189,8 +189,8 @@ func TestWALDir(t *testing.T) {
 
 func TestShouldDiscover(t *testing.T) {
 	tests := map[string]bool{
-		"":    false,
-		"foo": true,
+		"":                              false,
+		"foo":                           true,
 		"http://discovery.etcd.io/asdf": true,
 	}
 	for durl, w := range tests {

etcdserver/membership/cluster.go

@@ -36,6 +36,7 @@ import (
 	"github.com/coreos/etcd/version"
 
 	"github.com/coreos/go-semver/semver"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // RaftCluster is a list of Members that belong to the same raft cluster
@@ -368,6 +369,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version
 	} else {
 		plog.Noticef("set the initial cluster version to %v", version.Cluster(ver.String()))
 	}
+	oldVer := c.version
 	c.version = ver
 	mustDetectDowngrade(c.version)
 	if c.store != nil {
@@ -376,6 +378,10 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version
 	if c.be != nil {
 		mustSaveClusterVersionToBackend(c.be, ver)
 	}
+	if oldVer != nil {
+		ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(oldVer.String())}).Set(0)
+	}
+	ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(ver.String())}).Set(1)
 	onSet(ver)
 }
 

etcdserver/membership/metrics.go

@@ -0,0 +1,31 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package membership
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var (
+	ClusterVersionMetrics = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "etcd",
+		Subsystem: "cluster",
+		Name:      "version",
+		Help:      "Which version is running. 1 for 'cluster_version' label with current cluster version",
+	},
+		[]string{"cluster_version"})
+)
+
+func init() {
+	prometheus.MustRegister(ClusterVersionMetrics)
+}

etcdserver/raft.go

@@ -228,9 +228,19 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					r.transport.Send(r.processMessages(rd.Messages))
 				}
 
+				// Must save the snapshot file and WAL snapshot entry before saving any other entries or hardstate to
+				// ensure that recovery after a snapshot restore is possible.
+				if !raft.IsEmptySnap(rd.Snapshot) {
+					// gofail: var raftBeforeSaveSnap struct{}
+					if err := r.storage.SaveSnap(rd.Snapshot); err != nil {
+						plog.Fatalf("failed to save Raft snapshot %v", err)
+					}
+					// gofail: var raftAfterSaveSnap struct{}
+				}
+
 				// gofail: var raftBeforeSave struct{}
 				if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
-					plog.Fatalf("raft save state and entries error: %v", err)
+					plog.Fatalf("failed to raft save state and entries %v", err)
 				}
 				if !raft.IsEmptyHardState(rd.HardState) {
 					proposalsCommitted.Set(float64(rd.HardState.Commit))
@@ -238,10 +248,14 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 				// gofail: var raftAfterSave struct{}
 
 				if !raft.IsEmptySnap(rd.Snapshot) {
-					// gofail: var raftBeforeSaveSnap struct{}
-					if err := r.storage.SaveSnap(rd.Snapshot); err != nil {
-						plog.Fatalf("raft save snapshot error: %v", err)
+					// Force WAL to fsync its hard state before Release() releases
+					// old data from the WAL. Otherwise could get an error like:
+					// panic: tocommit(107) is out of range [lastIndex(84)]. Was the raft log corrupted, truncated, or lost?
+					// See https://github.com/etcd-io/etcd/issues/10219 for more details.
+					if err := r.storage.Sync(); err != nil {
+						plog.Fatalf("failed to sync Raft snapshot %v", err)
 					}
+
 					// etcdserver now claim the snapshot has been persisted onto the disk
 					notifyc <- struct{}{}
 
@@ -249,6 +263,10 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					r.raftStorage.ApplySnapshot(rd.Snapshot)
 					plog.Infof("raft applied incoming snapshot at index %d", rd.Snapshot.Metadata.Index)
 					// gofail: var raftAfterApplySnap struct{}
+
+					if err := r.storage.Release(rd.Snapshot); err != nil {
+						plog.Fatalf("failed to release Raft wal %v", err)
+					}
 				}
 
 				r.raftStorage.Append(rd.Entries)

etcdserver/server.go

@@ -375,7 +375,15 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 		if cfg.ShouldDiscover() {
 			plog.Warningf("discovery token ignored since a cluster has already been initialized. Valid log found at %q", cfg.WALDir())
 		}
-		snapshot, err = ss.Load()
+		
+		// Find a snapshot to start/restart a raft node
+		walSnaps, serr := wal.ValidSnapshotEntries(cfg.WALDir())
+		if serr != nil {
+			return nil, serr
+		}
+		// snapshot files can be orphaned if etcd crashes after writing them but before writing the corresponding
+		// wal log entries
+		snapshot, err = ss.LoadNewestAvailable(walSnaps)
 		if err != nil && err != snap.ErrNoSnapshot {
 			return nil, err
 		}
@@ -448,7 +456,19 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 	// always recover lessor before kv. When we recover the mvcc.KV it will reattach keys to its leases.
 	// If we recover mvcc.KV first, it will attach the keys to the wrong lessor before it recovers.
 	srv.lessor = lease.NewLessor(srv.be, int64(math.Ceil(minTTL.Seconds())))
-	srv.kv = mvcc.New(srv.be, srv.lessor, &srv.consistIndex)
+
+	tp, err := auth.NewTokenProvider(cfg.AuthToken,
+		func(index uint64) <-chan struct{} {
+			return srv.applyWait.Wait(index)
+		},
+	)
+	if err != nil {
+		plog.Warningf("failed to create token provider,err is %v", err)
+		return nil, err
+	}
+	srv.authStore = auth.NewAuthStore(srv.be, tp)
+
+	srv.kv = mvcc.New(srv.be, srv.lessor, srv.authStore, &srv.consistIndex)
 	if beExist {
 		kvindex := srv.kv.ConsistentIndex()
 		// TODO: remove kvindex != 0 checking when we do not expect users to upgrade
@@ -470,16 +490,6 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 	}()
 
 	srv.consistIndex.setConsistentIndex(srv.kv.ConsistentIndex())
-	tp, err := auth.NewTokenProvider(cfg.AuthToken,
-		func(index uint64) <-chan struct{} {
-			return srv.applyWait.Wait(index)
-		},
-	)
-	if err != nil {
-		plog.Errorf("failed to create token provider: %s", err)
-		return nil, err
-	}
-	srv.authStore = auth.NewAuthStore(srv.be, tp)
 	if num := cfg.AutoCompactionRetention; num != 0 {
 		srv.compactor, err = compactor.New(cfg.AutoCompactionMode, num, srv.kv, srv)
 		if err != nil {
@@ -602,6 +612,7 @@ func (s *EtcdServer) start() {
 	s.leaderChanged = make(chan struct{})
 	if s.ClusterVersion() != nil {
 		plog.Infof("starting server... [version: %v, cluster version: %v]", version.Version, version.Cluster(s.ClusterVersion().String()))
+		membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(s.ClusterVersion().String())}).Set(1)
 	} else {
 		plog.Infof("starting server... [version: %v, cluster version: to_be_decided]", version.Version)
 	}
@@ -612,12 +623,13 @@ func (s *EtcdServer) start() {
 
 func (s *EtcdServer) purgeFile() {
 	var dberrc, serrc, werrc <-chan error
+	var dbdonec, sdonec, wdonec <-chan struct{}
 	if s.Cfg.MaxSnapFiles > 0 {
-		dberrc = fileutil.PurgeFile(s.Cfg.SnapDir(), "snap.db", s.Cfg.MaxSnapFiles, purgeFileInterval, s.done)
-		serrc = fileutil.PurgeFile(s.Cfg.SnapDir(), "snap", s.Cfg.MaxSnapFiles, purgeFileInterval, s.done)
+		dbdonec, dberrc = fileutil.PurgeFileWithDoneNotify(s.Cfg.SnapDir(), "snap.db", s.Cfg.MaxSnapFiles, purgeFileInterval, s.stopping)
+		sdonec, serrc = fileutil.PurgeFileWithDoneNotify(s.Cfg.SnapDir(), "snap", s.Cfg.MaxSnapFiles, purgeFileInterval, s.stopping)
 	}
 	if s.Cfg.MaxWALFiles > 0 {
-		werrc = fileutil.PurgeFile(s.Cfg.WALDir(), "wal", s.Cfg.MaxWALFiles, purgeFileInterval, s.done)
+		wdonec, werrc = fileutil.PurgeFileWithDoneNotify(s.Cfg.WALDir(), "wal", s.Cfg.MaxWALFiles, purgeFileInterval, s.stopping)
 	}
 	select {
 	case e := <-dberrc:
@@ -627,6 +639,15 @@ func (s *EtcdServer) purgeFile() {
 	case e := <-werrc:
 		plog.Fatalf("failed to purge wal file %v", e)
 	case <-s.stopping:
+		if dbdonec != nil {
+			<-dbdonec
+		}
+		if sdonec != nil {
+			<-sdonec
+		}
+		if wdonec != nil {
+			<-wdonec
+		}
 		return
 	}
 }
@@ -1543,6 +1564,10 @@ func (s *EtcdServer) snapshot(snapi uint64, confState raftpb.ConfState) {
 		}
 		plog.Infof("saved snapshot at index %d", snap.Metadata.Index)
 
+		if err = s.r.storage.Release(snap); err != nil {
+				plog.Panicf("failed to release wal %v", err)
+		}
+
 		// When sending a snapshot, etcd will pause compaction.
 		// After receives a snapshot, the slow follower needs to get all the entries right after
 		// the snapshot sent to catch up. If we do not pause compaction, the log entries right after

etcdserver/server_test.go

@@ -916,17 +916,17 @@ func TestSnapshot(t *testing.T) {
 		r:     *r,
 		store: st,
 	}
-	srv.kv = mvcc.New(be, &lease.FakeLessor{}, &srv.consistIndex)
+	srv.kv = mvcc.New(be, &lease.FakeLessor{}, nil, &srv.consistIndex)
 	srv.be = be
 
 	ch := make(chan struct{}, 2)
 
 	go func() {
-		gaction, _ := p.Wait(1)
+		gaction, _ := p.Wait(2)
 		defer func() { ch <- struct{}{} }()
 
-		if len(gaction) != 1 {
-			t.Fatalf("len(action) = %d, want 1", len(gaction))
+		if len(gaction) != 2 {
+			t.Fatalf("len(action) = %d, want 2", len(gaction))
 		}
 		if !reflect.DeepEqual(gaction[0], testutil.Action{Name: "SaveSnap"}) {
 			t.Errorf("action = %s, want SaveSnap", gaction[0])
@@ -994,7 +994,7 @@ func TestSnapshotOrdering(t *testing.T) {
 
 	be, tmpPath := backend.NewDefaultTmpBackend()
 	defer os.RemoveAll(tmpPath)
-	s.kv = mvcc.New(be, &lease.FakeLessor{}, &s.consistIndex)
+	s.kv = mvcc.New(be, &lease.FakeLessor{}, nil, &s.consistIndex)
 	s.be = be
 
 	s.start()
@@ -1013,6 +1013,9 @@ func TestSnapshotOrdering(t *testing.T) {
 	if ac := <-p.Chan(); ac.Name != "Save" {
 		t.Fatalf("expected Save, got %+v", ac)
 	}
+	if ac := <-p.Chan(); ac.Name != "SaveSnap" {
+		t.Fatalf("expected Save, got %+v", ac)
+	}
 	if ac := <-p.Chan(); ac.Name != "Save" {
 		t.Fatalf("expected Save, got %+v", ac)
 	}
@@ -1022,7 +1025,10 @@ func TestSnapshotOrdering(t *testing.T) {
 		t.Fatalf("expected file %q, got missing", snapPath)
 	}
 	// unblock SaveSnapshot, etcdserver now permitted to move snapshot file
-	if ac := <-p.Chan(); ac.Name != "SaveSnap" {
+	if ac := <-p.Chan(); ac.Name != "Sync" {
+		t.Fatalf("expected SaveSnap, got %+v", ac)
+	}
+	if ac := <-p.Chan(); ac.Name != "Release" {
 		t.Fatalf("expected SaveSnap, got %+v", ac)
 	}
 }
@@ -1052,23 +1058,27 @@ func TestTriggerSnap(t *testing.T) {
 	}
 	srv.applyV2 = &applierV2store{store: srv.store, cluster: srv.cluster}
 
-	srv.kv = mvcc.New(be, &lease.FakeLessor{}, &srv.consistIndex)
+	srv.kv = mvcc.New(be, &lease.FakeLessor{}, nil, &srv.consistIndex)
 	srv.be = be
 
 	srv.start()
 
 	donec := make(chan struct{})
 	go func() {
-		wcnt := 2 + snapc
+		wcnt := 3 + snapc
 		gaction, _ := p.Wait(wcnt)
 
 		// each operation is recorded as a Save
-		// (SnapCount+1) * Puts + SaveSnap = (SnapCount+1) * Save + SaveSnap
+		// (SnapCount+1) * Puts + SaveSnap = (SnapCount+1) * Save + SaveSnap + Release
 		if len(gaction) != wcnt {
+			t.Logf("gaction: %v", gaction)
 			t.Fatalf("len(action) = %d, want %d", len(gaction), wcnt)
 		}
-		if !reflect.DeepEqual(gaction[wcnt-1], testutil.Action{Name: "SaveSnap"}) {
-			t.Errorf("action = %s, want SaveSnap", gaction[wcnt-1])
+		if !reflect.DeepEqual(gaction[wcnt-2], testutil.Action{Name: "SaveSnap"}) {
+			t.Errorf("action = %s, want SaveSnap", gaction[wcnt-2])
+		}
+		if !reflect.DeepEqual(gaction[wcnt-1], testutil.Action{Name: "Release"}) {
+			t.Errorf("action = %s, want Release", gaction[wcnt-1])
 		}
 		close(donec)
 	}()
@@ -1121,7 +1131,7 @@ func TestConcurrentApplyAndSnapshotV3(t *testing.T) {
 	defer func() {
 		os.RemoveAll(tmpPath)
 	}()
-	s.kv = mvcc.New(be, &lease.FakeLessor{}, &s.consistIndex)
+	s.kv = mvcc.New(be, &lease.FakeLessor{}, nil, &s.consistIndex)
 	s.be = be
 
 	s.start()

etcdserver/storage.go

@@ -34,6 +34,10 @@ type Storage interface {
 	SaveSnap(snap raftpb.Snapshot) error
 	// Close closes the Storage and performs finalization.
 	Close() error
+	// Release releases the locked wal files older than the provided snapshot.
+	Release(snap raftpb.Snapshot) error
+	// Sync WAL
+	Sync() error
 }
 
 type storage struct {
@@ -45,22 +49,32 @@ func NewStorage(w *wal.WAL, s *snap.Snapshotter) Storage {
 	return &storage{w, s}
 }
 
-// SaveSnap saves the snapshot to disk and release the locked
-// wal files since they will not be used.
+// SaveSnap saves the snapshot file to disk and writes the WAL snapshot entry.
 func (st *storage) SaveSnap(snap raftpb.Snapshot) error {
 	walsnap := walpb.Snapshot{
 		Index: snap.Metadata.Index,
 		Term:  snap.Metadata.Term,
 	}
-	err := st.WAL.SaveSnapshot(walsnap)
+
+	// save the snapshot file before writing the snapshot to the wal.
+	// This makes it possible for the snapshot file to become orphaned, but prevents
+	// a WAL snapshot entry from having no corresponding snapshot file.
+	err := st.Snapshotter.SaveSnap(snap)
 	if err != nil {
 		return err
 	}
-	err = st.Snapshotter.SaveSnap(snap)
-	if err != nil {
+
+	return st.WAL.SaveSnapshot(walsnap)
+}
+
+// Release releases resources older than the given snap and are no longer needed:
+// - releases the locks to the wal files that are older than the provided wal for the given snap.
+// - deletes any .snap.db files that are older than the given snap.
+func (st *storage) Release(snap raftpb.Snapshot) error {
+	if err := st.WAL.ReleaseLockTo(snap.Metadata.Index); err != nil {
 		return err
 	}
-	return st.WAL.ReleaseLockTo(snap.Metadata.Index)
+	return st.Snapshotter.ReleaseSnapDBs(snap)
 }
 
 func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) {

etcdserver/util.go

@@ -109,6 +109,15 @@ func warnOfExpensiveRequest(now time.Time, reqStringer fmt.Stringer, respMsg pro
 	warnOfExpensiveGenericRequest(now, reqStringer, "", resp, err)
 }
 
+func warnOfFailedRequest(now time.Time, reqStringer fmt.Stringer, respMsg proto.Message, err error) {
+	var resp string
+	if !isNil(respMsg) {
+		resp = fmt.Sprintf("size:%d", proto.Size(respMsg))
+	}
+	d := time.Since(now)
+	plog.Warningf("failed to apply request,took %v,request %s,resp %s,err is %v", d, reqStringer.String(), resp, err)
+}
+
 func warnOfExpensiveReadOnlyTxnRequest(now time.Time, r *pb.TxnRequest, txnResponse *pb.TxnResponse, err error) {
 	reqStringer := pb.NewLoggableTxnRequest(r)
 	var resp string

etcdserver/v3_server.go

@@ -513,39 +513,30 @@ func (s *EtcdServer) raftRequestOnce(ctx context.Context, r pb.InternalRaftReque
 }
 
 func (s *EtcdServer) raftRequest(ctx context.Context, r pb.InternalRaftRequest) (proto.Message, error) {
-	for {
-		resp, err := s.raftRequestOnce(ctx, r)
-		if err != auth.ErrAuthOldRevision {
-			return resp, err
-		}
-	}
+	return s.raftRequestOnce(ctx, r)
 }
 
 // doSerialize handles the auth logic, with permissions checked by "chk", for a serialized request "get". Returns a non-nil error on authentication failure.
 func (s *EtcdServer) doSerialize(ctx context.Context, chk func(*auth.AuthInfo) error, get func()) error {
-	for {
-		ai, err := s.AuthInfoFromCtx(ctx)
-		if err != nil {
-			return err
-		}
-		if ai == nil {
-			// chk expects non-nil AuthInfo; use empty credentials
-			ai = &auth.AuthInfo{}
-		}
-		if err = chk(ai); err != nil {
-			if err == auth.ErrAuthOldRevision {
-				continue
-			}
-			return err
-		}
-		// fetch response for serialized request
-		get()
-		//  empty credentials or current auth info means no need to retry
-		if ai.Revision == 0 || ai.Revision == s.authStore.Revision() {
-			return nil
-		}
-		// avoid TOCTOU error, retry of the request is required.
+	ai, err := s.AuthInfoFromCtx(ctx)
+	if err != nil {
+		return err
 	}
+	if ai == nil {
+		// chk expects non-nil AuthInfo; use empty credentials
+		ai = &auth.AuthInfo{}
+	}
+	if err = chk(ai); err != nil {
+		return err
+	}
+	// fetch response for serialized request
+	get()
+	// check for stale token revision in case the auth store was updated while
+	// the request has been handled.
+	if ai.Revision != 0 && ai.Revision != s.authStore.Revision() {
+		return auth.ErrAuthOldRevision
+	}
+	return nil
 }
 
 func (s *EtcdServer) processInternalRaftRequestOnce(ctx context.Context, r pb.InternalRaftRequest) (*applyResult, error) {

functional/agent/server.go

@@ -61,10 +61,10 @@ func NewServer(
 	address string,
 ) *Server {
 	return &Server{
-		lg:      lg,
-		network: network,
-		address: address,
-		last:    rpcpb.Operation_NOT_STARTED,
+		lg:                         lg,
+		network:                    network,
+		address:                    address,
+		last:                       rpcpb.Operation_NOT_STARTED,
 		advertiseClientPortToProxy: make(map[int]proxy.Server),
 		advertisePeerPortToProxy:   make(map[int]proxy.Server),
 	}

glide.lock

@@ -0,0 +1,230 @@
+hash: 87224cbe021ea625798ac508c8963f49ff4ceb852d99da50db9d6b7fe95282f4
+updated: 2020-05-04T21:27:57.570766+08:00
+imports:
+- name: github.com/beorn7/perks
+  version: 37c8de3658fcb183f997c4e13e8337516ab753e6
+  subpackages:
+  - quantile
+- name: github.com/bgentry/speakeasy
+  version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd
+- name: github.com/coreos/bbolt
+  version: a0458a2b35708eef59eb5f620ceb3cd1c01a824d
+- name: github.com/coreos/go-semver
+  version: 8ab6407b697782a06568d4b7f1db25550ec2e4c6
+  subpackages:
+  - semver
+- name: github.com/coreos/go-systemd
+  version: e64a0ec8b42a61e2a9801dc1d0abe539dea79197
+  subpackages:
+  - daemon
+  - journal
+  - util
+- name: github.com/coreos/pkg
+  version: 97fdf19511ea361ae1c100dd393cc47f8dcfa1e1
+  subpackages:
+  - capnslog
+  - dlopen
+- name: github.com/cpuguy83/go-md2man
+  version: 23709d0847197db6021a51fdb193e66e9222d4e7
+  subpackages:
+  - md2man
+- name: github.com/dgrijalva/jwt-go
+  version: d2709f9f1f31ebcda9651b03077758c1f3a0018c
+- name: github.com/dustin/go-humanize
+  version: 9f541cc9db5d55bce703bd99987c9d5cb8eea45e
+- name: github.com/ghodss/yaml
+  version: 0ca9ea5df5451ffdf184b4428c902747c2c11cd7
+- name: github.com/gogo/protobuf
+  version: ba06b47c162d49f2af050fb4c75bcbc86a159d5c
+  subpackages:
+  - gogoproto
+  - proto
+  - protoc-gen-gogo/descriptor
+- name: github.com/golang/groupcache
+  version: 869f871628b6baa9cfbc11732cdf6546b17c1298
+  subpackages:
+  - lru
+- name: github.com/golang/protobuf
+  version: 6c65a5562fc06764971b7c5d05c76c75e84bdbf7
+  subpackages:
+  - jsonpb
+  - proto
+  - protoc-gen-go/descriptor
+  - ptypes
+  - ptypes/any
+  - ptypes/duration
+  - ptypes/struct
+  - ptypes/timestamp
+- name: github.com/google/btree
+  version: 4030bb1f1f0c35b30ca7009e9ebd06849dd45306
+- name: github.com/google/uuid
+  version: d460ce9f8df2e77fb1ba55ca87fafed96c607494
+- name: github.com/gorilla/websocket
+  version: 4201258b820c74ac8e6922fc9e6b52f71fe46f8d
+- name: github.com/grpc-ecosystem/go-grpc-middleware
+  version: c250d6563d4d4c20252cd865923440e829844f4e
+- name: github.com/grpc-ecosystem/go-grpc-prometheus
+  version: 0dafe0d496ea71181bf2dd039e7e3f44b6bd11a7
+- name: github.com/grpc-ecosystem/grpc-gateway
+  version: 07f5e79768022f9a3265235f0db4ac8c3f675fec
+  subpackages:
+  - runtime
+  - runtime/internal
+  - utilities
+- name: github.com/inconshreveable/mousetrap
+  version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
+- name: github.com/jonboulle/clockwork
+  version: 2eee05ed794112d45db504eb05aa693efd2b8b09
+- name: github.com/json-iterator/go
+  version: 27518f6661eba504be5a7a9a9f6d9460d892ade3
+- name: github.com/kr/pty
+  version: 2c10821df3c3cf905230d078702dfbe9404c9b23
+- name: github.com/mattn/go-runewidth
+  version: 9e777a8366cce605130a531d2cd6363d07ad7317
+  subpackages:
+  - runewidth.go
+- name: github.com/matttproud/golang_protobuf_extensions
+  version: c182affec369e30f25d3eb8cd8a478dee585ae7d
+  subpackages:
+  - pbutil
+- name: github.com/modern-go/concurrent
+  version: bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94
+- name: github.com/modern-go/reflect2
+  version: 94122c33edd36123c84d5368cfb2b69df93a0ec8
+- name: github.com/olekukonko/tablewriter
+  version: a0225b3f23b5ce0cbec6d7a66a968f8a59eca9c4
+- name: github.com/prometheus/client_golang
+  version: 5cec1d0429b02e4323e042eb04dafdb079ddf568
+  subpackages:
+  - prometheus
+  - prometheus/promhttp
+- name: github.com/prometheus/client_model
+  version: 6f3806018612930941127f2a7c6c453ba2c527d2
+  subpackages:
+  - go
+- name: github.com/prometheus/common
+  version: e3fb1a1acd7605367a2b378bc2e2f893c05174b7
+  subpackages:
+  - expfmt
+  - internal/bitbucket.org/ww/goautoneg
+  - model
+- name: github.com/prometheus/procfs
+  version: a6e9df898b1336106c743392c48ee0b71f5c4efa
+  subpackages:
+  - xfs
+- name: github.com/russross/blackfriday
+  version: 4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c
+- name: github.com/sirupsen/logrus
+  version: f006c2ac4710855cf0f916dd6b77acf6b048dc6e
+- name: github.com/soheilhy/cmux
+  version: e09e9389d85d8492d313d73d1469c029e710623f
+- name: github.com/spf13/cobra
+  version: 1c44ec8d3f1552cac48999f9306da23c4d8a288b
+- name: github.com/spf13/pflag
+  version: e57e3eeb33f795204c1ca35f56c44f83227c6e66
+- name: github.com/tmc/grpc-websocket-proxy
+  version: 89b8d40f7ca833297db804fcb3be53a76d01c238
+  subpackages:
+  - wsproxy
+- name: github.com/ugorji/go
+  version: bdcc60b419d136a85cdf2e7cbcac34b3f1cd6e57
+  subpackages:
+  - codec
+- name: github.com/urfave/cli
+  version: 1efa31f08b9333f1bd4882d61f9d668a70cd902e
+- name: github.com/xiang90/probing
+  version: 07dd2e8dfe18522e9c447ba95f2fe95262f63bb2
+- name: go.uber.org/atomic
+  version: 845920076a298bdb984fb0f1b86052e4ca0a281c
+- name: go.uber.org/multierr
+  version: b587143a48b62b01d337824eab43700af6ffe222
+- name: go.uber.org/zap
+  version: 27376062155ad36be76b0f12cf1572a221d3a48c
+  subpackages:
+  - buffer
+  - internal/bufferpool
+  - internal/color
+  - internal/exit
+  - zapcore
+- name: golang.org/x/crypto
+  version: c2843e01d9a2bc60bb26ad24e09734fdc2d9ec58
+  subpackages:
+  - bcrypt
+  - blowfish
+  - ssh/terminal
+- name: golang.org/x/net
+  version: 74dc4d7220e7acc4e100824340f3e66577424772
+  subpackages:
+  - context
+  - http/httpguts
+  - http2
+  - http2/hpack
+  - idna
+  - internal/timeseries
+  - trace
+- name: golang.org/x/sys
+  version: fde4db37ae7ad8191b03d30d27f258b5291ae4e3
+  subpackages:
+  - unix
+  - windows
+- name: golang.org/x/text
+  version: 342b2e1fbaa52c93f31447ad2c6abc048c63e475
+  subpackages:
+  - secure/bidirule
+  - transform
+  - unicode/bidi
+  - unicode/norm
+- name: golang.org/x/time
+  version: c06e80d9300e4443158a03817b8a8cb37d230320
+  subpackages:
+  - rate
+- name: google.golang.org/genproto
+  version: 09f6ed296fc66555a25fe4ce95173148778dfa85
+  subpackages:
+  - googleapis/api/annotations
+  - googleapis/rpc/status
+- name: google.golang.org/grpc
+  version: 6eaf6f47437a6b4e2153a190160ef39a92c7eceb
+  subpackages:
+  - balancer
+  - balancer/base
+  - balancer/roundrobin
+  - binarylog/grpc_binarylog_v1
+  - codes
+  - connectivity
+  - credentials
+  - credentials/internal
+  - encoding
+  - encoding/proto
+  - grpclog
+  - health
+  - health/grpc_health_v1
+  - internal
+  - internal/backoff
+  - internal/balancerload
+  - internal/binarylog
+  - internal/channelz
+  - internal/envconfig
+  - internal/grpcrand
+  - internal/grpcsync
+  - internal/syscall
+  - internal/transport
+  - keepalive
+  - metadata
+  - naming
+  - peer
+  - resolver
+  - resolver/dns
+  - resolver/passthrough
+  - serviceconfig
+  - stats
+  - status
+  - tap
+  - transport
+- name: gopkg.in/cheggaaa/pb.v1
+  version: 226d21d43a305fac52b3a104ef83e721b15275e0
+- name: gopkg.in/yaml.v2
+  version: 51d6538a90f86fe93ac480b35f37b2be17fef232
+- name: sigs.k8s.io/yaml
+  version: fd68e9863619f6ec2fdd8625fe1f02e7c877e480
+testImports: []

glide.yaml

@@ -0,0 +1,168 @@
+package: github.com/coreos/etcd
+ignore:
+- google.golang.org/appengine
+import:
+- package: github.com/bgentry/speakeasy
+  version: v0.1.0
+- package: github.com/coreos/bbolt
+  version: v1.3.3
+- package: github.com/coreos/go-semver
+  version: v0.2.0
+  subpackages:
+  - semver
+- package: github.com/coreos/go-systemd
+  version: v20
+  subpackages:
+  - daemon
+  - journal
+  - util
+- package: go.uber.org/zap
+  version: v1.10.0
+- package: github.com/coreos/pkg
+  version: v4
+  subpackages:
+  - capnslog
+- package: github.com/cpuguy83/go-md2man
+  version: 23709d0847197db6021a51fdb193e66e9222d4e7
+- package: github.com/dustin/go-humanize
+  version: v1.0.0
+- package: github.com/ghodss/yaml
+  version: v1.0.0
+- package: github.com/gogo/protobuf
+  version: v1.2.1
+  subpackages:
+  - proto
+  - gogoproto
+- package: github.com/gorilla/websocket
+  version: 4201258b820c74ac8e6922fc9e6b52f71fe46f8d
+- package: github.com/golang/groupcache
+  version: 869f871628b6baa9cfbc11732cdf6546b17c1298
+  subpackages:
+  - lru
+- package: github.com/golang/protobuf
+  version: v1.3.2
+  subpackages:
+  - jsonpb
+  - proto
+- package: github.com/google/btree
+  version: v1.0.0
+- package: github.com/google/uuid
+  version: v1.0.0
+- package: github.com/grpc-ecosystem/grpc-gateway
+  version: v1.3.1
+- package: github.com/jonboulle/clockwork
+  version: v0.1.0
+- package: github.com/kr/pty
+  version: v1.0.0
+- package: github.com/olekukonko/tablewriter
+  version: a0225b3f23b5ce0cbec6d7a66a968f8a59eca9c4
+- package: github.com/mattn/go-runewidth
+  version: v0.0.2
+  subpackages:
+  - runewidth.go
+- package: github.com/prometheus/client_golang
+  version: 5cec1d0429b02e4323e042eb04dafdb079ddf568
+  subpackages:
+  - prometheus
+  - prometheus/promhttp
+- package: github.com/prometheus/client_model
+  version: 6f3806018612930941127f2a7c6c453ba2c527d2
+  subpackages:
+  - go
+- package: github.com/prometheus/common
+  version: e3fb1a1acd7605367a2b378bc2e2f893c05174b7
+- package: github.com/prometheus/procfs
+  version: a6e9df898b1336106c743392c48ee0b71f5c4efa
+  subpackages:
+  - xfs
+- package: github.com/grpc-ecosystem/go-grpc-middleware
+  version: v1.0.0
+- package: github.com/grpc-ecosystem/go-grpc-prometheus
+  version: 0dafe0d496ea71181bf2dd039e7e3f44b6bd11a7
+- package: github.com/spf13/cobra
+  version: 1c44ec8d3f1552cac48999f9306da23c4d8a288b
+- package: github.com/spf13/pflag
+  version: v1.0.0
+- package: github.com/ugorji/go
+  version: bdcc60b419d136a85cdf2e7cbcac34b3f1cd6e57
+  subpackages:
+  - codec
+- package: github.com/json-iterator/go
+  version: v1.1.7
+- package: github.com/modern-go/reflect2
+  version: v1.0.1
+- package: github.com/urfave/cli
+  version: v1.18.0
+- package: github.com/xiang90/probing
+  version: 0.0.1
+- package: golang.org/x/crypto
+  version: c2843e01d9a2bc60bb26ad24e09734fdc2d9ec58
+  subpackages:
+  - bcrypt
+  - blowfish
+- package: golang.org/x/net
+  version: 74dc4d7220e7acc4e100824340f3e66577424772
+  subpackages:
+  - context
+  - http2
+  - http2/hpack
+  - internal/timeseries
+  - trace
+- package: golang.org/x/sys
+  version: fde4db37ae7ad8191b03d30d27f258b5291ae4e3
+  subpackages:
+  - unix
+  - windows
+- package: golang.org/x/time
+  version: c06e80d9300e4443158a03817b8a8cb37d230320
+  subpackages:
+  - rate
+- package: google.golang.org/grpc
+  version: v1.23.0
+  subpackages:
+  - balancer
+  - codes
+  - connectivity
+  - credentials
+  - grpclog
+  - internal
+  - keepalive
+  - status
+  - metadata
+  - naming
+  - peer
+  - transport
+  - health
+  - health/grpc_health_v1
+  - resolver
+  - resolver/dns
+  - resolver/passthrough
+- package: gopkg.in/cheggaaa/pb.v1
+  version: v1.0.2
+- package: gopkg.in/yaml.v2
+  version: v2.2.2
+- package: sigs.k8s.io/yaml
+  version: v1.1.0
+- package: github.com/dgrijalva/jwt-go
+  version: v3.0.0
+- package: google.golang.org/genproto
+  version: 09f6ed296fc66555a25fe4ce95173148778dfa85
+  subpackages:
+  - googleapis/rpc/status
+- package: golang.org/x/text
+  version: 342b2e1fbaa52c93f31447ad2c6abc048c63e475
+  subpackages:
+  - secure/bidirule
+  - transform
+  - unicode/bidi
+  - unicode/norm
+- package: github.com/russross/blackfriday
+  version: 4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c
+- package: github.com/sirupsen/logrus
+  version: v1.0.3
+- package: github.com/soheilhy/cmux
+  version: v0.1.4
+- package: github.com/tmc/grpc-websocket-proxy
+  version: 89b8d40f7ca833297db804fcb3be53a76d01c238
+  subpackages:
+  - wsproxy

go.mod

@@ -1,62 +0,0 @@
-module github.com/coreos/etcd
-
-go 1.12
-
-require (
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
-	github.com/bgentry/speakeasy v0.1.0
-	github.com/coreos/bbolt v1.3.1-coreos.6
-	github.com/coreos/go-semver v0.2.0
-	github.com/coreos/go-systemd v0.0.0-20170731111925-d21964639418
-	github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf
-	github.com/cpuguy83/go-md2man v0.0.0-20170603125239-23709d084719 // indirect
-	github.com/dgrijalva/jwt-go v3.0.0+incompatible
-	github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4
-	github.com/fatih/color v1.7.0 // indirect
-	github.com/gogo/protobuf v1.2.1
-	github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903
-	github.com/golang/protobuf v1.3.2
-	github.com/google/btree v1.0.0
-	github.com/google/uuid v1.0.0
-	github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c // indirect
-	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
-	github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170826090648-0dafe0d496ea
-	github.com/grpc-ecosystem/grpc-gateway v1.3.0
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jonboulle/clockwork v0.1.0
-	github.com/json-iterator/go v1.1.7
-	github.com/kr/pty v1.0.0
-	github.com/mattn/go-colorable v0.1.2 // indirect
-	github.com/mattn/go-runewidth v0.0.2 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/modern-go/reflect2 v1.0.1
-	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
-	github.com/pkg/errors v0.8.1 // indirect
-	github.com/prometheus/client_golang v0.0.0-20171005112915-5cec1d0429b0
-	github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612
-	github.com/prometheus/common v0.0.0-20171104095907-e3fb1a1acd76 // indirect
-	github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13 // indirect
-	github.com/russross/blackfriday v0.0.0-20170728175326-4048872b16cc // indirect
-	github.com/sirupsen/logrus v1.0.3 // indirect
-	github.com/soheilhy/cmux v0.1.4
-	github.com/spf13/cobra v0.0.0-20151124153217-1c44ec8d3f15
-	github.com/spf13/pflag v1.0.0
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8
-	github.com/urfave/cli v1.18.0
-	github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18
-	go.uber.org/atomic v1.3.1 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
-	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
-	golang.org/x/net v0.0.0-20190813000000-74dc4d7220e7
-	golang.org/x/time v0.0.0-20170420181420-c06e80d9300e
-	google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8
-	google.golang.org/grpc v1.23.0
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/cheggaaa/pb.v1 v1.0.2
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
-	gopkg.in/yaml.v2 v2.2.2
-	sigs.k8s.io/yaml v1.1.0
-)

go.sum

@@ -1,169 +0,0 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A=
-github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20170731111925-d21964639418 h1:0QH6fTJVDpblGjjozilaO++YRbnQNnTYh3yuFJHH0o8=
-github.com/coreos/go-systemd v0.0.0-20170731111925-d21964639418/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man v0.0.0-20170603125239-23709d084719 h1:5igivQzpoMP+VGgMjEsCmitTIislsbKlA5V1u968mX0=
-github.com/cpuguy83/go-md2man v0.0.0-20170603125239-23709d084719/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.0.0+incompatible h1:nfVqwkkhaRUethVJaQf5TUFdFr3YUF4lJBTf/F2XwVI=
-github.com/dgrijalva/jwt-go v3.0.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170826090648-0dafe0d496ea h1:Bzd/0fcg24qAEJyr7pTtDOn806SRBtzyloCuLTEvSOo=
-github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170826090648-0dafe0d496ea/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.3.0 h1:HJtP6RRwj2EpPCD/mhAWzSvLL/dFTdPm1UrWwanoFos=
-github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/pty v1.0.0 h1:jR04h3bskdxb8xt+5B6MoxPwDhMCe0oEgxug4Ca1YSA=
-github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.0.0-20171005112915-5cec1d0429b0 h1:uEiENdm9N5Nj3ezfwdvwBGc2EHLiUgD3hUTOaMfBn5E=
-github.com/prometheus/client_golang v0.0.0-20171005112915-5cec1d0429b0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612 h1:13pIdM2tpaDi4OVe24fgoIS7ZTqMt0QI+bwQsX5hq+g=
-github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20171104095907-e3fb1a1acd76 h1:g2v6dZgmqj2wYGPgHYX5WVaQ9IwV1ylsSiD+f8RvS1Y=
-github.com/prometheus/common v0.0.0-20171104095907-e3fb1a1acd76/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13 h1:leRfx9kcgnSDkqAFhaaUcRqpAZgnFdwZkZcdRcea1h0=
-github.com/prometheus/procfs v0.0.0-20171017214025-a6e9df898b13/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/russross/blackfriday v0.0.0-20170728175326-4048872b16cc h1:Ng688TEbTGosxh0B0IQ7NqUMYZiERtWbGGGEvRLKjh4=
-github.com/russross/blackfriday v0.0.0-20170728175326-4048872b16cc/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/sirupsen/logrus v1.0.3 h1:B5C/igNWoiULof20pKfY4VntcIPqKuwEmoLZrabbUrc=
-github.com/sirupsen/logrus v1.0.3/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spf13/cobra v0.0.0-20151124153217-1c44ec8d3f15 h1:x2QTuHA5vXuhN7ZzzJNnGNItiOsvFZrq23mWI5qLjAE=
-github.com/spf13/cobra v0.0.0-20151124153217-1c44ec8d3f15/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/pflag v1.0.0 h1:oaPbdDe/x0UncahuwiPxW1GYJyilRAdsPnq3e1yaPcI=
-github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/urfave/cli v1.18.0 h1:m9MfmZWX7bwr9kUcs/Asr95j0IVXzGNNc+/5ku2m26Q=
-github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs=
-github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-go.uber.org/atomic v1.3.1 h1:U8WaWEmp56LGz7PReduqHRVF6zzs9GbMC2NEZ42dxSQ=
-go.uber.org/atomic v1.3.1/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190813000000-74dc4d7220e7 h1:HOTjhHFecQCpFnEhQ4MAH6Gf7yGklZnqaHvtezTKqnQ=
-golang.org/x/net v0.0.0-20190813000000-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/time v0.0.0-20170420181420-c06e80d9300e h1:J4S0GBcCoJ2pYYVewfgT7HU8SvahFdrkNK7FRuGCZdo=
-golang.org/x/time v0.0.0-20170420181420-c06e80d9300e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
-gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/cheggaaa/pb.v1 v1.0.2 h1:TBVEWjGePKAUdeFNbT3qP6Tt0I8GkybpMWmynFpeLd4=
-gopkg.in/cheggaaa/pb.v1 v1.0.2/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

integration/fixtures/ca-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvdN4pteH5d6f+xcxBg7rGKK6fwuuAIV79Q4ejh5A/VG6ErA+
+Z5aw30nOXDSVTmScHYCc/vdwLJ/cu3tUuOgrZE2J6X8AUW2wow78Ck4icrpwsQLR
+DN4/dU0T8Ff4d0AYucMkRdmgE0tOQX/8YvBAZr42sEwAmRVxW2V19znszZNOHJQj
+CXh6As6YBGqRvQSPr/LitmLQ8cQ77FFQ/6oC3/a5Biwdr2Pd14q1vU+nZVVkCfW5
+fbNDPSYNDlG6jU8abQhM5WslIqyIcy5kM6+rCnj+Ks2CyS0LqwZONTgQLfTfQJGc
+dwrztsTTm4+Ut4dPqouM6jQru3tC1hlNX3VFGwIDAQABAoIBADpvslGyQfyqtxcm
+AYp65zazSbQ4lT1F2m4LBa78c0dIdH6yUNO02Qg0AVnzOg9i+4g9gpce9yJVqC7y
+/Zbaqhj7obwGw8NNgDYCM+a8PPCSooRreI18kY57xuqTOkMDsVjmUPFL2HJ4GUQi
+sUCH81ttrQpCq2B08GuRZWpRI6v46DR9fUt+EQNIOgCyGUtDZWuSAtq+XwviqqLm
+nkfHpC3Zm1b69qsqPxDz0rplYc+lz/V/kcIa1uw9x/4bRBdUWgiCTY4HNLuP0ePm
+pFzOLRZGAo3uOidl/mR2ArFTKlUhjSEpRmFR1Gx7fH2cc2mszmUv0yvpA9JspYGT
+e/tH+qkCgYEA2j/t4Wv+KQ5KcCHUczPSzqS/9iwmFHWQm2jGNpvbfGNYsJk7MBku
+Lfe/OTGdxNuzA7UbQXN5e35eue2vGCiL0w8UA6bftA5hFDVFUM0YKlLj1Vy5lIJV
+HeV/DI3CNca6i6FYtD0mVLRrStj7geNlmStxPgd5dQZ//s8+OUQHOw8CgYEA3qjy
+HRv09h+A4ODsQIzqiO78YaMN0+sensFsm/OwH4Jxu7nYWp8X07FRCr+KZOqbsbbu
+tF4RHJuox4ZcB8ibUQzrD3ecKzd8p/py+ecmTkxSUOpjxMneL+aXN5znqjTuI4WP
+YCeeRtedSy3jyQ2pJDFXimM6az11SKSKr9RFRTUCgYBnLBODHe8fb28HBScOcrA0
+GbPZZtN1loIOxX/2LsWaTsiNa1KMkUrAVj7iha8Ecat1lDbXQQRrubiEAnVkYT6A
+Pr+CXm+gCbAgwnILGXlUAK1NnrDoIJimMmhWAemOTGzBNzvcsI+fOU8DKgHzTBEq
+UFwPK69h//mf9k5++cbFRwKBgB2fXm6u/H0OHehNJEFGPiGvodYfikRqYG2AkEGD
+PyhCA32VMQqFZfcc/QowB2p7p/ERxFostZwXvXGmF3JVpww2asNpB1bcj/INKOTE
+ct0x3DW1qUZSEQRQakfU0SFc313MdBG305/bKasJ1Oc3sQwGoH3hy7DewU7DzUut
+MvYNAoGBAMHxoqdTi/9ewLhKxKdwvhsyRa9Ll9d9Y85+VoI/eAGAEY2iFq9aPzBA
+pdFBkPLoApLqoPPH3m0/RHO3t/yLsyjDuF1xPlBRAfnT9GF4kTvw8sMMPQoWDLoO
+qLq31y4yzdlKQTT9SauXb7KRfBj0ClG863dM6ot9YWqVBPKuV5Yk
+-----END RSA PRIVATE KEY-----

integration/fixtures/ca.crt

@@ -1,23 +1,22 @@
 -----BEGIN CERTIFICATE-----
-MIID0jCCArqgAwIBAgIUGMTka1d/PO3J5ui12qLiCKjR1rMwDQYJKoZIhvcNAQEL
+MIIDrjCCApagAwIBAgIUeR3i5w/rlikvZJQTuUqZqOnoygswDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzEwMDUyMTU5MDBaFw0yNzEwMDMyMTU5
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMjM4MDBaFw0yOTEwMDUyMjM4
 MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
-AoIBAQC/oSHQqlc05uZYMHHxkkF/eTM4uh7Z/6TSjChMfnNk+5rZJzwSkLeL47Am
-kmcGnB9xo4tKkiDjq6MbHwgK2YpgdF5rRSC89BYBWPVvSMxMWV4Pvx/q3mdUjh7N
-tb9udJxG6c3rgI+2t2zcx1+qFGeKw0JSHKpcI6UKNm7E/xY0lqJ0ptHruBgFQaqy
-VdqVa5QkyF2imnFEdgakO2EuQR3vVAr5sM552LLnngNsfsxLStxXWDJZ/+34k2ON
-jJy5KEQ1KgYmVwi8843yQRDkvAjZ4Z1LzbcQQ83/6oT3kAfsOjy8bdsxMZBWSA53
-W9LuKUOk6WLRY0S3vUjzdw8To15RAgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIBBjAS
-BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBSt4lqZZ20BpzYG8GSym10Intbr
-EzAfBgNVHSMEGDAWgBSt4lqZZ20BpzYG8GSym10IntbrEzANBgkqhkiG9w0BAQsF
-AAOCAQEAA6TqqLkPdI6zuda71LL68myXbN2qdxzs7HK8jkGPM2cU6Ii0G66TvesM
-gf1k3VddbvF8mPCyhdLYRArQeDLxKDLq/efosOj/NiLtepXea6Ib7XEo2AKCe21S
-SpBAZ2Szx8mGa7IY3ISfxkY0PpGhe2G0Rf0kOX5SYhQ/4TdAoe2pr8jFX6b7Kbdl
-yBxq9nDYjSCA7fC5Yr3Pup7Uu9fh1TJr+62DMBeeQN1XFZ0hEdu5sk4jkNq3ijC7
-/vpDyzRduUGpbp8Jy4HzoyWrCmp+KEznEWNmXb/HoPHKBlAz0ovjzU+jnFYi9tVN
-WODbdeGuXqBBmdAGaWBEowVP8ZhdQg==
+AoIBAQC903im14fl3p/7FzEGDusYorp/C64AhXv1Dh6OHkD9UboSsD5nlrDfSc5c
+NJVOZJwdgJz+93Asn9y7e1S46CtkTYnpfwBRbbCjDvwKTiJyunCxAtEM3j91TRPw
+V/h3QBi5wyRF2aATS05Bf/xi8EBmvjawTACZFXFbZXX3OezNk04clCMJeHoCzpgE
+apG9BI+v8uK2YtDxxDvsUVD/qgLf9rkGLB2vY93XirW9T6dlVWQJ9bl9s0M9Jg0O
+UbqNTxptCEzlayUirIhzLmQzr6sKeP4qzYLJLQurBk41OBAt9N9AkZx3CvO2xNOb
+j5S3h0+qi4zqNCu7e0LWGU1fdUUbAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
+BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBT+Ax2zZdq5BEBtmtBecH/QP8ruZjAN
+BgkqhkiG9w0BAQsFAAOCAQEAtiJqHvQH3iqdXwThZjzaj+4evDY5QJVcHsGmZiYl
+g8gKsyWfE4tCuBHvHvoggnYm8BdToXVJ9cvUG+u6FWxdVXDlU17kVYf3SXM39/es
+Kf2HMZ+wc95xUol77f0Nmq6XXqIMpX5TSWHVucDWFHROTYjMDW9xau1fEV810b2r
+4CjieTyNtd6FUKFgdFNPTvse5fFp0W+DZmZIDeYv1wHqQsQObLsk/MOz7ybNqRBp
+JgNJKUr3Y9YaqH3jq4eokgPpzqnZ4t+teY/gAISwtQwx4Zspa8ZJ8ybyUvAM86hN
+rYa5usa2Obd13qwgPSYYuUkAwpYy/s/cQLgAQu68ZHxsKg==
 -----END CERTIFICATE-----

integration/fixtures/ca.csr

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIC1jCCAb4CAQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlh
+MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQL
+Ew1ldGNkIFNlY3VyaXR5MQswCQYDVQQDEwJjYTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAL3TeKbXh+Xen/sXMQYO6xiiun8LrgCFe/UOHo4eQP1RuhKw
+PmeWsN9Jzlw0lU5knB2AnP73cCyf3Lt7VLjoK2RNiel/AFFtsKMO/ApOInK6cLEC
+0QzeP3VNE/BX+HdAGLnDJEXZoBNLTkF//GLwQGa+NrBMAJkVcVtldfc57M2TThyU
+Iwl4egLOmARqkb0Ej6/y4rZi0PHEO+xRUP+qAt/2uQYsHa9j3deKtb1Pp2VVZAn1
+uX2zQz0mDQ5Ruo1PGm0ITOVrJSKsiHMuZDOvqwp4/irNgsktC6sGTjU4EC3030CR
+nHcK87bE05uPlLeHT6qLjOo0K7t7QtYZTV91RRsCAwEAAaAiMCAGCSqGSIb3DQEJ
+DjETMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAmUD/50fV
+uzw/NWXztsSDULzgCTxesWz33PaW237U6WuRfNAzz1NUtHkmo0rXm1r987ajhaBQ
+jD9aEpLUvmaqohRfnZLHsGrKYj+5DZ8j5UiVF34RjSw3FyPzimSzkHjfq+zl7kh0
+FbYTfTHsxhcICm62hBIWN7bgCSTPSTINa9VCQJTOorcFAUt0xTjfIFDm6EpSKwzh
+1jO/Q1NR6CvXYwd0uxIxtVvxYtqsTdSO+pI8sSsFcxBXu+5LbgSemy+y60tfXwZR
+hF7x4a95XVn0H6UyDISc3ZmlmPNB+K3wOJ/eTAu8lMda6izzoPR2E/1kkkfF8Zol
+LUSG/f8OEQ/AHg==
+-----END CERTIFICATE REQUEST-----

integration/fixtures/gencerts.sh

@@ -23,6 +23,15 @@ cfssl gencert \
 mv server.pem server.crt
 mv server-key.pem server.key.insecure
 
+# generate IPv6: [::1], CN: example.com certificates
+cfssl gencert \
+  --ca ./ca.crt \
+  --ca-key ./ca-key.pem \
+  --config ./gencert.json \
+  ./server-ca-csr-ipv6.json | cfssljson --bare ./server-ip
+mv server-ip.pem server-ipv6.crt
+mv server-ip-key.pem server-ipv6.key.insecure
+
 # generate DNS: localhost, IP: 127.0.0.1, CN: example2.com certificates
 cfssl gencert \
     --ca ./ca.crt \

integration/fixtures/server-ca-csr-ipv6.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "example.com",
+  "hosts": [
+    "::1"
+  ]
+}

integration/fixtures/server-ecdsa.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDRzCCAi+gAwIBAgIUKgQJ/CMaFxc4JcwwGyiT/7KpedIwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjO
+PQMBBwNCAARXbc8naiFZ3Y2LujrnDCScVNRks/TR+aXPmnuPGjDxbuHxSSbC8Q2z
+iTvCkgsIcsifmUIEQcI4v3Kbkj3qMF1so4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAd
+BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNV
+HQ4EFgQU3z1DifT82BfoU5DfMe08meeYmSUwHwYDVR0jBBgwFoAUZQe+r42mchS6
+xFBpug0H8o5jej4wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3
+DQEBCwUAA4IBAQAE3bhZcJuGrnMGMgebCFMuAXvoF9twYIHXpxNOg6u0HTIWOsMB
+njEJW/rfZFE/RAJ6JdOMNE2bq2LbJ8dUA25PX3uz6V4omm9B3EvEG9Hh3J+C77XQ
+P+ofiUd+j06SdewoxrmmQmjZZdotpFUQG3EEncs+v94jsamwGNLdq4yWDjFdmyuC
+hqzSkD48aGqP2Q93wfv8uIiCEmJS1vITTm2LxssCLfiYGortpCx32/DWme8nUlni
+1U/pRTx8Brx00dMeruTGjCCpwb8k453oNV6u0D1LsQ9y5DuyEwmZtBEHBN1kVPro
+yYW3/b1jcmZk8W9GXqcXy16LbWmpvJmTHPsj
+-----END CERTIFICATE-----

integration/fixtures/server-ecdsa.key.insecure

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIK3K2gimOw2P0pZ4soFAopriuORuqpRptllFXNRhCRV0oAoGCCqGSM49
+AwEHoUQDQgAEV23PJ2ohWd2Ni7o65wwknFTUZLP00fmlz5p7jxow8W7h8UkmwvEN
+s4k7wpILCHLIn5lCBEHCOL9ym5I96jBdbA==
+-----END EC PRIVATE KEY-----

integration/fixtures/server-ip.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEBzCCAu+gAwIBAgIUSvxuG1lgImYpnaK4sPaCiMAd0lgwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQC7mJOiyqWfmNM5ptQZ22plotVfgoBf9fHTzMw/ap2Vl0/0
+4V3GEyYCdPt6V87GWzjBSO9GAmlISBQQybMieZTaTm8KKW2066iJDKseBCv9m4nS
+mHv0oDqp3SHsZQ2xHis4lbi7ws2thdqpmjw4Dv96SUiCJUjhcBX4kBMRcOGgk1RF
+ENIOInTSKlAiwNF1NSnhj8wMNw7mjw90jpAGAuPuuiQ7+AYHJBJqtT9mRikR8ppw
+isjEE6kslCCg2RC45AiF4LXNp7A7Xwm6P34XJ6T9PJUh/r3pa0xHRuI2zQLaW8Z/
+b6NYkUGMbHR7AY/+2JzOfnnnQcSB8EYC9bHadvHnAgMBAAGjgZEwgY4wDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBSPaFA2Jh7s/IJN/Yw/QFFR4pO3nDAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3
+DQEBCwUAA4IBAQAO2EnUXDlZAzOJLmkzQQF/d88PjvzspFtBfj/jCGzK6bpjeZwq
+oM1fQOkjuFeNvVLA3WHVT0XEpZEM8lwAr/YwnBWMFlNd3Vb2Cho5VaQq0nVfhYoB
+tpzoWcf0Qx4cALesQZ3y2EnXePpzky1R4MfHqulYrmZKSBQsERob/7YgSBk+ucV9
+OHLzYxm4OvYvDoR54REq+vgZ3ohoDmBrNNv9OmUHLIrUi+nBpBgnww85Dc7cKB27
+EEKxqIfCNTeHSemvzfK/1M6manQX6eyGe48nOwQMV/ocfY6SeA7RABT0l/UsbeMp
+g/b2RU+liZ3e8FziW4/1VTt1pmFAN/2hnb0v
+-----END CERTIFICATE-----

integration/fixtures/server-ip.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAu5iTosqln5jTOabUGdtqZaLVX4KAX/Xx08zMP2qdlZdP9OFd
+xhMmAnT7elfOxls4wUjvRgJpSEgUEMmzInmU2k5vCilttOuoiQyrHgQr/ZuJ0ph7
+9KA6qd0h7GUNsR4rOJW4u8LNrYXaqZo8OA7/eklIgiVI4XAV+JATEXDhoJNURRDS
+DiJ00ipQIsDRdTUp4Y/MDDcO5o8PdI6QBgLj7rokO/gGByQSarU/ZkYpEfKacIrI
+xBOpLJQgoNkQuOQIheC1zaewO18Juj9+Fyek/TyVIf696WtMR0biNs0C2lvGf2+j
+WJFBjGx0ewGP/ticzn5550HEgfBGAvWx2nbx5wIDAQABAoIBAB0jBpM7TFwsfWov
+6jOV68Gbd+6cs1m0NnpCDdsvsQgh904+jrUMFlQ9XS3UY45Vbsw+isNh7n5Gi69L
+1KHfJmp90itO4fY+v++BYzaHSVnbhZ2LB32oQVROv00bKPRAjk/8mTO4fv+bkanU
+BdRjJ/UTWsq0BczV/uObZQrJcJHi6+sAMYw4b/kxzTALd+UuvmOP7Z/NoWW6x8Mm
+ahHgqaMwA0O1f4DsdKYnSUVMF9DNGsxKCUYSYR6RH93Bq/Eo0q1U2egmLIMcTVW9
+7QSWsJoZuXlzkq7Hb7mxGdppa6kSzA/VM26qPNE9Cjg4tCMu1RJSfgkcnv27Y8vZ
+fZSq3zkCgYEA68VjIqG6sj43SZSvD+Z+Dfuzc+lO4YBSI0Yru8B4ZZq0vfTVQdM/
+uf0Bpk/nMbqec/kfcPMHP8zznLe8rcmfZXNQFIaajOb6rzWhCRSgbX98MeGnUe/y
+9sG+zFSRrAPDaVRJZwSYILs6o6Hz4o6DBCvr8iKFfm26SLB7hIjwx8UCgYEAy7EL
+dIMdsGDzfmxAYqad3oy/N1KVp96zfdnHEiIC0oiXz3YfI7YLFj54yXxx5rHR2/AK
+wOo7b90Rc8R0PgtKedKrz5p/E0Bz723ToTxHjsqgVRZqYaEKUOp8wR2t2DJOF9b9
+0C/qp6iUy0IOTBYyu3BCMV0aB5kRW62jXJIsQbsCgYB6uO7mOurUFsBug38wNpjM
+rIR3RCz0Afg/NipTe1bwBDwqWEOdFNmp9QEj0ZmU7//EfBsajtXqJsNzgswqZbWb
+eA9p77qItz4rby3YbS0oceByknOmmdCNEsI+15JPyFGyBNaEUgbhmrNmM0mgVu/p
+fvc8vS1hZro9VeelUCaMxQKBgFDgnXHH1fQAqu4ZwX7qNWj2bb5jtjSPgqmH3Tlf
+88rwnYasmjStxb0xVPh7xyYYmQFBUKPE3ZDPMGzNJnK0PQAeHEY0TByyzNXWv98X
+djpGTl86pUbakKQMVzi+thZP8x4YKXOOcxfbIimKsu6XKdGvAzlihEFcD75dNa4+
+BACdAoGBAJevnrC7M/KyDDGW3ci4sFcn7MxRGqLBulwGoCuM+zecbG7NBvDynoaH
+NRGpASiboRJyCEoIQivvkZf+K7L/oB4bL/ThF2ZpJUe471tq0444xnXdHRDLG0Dw
+OnBl27e3iAiUctqR51ufXKOUaNEf4gcsS9duELMPBxM70GE2Q/2r
+-----END RSA PRIVATE KEY-----

integration/fixtures/server-ipv6.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEEzCCAvugAwIBAgIUYTkp3oUkde9wFRkJA1LlvwFrZ3MwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDcRWZxskwCNXhprj8XCtkxj9GP4z9hVgUxgquSBync1hic
+or6qNgrUztv6nlALdQdf+TbPKyGEwCgAlKU/hnJK6lAG3+riyShnyM74/ulV1wYS
+F3Rkeh0nNCo95TPNq4GLB+sMfzwoSsT0srPX7KzCqpGy+G7sB0JBNwkTZLkCuMZf
+dkkmcZJ3zqIiOzJPlcQa4iBa0L1nV3Uuv49kLZLMCLMslg//IZxC09fnmjn+XLcV
+4+RpOKIn7AMN1kqPqmaB6gk2aCbYTZZ8aS9+cOJmTERbynyX4y4sRV18ED3dRNvs
+HCedgPOp53nqDneSOqOhhg+Mb95tnMQq1on0+TRDAgMBAAGjgZ0wgZowDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBTFoXLQVq+Yg2AlRIirXj5ho0PMrjAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAbBgNVHREEFDAShxAAAAAAAAAAAAAAAAAA
+AAABMA0GCSqGSIb3DQEBCwUAA4IBAQB4bl4f8TI7k+nlHe4MhJuHP1BKHB5O5SeG
+wrgI2+qV38UrKvTag2Z3OVKw12ANGN1vcOUrDS7cCtIZ8Aar7JpBgWrYvVlhAtc5
+3syj74Iapg1Prc0PFRmMQTZ4mahRHEqUTm3rdzkwMjNDekBs9yyBsKa08Qrm9+Cz
+Z84k/cQTBc3Bg6Xw3vUiL4EmeRQudBQAvh/vdxj6X+fwKmvLbPpgogXuQS/lHhFQ
+/rZ+s22RHLlqzAMuordjxS4Nw91dqYFwdYVvEmsK89ZnSWqwLvFCJ4uNnAe8siS7
+53YTpGbpLdNkQKAQJdMQSyvcDbQoQ7FI19a1EtSwpg5qSMOTpQ/C
+-----END CERTIFICATE-----

integration/fixtures/server-ipv6.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA3EVmcbJMAjV4aa4/FwrZMY/Rj+M/YVYFMYKrkgcp3NYYnKK+
+qjYK1M7b+p5QC3UHX/k2zyshhMAoAJSlP4ZySupQBt/q4skoZ8jO+P7pVdcGEhd0
+ZHodJzQqPeUzzauBiwfrDH88KErE9LKz1+yswqqRsvhu7AdCQTcJE2S5ArjGX3ZJ
+JnGSd86iIjsyT5XEGuIgWtC9Z1d1Lr+PZC2SzAizLJYP/yGcQtPX55o5/ly3FePk
+aTiiJ+wDDdZKj6pmgeoJNmgm2E2WfGkvfnDiZkxEW8p8l+MuLEVdfBA93UTb7Bwn
+nYDzqed56g53kjqjoYYPjG/ebZzEKtaJ9Pk0QwIDAQABAoIBABBdY5gM3BLJ8DFB
+zdQjbTF+ct5SztGnd2lPQPnvaE/M5DU27h1tOG7JE5TSEDZZsnuR412O4cWgFRi9
+8mz+yxz/vYRVPHku4r6bL61WGvXSrNPJRE92txXDjWPd1HRySoSOyQq7pTeFHo7j
+e/MN1WP9EigOxwboHycDNLxpHkmyV1DIlAgNkCZV56//liU/b+4vAVIJrgWfwfGH
+NkFd9nkm93oCFOroJ2f30E1wLPlC+ZhIn4ysau+zlWDLYeils0xHwS2GD7gjp/if
+i/ibVPgMVW/WPb67olm3nMUsan6CLmKWTiG+yklJT2djoam/iCZWE8/SAZj3qsxy
+6W9rafkCgYEA+D8tPM8h0oHlKriFDQZx37EH1dfGJRqxr+SgQiJ03d9pGYEsT+jC
+yr/l5ntzTwEEJjp/biIRwCwSWPYQtN4dNqn+11ICQzjhQbfWTfeT6vhSoBNxkeTT
+R8tUM0fmoUNrXhPbGZ9XdIxDFgD1pJL96KtyaQGjIRAhyG+khIT7oIUCgYEA4yaM
+Mw65KDonnKSVfMiOuG0QNYf70UcIiLSH8USnhbQhzT/c2LG7tNmru9UtQhZtmrpc
+vezuOYTkfcAIUjwqm12Ra8Px8WMzwHwKx3C2SrFCLFgjNFyoQ+VIGjtAL1lNKvEx
+MObSX7kVIf5+gaO9+KRBEdu55R16yQpW/UVAwCcCgYEA8XdqRkLoED2/Ln3LFW9W
+ZpJpH61BlCfR/FhzNcEUUhiUv3UxKA0tJE/ijP05nPhNE+5Es1i6UWXM9vFqMLP4
+UIqsUr73anGyUd1CvBX8sEqY/BHNn26nwKbboQHoKKZOknTX4qVmSPyB6K5IQaul
+BKN3pwIrreZmJfPKYAiGRY0CgYAYgEbtFvB321X8enA5ZnSmhfUSoRlTaIMOI9Lp
+/krHjDd9KR9MLFef2T7B4uufzkWCRAnO3qiPgbsXqUf8fsrluUD/S8JkFBw37elH
+u+udwOLvX45kjn4D3M5bLfrtYIeHUz7IFI2qj48s/INuvle2Yxk1sOqrQPPGjZv2
+c6rZTwKBgQCHSa+ToxicPJBZ5E7ezgue0LyRGWIMsr2OR16PBL2lPPiCWPH8Ez+l
+mTClHll4KVZyqc0VOZDbjMjZBnTiAq/1lb8ZvwsXLi0ue1obkkEYfXLWcxYD3Yne
+iBCGhjkqaUA4rESb22j7yqB8WGT83qV0kB9JwElzE9SxnyR9iw2FmA==
+-----END RSA PRIVATE KEY-----

integration/fixtures/server-revoked.crt

@@ -1,4 +1,5 @@
 -----BEGIN CERTIFICATE-----
+<<<<<<< HEAD
 MIIEEjCCAvqgAwIBAgIUZf8MqK2zoEIlXqd8LqfVPpuEtLwwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
@@ -21,4 +22,28 @@ zI8J0/RtYCibCAcVKmNwmfhoUqMTERhSL4dcloU9n/45anZgQXqNCHXJk8+I6nAY
 ZLEJ2aGFhvNypPTYrr4BvHx+LnrUzPWcd7JwXGLXGJtDEF45HIMLgduof+azDp/X
 HJHVra4ChMbyJHiiC9nCJruGAtF2aJuwqrGG7KnPifDLPBsplE3zvDA6dtEPvGui
 l/IE15sZ++GqTgf4fn2CNJ0PK/xYCtcBejodus88SJviaEftEB0=
+=======
+MIIEEjCCAvqgAwIBAgIUBwoN2+J27JtT6IaqV9sWhsHii2IwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDHS/zscOjq013InbTlwsBVwasv8e5+ukZNGDQx5RNaXYxI
+NVUM/5Bai4R3CS+DTbr+jBDylKi55gPQ/UIDKlU/NQH+x6UJB050G+aLDWAuRmxi
+w8dq7kRw2QJvuMxI+quiZhWk2HYjtvZRZLCUGl//QTL/VCT1smXwXRBU19S2uOfy
+g9KgZL/DCkJ9VBUh3+bFVKXBDnIphY4N/0+B/sW71cvRj8zvW3iD0R5T1J+QVEFz
+sFRT99/OhV2kUEwMaAYOFv/mMIEO6qc7vf6pB91qdUfEP8AbsOlmiSuOOLuR6X/2
+FHUjc8JrFfMuOVHnedRR5quxXbP8o83ilat0tXeVAgMBAAGjgZwwgZkwDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBS7gBJSFrjAHryiQpe38OMTzCKH1TAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAE2tsLHtgF1T3d/anKf543q9uh61tr46HHf0
+RrGZF+RJuJY5XIAiCN514Z/I7i2wG/x1TDTKwZUebajbk4GvaI4nEnCXs05jwm/n
+wpdyRE1EUy5PkVFfXKCNQd096mpZu6EYXBGnQ2fQjg5zFvZSDnYaIf0vBF1WxE4W
+0a4a9na3N77OSamPEljM1RJ1Sk+Zg5yI+nwyKcWWk3OlD0j668Vp6/m5VZKyQEkx
+crfSj7kgRJWZRhMeh6li3xa9vDmzdF6ojGkgRN3Qljrs36JnmsTono2ETF8GIc+g
+eNByAQNppLJjMn+zsaG9J5pr0gDLubFA7oa8aAJgYgJMM/GecAg=
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END CERTIFICATE-----

integration/fixtures/server-revoked.key.insecure

@@ -1,4 +1,5 @@
 -----BEGIN RSA PRIVATE KEY-----
+<<<<<<< HEAD
 MIIEpQIBAAKCAQEA5BJlqSA8qtDvbgwdNfkTxRpXspQgqHJ2PyJuol2Z5Abhcw3L
 ZSCb361IM4pdrQejrJoIcliwqUVVsHCvTCwJEZ8nIYzyA3UmCRhRKprlD97mRgXB
 UZZvCRpCSCDvbxgUKVubvdNBc2ME+C5oyt/Ffs0eqhU0JJwjcDsdD0wv/8P99LO8
@@ -24,4 +25,31 @@ rzTd0hqEvwI8XDhYlDfOte+gYXgZeL6fqJXyUzoB/LCeysk+de8fQSmBk/qJ4dtI
 se7BWZUCgYEAvRn3UqhEVq6gZgJ48LKtCPAVdDH9I2gXf0ywa8ezRcuKSsCexKOH
 gM8/MuL6KeKZMj9X01fySx9KFGIAN7GQ6bm4kZLIAQCLVhBkG8YV6t0i44oVQbSz
 qTapBzKVPyuJPVE79adX+pOgQjIfnljFlrO7JCQ+XCfKGuU9MhJfuMU=
+=======
+MIIEpAIBAAKCAQEAx0v87HDo6tNdyJ205cLAVcGrL/HufrpGTRg0MeUTWl2MSDVV
+DP+QWouEdwkvg026/owQ8pSoueYD0P1CAypVPzUB/selCQdOdBvmiw1gLkZsYsPH
+au5EcNkCb7jMSPqromYVpNh2I7b2UWSwlBpf/0Ey/1Qk9bJl8F0QVNfUtrjn8oPS
+oGS/wwpCfVQVId/mxVSlwQ5yKYWODf9Pgf7Fu9XL0Y/M71t4g9EeU9SfkFRBc7BU
+U/ffzoVdpFBMDGgGDhb/5jCBDuqnO73+qQfdanVHxD/AG7DpZokrjji7kel/9hR1
+I3PCaxXzLjlR53nUUearsV2z/KPN4pWrdLV3lQIDAQABAoIBAQC2y+TVvY51bJ81
+lilJIIMnZTauCDqXdCVtKwkcxp8koG89/+Tdwj7WPeenAv7YcWBVf4U/6siDkgzo
+EJMOsjJ0ghstZFLkYBY+eyTPX9pbN27MfAQZ+Sc/VlxcuuRs/7aTgwzRIVXi1jtB
+Vph7j2GDj3rGJJit3w6PE90Z5MkPOhXwbPD+T2OCIhO0OQCv9YNrdHmQzFZJ8vn/
+FuKUjZuoKKnwgXvBVBKsUPvvSdPTWpavNYdA7WQtjpVYVjVHgEHZWtxUwQ43JHzb
+pABWqYp/XJNiGhZ+cEXsw5dBBWp/BPxbu1P2iagZTmNr/8EfGCq04fEkKhv22x0y
+FbQa+2e1AoGBAOCztIuf1Magca9mFD+3YZHgBv2TA2XSujwYBr/664dLjL/9NQIK
+00IBykiNykiWZ0ixcaJI1j+af7fWr5OuSzBVwdXMUZraKUEwrKI3hh764yX8aUYt
+JsqpAFhyro7smp3LaUyMCW2ZFVxayp60h8fQXcNepFwmK5o5BnsTsFHHAoGBAOMO
+ZooI0Yz/fzBKOEMM1Vdf3PpUqYnjCyJSXag8OeZn/OPgiYkwXWL0idfC49B0ArVZ
+/j2zMXJduIrwa3UIfd6tjPf8O24YOiO2SenkVkcsUwJgsB1nM1QlOamGw8BB+nbT
+O5V44r7vy3HldHHQgbPvjs0z5de3b3eBBTZC/2vDAoGAVNroSnYAV0YNyIwHB4zL
+9tegLDBRbylmFP2JxwQN39ji/Tm0w+Gsp9efOUj6Y/EQbf48iGlzJy/EHXugcGe4
+kzc/bOqswoqyW6DzAItxRc++6gBpDQxOAuhRbhVY4DZvqTlAuZyEjvPpgifzLn3E
+bOu+DOJ3tSjg/Guei+oCgs8CgYAUfwxKkZk4/SdiGJETnGj1xjWQc2wKgnBS3NSP
+h0BCyEhP2ckQlUkY0bJPw8wE2TQVYtZMg4yHImayRBmvKuER5ODA0ggbXByDdMUf
+U/ll215y7H95aAN+KQ4Xe47YIByX9WF/kLYHPmZDFc95JrVOpOVjKLgqzOhHBWKP
+D2U3OQKBgQDbmwsNr0mopOYiAp60KsKJmICUQO27RyL87UfdFysDrTZ+K1Pc6X1e
+HOFtma4zNftDym9Xjzz2eOXT6flHeJNu1qZwvurNV1g0JZdXnY83q4C130bAFJLt
+I0+I3vDpJt9wznYnC3jDI24gCbEJ2D//8dpeDNUPKk94rjsEjXxDFg==
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END RSA PRIVATE KEY-----

integration/fixtures/server-wildcard.crt

@@ -1,4 +1,5 @@
 -----BEGIN CERTIFICATE-----
+<<<<<<< HEAD
 MIIELDCCAxSgAwIBAgIUGtkVdLvghfSjGwEVSothEo9W2mcwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
@@ -22,4 +23,29 @@ vADeofmoiAA6a9HUJbDfcJYz0mUoPZPcN4emCMv9PNOOybxjRqDL2HeZWwHedPth
 kfJkOHM0NXwb+XyRY3uZHdRC5VkBBmI7H/Jo0kGYB3T7YlREfGAkPd9Iop9pfitY
 FfYVu2hcxQCmGYtLNC4csP8C/nL/0o/2pIz4ldFNsYqH1swRnZQ7A+xwo7oFhvfK
 RchIgJR6qcPHkR7oloYDxA==
+=======
+MIIELDCCAxSgAwIBAgIUEQuXXKtjueOgUpZjzr1ORDG/zHwwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDPAGzCcM5JSlitl/iHLYJ6eGO8MJ3S6R1qzAvmdB9+KsGD
+F99gWVTrJRzz/hJyo9Lt8GvNj9Ll3iT2QnXyyaSOX1+uT4cxBM2MFBBfERDh0WUY
+43uLQKY45H4zrS4tJTOuSGKM/LlK2ZMj++pQBqHsONrNG+nOhqe3qLqPDV3yBfmD
+PXfjASNvHINgxb9AwQWJydgjfGDiAwWHnKbnVScYBFgWfMG0Gm1wa8EfRfWD0NPd
+L61XwQwgb5VsYAs7XH7bxVbPm6E+/oQTOJXQHMzQYve9DFPy62KFSIkfvNwVRctL
+NE+k3HnyviDzbs387ys56ZjPG1/XpbFmeQuDRndJAgMBAAGjgbYwgbMwDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBRyAdJmiDFhC3CYXPLW8kufz4zp6DAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjA0BgNVHREELTArggwqLmV0Y2QubG9jYWyC
+CmV0Y2QubG9jYWyCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA
+sc0JnS1udx4HFStcPZXY+kyVwYPuRDv3GisO+TTrxzdupQQl4gPmuGa/cik4tKxQ
+o//XFgCBQCapO2cY+JerjMSOLHtt4YmdyYDSXeMjsRG0sP36njH3nHIYsFAoEyvg
+nJQX7iDWj/9bzMT4dU2ac3t9RgCtyABRoT4G/MNhWMlJt9XkwVTN2Pqf4TMV0GlV
+54GOScsWAIwoIDPOCoO8Q40jtFSSnehrlrW7x6B37gY/EbbYpZNrIDNckNfLJMvl
+dak9P/ovtjLk8GM11gE4s2ANWA3o5bIm17b1x7Fw122sB4Rxptbc/BDpv2GIY5Zm
+tqDculro2C7Ib5GyEBEl3Q==
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END CERTIFICATE-----

integration/fixtures/server-wildcard.key.insecure

@@ -1,4 +1,5 @@
 -----BEGIN RSA PRIVATE KEY-----
+<<<<<<< HEAD
 MIIEpQIBAAKCAQEAszEM04KJY6EdfcthECFGYVQQbXo+ndP5WB3PsiNBJlu+ShjH
 Lmw3RTmP4OnHoObfjCZFqbySeYdpe0lVd1Ewx7cScskHCXCexQUPUKsK6yvuF1NZ
 tlrAeqqs3ecf8DUap3KvUJej5zILXptRWmWEeSuR4nr09/tuL21hQFrWzmdSKCJ2
@@ -24,4 +25,31 @@ mJM4d4hgFhfCqsIjVpIs+z2/e91zadnbQx0BSO3KFk3L72evUzpQjHpfhBA/p/51
 dJ3NoMUCgYEAr1menJ2CogbPAy57h+0LST/w9tBQaYeu5krf3HTWrhzUaXVobkil
 3aVpO9Ia8Oo5SkeTSODJoa4U/oeuThJzrBJgGRxo8mXeELmpFCKjHOyj4h+8dgcK
 8KAamUqmT9WVDP+8RqTKbt/jA7HulC4ew76PMPty49Ln9t/o8BXBGVY=
+=======
+MIIEpAIBAAKCAQEAzwBswnDOSUpYrZf4hy2CenhjvDCd0ukdaswL5nQffirBgxff
+YFlU6yUc8/4ScqPS7fBrzY/S5d4k9kJ18smkjl9frk+HMQTNjBQQXxEQ4dFlGON7
+i0CmOOR+M60uLSUzrkhijPy5StmTI/vqUAah7DjazRvpzoant6i6jw1d8gX5gz13
+4wEjbxyDYMW/QMEFicnYI3xg4gMFh5ym51UnGARYFnzBtBptcGvBH0X1g9DT3S+t
+V8EMIG+VbGALO1x+28VWz5uhPv6EEziV0BzM0GL3vQxT8utihUiJH7zcFUXLSzRP
+pNx58r4g827N/O8rOemYzxtf16WxZnkLg0Z3SQIDAQABAoIBAQCd5u0PxY0WSygq
+A2sJcqW9Vmh9/XfmkvxloxDQ0nPTgjnrDiLPFFW6qazUUlMwL9eOuX8CZ1uxDSuU
+zk26ziZAlHAgP3oY4lkJKaTzX8lI+Lntqllrd/1UGLhMIya+OUqa/4xtj7qoZh/f
+qyKpuOV7lEMTgt9vMzhs2MC2rrOjEZxcpuwpnZLKvpuwBMcxD1ccRdCA1zHvKdQ9
+ukPTRVjz9WUEOgANRkndHTZKWMz2p4QC9Id35HlksZi0/M6oboz2Eg1mtZEpbgUX
+loMv1CPtWP1uj9PFWiOmnBC2/v/2MVGg2fJ1Lf4c72ZVFEIU4l3YNiV4IqFb38F+
+GJVcmiGhAoGBAOvrTjYYl5vodK57gRRT0UsaU6x64/IK2i0vbBGTATywuijJ313X
+vwZBU9I2rLZqr7FZ27g5ANorw8dUKn92otr/TVS/c/VZOSw/+gTM9Rl4ZGji+qKt
+4zY/dA38jlDJNWmFwK/9KNOfXNS+WLsA2QJlONgUfkFPb3yXJUGLsU8nAoGBAOCf
+AUcyDHjGwtYsLc/4aiKtQUIdeX0v6jCWtl9EI9cZ/o414iapE05sOGb724itSFN/
+EI4biQGw8CaMcaqMaRJ8+xVQQJ7qkXItzZEFVGqz0PKwiwYAwFp6raXuio9y+cTw
+savJIM8IDijph9ezRCalef4Qj6I0zFI8H7PmiwwPAoGABWvY1kFmanzDAadw5eiv
+LIykU5hXWJ6LOPKYBydbpethu8I30c49Y4VoybHb8i0tcGPiOq+Ep37N9uymNVui
+jmnDeykTHxY3zB6EPkv/beBoXkio/cgFKp/2qMOe+ZhGE/Cw5tpob8R/u5vMKi/w
+zK9KyRxfclzC8RgAESuGnY0CgYEAlHC/+Xrbvx0rOTps9Blomo4AqF6uIMr/ayjO
+UNrJDKfDD9wQHhhyB8uA4p3ikMpjF7rLB/6uZg22RuNdYqXz8iHiFE26xsqhX+Fh
+DkuFZBZ9KUT+OvNYKvMTuqqPqwkCguHFqI78PZVHNkZOXX+8tAV7PylWoo1d0aKm
+GM9saIUCgYBXO9TtUTiaoxIVTe+r+Abt5iwasAwxai/RvymdykwYFwq2NOEl02oi
+fU3gbqDV3oirHAsAKJipnrASc70hTn4SM9hUKTQrD3fNIABch811ZDd8vaHEzLZG
+pp9yKam09sPvQo6O4E7TJPccrddV286jZq+qO9YNIsRlYJiCnjzihw==
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END RSA PRIVATE KEY-----

integration/fixtures/server.crt

@@ -1,24 +1,24 @@
 -----BEGIN CERTIFICATE-----
-MIIEEjCCAvqgAwIBAgIURbT0TUoUtitOg0ell5xYd3mAiDUwDQYJKoZIhvcNAQEL
+MIIEEjCCAvqgAwIBAgIUemioNeSN8XEGGLLqfS4uF733NiMwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xNzEwMDUyMTU5MDBaFw0yNzEwMDMyMTU5
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMjM4MDBaFw0yOTEwMDUyMjM4
 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
-A4IBDwAwggEKAoIBAQC+k2hb+zgEk2WxfXtypI03y3HUo1muX7FZdHX1yKfm/TCl
-5OCR1Id8tZv3Rn3+hZEVvjNlOi/Ct5ic0SVa4OpeQTo7u5ku5/RiXE7c55I+wpmw
-o1/IUlgeq3nyDKX4RlBJfymSUD+lWHsmhXkpKdU5wcERwL5FnrkdbTQwo+4nBMcc
-UhLgWC9awpT7sAXW7OmTlg/szTIOzyJp2YRBoXp6mGsHF0rujVElQyCOBnAi/+zP
-hGGycCOoa1eVjZebpFgcisyIwZRO6KugfYIZrQ47Swcqsy4lfkbzXxZ6UMh+FtUz
-iXYSG2c8rzdpnHlFQcpfkNrbAu5Q2ObmQh21E18BAgMBAAGjgZwwgZkwDgYDVR0P
+A4IBDwAwggEKAoIBAQC58XVTmNuwAt6LstiaSQQro3IEjVaG3fiqxh0rjcWnWKEc
+6kZFxTZyRepfQtLSb52SDXTGf7kPJQX4kst/NR9qndrdT6gwwNcAXdEqVJIx9i92
+eMaYuAS0nIoslq/6ULPviK6TocTAIKJp2mrDU2ylU7PqvmRUuNxxieVCSIMAevNa
+ZFhucW/uGtbNd4sSzo2Xwav391DDWDIZYYvucGFitjY1mOHDyQb54ECVM4G57ITn
+t3fvPPjGYTSVDktgKIdcAZxd4pBDHJB8pB5IuvQPqvjtbbB4aubKEJVXaCFK22zJ
+wBTseo1S5hctdtswcqENKzgbqrtQTpHfB0XZXYKvAgMBAAGjgZwwgZkwDgYDVR0P
 AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
-Af8EAjAAMB0GA1UdDgQWBBTATssOHl8+T6AfYWGXqKHyCtUqMTAfBgNVHSMEGDAW
-gBSt4lqZZ20BpzYG8GSym10IntbrEzAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
-AAEwDQYJKoZIhvcNAQELBQADggEBAAzU+ZRqgGaVTGCl21QnfUY4Hj9aqt6uMhVv
-b+BG24gNUJgwkCIZODOsig7RWXjPAMkgeGmgyw1QbvV2AZo1NFJT111YhxjdqWLg
-ganDb7K2Jm5vm8mVvBVTe2y7ZBdmxJwfo9UYZkEXNQtlbvYcvYnzr0nr5QEc9v8X
-bkrbxG1DVB9wU+7hy6s4v9946xQavGUqOOC70wHUMj8gKGnGkd1mOTYaabx3VzPU
-uD82AsCQnxOIzk0qze3jCVVoTdzKt9iWpgLdFHY2pa0fdirTN1s80sLpXhUOihSP
-+gu2NGP3+C0I6SW6Wu35vpYI+uMWrggu6OCxomAC872d6CVdtcs=
+Af8EAjAAMB0GA1UdDgQWBBSj5OsDxzEYEqO3SBCEkegbA1MJ3TAfBgNVHSMEGDAW
+gBT+Ax2zZdq5BEBtmtBecH/QP8ruZjAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAKIhy98tSD8SgW2y6DA/CyennGzSYs5aj6r5
+BMCDAuaa7OH9MM6zk4EQPH6TU06/wrQ0Jhwk7BKTslEH+OFA5X0WYNfMQ+462v2Y
+QYGO68128domd2XSMNOsk2we1eN4J0RkAqua9/k1R5NwFpUki5f2G1zNOKFXklLg
+HleRsqGk5vTauICa79XPz3940K4f2oobSXHZzBOVyIFRTaUD7t8pF2wLF1tgwCDZ
+87UXkSpsR7nLTkEXEm/pk5gNPbyHMmPsXvxZ0lheAiZh2RgRMRSdOcvJcE4Vcbfw
+Vw+aQbo96GHHMU323DgSwbVLs4umBucvW1Ny2nZkrc7V56Qqdc8=
 -----END CERTIFICATE-----

integration/fixtures/server.csr

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIC6jCCAdICAQAweDEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlh
+MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQL
+Ew1ldGNkIFNlY3VyaXR5MRQwEgYDVQQDEwtleGFtcGxlLmNvbTCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBALnxdVOY27AC3ouy2JpJBCujcgSNVobd+KrG
+HSuNxadYoRzqRkXFNnJF6l9C0tJvnZINdMZ/uQ8lBfiSy381H2qd2t1PqDDA1wBd
+0SpUkjH2L3Z4xpi4BLSciiyWr/pQs++IrpOhxMAgomnaasNTbKVTs+q+ZFS43HGJ
+5UJIgwB681pkWG5xb+4a1s13ixLOjZfBq/f3UMNYMhlhi+5wYWK2NjWY4cPJBvng
+QJUzgbnshOe3d+88+MZhNJUOS2Aoh1wBnF3ikEMckHykHki69A+q+O1tsHhq5soQ
+lVdoIUrbbMnAFOx6jVLmFy122zByoQ0rOBuqu1BOkd8HRdldgq8CAwEAAaAtMCsG
+CSqGSIb3DQEJDjEeMBwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqG
+SIb3DQEBCwUAA4IBAQCwf7Q0Mkc6uMPFCa5W3cvMmRHSvfGGi7IcXDC2sdrEK+K3
+cOm4h7jCsWdKvE6+uxoPnDro0CgUeieHFDBpgZ613l9MCiYqBJwFHZeMoNqhItge
+GIx5TMF57Qt/TQ+5Mm/Oh8hbXEOwZUwSOW0sl1rPi+UWv8PXtmPb/di2dpbmRChq
+s/7yZQCNvQwwcdg21aWDB/rsYHlsD9InHy9NHdCzcA/QAiTTaTh15ej2s9/yo/8o
+lHOmytUAIhw44qXCg/ZLYX1Lb2LLKMSMlwFAnB9SV+EMvvrsAo07p6prjJk4pYp8
+PYDXZpnmR9zmtnKFN+2F2AWhknsdOmqaguLW/Z71
+-----END CERTIFICATE REQUEST-----

integration/fixtures/server.key.insecure

@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpQIBAAKCAQEAvpNoW/s4BJNlsX17cqSNN8tx1KNZrl+xWXR19cin5v0wpeTg
-kdSHfLWb90Z9/oWRFb4zZTovwreYnNElWuDqXkE6O7uZLuf0YlxO3OeSPsKZsKNf
-yFJYHqt58gyl+EZQSX8pklA/pVh7JoV5KSnVOcHBEcC+RZ65HW00MKPuJwTHHFIS
-4FgvWsKU+7AF1uzpk5YP7M0yDs8iadmEQaF6ephrBxdK7o1RJUMgjgZwIv/sz4Rh
-snAjqGtXlY2Xm6RYHIrMiMGUTuiroH2CGa0OO0sHKrMuJX5G818WelDIfhbVM4l2
-EhtnPK83aZx5RUHKX5Da2wLuUNjm5kIdtRNfAQIDAQABAoIBAQCDTcjfZw1Hic7N
-JWnCqUFrKc759Lo7fE8TFTyY5XFZoyS7iCB6GXZoJDCbhIQWsywtUOjUW+zAOgL6
-ONeF7+VKn6JhuXVnbgVhJ7xmU17dwvJlU4sQ2DtCll7kuHY5wyhaGzUnTAcuAvKG
-rfu2ss3oh2hgtO3jxeJBNhZ5VNknI+EycvW1JsMXG7qwySqLwtuUhptHmxyMTcXt
-LZi2zwmBKX7s44fCQrLq2CAo+GMJ7OFoUtZezu57ySki32VnL6uwM8ErKnvxUbHu
-H64erRRJ7Pw/qDQyYyo9pX/pKmcjGYIpt/ywYqbvszBY6ad3cdjtBHcS+9CeLeaN
-LrxAqNcpAoGBANr3dX2TMiCRiT6sTR+6ainzj/FHFA6xSU4ipe6uc/65zxh2xGaN
-+FrkDMmlvEEQLboEFdkiQikZK3xNHB/GEqAEmMsDgzYsg36JjsxLYLR9XQR3TDnu
-leNtjk/P05jiJchrL9m5xW1WM4QxwD7rf7TSRia/BrVyvQvhI3p8BKGbAoGBAN7O
-t/UajS8xjJmx1+u92FPXHnFi+tLuEfdd6ooKdw9rXUARVgeBsGgrLLrMKoCzdfWr
-txw7j5DjOlcx8ZXpyrcVyGJWuMboV3uf1IEiYZsMd7Le6yfnz4qgUPFmFy3JUQb1
-cbzc9dBuhCLQ2H7JU4EhlhtxyyY585kZtDaThmmTAoGBAIw9bVxuB+7gB1zCkeq+
-Q/x2aDyJ34jBd0e53TiPNu9wJflvJ77fMq9T2/TSV038hKzcrPmSfXlBC57i7B5V
-h9xA1XNA3qq1u8oxY+noZRl0KT0RAxsfeZRduIXZf5YtUTGZpN33o0CxsvD5xD0I
-K5SuEAwE0NEpmXagTU7HW1f9AoGAb+z0aEJQTjbb5JF8YEZcF7Hm7xrD2ZYSnGsn
-WPTs3mgWzgpnZxn1Hj8iFyxc5Y5BYYpDUAFzm1sqgYbrT13Eobhlk1DxPaqV19pw
-i/ZThen7b3WgN8mxbngecUXRuwR4mcBOxItTSMNbyYmUWAyW0DWpDFxbqvZNsslA
-yHHPgdUCgYEAuIRHMNanm5eZz7iXUMShAIUgaqcEIFOQ4W43zPQ1/F2beJBx+VoN
-u1Bvs7K9GBRDvJHcsjRxhYnwBwGu06M1NRG3QBW5VNuezpKzvchCgWR96ulzNOIe
-5C+j3zQmut4sOx2IY0zsJCfXnLJSoYwwtM1eVzY06uHwx+F4SMv1z8w=
+MIIEogIBAAKCAQEAufF1U5jbsALei7LYmkkEK6NyBI1Wht34qsYdK43Fp1ihHOpG
+RcU2ckXqX0LS0m+dkg10xn+5DyUF+JLLfzUfap3a3U+oMMDXAF3RKlSSMfYvdnjG
+mLgEtJyKLJav+lCz74iuk6HEwCCiadpqw1NspVOz6r5kVLjccYnlQkiDAHrzWmRY
+bnFv7hrWzXeLEs6Nl8Gr9/dQw1gyGWGL7nBhYrY2NZjhw8kG+eBAlTOBueyE57d3
+7zz4xmE0lQ5LYCiHXAGcXeKQQxyQfKQeSLr0D6r47W2weGrmyhCVV2ghSttsycAU
+7HqNUuYXLXbbMHKhDSs4G6q7UE6R3wdF2V2CrwIDAQABAoIBABdY9zdw+RzHr3Px
+F/t/cZOdKULjEFtM4d1nlOAUJB5YJZrA2+QE2EpO9xfRBzG/LBTbta76+JyrNG26
+2Ox3GiPEAGhLiT3d6OE15n6fMeAuHlWM4rroWEbpZPwmS6Bto7pO+kS1SkQplNsf
+vsoZ3ol/0haGo4hz0dI701qYSkKz9kldTYUp6iP1Lt9icSbIom95DP6wtKEOty9i
+j7WwZRRWWjmVQZEcL6CkB9B/XlWlOP6STqLwnN2pGmqHxxUfpWJYLZkuf9fYwNaf
+5xcUvztlMYHXHaCAD3MEZ0gh/YxlzgqdnCBQSO5w4niWIjCwi6Q/UANm3qzLk/7n
+HiUdwkkCgYEAwRjlrot5BCVwyF5f6qMz5SFKhE8xYUDfy2MkPb0tb7UoCj9/d9Ee
+Vu4ysPsyqALwG8gvDUhCBtEgqrs2QmlyTVlvZDOnN8JDU8DALeTitn0U+wUqHG8C
+fd9FqicsnUug9tKz92+npNvI6S/RmiEVid6OEUUxLxaIJVDjdL3xajUCgYEA9oP1
+Gd3C9yiInnQHZuhQzBxjar1IndrHiUV5t5VmUp+a2M7DDJupQ8o7903fabheWsHK
+ZLPiCFE2dOUBlAa15lNoQST4pW0jQaspWmljInjIbGBD1/3oNY2P5QsMVRpYR/jc
+zkEMu6UhccODobkDDKR1QuBXE3gUXB1NYKuONdMCgYA5che6gqHA+wZ/hZwRaPYi
+X3IUxJ6TKUKq9lasy1/+EK3Vxqg8VAkroXeRMVoTo7Qc/8QHtox2DLckM8fjoA0y
+N74s2DUSIIf5HanOvX44/iNEvneVt+zM16SZB0h9jydW5r5FiYEdSYmO3fwEV31Z
+48zuFGCaeqCMQXE5pNxyVQKBgHXQIN5ozpnCpHBd8X+r/Zle4+CKb98JRR2Et6QW
+YZ2RK7b8MdmftyhvyXLqo6Bp/aYQQcMY+SiSb500KSnufxaBvCyOF8svuo7S12Of
+REKgSOEKrMHYVhbp/eAwBIfTnGdhN13XVteDFtXA4/LxcdSj8GqyvsrOM6TGWEmg
+E8oDAoGAQqDV89Ed6r+IdyA2QY3IYDN+x7PJEt3xySVcfPGGFE2QNLEY5aNtwa5r
+y7uj33hg6Bkzacrw/9mzcm71wBv0RNX5C51hIKsdruzftOVDJFnK6qtoS5lQXtxd
+nngvlDI4CJRSj3O6d6uLT1cXjCQnT0hLjrpVIr72448w7M870mE=
 -----END RSA PRIVATE KEY-----

integration/fixtures/server2.crt

@@ -1,4 +1,5 @@
 -----BEGIN CERTIFICATE-----
+<<<<<<< HEAD
 MIIEEzCCAvugAwIBAgIUev7+NZl9RzdnsOGshKbMEHIxtD8wDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
@@ -21,4 +22,28 @@ R12Tnph2+RVsUaCcSmKNNcJxKjYXlU1bH3vyZ8/EmtGzsYnUlGtHfsHSHBdL/eHN
 g958qOHHYLtUWQsCf0az51mXO0yeaD+9pzTcnUX6tk2Er3OVKF51AdrdQVjQ9uub
 ST8onCbuICF6nRzXiF+sxv8h78ilIkdr3iCJw3TnIOLgXs8uh9PK3Du7Qh/2UD/5
 EucAVCeNgJQQ7Bvtcw+VIPLWwXnq71qu8p9Datir5dghK9FGmYeu
+=======
+MIIEEzCCAvugAwIBAgIUCHfFkPZDhLT2oK0fhu2TP4xZtcwwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHkxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEVMBMGA1UEAxMMZXhhbXBsZTIuY29tMIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAv6FwE/OeAwAmZutb+DTP+y7FKpONm0+aHtESlrmTRmDo
+iLGuoEXjYcFOCHr6gQJTYKEAivBK3TIN0dRirNBqLQxssDlsbU4ZXG++OciH/OoR
+8E+VmMsbqof/E0nhVFYDumnuoS+waX8elzuDjDX4u7F+d1/gIb8aYU1VDjtZxF2b
+vqaPyroOn5HvBs4MW+BpAB4guHfDXpK/oAnJDsq9JTUZqoG1xOZfHNhA/iVBSAJv
+hO6aBxDCjzwO1gT5kTbNELrBCJ0V2NXlHFcBOPhxyl4+DIPwa0Q8oExUEOzxnHFu
+U3GPoklbLp0RNNiqHHKfD4yaLI1rf8dH5AZG/QjK4wIDAQABo4GcMIGZMA4GA1Ud
+DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0T
+AQH/BAIwADAdBgNVHQ4EFgQUS6HUgF/GO55DtrxLN2dcyGACgnQwHwYDVR0jBBgw
+FoAUZQe+r42mchS6xFBpug0H8o5jej4wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/
+AAABMA0GCSqGSIb3DQEBCwUAA4IBAQC2pcpQZVkvZO4TC1zluGhG6Yhz+fzZuNlp
+jHa0U+WIdTPerqZZ98iU1hCm3F6voNjAQ0eHRV3z5q/rDph8AR7cVl9cT2rLO/zA
+F6M4QDMetddj7EIUq2/B1CWzHkOvwcJFgc0OfBWzJYAShFv/B7Ir1WpdtixOvyOH
+kWsWoy1WmatQvBQ2jDrvdGRWhqsPmg2uGbJrUjABeYtc5whQr0zscy+jEIrDpqPT
+VVuUu19/ALvdv2kOC+ayhH+vTAvEA38P6wlavDlgsv/M902ORWahdLQ/H0XX+gVP
+QDe3MyrBR6QkjAfKJvnMv/5x8mj+AFWfnALnRb9j3/q1UOwITWN2
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END CERTIFICATE-----

integration/fixtures/server2.key.insecure

@@ -1,4 +1,5 @@
 -----BEGIN RSA PRIVATE KEY-----
+<<<<<<< HEAD
 MIIEpAIBAAKCAQEA3VZyO4MoZjGpPU9hbb6SDV3ESNjUiS+pojigIVJi7C4bW1eJ
 vPDpodetKQg22Htq+Vj8mpCmnqn7/8pegbxV9Fc3Fyy0kxiEfHnaOpzo8GWhpReu
 bHJ2+kj2I6gY7HnKPJ3KP3Z9txKECNm9CtHuoOxjRvHR8SThcVYZqxz6cjkgZB/A
@@ -24,4 +25,31 @@ B3axr3feZ12lpqFGSEJAXqtN+UKgrx7oE4jibIDdPE78jW3YgIhagc0d4MhE6FPW
 Y+dqLQKBgQCUQEMiTwGadmDy0VmzAYVmKQV0qqb+hHswPt2Nyr5tNs7ZGJXaiW1+
 t2AtWeFFvpvdCaN9q7G5sZedIeB/zAByNTGF8ztBB3dSyv7IiMhthAWaC1UZefD6
 zw7N2UCuvjwvh+T1fi8BpMvK4YizmSyw04tEvoZMnhK1fQ2r/PWsjw==
+=======
+MIIEpAIBAAKCAQEAv6FwE/OeAwAmZutb+DTP+y7FKpONm0+aHtESlrmTRmDoiLGu
+oEXjYcFOCHr6gQJTYKEAivBK3TIN0dRirNBqLQxssDlsbU4ZXG++OciH/OoR8E+V
+mMsbqof/E0nhVFYDumnuoS+waX8elzuDjDX4u7F+d1/gIb8aYU1VDjtZxF2bvqaP
+yroOn5HvBs4MW+BpAB4guHfDXpK/oAnJDsq9JTUZqoG1xOZfHNhA/iVBSAJvhO6a
+BxDCjzwO1gT5kTbNELrBCJ0V2NXlHFcBOPhxyl4+DIPwa0Q8oExUEOzxnHFuU3GP
+oklbLp0RNNiqHHKfD4yaLI1rf8dH5AZG/QjK4wIDAQABAoIBAE5QTHxq4BV71zXS
+U7ig5KpTV9JpkMJ7CpIzgTRFzNFDQ2SxsJrhVOabWCeREpTsfWSNB6rAPugcz5cE
+A/t6BRo57KUsIoqdEzI6nHQC5shOZFxgOdPClaDgiTa5x7Nun4FsT1BiK+dBQyAs
++zqux+L0y6k/blp8Peyr7OmvCaV8osB4/JLLH/WHt2wWgqFWisyIT7/D/gQlQn81
+Hvv84BAL9y8iyCmCzWhQL0YisLPyaFkGkb7DK4wznWxfQn3jRAkZDQMH675o/OHj
+8nL0NSdCA/MGLEtPAXegM7kMPCf68JwZV3gPZDyEK0JES1oT1z+op7JHuatlhgdL
+WTA10fkCgYEA1QXRRpOeZvHzGVMzrXrrgS5GeaR+XgjUalBGgu0w8nSET195oXu6
+Y8dVco4FlEZ0Wq7evA4M9XVJyKQkkEGR7Nkv922p8RhG+U73ajODANAQURIwqOPJ
+01IrfMIK2mkXDZwzkAxPaOnny2OMZtUznmZnNdJ/vLd7U0sScNPVQp8CgYEA5krD
+ImQ8U9/S4VOK78i3FMWMoutffXpW71lEc9tsz1YWUPf170raujjF5mqtBBXup9ko
+37CmVk6mOO2TdXLg1feMaVBsoblL6iPoBZot/fLdzgmICccpimst2yrUZEHwJpdk
+9k95xEZUQhN73eY/Ih6b5HZZ/ygxfAVvhDFzNT0CgYEAru5aDvUGbU9e7HsQwvNg
+FfMkWJwmUZ46oRtO7BFP0qqwRGYJAf0S8QEuQCY0mrDIt/dGXXPEXIV2k9eHVxch
+eDhaVXuuxJfFINIiBwpKGA7Ed27Smr6EbI7bu1W1h+oozjppdW9GfscmXDVhhMir
+3PYG54H298hM8/eAKzsps80CgYEAzk5Vp76ySNVv3spv4kYmtaYQSnef8RIjRYLs
+DvqY7NmLXnf0y618a22m5LfWTZ20Uov50QM40ILe6Ir1GjeS8jw1frc8yljsiFIo
+brRj1We4ivcA9vmD3mwMBZbF9RcZJAlmuj4SsOHsY9F+mxjEoDVZpP7duvbv9dIM
+yBlgw2UCgYB27Zwd3ta3x9mx2wRHaFMlcnwh+ERRGO6yAKndsr3L80Gkb2HxHlR3
+cA3dIg/YAGocjROOIO8MO5rclZm30qM5rlv0SH6gQ4nW16PZgHSuakgeX5fVHb4v
+nT8TKt0PY9r5e5cdHzpwlrDD1Q8GiAHvn0oH6oioGxlIV190YAHvGQ==
+>>>>>>> Replace endpoint.ParseHostPort with net.SplitHostPort to fix IPv6 client endpoints
 -----END RSA PRIVATE KEY-----

integration/fixtures/server3.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID/DCCAuSgAwIBAgIUMgfWSv3FrD0dRcHmPNxLr7p9BXcwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMGIxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKKf9zhwcgu
+S53Vd92PLqHtDNpTbl4nM/5qYy7nHV5FHW7uolQHFA0BjJwxz7/LC39G950PpSIs
+K7Y99P7aMPVqVH5FGL+uoDbHMzt/gIUGgWj51J38+x6zN/9vIvAVIBClBhzEuB2k
+WJU6KyB1V5G+1wnnKRXLB9QC0f/7vqd9f21O7sJmWeVhGJuEUwwAp1p5WDGM2Tn8
+Fjy57O9f1nT4WVqWhB5EbvYGDF2Z5DWyKz90EWOwVw30ThcQHF57X6WJdlNiQOrY
+KIWO475QKPwbUSpRkvw1jwvllU8s4l6pB1hKTAcUK3UsWrzpf2+m8v8ou35uFD/c
+pd2bBx5MVlkCAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYI
+KwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFAcMdmvX
+8NC4V4Z67Jb/pNUwai7cMB8GA1UdIwQYMBaAFGUHvq+NpnIUusRQaboNB/KOY3o+
+MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA
+btQjJEWHD/0gYsNLFg3tDxZ64U/HfNlh4USGOK02VL2LteMcV8AoYlZ3jwmp4+33
+D3HlqLclJNABax2pOvTHVnQlf25TSNwJRtmzOvcg+6xYbPdgRoeEVsWbmgbpX7Vi
+P8FYelYCiYTPezjqZgPG1gmq0Uf/drlTrjwsG2njEcuK7hip+LdJnIrtpIrabpIk
+lZRa7Y/JBM3gP/rR1fu9lhzJ97s3NabuHzPwyouSTTknaaiGwSV8F5frh9NGcFhd
+G7giCLZLKklQB4IUTOFcVFSZmeAGy6KBqyT10N2kkBrsrcWhyMKIU9X0+6hh3Tlc
+JEla9as6qFvt1dFGp+qeww==
+-----END CERTIFICATE-----

integration/fixtures/server3.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAsop/3OHByC5LndV33Y8uoe0M2lNuXicz/mpjLucdXkUdbu6i
+VAcUDQGMnDHPv8sLf0b3nQ+lIiwrtj30/tow9WpUfkUYv66gNsczO3+AhQaBaPnU
+nfz7HrM3/28i8BUgEKUGHMS4HaRYlTorIHVXkb7XCecpFcsH1ALR//u+p31/bU7u
+wmZZ5WEYm4RTDACnWnlYMYzZOfwWPLns71/WdPhZWpaEHkRu9gYMXZnkNbIrP3QR
+Y7BXDfROFxAcXntfpYl2U2JA6tgohY7jvlAo/BtRKlGS/DWPC+WVTyziXqkHWEpM
+BxQrdSxavOl/b6by/yi7fm4UP9yl3ZsHHkxWWQIDAQABAoIBAQCVyfrCDqlsT+Li
+1UBOIp0l/uIEnXCAD3XgodL6e6249FVgR1brFlEtJDqapHO+XhQUQS7ml0ScqeA2
+cj6EPfxLOV0P3tqHnnMN4gvKhAsID9AsiUVnEuJ//C4j4FK4h5CyRjEdm7E4NTSY
+ZgfeoHPKdAinZ0eh4Ad+SKt0jvmCPDD1L+6bxpJ6E258xPDxB71rHTCgnwZZmXrN
+rHDg07tVVU6lYtXEsZAsyIBIxXV/RaCt4xSijM1C7kuSsUE3+CCw+pzQUvHDxkuk
+FPxE5hONzkUaCSBKTv4L6gVaiYa30Jo9THuTRWvzDJmcnNwlYgRLeIR8PJY0eHqv
+FYJbLdQBAoGBAOeiW1aHpNO+K4BqVMzm7fvz0a3DBkml/wIKSvw3dbTYnyIXFvDB
+Be7OZLhiWPwaE/58aQWh+/OGCr88yyCLAt2mOQ5aMalKpdSCaY5swEnzt923pY9z
+jt3DnhX8aXggPbqM6eJjaxJY7jIMSDNKUQZKJeSesr/EzFo0CRkkepyBAoGBAMVS
+Z2nIY+G2+P1VSUEbI3dbaZ3ciMEDLc+rZQs0fx8+xQCD71eCU9ggxf+O0R1/0smm
+9so65KrmKl3yOt8OQW3YgUpQqIQdJPHnfuTnsU7y/+zRl6k119gV89LLDlCM7nfW
+5/ey/iJLXQxfC7OoFF/hQM0odmorA8jBuqKDWy3ZAoGAZL6gq0njzpRvpzKYH2Zx
+K5woHkMsgOvJtcF0S65za2ysCc+xEpVhVzQ9alScD0noWE8T/nctdgVetz5huo27
+eVvKhQuFffQRnBP8hQ2XtJJj7fLp9zJzeNCT+UwHM1ASiQiw0N4cu6YiM3JUFLrF
+8s5dHMpJRE778l+fdWgATAECgYEAmJ3osE+2uUCtCjvpwbp8zvdcFCYbe7W6vBGj
+wGvlGsSQ2JozB2sc8GBA5C2RHhDcdu11meq9LFWDVVBiKl27S3uWXGVQQYbNKXDU
+m7V8VUTrnz5o4A5uGIq6IEK/mpu2YehNWC8QEnRZzpTA1z7cK2Bsn4F5PRpx/deh
+Q8r3PdkCgYBd2rT+S4/51C5AnIhgMF/PYl0+DYMFD8HAfsx7VTaIUmFQ4devHMOz
+J6lbqRyEITZtXgna1n35LkyBDcPwsEtntjJOP+xneCtKzozdzhXyoAw2xQR3gqvV
+7YtV3miYQiOTqjOefViMhR/XiOV2zng3OId1AQObfOUZODJPfSN26g==
+-----END RSA PRIVATE KEY-----

integration/v3_auth_test.go

@@ -347,6 +347,7 @@ func TestV3AuthNonAuthorizedRPCs(t *testing.T) {
 }
 
 func TestV3AuthOldRevConcurrent(t *testing.T) {
+	t.Skip() // TODO(jingyih): re-enable the test when #10408 is fixed.
 	defer testutil.AfterTest(t)
 	clus := NewClusterV3(t, &ClusterConfig{Size: 1})
 	defer clus.Terminate(t)

integration/v3_watch_test.go

@@ -781,9 +781,11 @@ func testV3WatchMultipleEventsTxn(t *testing.T, startRev int64) {
 
 type eventsSortByKey []*mvccpb.Event
 
-func (evs eventsSortByKey) Len() int           { return len(evs) }
-func (evs eventsSortByKey) Swap(i, j int)      { evs[i], evs[j] = evs[j], evs[i] }
-func (evs eventsSortByKey) Less(i, j int) bool { return bytes.Compare(evs[i].Kv.Key, evs[j].Kv.Key) < 0 }
+func (evs eventsSortByKey) Len() int      { return len(evs) }
+func (evs eventsSortByKey) Swap(i, j int) { evs[i], evs[j] = evs[j], evs[i] }
+func (evs eventsSortByKey) Less(i, j int) bool {
+	return bytes.Compare(evs[i].Kv.Key, evs[j].Kv.Key) < 0
+}
 
 func TestV3WatchMultipleEventsPutUnsynced(t *testing.T) {
 	defer testutil.AfterTest(t)

mvcc/backend/backend.go

@@ -292,7 +292,7 @@ func (b *backend) Defrag() error {
 
 func (b *backend) defrag() error {
 	now := time.Now()
-	
+
 	// TODO: make this non-blocking?
 	// lock batchTx to ensure nobody is using previous tx, and then
 	// close previous ongoing tx.
@@ -310,21 +310,38 @@ func (b *backend) defrag() error {
 	b.batchTx.unsafeCommit(true)
 	b.batchTx.tx = nil
 
-	tmpdb, err := bolt.Open(b.db.Path()+".tmp", 0600, boltOpenOptions)
+	// Create a temporary file to ensure we start with a clean slate.
+	// Snapshotter.cleanupSnapdir cleans up any of these that are found during startup.
+	dir := filepath.Dir(b.db.Path())
+	temp, err := ioutil.TempFile(dir, "db.tmp.*")
+	if err != nil {
+		return err
+	}
+	options := bolt.Options{}
+	if boltOpenOptions != nil {
+		options = *boltOpenOptions
+	}
+	options.OpenFile = func(path string, i int, mode os.FileMode) (file *os.File, err error) {
+		return temp, nil
+	}
+	tdbp := temp.Name()
+	tmpdb, err := bolt.Open(tdbp, 0600, &options)
 	if err != nil {
 		return err
 	}
 
+	// gofail: var defragBeforeCopy struct{}
 	err = defragdb(b.db, tmpdb, defragLimit)
 
 	if err != nil {
 		tmpdb.Close()
-		os.RemoveAll(tmpdb.Path())
+		if rmErr := os.RemoveAll(tmpdb.Path()); rmErr != nil {
+			plog.Fatalf("failed to remove db.tmp after defragmentation completed: %v", rmErr)
+		}
 		return err
 	}
 
 	dbp := b.db.Path()
-	tdbp := tmpdb.Path()
 
 	err = b.db.Close()
 	if err != nil {
@@ -334,6 +351,7 @@ func (b *backend) defrag() error {
 	if err != nil {
 		plog.Fatalf("cannot close database (%s)", err)
 	}
+	// gofail: var defragBeforeRename struct{}
 	err = os.Rename(tdbp, dbp)
 	if err != nil {
 		plog.Fatalf("cannot rename database (%s)", err)

mvcc/kv_test.go

@@ -710,7 +710,7 @@ func TestKVSnapshot(t *testing.T) {
 
 func TestWatchableKVWatch(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()

mvcc/kvstore.go

@@ -142,14 +142,18 @@ func NewStore(b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) *sto
 
 func (s *store) compactBarrier(ctx context.Context, ch chan struct{}) {
 	if ctx == nil || ctx.Err() != nil {
-		s.mu.Lock()
 		select {
 		case <-s.stopc:
 		default:
+			// fix deadlock in mvcc,for more information, please refer to pr 11817.
+			// s.stopc is only updated in restore operation, which is called by apply
+			// snapshot call, compaction and apply snapshot requests are serialized by
+			// raft, and do not happen at the same time.
+			s.mu.Lock()
 			f := func(ctx context.Context) { s.compactBarrier(ctx, ch) }
 			s.fifoSched.Schedule(f)
+			s.mu.Unlock()
 		}
-		s.mu.Unlock()
 		return
 	}
 	close(ch)
@@ -309,14 +313,7 @@ func (s *store) Restore(b backend.Backend) error {
 }
 
 func (s *store) restore() error {
-	b := s.b
-
-	reportDbTotalSizeInBytesMu.Lock()
-	reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) }
-	reportDbTotalSizeInBytesMu.Unlock()
-	reportDbTotalSizeInUseInBytesMu.Lock()
-	reportDbTotalSizeInUseInBytes = func() float64 { return float64(b.SizeInUse()) }
-	reportDbTotalSizeInUseInBytesMu.Unlock()
+	s.setupMetricsReporter()
 
 	min, max := newRevBytes(), newRevBytes()
 	revToBytes(revision{main: 1}, min)
@@ -496,6 +493,30 @@ func (s *store) ConsistentIndex() uint64 {
 	return v
 }
 
+func (s *store) setupMetricsReporter() {
+	b := s.b
+	reportDbTotalSizeInBytesMu.Lock()
+	reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) }
+	reportDbTotalSizeInBytesMu.Unlock()
+	reportDbTotalSizeInUseInBytesMu.Lock()
+	reportDbTotalSizeInUseInBytes = func() float64 { return float64(b.SizeInUse()) }
+	reportDbTotalSizeInUseInBytesMu.Unlock()
+	reportCurrentRevMu.Lock()
+	reportCurrentRev = func() float64 {
+		s.revMu.RLock()
+		defer s.revMu.RUnlock()
+		return float64(s.currentRev)
+	}
+	reportCurrentRevMu.Unlock()
+	reportCompactRevMu.Lock()
+	reportCompactRev = func() float64 {
+		s.revMu.RLock()
+		defer s.revMu.RUnlock()
+		return float64(s.compactMainRev)
+	}
+	reportCompactRevMu.Unlock()
+}
+
 // appendMarkTombstone appends tombstone mark to normal revision bytes.
 func appendMarkTombstone(b []byte) []byte {
 	if len(b) != revBytesLen {

mvcc/metrics.go

@@ -206,6 +206,46 @@ var (
 		// highest bucket start of 0.01 sec * 2^14 == 163.84 sec
 		Buckets: prometheus.ExponentialBuckets(.01, 2, 15),
 	})
+
+	currentRev = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "etcd_debugging",
+		Subsystem: "mvcc",
+		Name:      "current_revision",
+		Help:      "The current revision of store.",
+	},
+		func() float64 {
+			reportCurrentRevMu.RLock()
+			defer reportCurrentRevMu.RUnlock()
+			return reportCurrentRev()
+		},
+	)
+	// overridden by mvcc initialization
+	reportCurrentRevMu sync.RWMutex
+	reportCurrentRev   = func() float64 { return 0 }
+
+	compactRev = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Namespace: "etcd_debugging",
+		Subsystem: "mvcc",
+		Name:      "compact_revision",
+		Help:      "The revision of the last compaction in store.",
+	},
+		func() float64 {
+			reportCompactRevMu.RLock()
+			defer reportCompactRevMu.RUnlock()
+			return reportCompactRev()
+		},
+	)
+	// overridden by mvcc initialization
+	reportCompactRevMu sync.RWMutex
+	reportCompactRev   = func() float64 { return 0 }
+
+	totalPutSizeGauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "etcd_debugging",
+			Subsystem: "mvcc",
+			Name:      "total_put_size_in_bytes",
+			Help:      "The total size of put kv pairs seen by this member.",
+		})
 )
 
 func init() {
@@ -228,6 +268,9 @@ func init() {
 	prometheus.MustRegister(dbTotalSizeInUse)
 	prometheus.MustRegister(hashDurations)
 	prometheus.MustRegister(hashRevDurations)
+	prometheus.MustRegister(currentRev)
+	prometheus.MustRegister(compactRev)
+	prometheus.MustRegister(totalPutSizeGauge)
 }
 
 // ReportEventReceived reports that an event is received.

mvcc/metrics_txn.go

@@ -23,14 +23,15 @@ type metricsTxnWrite struct {
 	ranges  uint
 	puts    uint
 	deletes uint
+	putSize int64
 }
 
 func newMetricsTxnRead(tr TxnRead) TxnRead {
-	return &metricsTxnWrite{&txnReadWrite{tr}, 0, 0, 0}
+	return &metricsTxnWrite{&txnReadWrite{tr}, 0, 0, 0, 0}
 }
 
 func newMetricsTxnWrite(tw TxnWrite) TxnWrite {
-	return &metricsTxnWrite{tw, 0, 0, 0}
+	return &metricsTxnWrite{tw, 0, 0, 0, 0}
 }
 
 func (tw *metricsTxnWrite) Range(key, end []byte, ro RangeOptions) (*RangeResult, error) {
@@ -45,6 +46,8 @@ func (tw *metricsTxnWrite) DeleteRange(key, end []byte) (n, rev int64) {
 
 func (tw *metricsTxnWrite) Put(key, value []byte, lease lease.LeaseID) (rev int64) {
 	tw.puts++
+	size := int64(len(key) + len(value))
+	tw.putSize += size
 	return tw.TxnWrite.Put(key, value, lease)
 }
 
@@ -55,5 +58,6 @@ func (tw *metricsTxnWrite) End() {
 	}
 	rangeCounter.Add(float64(tw.ranges))
 	putCounter.Add(float64(tw.puts))
+	totalPutSizeGauge.Add(float64(tw.putSize))
 	deleteCounter.Add(float64(tw.deletes))
 }

mvcc/watchable_store.go

@@ -18,6 +18,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/coreos/etcd/auth"
 	"github.com/coreos/etcd/lease"
 	"github.com/coreos/etcd/mvcc/backend"
 	"github.com/coreos/etcd/mvcc/mvccpb"
@@ -67,11 +68,11 @@ type watchableStore struct {
 // cancel operations.
 type cancelFunc func()
 
-func New(b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) ConsistentWatchableKV {
-	return newWatchableStore(b, le, ig)
+func New(b backend.Backend, le lease.Lessor, as auth.AuthStore, ig ConsistentIndexGetter) ConsistentWatchableKV {
+	return newWatchableStore(b, le, as, ig)
 }
 
-func newWatchableStore(b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) *watchableStore {
+func newWatchableStore(b backend.Backend, le lease.Lessor, as auth.AuthStore, ig ConsistentIndexGetter) *watchableStore {
 	s := &watchableStore{
 		store:    NewStore(b, le, ig),
 		victimc:  make(chan struct{}, 1),
@@ -85,6 +86,10 @@ func newWatchableStore(b backend.Backend, le lease.Lessor, ig ConsistentIndexGet
 		// use this store as the deleter so revokes trigger watch events
 		s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write() })
 	}
+	if as != nil {
+		// TODO: encapsulating consistentindex into a separate package
+		as.SetConsistentIndexSyncer(s.store.saveIndex)
+	}
 	s.wg.Add(2)
 	go s.syncWatchersLoop()
 	go s.syncVictimsLoop()

mvcc/watchable_store_bench_test.go

@@ -25,7 +25,7 @@ import (
 
 func BenchmarkWatchableStorePut(b *testing.B) {
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	s := New(be, &lease.FakeLessor{}, nil)
+	s := New(be, &lease.FakeLessor{}, nil, nil)
 	defer cleanup(s, be, tmpPath)
 
 	// arbitrary number of bytes
@@ -46,7 +46,7 @@ func BenchmarkWatchableStorePut(b *testing.B) {
 func BenchmarkWatchableStoreTxnPut(b *testing.B) {
 	var i fakeConsistentIndex
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	s := New(be, &lease.FakeLessor{}, &i)
+	s := New(be, &lease.FakeLessor{}, nil, &i)
 	defer cleanup(s, be, tmpPath)
 
 	// arbitrary number of bytes
@@ -67,7 +67,7 @@ func BenchmarkWatchableStoreTxnPut(b *testing.B) {
 // many synced watchers receiving a Put notification.
 func BenchmarkWatchableStoreWatchSyncPut(b *testing.B) {
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(be, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(be, &lease.FakeLessor{}, nil, nil)
 	defer cleanup(s, be, tmpPath)
 
 	k := []byte("testkey")
@@ -162,7 +162,7 @@ func BenchmarkWatchableStoreUnsyncedCancel(b *testing.B) {
 
 func BenchmarkWatchableStoreSyncedCancel(b *testing.B) {
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(be, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(be, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()

mvcc/watchable_store_test.go

@@ -30,7 +30,7 @@ import (
 
 func TestWatch(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -52,7 +52,7 @@ func TestWatch(t *testing.T) {
 
 func TestNewWatcherCancel(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -222,7 +222,7 @@ func TestSyncWatchers(t *testing.T) {
 // TestWatchCompacted tests a watcher that watches on a compacted revision.
 func TestWatchCompacted(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -259,7 +259,7 @@ func TestWatchCompacted(t *testing.T) {
 
 func TestWatchFutureRev(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -300,7 +300,7 @@ func TestWatchRestore(t *testing.T) {
 	test := func(delay time.Duration) func(t *testing.T) {
 		return func(t *testing.T) {
 			b, tmpPath := backend.NewDefaultTmpBackend()
-			s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+			s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 			defer cleanup(s, b, tmpPath)
 
 			testKey := []byte("foo")
@@ -308,7 +308,7 @@ func TestWatchRestore(t *testing.T) {
 			rev := s.Put(testKey, testValue, lease.NoLease)
 
 			newBackend, newPath := backend.NewDefaultTmpBackend()
-			newStore := newWatchableStore(newBackend, &lease.FakeLessor{}, nil)
+			newStore := newWatchableStore(newBackend, &lease.FakeLessor{}, nil, nil)
 			defer cleanup(newStore, newBackend, newPath)
 
 			w := newStore.NewWatchStream()
@@ -346,11 +346,11 @@ func TestWatchRestore(t *testing.T) {
 //   5. choose the watcher from step 1, without panic
 func TestWatchRestoreSyncedWatcher(t *testing.T) {
 	b1, b1Path := backend.NewDefaultTmpBackend()
-	s1 := newWatchableStore(b1, &lease.FakeLessor{}, nil)
+	s1 := newWatchableStore(b1, &lease.FakeLessor{}, nil, nil)
 	defer cleanup(s1, b1, b1Path)
 
 	b2, b2Path := backend.NewDefaultTmpBackend()
-	s2 := newWatchableStore(b2, &lease.FakeLessor{}, nil)
+	s2 := newWatchableStore(b2, &lease.FakeLessor{}, nil, nil)
 	defer cleanup(s2, b2, b2Path)
 
 	testKey, testValue := []byte("foo"), []byte("bar")
@@ -397,7 +397,7 @@ func TestWatchRestoreSyncedWatcher(t *testing.T) {
 // TestWatchBatchUnsynced tests batching on unsynced watchers
 func TestWatchBatchUnsynced(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	oldMaxRevs := watchBatchMaxRevs
 	defer func() {
@@ -531,7 +531,7 @@ func TestWatchVictims(t *testing.T) {
 	oldChanBufLen, oldMaxWatchersPerSync := chanBufLen, maxWatchersPerSync
 
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -609,7 +609,7 @@ func TestWatchVictims(t *testing.T) {
 // canceling its watches.
 func TestStressWatchCancelClose(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()

mvcc/watcher_bench_test.go

@@ -24,7 +24,7 @@ import (
 
 func BenchmarkKVWatcherMemoryUsage(b *testing.B) {
 	be, tmpPath := backend.NewDefaultTmpBackend()
-	watchable := newWatchableStore(be, &lease.FakeLessor{}, nil)
+	watchable := newWatchableStore(be, &lease.FakeLessor{}, nil, nil)
 
 	defer cleanup(watchable, be, tmpPath)
 

mvcc/watcher_test.go

@@ -31,7 +31,7 @@ import (
 // and the watched event attaches the correct watchID.
 func TestWatcherWatchID(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()
@@ -83,7 +83,7 @@ func TestWatcherWatchID(t *testing.T) {
 // and returns events with matching prefixes.
 func TestWatcherWatchPrefix(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()
@@ -157,7 +157,7 @@ func TestWatcherWatchPrefix(t *testing.T) {
 // does not create watcher, which panics when canceling in range tree.
 func TestWatcherWatchWrongRange(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()
@@ -177,7 +177,7 @@ func TestWatcherWatchWrongRange(t *testing.T) {
 
 func TestWatchDeleteRange(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+	s := newWatchableStore(b, &lease.FakeLessor{}, nil, nil)
 
 	defer func() {
 		s.store.Close()
@@ -216,7 +216,7 @@ func TestWatchDeleteRange(t *testing.T) {
 // with given id inside watchStream.
 func TestWatchStreamCancelWatcherByID(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()
@@ -308,7 +308,7 @@ func TestWatcherRequestProgress(t *testing.T) {
 
 func TestWatcherWatchWithFilter(t *testing.T) {
 	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil))
+	s := WatchableKV(newWatchableStore(b, &lease.FakeLessor{}, nil, nil))
 	defer cleanup(s, b, tmpPath)
 
 	w := s.NewWatchStream()

pkg/fileutil/purge.go

@@ -23,13 +23,23 @@ import (
 )
 
 func PurgeFile(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) <-chan error {
-	return purgeFile(dirname, suffix, max, interval, stop, nil)
+	return purgeFile(dirname, suffix, max, interval, stop, nil, nil)
+}
+
+func PurgeFileWithDoneNotify(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) (<-chan struct{}, <-chan error) {
+	doneC := make(chan struct{})
+	errC := purgeFile(dirname, suffix, max, interval, stop, nil, doneC)
+	return doneC, errC
 }
 
 // purgeFile is the internal implementation for PurgeFile which can post purged files to purgec if non-nil.
-func purgeFile(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string) <-chan error {
+// if donec is non-nil, the function closes it to notify its exit.
+func purgeFile(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}, purgec chan<- string, donec chan<- struct{}) <-chan error {
 	errC := make(chan error, 1)
 	go func() {
+		if donec != nil {
+			defer close(donec)
+		}
 		for {
 			fnames, err := ReadDir(dirname)
 			if err != nil {

pkg/fileutil/purge_test.go

@@ -43,7 +43,7 @@ func TestPurgeFile(t *testing.T) {
 	stop, purgec := make(chan struct{}), make(chan string, 10)
 
 	// keep 3 most recent files
-	errch := purgeFile(dir, "test", 3, time.Millisecond, stop, purgec)
+	errch := purgeFile(dir, "test", 3, time.Millisecond, stop, purgec, nil)
 	select {
 	case f := <-purgec:
 		t.Errorf("unexpected purge on %q", f)
@@ -114,7 +114,7 @@ func TestPurgeFileHoldingLockFile(t *testing.T) {
 	}
 
 	stop, purgec := make(chan struct{}), make(chan string, 10)
-	errch := purgeFile(dir, "test", 3, time.Millisecond, stop, purgec)
+	errch := purgeFile(dir, "test", 3, time.Millisecond, stop, purgec, nil)
 
 	for i := 0; i < 5; i++ {
 		select {

pkg/ioutil/pagewriter.go

@@ -95,12 +95,23 @@ func (pw *PageWriter) Write(p []byte) (n int, err error) {
 	return n, werr
 }
 
+// Flush flushes buffered data.
 func (pw *PageWriter) Flush() error {
-	if pw.bufferedBytes == 0 {
-		return nil
-	}
-	_, err := pw.w.Write(pw.buf[:pw.bufferedBytes])
-	pw.pageOffset = (pw.pageOffset + pw.bufferedBytes) % pw.pageBytes
-	pw.bufferedBytes = 0
+	_, err := pw.flush()
 	return err
 }
+
+// FlushN flushes buffered data and returns the number of written bytes.
+func (pw *PageWriter) FlushN() (int, error) {
+	return pw.flush()
+}
+
+func (pw *PageWriter) flush() (int, error) {
+	if pw.bufferedBytes == 0 {
+		return 0, nil
+	}
+	n, err := pw.w.Write(pw.buf[:pw.bufferedBytes])
+	pw.pageOffset = (pw.pageOffset + pw.bufferedBytes) % pw.pageBytes
+	pw.bufferedBytes = 0
+	return n, err
+}

pkg/mock/mockstorage/storage_recorder.go

@@ -45,4 +45,16 @@ func (p *storageRecorder) SaveSnap(st raftpb.Snapshot) error {
 	return nil
 }
 
+func (p *storageRecorder) Release(st raftpb.Snapshot) error {
+	if !raft.IsEmptySnap(st) {
+		p.Record(testutil.Action{Name: "Release"})
+	}
+	return nil
+}
+
+func (p *storageRecorder) Sync() error {
+	p.Record(testutil.Action{Name: "Sync"})
+	return nil
+}
+
 func (p *storageRecorder) Close() error { return nil }

pkg/testutil/leak.go

@@ -70,13 +70,13 @@ func CheckAfterTest(d time.Duration) error {
 	}
 	var bad string
 	badSubstring := map[string]string{
-		").writeLoop(":                                 "a Transport",
+		").writeLoop(": "a Transport",
 		"created by net/http/httptest.(*Server).Start": "an httptest.Server",
-		"timeoutHandler":                               "a TimeoutHandler",
-		"net.(*netFD).connect(":                        "a timing out dial",
-		").noteClientGone(":                            "a closenotifier sender",
-		").readLoop(":                                  "a Transport",
-		".grpc":                                        "a gRPC resource",
+		"timeoutHandler":        "a TimeoutHandler",
+		"net.(*netFD).connect(": "a timing out dial",
+		").noteClientGone(":     "a closenotifier sender",
+		").readLoop(":           "a Transport",
+		".grpc":                 "a gRPC resource",
 	}
 
 	var stacks string

pkg/transport/listener.go

@@ -53,6 +53,9 @@ func wrapTLS(addr, scheme string, tlsinfo *TLSInfo, l net.Listener) (net.Listene
 	if scheme != "https" && scheme != "unixs" {
 		return l, nil
 	}
+	if tlsinfo != nil && tlsinfo.SkipClientSANVerify {
+		return NewTLSListener(l, tlsinfo)
+	}
 	return newTLSListener(l, tlsinfo, checkSAN)
 }
 
@@ -65,6 +68,8 @@ type TLSInfo struct {
 	CRLFile            string
 	InsecureSkipVerify bool
 
+	SkipClientSANVerify bool
+
 	// ServerName ensures the cert matches the given host in case of discovery / virtual hosting
 	ServerName string
 
@@ -95,7 +100,7 @@ func (info TLSInfo) Empty() bool {
 	return info.CertFile == "" && info.KeyFile == ""
 }
 
-func SelfCert(dirpath string, hosts []string) (info TLSInfo, err error) {
+func SelfCert(dirpath string, hosts []string, additionalUsages ...x509.ExtKeyUsage) (info TLSInfo, err error) {
 	if err = os.MkdirAll(dirpath, 0700); err != nil {
 		return
 	}
@@ -124,7 +129,7 @@ func SelfCert(dirpath string, hosts []string) (info TLSInfo, err error) {
 		NotAfter:     time.Now().Add(365 * (24 * time.Hour)),
 
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		ExtKeyUsage:           append([]x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, additionalUsages...),
 		BasicConstraintsValid: true,
 	}
 

pkg/transport/listener_test.go

@@ -16,20 +16,26 @@ package transport
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"os"
 	"testing"
 	"time"
 )
 
-func createSelfCert() (*TLSInfo, func(), error) {
+func createSelfCert(hosts ...string) (*TLSInfo, func(), error) {
+	return createSelfCertEx("127.0.0.1")
+}
+
+func createSelfCertEx(host string, additionalUsages ...x509.ExtKeyUsage) (*TLSInfo, func(), error) {
 	d, terr := ioutil.TempDir("", "etcd-test-tls-")
 	if terr != nil {
 		return nil, nil, terr
 	}
-	info, err := SelfCert(d, []string{"127.0.0.1"})
+	info, err := SelfCert(d, []string{host + ":0"}, additionalUsages...)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -70,10 +76,108 @@ func testNewListenerTLSInfoAccept(t *testing.T, tlsInfo TLSInfo) {
 	}
 	defer conn.Close()
 	if _, ok := conn.(*tls.Conn); !ok {
-		t.Errorf("failed to accept *tls.Conn")
+		t.Error("failed to accept *tls.Conn")
 	}
 }
 
+// TestNewListenerTLSInfoSkipClientSANVerify tests that if client IP address mismatches
+// with specified address in its certificate the connection is still accepted
+// if the flag SkipClientSANVerify is set (i.e. checkSAN() is disabled for the client side)
+func TestNewListenerTLSInfoSkipClientSANVerify(t *testing.T) {
+	tests := []struct {
+		skipClientSANVerify bool
+		goodClientHost      bool
+		acceptExpected      bool
+	}{
+		{false, true, true},
+		{false, false, false},
+		{true, true, true},
+		{true, false, true},
+	}
+	for _, test := range tests {
+		testNewListenerTLSInfoClientCheck(t, test.skipClientSANVerify, test.goodClientHost, test.acceptExpected)
+	}
+}
+
+func testNewListenerTLSInfoClientCheck(t *testing.T, skipClientSANVerify, goodClientHost, acceptExpected bool) {
+	tlsInfo, del, err := createSelfCert()
+	if err != nil {
+		t.Fatalf("unable to create cert: %v", err)
+	}
+	defer del()
+
+	host := "127.0.0.222"
+	if goodClientHost {
+		host = "127.0.0.1"
+	}
+	clientTLSInfo, del2, err := createSelfCertEx(host, x509.ExtKeyUsageClientAuth)
+	if err != nil {
+		t.Fatalf("unable to create cert: %v", err)
+	}
+	defer del2()
+
+	tlsInfo.SkipClientSANVerify = skipClientSANVerify
+	tlsInfo.CAFile = clientTLSInfo.CertFile
+
+	rootCAs := x509.NewCertPool()
+	loaded, err := ioutil.ReadFile(tlsInfo.CertFile)
+	if err != nil {
+		t.Fatalf("unexpected missing certfile: %v", err)
+	}
+	rootCAs.AppendCertsFromPEM(loaded)
+
+	clientCert, err := tls.LoadX509KeyPair(clientTLSInfo.CertFile, clientTLSInfo.KeyFile)
+	if err != nil {
+		t.Fatalf("unable to create peer cert: %v", err)
+	}
+
+	tlsConfig := &tls.Config{}
+	tlsConfig.InsecureSkipVerify = false
+	tlsConfig.Certificates = []tls.Certificate{clientCert}
+	tlsConfig.RootCAs = rootCAs
+
+	ln, err := NewListener("127.0.0.1:0", "https", tlsInfo)
+	if err != nil {
+		t.Fatalf("unexpected NewListener error: %v", err)
+	}
+	defer ln.Close()
+
+	tr := &http.Transport{TLSClientConfig: tlsConfig}
+	cli := &http.Client{Transport: tr}
+	chClientErr := make(chan error)
+	go func() {
+		_, err := cli.Get("https://" + ln.Addr().String())
+		chClientErr <- err
+	}()
+
+	chAcceptErr := make(chan error)
+	chAcceptConn := make(chan net.Conn)
+	go func() {
+		conn, err := ln.Accept()
+		if err != nil {
+			chAcceptErr <- err
+		} else {
+			chAcceptConn <- conn
+		}
+	}()
+
+	select {
+	case <-chClientErr:
+		if acceptExpected {
+			t.Errorf("accepted for good client address: skipClientSANVerify=%v, goodClientHost=%v", skipClientSANVerify, goodClientHost)
+		}
+	case acceptErr := <-chAcceptErr:
+		t.Fatalf("unexpected Accept error: %v", acceptErr)
+	case conn := <-chAcceptConn:
+		defer conn.Close()
+		if _, ok := conn.(*tls.Conn); !ok {
+			t.Errorf("failed to accept *tls.Conn")
+		}
+		if !acceptExpected {
+			t.Errorf("accepted for bad client address: skipClientSANVerify=%v, goodClientHost=%v", skipClientSANVerify, goodClientHost)
+		}
+	}
+}
 func TestNewListenerTLSEmptyInfo(t *testing.T) {
 	_, err := NewListener("127.0.0.1:0", "https", nil)
 	if err == nil {

proxy/grpcproxy/metrics.go

@@ -89,6 +89,7 @@ func HandleMetrics(mux *http.ServeMux, c *http.Client, eps []string) {
 		resp, err := c.Get(target)
 		if err != nil {
 			http.Error(w, "Internal server error", http.StatusInternalServerError)
+			return
 		}
 		defer resp.Body.Close()
 		w.Header().Set("Content-Type", "text/plain; version=0.0.4")

proxy/httpproxy/reverse_test.go

@@ -175,10 +175,10 @@ func TestRemoveSingleHopHeaders(t *testing.T) {
 		"Keep-Alive":          {"foo"},
 		"Proxy-Authenticate":  {"Basic realm=example.com"},
 		"Proxy-Authorization": {"foo"},
-		"Te":                {"deflate,gzip"},
-		"Trailers":          {"ETag"},
-		"Transfer-Encoding": {"chunked"},
-		"Upgrade":           {"WebSocket"},
+		"Te":                  {"deflate,gzip"},
+		"Trailers":            {"ETag"},
+		"Transfer-Encoding":   {"chunked"},
+		"Upgrade":             {"WebSocket"},
 
 		// headers that should persist
 		"Accept": {"application/json"},

rafthttp/http.go

@@ -29,6 +29,7 @@ import (
 	"github.com/coreos/etcd/raft/raftpb"
 	"github.com/coreos/etcd/snap"
 	"github.com/coreos/etcd/version"
+	"github.com/dustin/go-humanize"
 )
 
 const (
@@ -194,7 +195,9 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	receivedBytes.WithLabelValues(from).Add(float64(m.Size()))
+	msgSizeVal := m.Size()
+	msgSize := humanize.Bytes(uint64(msgSizeVal))
+	receivedBytes.WithLabelValues(from).Add(float64(msgSizeVal))
 
 	if m.Type != raftpb.MsgSnap {
 		plog.Errorf("unexpected raft message type %s on snapshot path", m.Type)
@@ -207,7 +210,7 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer func() {
 		snapshotReceiveInflights.WithLabelValues(from).Dec()
 	}()
-	plog.Infof("receiving database snapshot [index:%d, from %s] ...", m.Snapshot.Metadata.Index, types.ID(m.From))
+	plog.Infof("receiving database snapshot [index: %d, from: %s, raft message size: %s]", m.Snapshot.Metadata.Index, types.ID(m.From), msgSize)
 	// save incoming database snapshot.
 	n, err := h.snapshotter.SaveDBFrom(r.Body, m.Snapshot.Metadata.Index)
 	if err != nil {
@@ -217,8 +220,11 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		snapshotReceiveFailures.WithLabelValues(from).Inc()
 		return
 	}
+
+	downloadTook := time.Since(start)
+	dbSize := humanize.Bytes(uint64(n))
 	receivedBytes.WithLabelValues(from).Add(float64(n))
-	plog.Infof("received and saved database snapshot [index: %d, from: %s] successfully", m.Snapshot.Metadata.Index, types.ID(m.From))
+	plog.Infof("successfully received and saved database snapshot [index: %d, from: %s, raft message size: %s, db size: %s, took: %s]", m.Snapshot.Metadata.Index, types.ID(m.From), msgSize, dbSize, downloadTook)
 
 	if err := h.r.Process(context.TODO(), m); err != nil {
 		switch v := err.(type) {

rafthttp/snapshot_sender.go

@@ -27,6 +27,7 @@ import (
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft"
 	"github.com/coreos/etcd/snap"
+	"github.com/dustin/go-humanize"
 )
 
 var (
@@ -75,7 +76,9 @@ func (s *snapshotSender) send(merged snap.Message) {
 	u := s.picker.pick()
 	req := createPostRequest(u, RaftSnapshotPrefix, body, "application/octet-stream", s.tr.URLs, s.from, s.cid)
 
-	plog.Infof("start to send database snapshot [index: %d, to %s]...", m.Snapshot.Metadata.Index, types.ID(m.To))
+	snapshotTotalSizeVal := uint64(merged.TotalSize)
+	snapshotTotalSize := humanize.Bytes(snapshotTotalSizeVal)
+	plog.Infof("start to send database snapshot [index: %d, to %s, size %s]...", m.Snapshot.Metadata.Index, types.ID(m.To), snapshotTotalSize)
 	snapshotSendInflights.WithLabelValues(to).Inc()
 	defer func() {
 		snapshotSendInflights.WithLabelValues(to).Dec()

scripts/build-binary

@@ -57,6 +57,11 @@ function main {
 	cd release
 	setup_env "${PROJ}" "${VER}"
 
+	if [[ $(go env GOOS) == "darwin" ]]; then
+		echo "Please use linux machine for release builds."
+		exit 1
+	fi
+
 	for os in darwin windows linux; do
 		export GOOS=${os}
 		TARGET_ARCHS=("amd64")

scripts/release

@@ -37,12 +37,6 @@ main() {
     exit 1
   fi
 
-  KEYID=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }')
-  if [[ -z "${KEYID}" ]]; then
-    echo "Failed to load gpg key. Is gpg set up correctly for etcd releases?"
-    exit 1
-  fi
-
   # Expected umask for etcd release artifacts
   umask 022
 
@@ -51,7 +45,7 @@ main() {
   if [ ! -d "${reldir}/etcd" ]; then
     mkdir -p "${reldir}"
     cd "${reldir}"
-    git clone git@github.com:etcd-io/etcd.git --branch "${BRANCH}"
+    git clone https://github.com/etcd-io/etcd.git --branch "${BRANCH}"
   fi
   cd "${reldir}/etcd"
 
@@ -112,9 +106,28 @@ main() {
       echo "Skipping tag step. git tag ${RELEASE_VERSION} already exists."
     else
       echo "Tagging release..."
+      KEYID=$(gpg --list-keys --with-colons| awk -F: '/^pub:/ { print $5 }')
+      if [[ -z "${KEYID}" ]]; then
+        echo "Failed to load gpg key. Is gpg set up correctly for etcd releases?"
+        exit 1
+      fi
       git tag --local-user "${KEYID}" --sign "${RELEASE_VERSION}" --message "${RELEASE_VERSION}"
     fi
 
+    # Verify the latest commit has the version tag
+    local tag="$(git describe --exact-match HEAD)"
+    if [ "${tag}" != "${RELEASE_VERSION}" ]; then
+      echo "Error: Expected HEAD to be tagged with ${RELEASE_VERSION}, but 'git describe --exact-match HEAD' reported: ${tag}"
+      exit 1
+    fi
+
+    # Verify the version tag is on the right branch
+    local branch=$(git branch --contains "${RELEASE_VERSION}")
+    if [ "${branch}" != "release-${MINOR_VERSION}" ]; then
+      echo "Error: Git tag ${RELEASE_VERSION} should be on branch release-${MINOR_VERSION} but is on ${branch}"
+      exit 1
+    fi
+
     # Push the tag change if it's not already been pushed.
     read -p "Push etcd ${RELEASE_VERSION} tag [y/N]? " confirm
     [[ "${confirm,,}" == "y" ]] || exit 1
@@ -181,6 +194,28 @@ main() {
     gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
   fi
 
+  ### Release validation
+  mkdir -p downloads
+
+  # Check image versions
+  for IMAGE in "quay.io/coreos/etcd:${RELEASE_VERSION}" "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"; do
+    local image_version=$(docker run --rm "${IMAGE}" etcd --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]')
+    if [ "${image_version}" != "${VERSION}" ]; then
+      echo "Check failed: etcd --version output for ${IMAGE} is incorrect: ${image_version}"
+      exit 1
+    fi
+  done
+
+  # Check gsutil binary versions
+  local BINARY_TGZ="etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64.tar.gz"
+  gsutil cp "gs://etcd/${RELEASE_VERSION}/${BINARY_TGZ}" downloads
+  tar -zx -C downloads -f "downloads/${BINARY_TGZ}"
+  local binary_version=$("./downloads/etcd-${RELEASE_VERSION}-$(go env GOOS)-amd64/etcd" --version | grep "etcd Version" | awk -F: '{print $2}' | tr -d '[:space:]')
+  if [ "${binary_version}" != "${VERSION}" ]; then
+    echo "Check failed: etcd --version output for ${BINARY_TGZ} from gs://etcd/${RELEASE_VERSION} is incorrect: ${binary_version}"
+    exit 1
+  fi
+
   # TODO: signing process
   echo ""
   echo "WARNING: The release has not been signed and published to github. This must be done manually."

scripts/updatebom.sh

@@ -10,19 +10,9 @@ fi
 echo "installing 'bill-of-materials.json'"
 go get -v -u github.com/coreos/license-bill-of-materials
 
-echo "setting up GOPATH"
-rm -rf ./gopath
-mkdir ./gopath
-mv ./cmd/vendor ./gopath/src
-
 echo "generating bill-of-materials.json"
-GOPATH=$(pwd)/gopath license-bill-of-materials \
+license-bill-of-materials \
     --override-file ./bill-of-materials.override.json \
     github.com/coreos/etcd github.com/coreos/etcd/etcdctl > bill-of-materials.json
 
-echo "reverting GOPATH,vendor"
-mv ./gopath/src ./cmd/vendor
-rm -rf ./gopath
-
 echo "generated bill-of-materials.json"
-

scripts/updatedep.sh

@@ -6,6 +6,17 @@ if ! [[ "$0" =~ scripts/updatedep.sh ]]; then
   exit 255
 fi
 
+# NOTE: just run
+rm -rf ./vendor
+glide install --strip-vendor --skip-test .
+glide vc --only-code --no-tests
+
+# ref. https://github.com/kubernetes/kubernetes/pull/81434
+exit 0
+
+
+
+
 if [ -d "gopath.proto" ]; then
   # gopath.proto is created by genproto.sh and it thoroughly messes
   # with go mod.

snap/snapshotter.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -31,7 +32,7 @@ import (
 	"github.com/coreos/etcd/raft"
 	"github.com/coreos/etcd/raft/raftpb"
 	"github.com/coreos/etcd/snap/snappb"
-
+	"github.com/coreos/etcd/wal/walpb"
 	"github.com/coreos/pkg/capnslog"
 )
 
@@ -80,9 +81,8 @@ func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
 	d, err := snap.Marshal()
 	if err != nil {
 		return err
-	} else {
-		marshallingDurations.Observe(float64(time.Since(start)) / float64(time.Second))
 	}
+	marshallingDurations.Observe(float64(time.Since(start)) / float64(time.Second))
 
 	err = pioutil.WriteAndSyncFile(filepath.Join(s.dir, fname), d, 0666)
 	if err == nil {
@@ -97,20 +97,35 @@ func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
 }
 
 func (s *Snapshotter) Load() (*raftpb.Snapshot, error) {
+	return s.loadMatching(func(*raftpb.Snapshot) bool { return true })
+}
+
+// LoadNewestAvailable loads the newest snapshot available that is in walSnaps.
+func (s *Snapshotter) LoadNewestAvailable(walSnaps []walpb.Snapshot) (*raftpb.Snapshot, error) {
+	return s.loadMatching(func(snapshot *raftpb.Snapshot) bool {
+		m := snapshot.Metadata
+		for i := len(walSnaps) - 1; i >= 0; i-- {
+			if m.Term == walSnaps[i].Term && m.Index == walSnaps[i].Index {
+				return true
+			}
+		}
+		return false
+	})
+}
+
+// loadMatching returns the newest snapshot where matchFn returns true.
+func (s *Snapshotter) loadMatching(matchFn func(*raftpb.Snapshot) bool) (*raftpb.Snapshot, error) {
 	names, err := s.snapNames()
 	if err != nil {
 		return nil, err
 	}
 	var snap *raftpb.Snapshot
 	for _, name := range names {
-		if snap, err = loadSnap(s.dir, name); err == nil {
-			break
+		if snap, err = loadSnap(s.dir, name); err == nil && matchFn(snap) {
+			return snap, nil
 		}
 	}
-	if err != nil {
-		return nil, ErrNoSnapshot
-	}
-	return snap, nil
+	return nil, ErrNoSnapshot
 }
 
 func loadSnap(dir, name string) (*raftpb.Snapshot, error) {
@@ -172,6 +187,10 @@ func (s *Snapshotter) snapNames() ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
+	names, err = s.cleanupSnapdir(names)
+	if err != nil {
+		return nil, err
+	}
 	snaps := checkSuffix(names)
 	if len(snaps) == 0 {
 		return nil, ErrNoSnapshot
@@ -202,3 +221,48 @@ func renameBroken(path string) {
 		plog.Warningf("cannot rename broken snapshot file %v to %v: %v", path, brokenPath, err)
 	}
 }
+
+// cleanupSnapdir removes any files that should not be in the snapshot directory:
+// - db.tmp prefixed files that can be orphaned by defragmentation
+func (s *Snapshotter) cleanupSnapdir(filenames []string) (names []string, err error) {
+	for _, filename := range filenames {
+		if strings.HasPrefix(filename, "db.tmp") {
+			plog.Infof("found orphaned defragmentation file; deleting: %s", filename)
+			if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) {
+				return nil, fmt.Errorf("failed to remove orphaned defragmentation file %s: %v", filename, rmErr)
+			}
+			continue
+		}
+		names = append(names, filename)
+	}
+	return names, nil
+}
+
+func (s *Snapshotter) ReleaseSnapDBs(snap raftpb.Snapshot) error {
+	dir, err := os.Open(s.dir)
+	if err != nil {
+		return err
+	}
+	defer dir.Close()
+	filenames, err := dir.Readdirnames(-1)
+	if err != nil {
+		return err
+	}
+	for _, filename := range filenames {
+		if strings.HasSuffix(filename, ".snap.db") {
+			hexIndex := strings.TrimSuffix(filepath.Base(filename), ".snap.db")
+			index, err := strconv.ParseUint(hexIndex, 16, 64)
+			if err != nil {
+				plog.Warningf("failed to parse index from filename: %s (%v)", filename, err)
+				continue
+			}
+			if index < snap.Metadata.Index {
+				plog.Infof("found orphaned .snap.db file; deleting %q", filename)
+				if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) {
+					plog.Warningf("failed to remove orphaned .snap.db file: %s (%v)", filename, rmErr)
+				}
+			}
+		}
+	}
+	return nil
+}

snap/snapshotter_test.go

@@ -23,7 +23,9 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/coreos/etcd/raft/raftpb"
+	"github.com/coreos/etcd/wal/walpb"
 )
 
 var testSnap = &raftpb.Snapshot{
@@ -165,12 +167,48 @@ func TestLoadNewestSnap(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	g, err := ss.Load()
-	if err != nil {
-		t.Errorf("err = %v, want nil", err)
+	cases := []struct {
+		name              string
+		availableWalSnaps []walpb.Snapshot
+		expected          *raftpb.Snapshot
+	}{
+		{
+			name:     "load-newest",
+			expected: &newSnap,
+		},
+		{
+			name:              "loadnewestavailable-newest",
+			availableWalSnaps: []walpb.Snapshot{{Index: 0, Term: 0}, {Index: 1, Term: 1}, {Index: 5, Term: 1}},
+			expected:          &newSnap,
+		},
+		{
+			name:              "loadnewestavailable-newest-unsorted",
+			availableWalSnaps: []walpb.Snapshot{{Index: 5, Term: 1}, {Index: 1, Term: 1}, {Index: 0, Term: 0}},
+			expected:          &newSnap,
+		},
+		{
+			name:              "loadnewestavailable-previous",
+			availableWalSnaps: []walpb.Snapshot{{Index: 0, Term: 0}, {Index: 1, Term: 1}},
+			expected:          testSnap,
+		},
 	}
-	if !reflect.DeepEqual(g, &newSnap) {
-		t.Errorf("snap = %#v, want %#v", g, &newSnap)
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			var err error
+			var g *raftpb.Snapshot
+			if tc.availableWalSnaps != nil {
+				g, err = ss.LoadNewestAvailable(tc.availableWalSnaps)
+			} else {
+				g, err = ss.Load()
+			}
+			if err != nil {
+				t.Errorf("err = %v, want nil", err)
+			}
+			if !reflect.DeepEqual(g, tc.expected) {
+				t.Errorf("snap = %#v, want %#v", g, tc.expected)
+			}
+		})
 	}
 }
 
@@ -228,3 +266,42 @@ func TestAllSnapshotBroken(t *testing.T) {
 		t.Errorf("err = %v, want %v", err, ErrNoSnapshot)
 	}
 }
+
+func TestReleaseSnapDBs(t *testing.T) {
+	dir := filepath.Join(os.TempDir(), "snapshot")
+	err := os.Mkdir(dir, 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	snapIndices := []uint64{100, 200, 300, 400}
+	for _, index := range snapIndices {
+		filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index))
+		if err := ioutil.WriteFile(filename, []byte("snap file\n"), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	ss := New(dir)
+
+	if err := ss.ReleaseSnapDBs(raftpb.Snapshot{Metadata: raftpb.SnapshotMetadata{Index: 300}}); err != nil {
+		t.Fatal(err)
+	}
+
+	deleted := []uint64{100, 200}
+	for _, index := range deleted {
+		filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index))
+		if fileutil.Exist(filename) {
+			t.Errorf("expected %s (index: %d)  to be deleted, but it still exists", filename, index)
+		}
+	}
+
+	retained := []uint64{300, 400}
+	for _, index := range retained {
+		filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index))
+		if !fileutil.Exist(filename) {
+			t.Errorf("expected %s (index: %d) to be retained, but it no longer exists", filename, index)
+		}
+	}
+}

tests/e2e/etcd_release_upgrade_test.go

@@ -103,6 +103,11 @@ func TestReleaseUpgrade(t *testing.T) {
 			}
 		}
 	}
+
+	// expect upgraded cluster version
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
 }
 
 func TestReleaseUpgradeWithRestart(t *testing.T) {

tests/e2e/metrics_test.go

@@ -0,0 +1,59 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/coreos/etcd/version"
+)
+
+func TestV3MetricsSecure(t *testing.T) {
+	cfg := configTLS
+	cfg.clusterSize = 1
+	cfg.metricsURLScheme = "https"
+	testCtl(t, metricsTest)
+}
+
+func TestV3MetricsInsecure(t *testing.T) {
+	cfg := configTLS
+	cfg.clusterSize = 1
+	cfg.metricsURLScheme = "http"
+	testCtl(t, metricsTest)
+}
+
+func metricsTest(cx ctlCtx) {
+	if err := ctlV3Put(cx, "k", "v", ""); err != nil {
+		cx.t.Fatal(err)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: `etcd_debugging_mvcc_keys_total 1`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+	ver := version.Version
+	if strings.HasSuffix(ver, "+git") {
+		ver = strings.Replace(ver, "+git", "", 1)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+	if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health":"true"}`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil {
+		cx.t.Fatalf("failed get with curl (%v)", err)
+	}
+}

vendor/github.com/bgentry/speakeasy/.gitignore

@@ -1,2 +0,0 @@
-example/example
-example/example.exe

vendor/github.com/bgentry/speakeasy/Readme.md

@@ -1,30 +0,0 @@
-# Speakeasy
-
-This package provides cross-platform Go (#golang) helpers for taking user input
-from the terminal while not echoing the input back (similar to `getpasswd`). The
-package uses syscalls to avoid any dependence on cgo, and is therefore
-compatible with cross-compiling.
-
-[![GoDoc](https://godoc.org/github.com/bgentry/speakeasy?status.png)][godoc]
-
-## Unicode
-
-Multi-byte unicode characters work successfully on Mac OS X. On Windows,
-however, this may be problematic (as is UTF in general on Windows). Other
-platforms have not been tested.
-
-## License
-
-The code herein was not written by me, but was compiled from two separate open
-source packages. Unix portions were imported from [gopass][gopass], while
-Windows portions were imported from the [CloudFoundry Go CLI][cf-cli]'s
-[Windows terminal helpers][cf-ui-windows].
-
-The [license for the windows portion](./LICENSE_WINDOWS) has been copied exactly
-from the source (though I attempted to fill in the correct owner in the
-boilerplate copyright notice).
-
-[cf-cli]: https://github.com/cloudfoundry/cli "CloudFoundry Go CLI"
-[cf-ui-windows]: https://github.com/cloudfoundry/cli/blob/master/src/cf/terminal/ui_windows.go "CloudFoundry Go CLI Windows input helpers"
-[godoc]: https://godoc.org/github.com/bgentry/speakeasy "speakeasy on Godoc.org"
-[gopass]: https://code.google.com/p/gopass "gopass"

vendor/github.com/coreos/bbolt/.gitignore

@@ -1,5 +0,0 @@
-*.prof
-*.test
-*.swp
-/bin/
-cmd/bolt/bolt

vendor/github.com/coreos/bbolt/Makefile

@@ -1,30 +0,0 @@
-BRANCH=`git rev-parse --abbrev-ref HEAD`
-COMMIT=`git rev-parse --short HEAD`
-GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)"
-
-default: build
-
-race:
-	@go test -v -race -test.run="TestSimulate_(100op|1000op)"
-
-fmt:
-	!(gofmt -l -s -d $(shell find . -name \*.go) | grep '[a-z]')
-
-# go get honnef.co/go/tools/simple
-gosimple:
-	gosimple ./...
-
-# go get honnef.co/go/tools/unused
-unused:
-	unused ./...
-
-# go get github.com/kisielk/errcheck
-errcheck:
-	@errcheck -ignorepkg=bytes -ignore=os:Remove github.com/coreos/bbolt
-
-test:
-	go test -timeout 20m -v -coverprofile cover.out -covermode atomic
-	# Note: gets "program not an importable package" in out of path builds
-	go test -v ./cmd/bolt
-
-.PHONY: race fmt errcheck test gosimple unused