version: 3.3.3

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

.gitignore

@@ -7,7 +7,6 @@
 /release
 /machine*
 /bin
-.Dockerfile-test
 .vagrant
 *.etcd
 *.log
@@ -15,8 +14,6 @@
 *.swp
 /hack/insta-discovery/.env
 *.test
-tools/functional-tester/docker/bin
-hack/scripts-dev/docker-dns/.Dockerfile
-hack/scripts-dev/docker-dns-srv/.Dockerfile
 hack/tls-setup/certs
 .idea
+*.bak

.semaphore.sh

@@ -2,7 +2,7 @@
 
 TEST_SUFFIX=$(date +%s | base64 | head -c 15)
 
-TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional' MANUAL_VER=v3.3.0-rc.0"
+TEST_OPTS="PASSES='build unit release integration_e2e functional' MANUAL_VER=v3.3.1"
 if [ "$TEST_ARCH" == "386" ]; then
 	TEST_OPTS="GOARCH=386 PASSES='build unit integration_e2e'"
 fi
@@ -10,7 +10,7 @@ fi
 docker run \
 	--rm \
 	--volume=`pwd`:/go/src/github.com/coreos/etcd \
-	gcr.io/etcd-development/etcd-test:go1.9.2 \
+	gcr.io/etcd-development/etcd-test:go1.9.4 \
 	/bin/bash -c "${TEST_OPTS} ./test 2>&1 | tee test-${TEST_SUFFIX}.log"
 
 ! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-${TEST_SUFFIX}.log

.travis.yml

@@ -6,7 +6,7 @@ sudo: required
 services: docker
 
 go:
-- 1.9.2
+- "1.9.4"
 - tip
 
 notifications:
@@ -30,7 +30,7 @@ matrix:
   - go: tip
     env: TARGET=amd64-go-tip
   exclude:
-  - go: 1.9.2
+  - go: "1.9.4"
     env: TARGET=amd64-go-tip
   - go: tip
     env: TARGET=amd64
@@ -48,17 +48,18 @@ matrix:
     env: TARGET=ppc64le
 
 before_install:
-- docker pull gcr.io/etcd-development/etcd-test:go1.9.2
+- if [[ $TRAVIS_GO_VERSION == 1.* ]]; then docker pull gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION}; fi
 
 install:
 - pushd cmd/etcd && go get -t -v ./... && popd
 
 script:
+ - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
  - >
     case "${TARGET}" in
       amd64)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GOARCH=amd64 ./test"
         ;;
       amd64-go-tip)
@@ -66,23 +67,23 @@ script:
         ;;
       darwin-amd64)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOOS=darwin GOARCH=amd64 ./build"
         ;;
       windows-amd64)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOOS=windows GOARCH=amd64 ./build"
         ;;
       386)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GOARCH=386 PASSES='build unit' ./test"
         ;;
       *)
         # test building out of gopath
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.2 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOARCH='${TARGET}' ./build"
         ;;
     esac

Dockerfile-functional-tester

@@ -0,0 +1,53 @@
+FROM ubuntu:17.10
+
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
+RUN apt-get -y update \
+  && apt-get -y install \
+  build-essential \
+  gcc \
+  apt-utils \
+  pkg-config \
+  software-properties-common \
+  apt-transport-https \
+  libssl-dev \
+  sudo \
+  bash \
+  curl \
+  wget \
+  tar \
+  git \
+  && apt-get -y update \
+  && apt-get -y upgrade \
+  && apt-get -y autoremove \
+  && apt-get -y autoclean
+
+ENV GOROOT /usr/local/go
+ENV GOPATH /go
+ENV PATH ${GOPATH}/bin:${GOROOT}/bin:${PATH}
+ENV GO_VERSION REPLACE_ME_GO_VERSION
+ENV GO_DOWNLOAD_URL https://storage.googleapis.com/golang
+RUN rm -rf ${GOROOT} \
+  && curl -s ${GO_DOWNLOAD_URL}/go${GO_VERSION}.linux-amd64.tar.gz | tar -v -C /usr/local/ -xz \
+  && mkdir -p ${GOPATH}/src ${GOPATH}/bin \
+  && go version
+
+RUN mkdir -p ${GOPATH}/src/github.com/coreos/etcd
+ADD . ${GOPATH}/src/github.com/coreos/etcd
+
+RUN go get -v github.com/coreos/gofail \
+  && pushd ${GOPATH}/src/github.com/coreos/etcd \
+  && GO_BUILD_FLAGS="-v" ./build \
+  && cp ./bin/etcd /etcd \
+  && cp ./bin/etcdctl /etcdctl \
+  && GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build \
+  && cp ./bin/etcd /etcd-failpoints \
+  && ./tools/functional-tester/build \
+  && cp ./bin/etcd-agent /etcd-agent \
+  && cp ./bin/etcd-tester /etcd-tester \
+  && cp ./bin/etcd-runner /etcd-runner \
+  && go build -v -o /benchmark ./cmd/tools/benchmark \
+  && go build -v -o /etcd-test-proxy ./cmd/tools/etcd-test-proxy \
+  && popd \
+  && rm -rf ${GOPATH}/src/github.com/coreos/etcd

Documentation/dev-guide/limit.md

@@ -6,5 +6,4 @@ etcd is designed to handle small key value pairs typical for metadata. Larger re
 
 ## Storage size limit
 
-The default storage size limit is 2GB, configurable with `--quota-backend-bytes` flag; supports up to 8GB.
-
+The default storage size limit is 2GB, configurable with `--quota-backend-bytes` flag. 8GB is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.

Documentation/faq.md

@@ -22,7 +22,7 @@ A member's advertised peer URLs come from `--initial-advertise-peer-urls` on ini
 
 ### System requirements
 
-Since etcd writes data to disk, SSD is highly recommended. To prevent performance degradation or unintentionally overloading the key-value store, etcd enforces a 2GB default storage size quota, configurable up to 8GB. To avoid swapping or running out of memory, the machine should have at least as much RAM to cover the quota. At CoreOS, an etcd cluster is usually deployed on dedicated CoreOS Container Linux machines with dual-core processors, 2GB of RAM, and 80GB of SSD *at the very least*. **Note that performance is intrinsically workload dependent; please test before production deployment**. See [hardware][hardware-setup] for more recommendations.
+Since etcd writes data to disk, SSD is highly recommended. To prevent performance degradation or unintentionally overloading the key-value store, etcd enforces a configurable storage size quota set to 2GB by default. To avoid swapping or running out of memory, the machine should have at least as much RAM to cover the quota. 8GB is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it. At CoreOS, an etcd cluster is usually deployed on dedicated CoreOS Container Linux machines with dual-core processors, 2GB of RAM, and 80GB of SSD *at the very least*. **Note that performance is intrinsically workload dependent; please test before production deployment**. See [hardware][hardware-setup] for more recommendations.
 
 Most stable production environment is Linux operating system with amd64 architecture; see [supported platform][supported-platform] for more.
 
@@ -102,6 +102,12 @@ To recover from the low space quota alarm:
 2. [Defragment][maintenance-defragment] every etcd endpoint.
 3. [Disarm][maintenance-disarm] the alarm.
 
+### What does the etcd warning "etcdserver/api/v3rpc: transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:2379->127.0.0.1:43020: read: connection reset by peer" mean?
+
+This is gRPC-side warning when a server receives a TCP RST flag with client-side streams being prematurely closed. For example, a client closes its connection, while gRPC server has not yet processed all HTTP/2 frames in the TCP queue. Some data may have been lost in server side, but it is ok so long as client connection has already been closed.
+
+Only [old versions of gRPC](https://github.com/grpc/grpc-go/issues/1362) log this. etcd [>=v3.2.13 by default log this with DEBUG level](https://github.com/coreos/etcd/pull/9080), thus only visible with `--debug` flag enabled.
+
 ## Performance
 
 ### How should I benchmark etcd?

Documentation/integrations.md

@@ -152,7 +152,6 @@
 - [mattn/etcdenv](https://github.com/mattn/etcdenv) - "env" shebang with etcd integration
 - [kelseyhightower/confd](https://github.com/kelseyhightower/confd) - Manage local app config files using templates and data from etcd
 - [configdb](https://git.autistici.org/ai/configdb/tree/master) - A REST relational abstraction on top of arbitrary database backends, aimed at storing configs and inventories.
-- [fleet](https://github.com/coreos/fleet) - Distributed init system
 - [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) - Container cluster manager introduced by Google.
 - [mailgun/vulcand](https://github.com/mailgun/vulcand) - HTTP proxy that uses etcd as a configuration backend.
 - [duedil-ltd/discodns](https://github.com/duedil-ltd/discodns) - Simple DNS nameserver using etcd as a database for names and records.

Documentation/learning/why.md

@@ -47,7 +47,7 @@ When considering features, support, and stability, new applications planning to
 
 ### Consul
 
-Consul bills itself as an end-to-end service discovery framework. To wit, it includes services such as health checking, failure detection, and DNS. Incidentally, Consul also exposes a key value store with mediocre performance and an intricate API. As it stands in Consul 0.7, the storage system does not scales well; systems requiring millions of keys will suffer from high latencies and memory pressure. The key value API is missing, most notably, multi-version keys, conditional transactions, and reliable streaming watches.
+Consul is an end-to-end service discovery framework. It provides built-in health checking, failure detection, and DNS services. In addition, Consul exposes a key value store with RESTful HTTP APIs. [As it stands in Consul 1.0][dbtester-comparison-results], the storage system does not scale as well as other systems like etcd or Zookeeper in key-value operations; systems requiring millions of keys will suffer from high latencies and memory pressure. The key value API is missing, most notably, multi-version keys, conditional transactions, and reliable streaming watches.
 
 etcd and Consul solve different problems. If looking for a distributed consistent key value store, etcd is a better choice over Consul. If looking for end-to-end cluster service discovery, etcd will not have enough features; choose Kubernetes, Consul, or SmartStack.
 
@@ -113,3 +113,4 @@ For distributed coordination, choosing etcd can help prevent operational headach
 [container-linux]: https://coreos.com/why
 [locksmith]: https://github.com/coreos/locksmith
 [kubernetes]: http://kubernetes.io/docs/whatisk8s
+[dbtester-comparison-results]: https://github.com/coreos/dbtester/tree/master/test-results/2018Q1-02-etcd-zookeeper-consul

Documentation/op-guide/configuration.md

@@ -1,6 +1,11 @@
 # Configuration flags
 
-etcd is configurable through command-line flags and environment variables. Options set on the command line take precedence over those from the environment.
+etcd is configurable through a configuration file, various command-line flags, and environment variables.
+
+A reusable configuration file is a YAML file made with name and value of one or more command-line flags described below. In order to use this file, specify the file path as a value to the `--config-file` flag. The [sample configuration file][sample-config-file] can be used as a starting point to create a new configuration file as needed.
+
+Options set on the command line take precedence over those from the environment. If a configuration file is provided, other command line flags and environment variables will be ignored.
+For example, `etcd --config-file etcd.conf.yml.sample --data-dir /tmp` will ignore the `--data-dir` flag.
 
 The format of environment variable for flag `--my-flag` is `ETCD_MY_FLAG`. It applies to all flags.
 
@@ -266,12 +271,12 @@ The security flags help to [build a secure etcd cluster][security].
 + env variable: ETCD_PEER_CA_FILE
 
 ### --peer-cert-file
-+ Path to the peer server TLS cert file.
++ Path to the peer server TLS cert file. This is the cert for peer-to-peer traffic, used both for server and client.
 + default: ""
 + env variable: ETCD_PEER_CERT_FILE
 
 ### --peer-key-file
-+ Path to the peer server TLS key file.
++ Path to the peer server TLS key file. This is the key for peer-to-peer traffic, used both for server and client.
 + default: ""
 + env variable: ETCD_PEER_KEY_FILE
 
@@ -332,6 +337,7 @@ Follow the instructions when using these flags.
 ### --config-file
 + Load server configuration from a file.
 + default: ""
++ example: [sample configuration file][sample-config-file]
 
 ## Profiling flags
 
@@ -369,3 +375,4 @@ Follow the instructions when using these flags.
 [security]: security.md
 [systemd-intro]: http://freedesktop.org/wiki/Software/systemd/
 [tuning]: ../tuning.md#time-parameters
+[sample-config-file]: ../../etcd.conf.yml.sample

Documentation/op-guide/container.md

@@ -17,14 +17,14 @@ export NODE1=192.168.1.21
 Trust the CoreOS [App Signing Key](https://coreos.com/security/app-signing-key/).
 
 ```
-sudo rkt trust --prefix coreos.com/etcd
+sudo rkt trust --prefix quay.io/coreos/etcd
 # gpg key fingerprint is: 18AD 5014 C99E F7E3 BA5F  6CE9 50BD D3E0 FC8A 365E
 ```
 
-Run the `v3.1.2` version of etcd or specify another release version.
+Run the `v3.2` version of etcd or specify another release version.
 
 ```
-sudo rkt run --net=default:IP=${NODE1} coreos.com/etcd:v3.1.2 -- -name=node1 -advertise-client-urls=http://${NODE1}:2379 -initial-advertise-peer-urls=http://${NODE1}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE1}:2380 -initial-cluster=node1=http://${NODE1}:2380
+sudo rkt run --net=default:IP=${NODE1} quay.io/coreos/etcd:v3.2 -- -name=node1 -advertise-client-urls=http://${NODE1}:2379 -initial-advertise-peer-urls=http://${NODE1}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE1}:2380 -initial-cluster=node1=http://${NODE1}:2380
 ```
 
 List the cluster member.
@@ -45,13 +45,13 @@ export NODE3=172.16.28.23
 
 ```
 # node 1
-sudo rkt run --net=default:IP=${NODE1} coreos.com/etcd:v3.1.2 -- -name=node1 -advertise-client-urls=http://${NODE1}:2379 -initial-advertise-peer-urls=http://${NODE1}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE1}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
+sudo rkt run --net=default:IP=${NODE1} quay.io/coreos/etcd:v3.2 -- -name=node1 -advertise-client-urls=http://${NODE1}:2379 -initial-advertise-peer-urls=http://${NODE1}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE1}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
 
 # node 2
-sudo rkt run --net=default:IP=${NODE2} coreos.com/etcd:v3.1.2 -- -name=node2 -advertise-client-urls=http://${NODE2}:2379 -initial-advertise-peer-urls=http://${NODE2}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE2}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
+sudo rkt run --net=default:IP=${NODE2} quay.io/coreos/etcd:v3.2 -- -name=node2 -advertise-client-urls=http://${NODE2}:2379 -initial-advertise-peer-urls=http://${NODE2}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE2}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
 
 # node 3
-sudo rkt run --net=default:IP=${NODE3} coreos.com/etcd:v3.1.2 -- -name=node3 -advertise-client-urls=http://${NODE3}:2379 -initial-advertise-peer-urls=http://${NODE3}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE3}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
+sudo rkt run --net=default:IP=${NODE3} quay.io/coreos/etcd:v3.2 -- -name=node3 -advertise-client-urls=http://${NODE3}:2379 -initial-advertise-peer-urls=http://${NODE3}:2380 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://${NODE3}:2380 -initial-cluster=node1=http://${NODE1}:2380,node2=http://${NODE2}:2380,node3=http://${NODE3}:2380
 ```
 
 Verify the cluster is healthy and can be reached.

Documentation/op-guide/etcd3_alert.rules

@@ -43,8 +43,8 @@ ANNOTATIONS {
 
 # alert if more than 1% of gRPC method calls have failed within the last 5 minutes
 ALERT HighNumberOfFailedGRPCRequests
-IF sum by(grpc_method) (rate(etcd_grpc_requests_failed_total{job="etcd"}[5m]))
-  / sum by(grpc_method) (rate(etcd_grpc_total{job="etcd"}[5m])) > 0.01
+IF 100 * (sum by(grpc_method) (rate(etcd_grpc_requests_failed_total{job="etcd"}[5m]))
+  / sum by(grpc_method) (rate(etcd_grpc_total{job="etcd"}[5m]))) > 1
 FOR 10m
 LABELS {
   severity = "warning"
@@ -56,8 +56,8 @@ ANNOTATIONS {
 
 # alert if more than 5% of gRPC method calls have failed within the last 5 minutes
 ALERT HighNumberOfFailedGRPCRequests
-IF sum by(grpc_method) (rate(etcd_grpc_requests_failed_total{job="etcd"}[5m]))
-  / sum by(grpc_method) (rate(etcd_grpc_total{job="etcd"}[5m])) > 0.05
+IF 100 * (sum by(grpc_method) (rate(etcd_grpc_requests_failed_total{job="etcd"}[5m]))
+  / sum by(grpc_method) (rate(etcd_grpc_total{job="etcd"}[5m]))) > 5
 FOR 5m
 LABELS {
   severity = "critical"
@@ -84,8 +84,8 @@ ANNOTATIONS {
 
 # alert if more than 1% of requests to an HTTP endpoint have failed within the last 5 minutes
 ALERT HighNumberOfFailedHTTPRequests
-IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
-  / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
+IF 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+  / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method)) > 1
 FOR 10m
 LABELS {
   severity = "warning"
@@ -97,8 +97,8 @@ ANNOTATIONS {
 
 # alert if more than 5% of requests to an HTTP endpoint have failed within the last 5 minutes
 ALERT HighNumberOfFailedHTTPRequests
-IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
-  / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
+IF 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+  / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method))  > 5
 FOR 5m
 LABELS {
   severity = "critical"

Documentation/op-guide/etcd3_alert.rules.yml

@@ -26,8 +26,8 @@ groups:
         changes within the last hour
       summary: a high number of leader changes within the etcd cluster are happening
   - alert: HighNumberOfFailedGRPCRequests
-    expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
-      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
+    expr: 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method)) > 1
     for: 10m
     labels:
       severity: warning
@@ -36,8 +36,8 @@ groups:
         on etcd instance {{ $labels.instance }}'
       summary: a high number of gRPC requests are failing
   - alert: HighNumberOfFailedGRPCRequests
-    expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
-      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
+    expr: 100 * (sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method)) > 5
     for: 5m
     labels:
       severity: critical
@@ -56,8 +56,8 @@ groups:
         }} are slow
       summary: slow gRPC requests
   - alert: HighNumberOfFailedHTTPRequests
-    expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
-      BY (method) > 0.01
+    expr: 100 * (sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
+      BY (method)) > 1
     for: 10m
     labels:
       severity: warning
@@ -66,8 +66,8 @@ groups:
         instance {{ $labels.instance }}'
       summary: a high number of HTTP requests are failing
   - alert: HighNumberOfFailedHTTPRequests
-    expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
-      BY (method) > 0.05
+    expr: 100 * (sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
+      BY (method)) > 5
     for: 5m
     labels:
       severity: critical

Documentation/op-guide/hardware.md

@@ -48,7 +48,7 @@ Example application workload: A 50-node Kubernetes cluster
 | Provider | Type | vCPUs | Memory (GB) | Max concurrent IOPS | Disk bandwidth (MB/s) |
 |----------|------|-------|--------|------|----------------|
 | AWS | m4.large | 2 | 8 | 3600 | 56.25 |
-| GCE | n1-standard-1 + 50GB PD SSD | 2 | 7.5 | 1500 | 25 |
+| GCE | n1-standard-2 + 50GB PD SSD | 2 | 7.5 | 1500 | 25 |
 
 
 ### Medium cluster

Documentation/op-guide/maintenance.md

@@ -36,9 +36,9 @@ Error:  rpc error: code = 11 desc = etcdserver: mvcc: required revision has been
 
 ## Defragmentation
 
-After compacting the keyspace, the backend database may exhibit internal fragmentation. Any internal fragmentation is space that is free to use by the backend but still consumes storage space. The process of defragmentation releases this storage space back to the file system. Defragmentation is issued on a per-member so that cluster-wide latency spikes may be avoided.
+After compacting the keyspace, the backend database may exhibit internal fragmentation. Any internal fragmentation is space that is free to use by the backend but still consumes storage space. Compacting old revisions internally fragments `etcd` by leaving gaps in backend database. Fragmented space is available for use by `etcd` but unavailable to the host filesystem. In other words, deleting application data does not reclaim the space on disk.
 
-Compacting old revisions internally fragments `etcd` by leaving gaps in backend database. Fragmented space is available for use by `etcd` but unavailable to the host filesystem.
+The process of defragmentation releases this storage space back to the file system. Defragmentation is issued on a per-member so that cluster-wide latency spikes may be avoided.
 
 To defragment an etcd member, use the `etcdctl defrag` command:
 
@@ -47,6 +47,10 @@ $ etcdctl defrag
 Finished defragmenting etcd member[127.0.0.1:2379]
 ```
 
+**Note that defragmentation to a live member blocks the system from reading and writing data while rebuilding its states**.
+
+**Note that defragmentation request does not get replicated over cluster. That is, the request is only applied to the local node. Specify all members in `--endpoints` flag.**
+
 To defragment an etcd data directory directly, while etcd is not running, use the command:
 
 ``` sh
@@ -80,7 +84,7 @@ $ ETCDCTL_API=3 etcdctl --write-out=table endpoint status
 +----------------+------------------+-----------+---------+-----------+-----------+------------+
 # confirm alarm is raised
 $ ETCDCTL_API=3 etcdctl alarm list
-memberID:13803658152347727308 alarm:NOSPACE 
+memberID:13803658152347727308 alarm:NOSPACE
 ```
 
 Removing excessive keyspace data and defragmenting the backend database will put the cluster back within the quota limits:
@@ -96,7 +100,7 @@ $ ETCDCTL_API=3 etcdctl defrag
 Finished defragmenting etcd member[127.0.0.1:2379]
 # disarm alarm
 $ ETCDCTL_API=3 etcdctl alarm disarm
-memberID:13803658152347727308 alarm:NOSPACE 
+memberID:13803658152347727308 alarm:NOSPACE
 # test puts are allowed again
 $ ETCDCTL_API=3 etcdctl put newkey 123
 OK

Documentation/op-guide/security.md

@@ -195,9 +195,9 @@ When client authentication is enabled for an etcd member, the administrator must
 
 ## Notes for TLS authentication
 
-Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017-06-09), [TLS certificates get reloaded on every client connection](https://github.com/coreos/etcd/pull/7829). This is useful when replacing expiry certs without stopping etcd servers; it can be done by overwriting old certs with new ones. Refreshing certs for every connection should not have too much overhead, but can be improved in the future, with caching layer. Example tests can be found [here](https://github.com/coreos/etcd/blob/b041ce5d514a4b4aaeefbffb008f0c7570a18986/integration/v3_grpc_test.go#L1601-L1757).
+Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v320-2017-06-09), [TLS certificates get reloaded on every client connection](https://github.com/coreos/etcd/pull/7829). This is useful when replacing expiry certs without stopping etcd servers; it can be done by overwriting old certs with new ones. Refreshing certs for every connection should not have too much overhead, but can be improved in the future, with caching layer. Example tests can be found [here](https://github.com/coreos/etcd/blob/b041ce5d514a4b4aaeefbffb008f0c7570a18986/integration/v3_grpc_test.go#L1601-L1757).
 
-Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017-06-09), [server denies incoming peer certs with wrong IP `SAN`](https://github.com/coreos/etcd/pull/7687). For instance, if peer cert contains any IP addresses in Subject Alternative Name (SAN) field, server authenticates a peer only when the remote IP address matches one of those IP addresses. This is to prevent unauthorized endpoints from joining the cluster. For example, peer B's CSR (with `cfssl`) is:
+Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v320-2017-06-09), [server denies incoming peer certs with wrong IP `SAN`](https://github.com/coreos/etcd/pull/7687). For instance, if peer cert contains any IP addresses in Subject Alternative Name (SAN) field, server authenticates a peer only when the remote IP address matches one of those IP addresses. This is to prevent unauthorized endpoints from joining the cluster. For example, peer B's CSR (with `cfssl`) is:
 
 ```json
 {
@@ -223,7 +223,7 @@ Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017
 
 when peer B's actual IP address is `10.138.0.2`, not `10.138.0.27`. When peer B tries to join the cluster, peer A will reject B with the error `x509: certificate is valid for 10.138.0.27, not 10.138.0.2`, because B's remote IP address does not match the one in Subject Alternative Name (SAN) field.
 
-Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017-06-09), [server resolves TLS `DNSNames` when checking `SAN`](https://github.com/coreos/etcd/pull/7767). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server authenticates a peer only when forward-lookups (`dig b.com`) on those DNS names have matching IP with the remote IP address. For example, peer B's CSR (with `cfssl`) is:
+Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v320-2017-06-09), [server resolves TLS `DNSNames` when checking `SAN`](https://github.com/coreos/etcd/pull/7767). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server authenticates a peer only when forward-lookups (`dig b.com`) on those DNS names have matching IP with the remote IP address. For example, peer B's CSR (with `cfssl`) is:
 
 ```json
 {
@@ -235,7 +235,7 @@ Since [v3.2.0](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v320-2017
 
 when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A looks up the incoming host `b.com` to get the list of IP addresses (e.g. `dig b.com`). And rejects B if the list does not contain the IP `10.138.0.2`, with the error `tls: 10.138.0.2 does not match any of DNSNames ["b.com"]`.
 
-Since [v3.2.2](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v322-2017-07-07), [server accepts connections if IP matches, without checking DNS entries](https://github.com/coreos/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. For example, peer B's CSR (with `cfssl`) is:
+Since [v3.2.2](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v322-2017-07-07), [server accepts connections if IP matches, without checking DNS entries](https://github.com/coreos/etcd/pull/8223). For instance, if peer cert contains IP addresses and DNS names in Subject Alternative Name (SAN) field, and the remote IP address matches one of those IP addresses, server just accepts connection without further checking the DNS names. For example, peer B's CSR (with `cfssl`) is:
 
 ```json
 {
@@ -248,7 +248,7 @@ Since [v3.2.2](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v322-2017
 
 when peer B's remote IP address is `10.138.0.2` and `invalid.domain` is a invalid host. When peer B tries to join the cluster, peer A successfully authenticates B, since Subject Alternative Name (SAN) field has a valid matching IP address. See [issue#8206](https://github.com/coreos/etcd/issues/8206) for more detail.
 
-Since [v3.2.5](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v325-2017-08-04), [server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/coreos/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. For example, peer B's CSR (with `cfssl`) is:
+Since [v3.2.5](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.2.md#v325-2017-08-04), [server supports reverse-lookup on wildcard DNS `SAN`](https://github.com/coreos/etcd/pull/8281). For instance, if peer cert contains only DNS names (no IP addresses) in Subject Alternative Name (SAN) field, server first reverse-lookups the remote IP address to get a list of names mapping to that address (e.g. `nslookup IPADDR`). Then accepts the connection if those names have a matching name with peer cert's DNS names (either by exact or wildcard match). If none is matched, server forward-lookups each DNS entry in peer cert (e.g. look up `example.default.svc` when the entry is `*.example.default.svc`), and accepts connection only when the host's resolved addresses have the matching IP address with the peer's remote IP address. For example, peer B's CSR (with `cfssl`) is:
 
 ```json
 {
@@ -261,6 +261,66 @@ Since [v3.2.5](https://github.com/coreos/etcd/blob/master/CHANGELOG.md#v325-2017
 
 when peer B's remote IP address is `10.138.0.2`. When peer B tries to join the cluster, peer A reverse-lookup the IP `10.138.0.2` to get the list of host names. And either exact or wildcard match the host names with peer B's cert DNS names in Subject Alternative Name (SAN) field. If none of reverse/forward lookups worked, it returns an error `"tls: "10.138.0.2" does not match any of DNSNames ["*.example.default.svc","*.example.default.svc.cluster.local"]`. See [issue#8268](https://github.com/coreos/etcd/issues/8268) for more detail.
 
+[v3.3.0](https://github.com/coreos/etcd/blob/master/CHANGELOG-3.3.md) adds [`etcd --peer-cert-allowed-cn`](https://github.com/coreos/etcd/pull/8616) flag to support [CN(Common Name)-based auth for inter-peer connections](https://github.com/coreos/etcd/issues/8262). Kubernetes TLS bootstrapping involves generating dynamic certificates for etcd members and other system components (e.g. API server, kubelet, etc.). Maintaining different CAs for each component provides tighter access control to etcd cluster but often tedious. When `--peer-cert-allowed-cn` flag is specified, node can only join with matching common name even with shared CAs. For example, each member in 3-node cluster is set up with CSRs (with `cfssl`) as below:
+
+```json
+{
+  "CN": "etcd.local",
+  "hosts": [
+    "m1.etcd.local",
+    "127.0.0.1",
+    "localhost"
+  ],
+```
+
+```json
+{
+  "CN": "etcd.local",
+  "hosts": [
+    "m2.etcd.local",
+    "127.0.0.1",
+    "localhost"
+  ],
+```
+
+```json
+{
+  "CN": "etcd.local",
+  "hosts": [
+    "m3.etcd.local",
+    "127.0.0.1",
+    "localhost"
+  ],
+```
+
+Then only peers with matching common names will be authenticated if `--peer-cert-allowed-cn etcd.local` is given. And nodes with different CNs in CSRs or different `--peer-cert-allowed-cn` will be rejected:
+
+```bash
+$ etcd --peer-cert-allowed-cn m1.etcd.local
+
+I | embed: rejected connection from "127.0.0.1:48044" (error "CommonName authentication failed", ServerName "m1.etcd.local")
+I | embed: rejected connection from "127.0.0.1:55702" (error "remote error: tls: bad certificate", ServerName "m3.etcd.local")
+```
+
+Each process should be started with:
+
+```bash
+etcd --peer-cert-allowed-cn etcd.local
+
+I | pkg/netutil: resolving m3.etcd.local:32380 to 127.0.0.1:32380
+I | pkg/netutil: resolving m2.etcd.local:22380 to 127.0.0.1:22380
+I | pkg/netutil: resolving m1.etcd.local:2380 to 127.0.0.1:2380
+I | etcdserver: published {Name:m3 ClientURLs:[https://m3.etcd.local:32379]} to cluster 9db03f09b20de32b
+I | embed: ready to serve client requests
+I | etcdserver: published {Name:m1 ClientURLs:[https://m1.etcd.local:2379]} to cluster 9db03f09b20de32b
+I | embed: ready to serve client requests
+I | etcdserver: published {Name:m2 ClientURLs:[https://m2.etcd.local:22379]} to cluster 9db03f09b20de32b
+I | embed: ready to serve client requests
+I | embed: serving client requests on 127.0.0.1:32379
+I | embed: serving client requests on 127.0.0.1:22379
+I | embed: serving client requests on 127.0.0.1:2379
+```
+
 ## Frequently asked questions
 
 ### I'm seeing a SSLv3 alert handshake failure when using TLS client authentication?

Documentation/upgrades/upgrade_3_0.md

@@ -8,6 +8,8 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 ### Upgrade checklists
 
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
 #### Upgrade requirements
 
 To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.8) before upgrading to 3.0.

Documentation/upgrades/upgrade_3_1.md

@@ -8,6 +8,8 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 ### Upgrade checklists
 
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
 #### Monitoring
 
 Following metrics from v3.0.x have been deprecated in favor of [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus):

Documentation/upgrades/upgrade_3_2.md

@@ -8,8 +8,14 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 ### Upgrade checklists
 
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
 Highlighted breaking changes in 3.2.
 
+#### Change in default `snapshot-count` value
+
+The default value of `--snapshot-count` has [changed from from 10,000 to 100,000](https://github.com/coreos/etcd/pull/7160). Higher snapshot count means it holds Raft entries in memory for longer before discarding old entries. It is a trade-off between less frequent snapshotting and [higher memory usage](https://github.com/kubernetes/kubernetes/issues/60589#issuecomment-371977156). Higher `--snapshot-count` will be manifested with higher memory usage, while retaining more Raft entries helps with the availabilities of slow followers: leader is still able to replicate its logs to followers, rather than forcing followers to rebuild its stores from leader snapshots.
+
 #### Change in gRPC dependency (>=3.2.10)
 
 3.2.10 or later now requires [grpc/grpc-go](https://github.com/grpc/grpc-go/releases) `v1.7.5` (<=3.2.9 requires `v1.2.1`).
@@ -66,7 +72,7 @@ if err == context.DeadlineExceeded {
 
 #### Change in maximum request size limits (>=3.2.10)
 
-3.2.10 and 3.2.11 allow custom request size limits in server side. >=3.2.12 allows custom request size limits for both server and **client side**.
+3.2.10 and 3.2.11 allow custom request size limits in server side. >=3.2.12 allows custom request size limits for both server and **client side**. In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB.
 
 Server-side request limits can be configured with `--max-request-bytes` flag:
 
@@ -160,12 +166,6 @@ Before and after
 +func NewWatchFromWatchClient(wc pb.WatchClient, c *Client) Watcher {
 ```
 
-#### Change in `--listen-peer-urls` and `--listen-client-urls`
-
-3.2 now rejects domains names for `--listen-peer-urls` and `--listen-client-urls` (3.1 only prints out warnings), since domain name is invalid for network interface binding. Make sure that those URLs are properly formated as `scheme://IP:port`.
-
-See [issue #6336](https://github.com/coreos/etcd/issues/6336) for more contexts.
-
 #### Change in `clientv3.Lease.TimeToLive` API
 
 Previously, `clientv3.Lease.TimeToLive` API returned `lease.ErrLeaseNotFound` on non-existent lease ID. 3.2 instead returns TTL=-1 in its response and no error (see [#7305](https://github.com/coreos/etcd/pull/7305)).
@@ -206,6 +206,12 @@ import clientv3yaml "github.com/coreos/etcd/clientv3/yaml"
 clientv3yaml.NewConfig
 ```
 
+#### Change in `--listen-peer-urls` and `--listen-client-urls`
+
+3.2 now rejects domains names for `--listen-peer-urls` and `--listen-client-urls` (3.1 only prints out warnings), since domain name is invalid for network interface binding. Make sure that those URLs are properly formated as `scheme://IP:port`.
+
+See [issue #6336](https://github.com/coreos/etcd/issues/6336) for more contexts.
+
 ### Server upgrade checklists
 
 #### Upgrade requirements

Documentation/upgrades/upgrade_3_3.md

@@ -8,6 +8,8 @@ Before [starting an upgrade](#upgrade-procedure), read through the rest of this
 
 ### Upgrade checklists
 
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
 Highlighted breaking changes in 3.3.
 
 #### Change in `etcdserver.EtcdServer` struct
@@ -72,27 +74,17 @@ cfg.SetupLogging()
 
 Set `embed.Config.Debug` field to `true` to enable gRPC server logs.
 
-#### Change in `/health` endpoint response value
+#### Change in `/health` endpoint response
 
-Previously, `[endpoint]:[client-port]/health` returned manually marshaled JSON value. 3.3 now defines [`etcdhttp.Health`](https://godoc.org/github.com/coreos/etcd/etcdserver/api/etcdhttp#Health) struct and includes errors, if any.
+Previously, `[endpoint]:[client-port]/health` returned manually marshaled JSON value. 3.3 now defines [`etcdhttp.Health`](https://godoc.org/github.com/coreos/etcd/etcdserver/api/etcdhttp#Health) struct.
 
-Before
+Note that in v3.3.0-rc.0, v3.3.0-rc.1, and v3.3.0-rc.2, `etcdhttp.Health` has boolean type `"health"` and `"errors"` fields. For backward compatibilities, we reverted `"health"` field to `string` type and removed `"errors"` field. Further health information will be provided in separate APIs.
 
 ```bash
 $ curl http://localhost:2379/health
 {"health":"true"}
 ```
 
-After
-
-```bash
-$ curl http://localhost:2379/health
-{"health":"true"}
-
-# Or
-{"health":"false","errors":["NOSPACE"]}
-```
-
 #### Change in gRPC gateway HTTP endpoints (replaced `/v3alpha` with `/v3beta`)
 
 Before
@@ -113,7 +105,7 @@ Requests to `/v3alpha` endpoints will redirect to `/v3beta`, and `/v3alpha` will
 
 #### Change in maximum request size limits
 
-3.3 now allows custom request size limits for both server and **client side**.
+3.3 now allows custom request size limits for both server and **client side**. In previous versions(v3.2.10, v3.2.11), client response size was limited to only 4 MiB.
 
 Server-side request limits can be configured with `--max-request-bytes` flag:
 

Documentation/upgrades/upgrade_3_4.md

@@ -0,0 +1,171 @@
+## Upgrade etcd from 3.3 to 3.4
+
+In the general case, upgrading from etcd 3.3 to 3.4 can be a zero-downtime, rolling upgrade:
+ - one by one, stop the etcd v3.3 processes and replace them with etcd v3.4 processes
+ - after running all v3.4 processes, new features in v3.4 are available to the cluster
+
+Before [starting an upgrade](#upgrade-procedure), read through the rest of this guide to prepare.
+
+### Upgrade checklists
+
+**NOTE:** When [migrating from v2 with no v3 data](https://github.com/coreos/etcd/issues/9480), etcd server v3.2+ panics when etcd restores from existing snapshots but no v3 `ETCD_DATA_DIR/member/snap/db` file. This happens when the server had migrated from v2 with no previous v3 data. This also prevents accidental v3 data loss (e.g. `db` file might have been moved). etcd requires that post v3 migration can only happen with v3 data. Do not upgrade to newer v3 versions until v3.0 server contains v3 data.
+
+Highlighted breaking changes in 3.4.
+
+#### Change in `etcd` flags
+
+`--ca-file` and `--peer-ca-file` flags are deprecated; they have been deprecated since v2.1.
+
+```diff
+-etcd --ca-file ca-client.crt
++etcd --trusted-ca-file ca-client.crt
+```
+
+```diff
+-etcd --peer-ca-file ca-peer.crt
++etcd --peer-trusted-ca-file ca-peer.crt
+```
+
+#### Change in ``pkg/transport`
+
+Deprecated `pkg/transport.TLSInfo.CAFile` field.
+
+```diff
+import "github.com/coreos/etcd/pkg/transport"
+
+tlsInfo := transport.TLSInfo{
+    CertFile: "/tmp/test-certs/test.pem",
+    KeyFile: "/tmp/test-certs/test-key.pem",
+-   CAFile: "/tmp/test-certs/trusted-ca.pem",
++   TrustedCAFile: "/tmp/test-certs/trusted-ca.pem",
+}
+tlsConfig, err := tlsInfo.ClientConfig()
+if err != nil {
+    panic(err)
+}
+```
+
+### Server upgrade checklists
+
+#### Upgrade requirements
+
+To upgrade an existing etcd deployment to 3.4, the running cluster must be 3.3 or greater. If it's before 3.3, please [upgrade to 3.3](upgrade_3_3.md) before upgrading to 3.4.
+
+Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding.
+
+#### Preparation
+
+Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment.
+
+Before beginning, [backup the etcd data](../op-guide/maintenance.md#snapshot-backup). Should something go wrong with the upgrade, it is possible to use this backup to [downgrade](#downgrade) back to existing etcd version. Please note that the `snapshot` command only backs up the v3 data. For v2 data, see [backing up v2 datastore](../v2/admin_guide.md#backing-up-the-datastore).
+
+#### Mixed versions
+
+While upgrading, an etcd cluster supports mixed versions of etcd members, and operates with the protocol of the lowest common version. The cluster is only considered upgraded once all of its members are upgraded to version 3.4. Internally, etcd members negotiate with each other to determine the overall cluster version, which controls the reported version and the supported features.
+
+#### Limitations
+
+Note: If the cluster only has v3 data and no v2 data, it is not subject to this limitation.
+
+If the cluster is serving a v2 data set larger than 50MB, each newly upgraded member may take up to two minutes to catch up with the existing cluster. Check the size of a recent snapshot to estimate the total data size. In other words, it is safest to wait for 2 minutes between upgrading each member.
+
+For a much larger total data size, 100MB or more , this one-time process might take even more time. Administrators of very large etcd clusters of this magnitude can feel free to contact the [etcd team][etcd-contact] before upgrading, and we'll be happy to provide advice on the procedure.
+
+#### Downgrade
+
+If all members have been upgraded to v3.4, the cluster will be upgraded to v3.4, and downgrade from this completed state is **not possible**. If any single member is still v3.3, however, the cluster and its operations remains "v3.3", and it is possible from this mixed cluster state to return to using a v3.3 etcd binary on all members.
+
+Please [backup the data directory](../op-guide/maintenance.md#snapshot-backup) of all etcd members to make downgrading the cluster possible even after it has been completely upgraded.
+
+### Upgrade procedure
+
+This example shows how to upgrade a 3-member v3.3 ectd cluster running on a local machine.
+
+#### 1. Check upgrade requirements
+
+Is the cluster healthy and running v3.3.x?
+
+```
+$ ETCDCTL_API=3 etcdctl endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+localhost:2379 is healthy: successfully committed proposal: took = 6.600684ms
+localhost:22379 is healthy: successfully committed proposal: took = 8.540064ms
+localhost:32379 is healthy: successfully committed proposal: took = 8.763432ms
+
+$ curl http://localhost:2379/version
+{"etcdserver":"3.3.0","etcdcluster":"3.3.0"}
+```
+
+#### 2. Stop the existing etcd process
+
+When each etcd process is stopped, expected errors will be logged by other cluster members. This is normal since a cluster member connection has been (temporarily) broken:
+
+```
+14:13:31.491746 I | raft: c89feb932daef420 [term 3] received MsgTimeoutNow from 6d4f535bae3ab960 and starts an election to get leadership.
+14:13:31.491769 I | raft: c89feb932daef420 became candidate at term 4
+14:13:31.491788 I | raft: c89feb932daef420 received MsgVoteResp from c89feb932daef420 at term 4
+14:13:31.491797 I | raft: c89feb932daef420 [logterm: 3, index: 9] sent MsgVote request to 6d4f535bae3ab960 at term 4
+14:13:31.491805 I | raft: c89feb932daef420 [logterm: 3, index: 9] sent MsgVote request to 9eda174c7df8a033 at term 4
+14:13:31.491815 I | raft: raft.node: c89feb932daef420 lost leader 6d4f535bae3ab960 at term 4
+14:13:31.524084 I | raft: c89feb932daef420 received MsgVoteResp from 6d4f535bae3ab960 at term 4
+14:13:31.524108 I | raft: c89feb932daef420 [quorum:2] has received 2 MsgVoteResp votes and 0 vote rejections
+14:13:31.524123 I | raft: c89feb932daef420 became leader at term 4
+14:13:31.524136 I | raft: raft.node: c89feb932daef420 elected leader c89feb932daef420 at term 4
+14:13:31.592650 W | rafthttp: lost the TCP streaming connection with peer 6d4f535bae3ab960 (stream MsgApp v2 reader)
+14:13:31.592825 W | rafthttp: lost the TCP streaming connection with peer 6d4f535bae3ab960 (stream Message reader)
+14:13:31.693275 E | rafthttp: failed to dial 6d4f535bae3ab960 on stream Message (dial tcp [::1]:2380: getsockopt: connection refused)
+14:13:31.693289 I | rafthttp: peer 6d4f535bae3ab960 became inactive
+14:13:31.936678 W | rafthttp: lost the TCP streaming connection with peer 6d4f535bae3ab960 (stream Message writer)
+```
+
+It's a good idea at this point to [backup the etcd data](../op-guide/maintenance.md#snapshot-backup) to provide a downgrade path should any problems occur:
+
+```
+$ etcdctl snapshot save backup.db
+```
+
+#### 3. Drop-in etcd v3.4 binary and start the new etcd process
+
+The new v3.4 etcd will publish its information to the cluster:
+
+```
+14:14:25.363225 I | etcdserver: published {Name:s1 ClientURLs:[http://localhost:2379]} to cluster a9ededbffcb1b1f1
+```
+
+Verify that each member, and then the entire cluster, becomes healthy with the new v3.4 etcd binary:
+
+```
+$ ETCDCTL_API=3 /etcdctl endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+localhost:22379 is healthy: successfully committed proposal: took = 5.540129ms
+localhost:32379 is healthy: successfully committed proposal: took = 7.321771ms
+localhost:2379 is healthy: successfully committed proposal: took = 10.629901ms
+```
+
+Upgraded members will log warnings like the following until the entire cluster is upgraded. This is expected and will cease after all etcd cluster members are upgraded to v3.4:
+
+```
+14:15:17.071804 W | etcdserver: member c89feb932daef420 has a higher version 3.4.0
+14:15:21.073110 W | etcdserver: the local etcd version 3.3.0 is not up-to-date
+14:15:21.073142 W | etcdserver: member 6d4f535bae3ab960 has a higher version 3.4.0
+14:15:21.073157 W | etcdserver: the local etcd version 3.3.0 is not up-to-date
+14:15:21.073164 W | etcdserver: member c89feb932daef420 has a higher version 3.4.0
+```
+
+#### 4. Repeat step 2 to step 3 for all other members
+
+#### 5. Finish
+
+When all members are upgraded, the cluster will report upgrading to 3.4 successfully:
+
+```
+14:15:54.536901 N | etcdserver/membership: updated the cluster version from 3.3 to 3.4
+14:15:54.537035 I | etcdserver/api: enabled capabilities for version 3.4
+```
+
+```
+$ ETCDCTL_API=3 /etcdctl endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+localhost:2379 is healthy: successfully committed proposal: took = 2.312897ms
+localhost:22379 is healthy: successfully committed proposal: took = 2.553476ms
+localhost:32379 is healthy: successfully committed proposal: took = 2.517902ms
+```
+
+[etcd-contact]: https://groups.google.com/forum/#!forum/etcd-dev

clientv3/integration/lease_test.go

@@ -55,6 +55,11 @@ func TestLeaseGrant(t *testing.T) {
 
 	kv := clus.RandClient()
 
+	_, merr := lapi.Grant(context.Background(), clientv3.MaxLeaseTTL+1)
+	if merr != rpctypes.ErrLeaseTTLTooLarge {
+		t.Fatalf("err = %v, want %v", merr, rpctypes.ErrLeaseTTLTooLarge)
+	}
+
 	resp, err := lapi.Grant(context.Background(), 10)
 	if err != nil {
 		t.Errorf("failed to create lease %v", err)

clientv3/options.go

@@ -44,3 +44,6 @@ var (
 // Some options are exposed to "clientv3.Config".
 // Defaults will be overridden by the settings in "clientv3.Config".
 var defaultCallOpts = []grpc.CallOption{defaultFailFast, defaultMaxCallSendMsgSize, defaultMaxCallRecvMsgSize}
+
+// MaxLeaseTTL is the maximum lease TTL value
+const MaxLeaseTTL = 9000000000

compactor/compactor.go

@@ -29,8 +29,6 @@ var (
 )
 
 const (
-	checkCompactionInterval = 5 * time.Minute
-
 	ModePeriodic = "periodic"
 	ModeRevision = "revision"
 )

compactor/periodic.go

@@ -46,30 +46,74 @@ type Periodic struct {
 // NewPeriodic creates a new instance of Periodic compactor that purges
 // the log older than h Duration.
 func NewPeriodic(h time.Duration, rg RevGetter, c Compactable) *Periodic {
-	return &Periodic{
-		clock:  clockwork.NewRealClock(),
+	return newPeriodic(clockwork.NewRealClock(), h, rg, c)
+}
+
+func newPeriodic(clock clockwork.Clock, h time.Duration, rg RevGetter, c Compactable) *Periodic {
+	t := &Periodic{
+		clock:  clock,
 		period: h,
 		rg:     rg,
 		c:      c,
+		revs:   make([]int64, 0),
 	}
+	t.ctx, t.cancel = context.WithCancel(context.Background())
+	return t
 }
 
-// periodDivisor divides Periodic.period in into checkCompactInterval duration
-const periodDivisor = 10
+/*
+Compaction period 1-hour:
+  1. compute compaction period, which is 1-hour
+  2. record revisions for every 1/10 of 1-hour (6-minute)
+  3. keep recording revisions with no compaction for first 1-hour
+  4. do compact with revs[0]
+	- success? contiue on for-loop and move sliding window; revs = revs[1:]
+	- failure? update revs, and retry after 1/10 of 1-hour (6-minute)
 
+Compaction period 24-hour:
+  1. compute compaction period, which is 1-hour
+  2. record revisions for every 1/10 of 1-hour (6-minute)
+  3. keep recording revisions with no compaction for first 24-hour
+  4. do compact with revs[0]
+	- success? contiue on for-loop and move sliding window; revs = revs[1:]
+	- failure? update revs, and retry after 1/10 of 1-hour (6-minute)
+
+Compaction period 59-min:
+  1. compute compaction period, which is 59-min
+  2. record revisions for every 1/10 of 59-min (5.9-min)
+  3. keep recording revisions with no compaction for first 59-min
+  4. do compact with revs[0]
+	- success? contiue on for-loop and move sliding window; revs = revs[1:]
+	- failure? update revs, and retry after 1/10 of 59-min (5.9-min)
+
+Compaction period 5-sec:
+  1. compute compaction period, which is 5-sec
+  2. record revisions for every 1/10 of 5-sec (0.5-sec)
+  3. keep recording revisions with no compaction for first 5-sec
+  4. do compact with revs[0]
+	- success? contiue on for-loop and move sliding window; revs = revs[1:]
+	- failure? update revs, and retry after 1/10 of 5-sec (0.5-sec)
+*/
+
+// Run runs periodic compactor.
 func (t *Periodic) Run() {
-	t.ctx, t.cancel = context.WithCancel(context.Background())
-	t.revs = make([]int64, 0)
-	clock := t.clock
-	checkCompactInterval := t.period / time.Duration(periodDivisor)
+	compactInterval := t.getCompactInterval()
+	retryInterval := t.getRetryInterval()
+	retentions := t.getRetentions()
+
 	go func() {
-		last := clock.Now()
+		lastSuccess := t.clock.Now()
+		baseInterval := t.period
 		for {
 			t.revs = append(t.revs, t.rg.Rev())
+			if len(t.revs) > retentions {
+				t.revs = t.revs[1:] // t.revs[0] is always the rev at t.period ago
+			}
+
 			select {
 			case <-t.ctx.Done():
 				return
-			case <-clock.After(checkCompactInterval):
+			case <-t.clock.After(retryInterval):
 				t.mu.Lock()
 				p := t.paused
 				t.mu.Unlock()
@@ -77,46 +121,71 @@ func (t *Periodic) Run() {
 					continue
 				}
 			}
-			if clock.Now().Sub(last) < t.period {
+
+			if t.clock.Now().Sub(lastSuccess) < baseInterval {
 				continue
 			}
-			rev, remaining := t.getRev()
-			if rev < 0 {
-				continue
+
+			// wait up to initial given period
+			if baseInterval == t.period {
+				baseInterval = compactInterval
 			}
+			rev := t.revs[0]
+
 			plog.Noticef("Starting auto-compaction at revision %d (retention: %v)", rev, t.period)
 			_, err := t.c.Compact(t.ctx, &pb.CompactionRequest{Revision: rev})
 			if err == nil || err == mvcc.ErrCompacted {
-				t.revs = remaining
+				lastSuccess = t.clock.Now()
 				plog.Noticef("Finished auto-compaction at revision %d", rev)
 			} else {
 				plog.Noticef("Failed auto-compaction at revision %d (%v)", rev, err)
-				plog.Noticef("Retry after %v", checkCompactInterval)
+				plog.Noticef("Retry after %v", retryInterval)
 			}
 		}
 	}()
 }
 
+// if given compaction period x is <1-hour, compact every x duration.
+// (e.g. --auto-compaction-mode 'periodic' --auto-compaction-retention='10m', then compact every 10-minute)
+// if given compaction period x is >1-hour, compact every hour.
+// (e.g. --auto-compaction-mode 'periodic' --auto-compaction-retention='2h', then compact every 1-hour)
+func (t *Periodic) getCompactInterval() time.Duration {
+	itv := t.period
+	if itv > time.Hour {
+		itv = time.Hour
+	}
+	return itv
+}
+
+func (t *Periodic) getRetentions() int {
+	return int(t.period/t.getRetryInterval()) + 1
+}
+
+const retryDivisor = 10
+
+func (t *Periodic) getRetryInterval() time.Duration {
+	itv := t.period
+	if itv > time.Hour {
+		itv = time.Hour
+	}
+	return itv / retryDivisor
+}
+
+// Stop stops periodic compactor.
 func (t *Periodic) Stop() {
 	t.cancel()
 }
 
+// Pause pauses periodic compactor.
 func (t *Periodic) Pause() {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	t.paused = true
 }
 
+// Resume resumes periodic compactor.
 func (t *Periodic) Resume() {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	t.paused = false
 }
-
-func (t *Periodic) getRev() (int64, []int64) {
-	i := len(t.revs) - periodDivisor
-	if i < 0 {
-		return -1, t.revs
-	}
-	return t.revs[i], t.revs[i+1:]
-}

compactor/periodic_test.go

@@ -21,76 +21,129 @@ import (
 
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/testutil"
+
 	"github.com/jonboulle/clockwork"
 )
 
-func TestPeriodic(t *testing.T) {
+func TestPeriodicHourly(t *testing.T) {
 	retentionHours := 2
 	retentionDuration := time.Duration(retentionHours) * time.Hour
 
 	fc := clockwork.NewFakeClock()
 	rg := &fakeRevGetter{testutil.NewRecorderStream(), 0}
 	compactable := &fakeCompactable{testutil.NewRecorderStream()}
-	tb := &Periodic{
-		clock:  fc,
-		period: retentionDuration,
-		rg:     rg,
-		c:      compactable,
-	}
+	tb := newPeriodic(fc, retentionDuration, rg, compactable)
 
 	tb.Run()
 	defer tb.Stop()
-	checkCompactInterval := retentionDuration / time.Duration(periodDivisor)
-	n := periodDivisor
-	// simulate 5 hours worth of intervals.
-	for i := 0; i < n/retentionHours*5; i++ {
+
+	initialIntervals, intervalsPerPeriod := tb.getRetentions(), 10
+
+	// compaction doesn't happen til 2 hours elapse
+	for i := 0; i < initialIntervals; i++ {
 		rg.Wait(1)
-		fc.Advance(checkCompactInterval)
-		// compaction doesn't happen til 2 hours elapses.
-		if i < n {
-			continue
+		fc.Advance(tb.getRetryInterval())
+	}
+
+	// very first compaction
+	a, err := compactable.Wait(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedRevision := int64(1)
+	if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+		t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+	}
+
+	// simulate 3 hours
+	// now compactor kicks in, every hour
+	for i := 0; i < 3; i++ {
+		// advance one hour, one revision for each interval
+		for j := 0; j < intervalsPerPeriod; j++ {
+			rg.Wait(1)
+			fc.Advance(tb.getRetryInterval())
 		}
-		// after 2 hours, compaction happens at every checkCompactInterval.
-		a, err := compactable.Wait(1)
+
+		a, err = compactable.Wait(1)
 		if err != nil {
 			t.Fatal(err)
 		}
-		expectedRevision := int64(i + 1 - n)
+
+		expectedRevision = int64((i + 1) * 10)
 		if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
 			t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
 		}
 	}
+}
 
-	// unblock the rev getter, so we can stop the compactor routine.
-	_, err := rg.Wait(1)
+func TestPeriodicMinutes(t *testing.T) {
+	retentionMinutes := 5
+	retentionDuration := time.Duration(retentionMinutes) * time.Minute
+
+	fc := clockwork.NewFakeClock()
+	rg := &fakeRevGetter{testutil.NewRecorderStream(), 0}
+	compactable := &fakeCompactable{testutil.NewRecorderStream()}
+	tb := newPeriodic(fc, retentionDuration, rg, compactable)
+
+	tb.Run()
+	defer tb.Stop()
+
+	initialIntervals, intervalsPerPeriod := tb.getRetentions(), 10
+
+	// compaction doesn't happen til 5 minutes elapse
+	for i := 0; i < initialIntervals; i++ {
+		rg.Wait(1)
+		fc.Advance(tb.getRetryInterval())
+	}
+
+	// very first compaction
+	a, err := compactable.Wait(1)
 	if err != nil {
 		t.Fatal(err)
 	}
+	expectedRevision := int64(1)
+	if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+		t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+	}
+
+	// compaction happens at every interval
+	for i := 0; i < 5; i++ {
+		// advance 5-minute, one revision for each interval
+		for j := 0; j < intervalsPerPeriod; j++ {
+			rg.Wait(1)
+			fc.Advance(tb.getRetryInterval())
+		}
+
+		a, err := compactable.Wait(1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expectedRevision = int64((i + 1) * 10)
+		if !reflect.DeepEqual(a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision}) {
+			t.Errorf("compact request = %v, want %v", a[0].Params[0], &pb.CompactionRequest{Revision: expectedRevision})
+		}
+	}
 }
 
 func TestPeriodicPause(t *testing.T) {
 	fc := clockwork.NewFakeClock()
-	compactable := &fakeCompactable{testutil.NewRecorderStream()}
-	rg := &fakeRevGetter{testutil.NewRecorderStream(), 0}
 	retentionDuration := time.Hour
-	tb := &Periodic{
-		clock:  fc,
-		period: retentionDuration,
-		rg:     rg,
-		c:      compactable,
-	}
+	rg := &fakeRevGetter{testutil.NewRecorderStream(), 0}
+	compactable := &fakeCompactable{testutil.NewRecorderStream()}
+	tb := newPeriodic(fc, retentionDuration, rg, compactable)
 
 	tb.Run()
 	tb.Pause()
 
+	n := tb.getRetentions()
+
 	// tb will collect 3 hours of revisions but not compact since paused
-	checkCompactInterval := retentionDuration / time.Duration(periodDivisor)
-	n := periodDivisor
-	for i := 0; i < 3*n; i++ {
+	for i := 0; i < n*3; i++ {
 		rg.Wait(1)
-		fc.Advance(checkCompactInterval)
+		fc.Advance(tb.getRetryInterval())
 	}
-	// tb ends up waiting for the clock
+	// t.revs = [21 22 23 24 25 26 27 28 29 30]
 
 	select {
 	case a := <-compactable.Chan():
@@ -100,14 +153,17 @@ func TestPeriodicPause(t *testing.T) {
 
 	// tb resumes to being blocked on the clock
 	tb.Resume()
-
-	// unblock clock, will kick off a compaction at hour 3:06
 	rg.Wait(1)
-	fc.Advance(checkCompactInterval)
+
+	// unblock clock, will kick off a compaction at T=3h6m by retry
+	fc.Advance(tb.getRetryInterval())
+
+	// T=3h6m
 	a, err := compactable.Wait(1)
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	// compact the revision from hour 2:06
 	wreq := &pb.CompactionRequest{Revision: int64(1 + 2*n + 1)}
 	if !reflect.DeepEqual(a[0].Params[0], wreq) {

compactor/revision.go

@@ -17,6 +17,7 @@ package compactor
 import (
 	"context"
 	"sync"
+	"time"
 
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/mvcc"
@@ -43,25 +44,31 @@ type Revision struct {
 // NewRevision creates a new instance of Revisonal compactor that purges
 // the log older than retention revisions from the current revision.
 func NewRevision(retention int64, rg RevGetter, c Compactable) *Revision {
-	return &Revision{
-		clock:     clockwork.NewRealClock(),
+	return newRevision(clockwork.NewRealClock(), retention, rg, c)
+}
+
+func newRevision(clock clockwork.Clock, retention int64, rg RevGetter, c Compactable) *Revision {
+	t := &Revision{
+		clock:     clock,
 		retention: retention,
 		rg:        rg,
 		c:         c,
 	}
+	t.ctx, t.cancel = context.WithCancel(context.Background())
+	return t
 }
 
-func (t *Revision) Run() {
-	t.ctx, t.cancel = context.WithCancel(context.Background())
-	clock := t.clock
-	previous := int64(0)
+const revInterval = 5 * time.Minute
 
+// Run runs revision-based compactor.
+func (t *Revision) Run() {
+	prev := int64(0)
 	go func() {
 		for {
 			select {
 			case <-t.ctx.Done():
 				return
-			case <-clock.After(checkCompactionInterval):
+			case <-t.clock.After(revInterval):
 				t.mu.Lock()
 				p := t.paused
 				t.mu.Unlock()
@@ -71,34 +78,36 @@ func (t *Revision) Run() {
 			}
 
 			rev := t.rg.Rev() - t.retention
-
-			if rev <= 0 || rev == previous {
+			if rev <= 0 || rev == prev {
 				continue
 			}
 
 			plog.Noticef("Starting auto-compaction at revision %d (retention: %d revisions)", rev, t.retention)
 			_, err := t.c.Compact(t.ctx, &pb.CompactionRequest{Revision: rev})
 			if err == nil || err == mvcc.ErrCompacted {
-				previous = rev
+				prev = rev
 				plog.Noticef("Finished auto-compaction at revision %d", rev)
 			} else {
 				plog.Noticef("Failed auto-compaction at revision %d (%v)", rev, err)
-				plog.Noticef("Retry after %v", checkCompactionInterval)
+				plog.Noticef("Retry after %v", revInterval)
 			}
 		}
 	}()
 }
 
+// Stop stops revision-based compactor.
 func (t *Revision) Stop() {
 	t.cancel()
 }
 
+// Pause pauses revision-based compactor.
 func (t *Revision) Pause() {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	t.paused = true
 }
 
+// Resume resumes revision-based compactor.
 func (t *Revision) Resume() {
 	t.mu.Lock()
 	defer t.mu.Unlock()

compactor/revision_test.go

@@ -21,6 +21,7 @@ import (
 
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/testutil"
+
 	"github.com/jonboulle/clockwork"
 )
 
@@ -28,23 +29,18 @@ func TestRevision(t *testing.T) {
 	fc := clockwork.NewFakeClock()
 	rg := &fakeRevGetter{testutil.NewRecorderStream(), 0}
 	compactable := &fakeCompactable{testutil.NewRecorderStream()}
-	tb := &Revision{
-		clock:     fc,
-		retention: 10,
-		rg:        rg,
-		c:         compactable,
-	}
+	tb := newRevision(fc, 10, rg, compactable)
 
 	tb.Run()
 	defer tb.Stop()
 
-	fc.Advance(checkCompactionInterval)
+	fc.Advance(revInterval)
 	rg.Wait(1)
 	// nothing happens
 
 	rg.SetRev(99) // will be 100
 	expectedRevision := int64(90)
-	fc.Advance(checkCompactionInterval)
+	fc.Advance(revInterval)
 	rg.Wait(1)
 	a, err := compactable.Wait(1)
 	if err != nil {
@@ -61,7 +57,7 @@ func TestRevision(t *testing.T) {
 
 	rg.SetRev(199) // will be 200
 	expectedRevision = int64(190)
-	fc.Advance(checkCompactionInterval)
+	fc.Advance(revInterval)
 	rg.Wait(1)
 	a, err = compactable.Wait(1)
 	if err != nil {
@@ -74,22 +70,17 @@ func TestRevision(t *testing.T) {
 
 func TestRevisionPause(t *testing.T) {
 	fc := clockwork.NewFakeClock()
-	compactable := &fakeCompactable{testutil.NewRecorderStream()}
 	rg := &fakeRevGetter{testutil.NewRecorderStream(), 99} // will be 100
-	tb := &Revision{
-		clock:     fc,
-		retention: 10,
-		rg:        rg,
-		c:         compactable,
-	}
+	compactable := &fakeCompactable{testutil.NewRecorderStream()}
+	tb := newRevision(fc, 10, rg, compactable)
 
 	tb.Run()
 	tb.Pause()
 
 	// tb will collect 3 hours of revisions but not compact since paused
-	n := int(time.Hour / checkCompactionInterval)
+	n := int(time.Hour / revInterval)
 	for i := 0; i < 3*n; i++ {
-		fc.Advance(checkCompactionInterval)
+		fc.Advance(revInterval)
 	}
 	// tb ends up waiting for the clock
 
@@ -103,7 +94,7 @@ func TestRevisionPause(t *testing.T) {
 	tb.Resume()
 
 	// unblock clock, will kick off a compaction at hour 3:05
-	fc.Advance(checkCompactionInterval)
+	fc.Advance(revInterval)
 	rg.Wait(1)
 	a, err := compactable.Wait(1)
 	if err != nil {

e2e/util.go

@@ -42,9 +42,14 @@ func spawnWithExpect(args []string, expected string) error {
 }
 
 func spawnWithExpects(args []string, xs ...string) error {
+	_, err := spawnWithExpectLines(args, xs...)
+	return err
+}
+
+func spawnWithExpectLines(args []string, xs ...string) ([]string, error) {
 	proc, err := spawnCmd(args)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	// process until either stdout or stderr contains
 	// the expected string
@@ -57,7 +62,7 @@ func spawnWithExpects(args []string, xs ...string) error {
 			l, lerr := proc.ExpectFunc(lineFunc)
 			if lerr != nil {
 				proc.Close()
-				return fmt.Errorf("%v (expected %q, got %q)", lerr, txt, lines)
+				return nil, fmt.Errorf("%v (expected %q, got %q)", lerr, txt, lines)
 			}
 			lines = append(lines, l)
 			if strings.Contains(l, txt) {
@@ -67,9 +72,9 @@ func spawnWithExpects(args []string, xs ...string) error {
 	}
 	perr := proc.Close()
 	if len(xs) == 0 && proc.LineCount() != noOutputLineCount { // expect no output
-		return fmt.Errorf("unexpected output (got lines %q, line count %d)", lines, proc.LineCount())
+		return nil, fmt.Errorf("unexpected output (got lines %q, line count %d)", lines, proc.LineCount())
 	}
-	return perr
+	return lines, perr
 }
 
 func closeWithTimeout(p *expect.ExpectProcess, d time.Duration) error {

e2e/v3_curl_test.go

@@ -15,10 +15,13 @@
 package e2e
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"path"
+	"strconv"
 	"testing"
 
+	epb "github.com/coreos/etcd/etcdserver/api/v3election/v3electionpb"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/testutil"
 
@@ -271,3 +274,119 @@ func testV3CurlAuth(t *testing.T, pathPrefix string) {
 		t.Fatalf("failed auth put with curl (%v)", err)
 	}
 }
+
+func TestV3CurlCampaignAlpha(t *testing.T) { testV3CurlCampaign(t, "/v3alpha") }
+func TestV3CurlCampaignBeta(t *testing.T)  { testV3CurlCampaign(t, "/v3beta") }
+func testV3CurlCampaign(t *testing.T, pathPrefix string) {
+	defer testutil.AfterTest(t)
+
+	epc, err := newEtcdProcessCluster(&configNoTLS)
+	if err != nil {
+		t.Fatalf("could not start etcd process cluster (%v)", err)
+	}
+	defer func() {
+		if cerr := epc.Close(); err != nil {
+			t.Fatalf("error closing etcd processes (%v)", cerr)
+		}
+	}()
+
+	cdata, err := json.Marshal(&epb.CampaignRequest{
+		Name:  []byte("/election-prefix"),
+		Value: []byte("v1"),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cargs := cURLPrefixArgs(epc, "POST", cURLReq{
+		endpoint: path.Join(pathPrefix, "/election/campaign"),
+		value:    string(cdata),
+	})
+	lines, err := spawnWithExpectLines(cargs, `"leader":{"name":"`)
+	if err != nil {
+		t.Fatalf("failed post campaign request (%s) (%v)", pathPrefix, err)
+	}
+	if len(lines) != 1 {
+		t.Fatalf("len(lines) expected 1, got %+v", lines)
+	}
+
+	var cresp campaignResponse
+	if err = json.Unmarshal([]byte(lines[0]), &cresp); err != nil {
+		t.Fatalf("failed to unmarshal campaign response %v", err)
+	}
+	ndata, err := base64.StdEncoding.DecodeString(cresp.Leader.Name)
+	if err != nil {
+		t.Fatalf("failed to decode leader key %v", err)
+	}
+	kdata, err := base64.StdEncoding.DecodeString(cresp.Leader.Key)
+	if err != nil {
+		t.Fatalf("failed to decode leader key %v", err)
+	}
+
+	rev, _ := strconv.ParseInt(cresp.Leader.Rev, 10, 64)
+	lease, _ := strconv.ParseInt(cresp.Leader.Lease, 10, 64)
+	pdata, err := json.Marshal(&epb.ProclaimRequest{
+		Leader: &epb.LeaderKey{
+			Name:  ndata,
+			Key:   kdata,
+			Rev:   rev,
+			Lease: lease,
+		},
+		Value: []byte("v2"),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = cURLPost(epc, cURLReq{
+		endpoint: path.Join(pathPrefix, "/election/proclaim"),
+		value:    string(pdata),
+		expected: `"revision":`,
+	}); err != nil {
+		t.Fatalf("failed post proclaim request (%s) (%v)", pathPrefix, err)
+	}
+}
+
+func TestV3CurlProclaimMissiongLeaderKeyNoTLS(t *testing.T) {
+	testCtl(t, testV3CurlProclaimMissiongLeaderKey, withCfg(configNoTLS))
+}
+
+func testV3CurlProclaimMissiongLeaderKey(cx ctlCtx) {
+	pdata, err := json.Marshal(&epb.ProclaimRequest{Value: []byte("v2")})
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+	if err = cURLPost(cx.epc, cURLReq{
+		endpoint: path.Join("/v3beta", "/election/proclaim"),
+		value:    string(pdata),
+		expected: `{"error":"\"leader\" field must be provided","code":2}`,
+	}); err != nil {
+		cx.t.Fatalf("failed post proclaim request (%s) (%v)", "/v3beta", err)
+	}
+}
+
+func TestV3CurlResignMissiongLeaderKeyNoTLS(t *testing.T) {
+	testCtl(t, testV3CurlResignMissiongLeaderKey, withCfg(configNoTLS))
+}
+
+func testV3CurlResignMissiongLeaderKey(cx ctlCtx) {
+	if err := cURLPost(cx.epc, cURLReq{
+		endpoint: path.Join("/v3beta", "/election/resign"),
+		value:    `{}`,
+		expected: `{"error":"\"leader\" field must be provided","code":2}`,
+	}); err != nil {
+		cx.t.Fatalf("failed post resign request (%s) (%v)", "/v3beta", err)
+	}
+}
+
+// to manually decode; JSON marshals integer fields with
+// string types, so can't unmarshal with epb.CampaignResponse
+type campaignResponse struct {
+	Leader struct {
+		Name  string `json:"name,omitempty"`
+		Key   string `json:"key,omitempty"`
+		Rev   string `json:"rev,omitempty"`
+		Lease string `json:"lease,omitempty"`
+	} `json:"leader,omitempty"`
+}

embed/config.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/coreos/etcd/compactor"
 	"github.com/coreos/etcd/etcdserver"
 	"github.com/coreos/etcd/pkg/cors"
 	"github.com/coreos/etcd/pkg/netutil"
@@ -90,16 +91,22 @@ func init() {
 type Config struct {
 	// member
 
-	CorsInfo                *cors.CORSInfo
-	LPUrls, LCUrls          []url.URL
-	Dir                     string `json:"data-dir"`
-	WalDir                  string `json:"wal-dir"`
-	MaxSnapFiles            uint   `json:"max-snapshots"`
-	MaxWalFiles             uint   `json:"max-wals"`
-	Name                    string `json:"name"`
-	SnapCount               uint64 `json:"snapshot-count"`
+	CorsInfo       *cors.CORSInfo
+	LPUrls, LCUrls []url.URL
+	Dir            string `json:"data-dir"`
+	WalDir         string `json:"wal-dir"`
+	MaxSnapFiles   uint   `json:"max-snapshots"`
+	MaxWalFiles    uint   `json:"max-wals"`
+	Name           string `json:"name"`
+	SnapCount      uint64 `json:"snapshot-count"`
+
+	// AutoCompactionMode is either 'periodic' or 'revision'.
+	AutoCompactionMode string `json:"auto-compaction-mode"`
+	// AutoCompactionRetention is either duration string with time unit
+	// (e.g. '5m' for 5-minute), or revision unit (e.g. '5000').
+	// If no time unit is provided and compaction mode is 'periodic',
+	// the unit defaults to hour. For example, '5' translates into 5-hour.
 	AutoCompactionRetention string `json:"auto-compaction-retention"`
-	AutoCompactionMode      string `json:"auto-compaction-mode"`
 
 	// TickMs is the number of milliseconds between heartbeat ticks.
 	// TODO: decouple tickMs and heartbeat tick (current heartbeat tick = 1).
@@ -388,6 +395,7 @@ func (cfg *configYAML) configFromFile(path string) error {
 	return cfg.Validate()
 }
 
+// Validate ensures that '*embed.Config' fields are properly configured.
 func (cfg *Config) Validate() error {
 	if err := checkBindURLs(cfg.LPUrls); err != nil {
 		return err
@@ -449,6 +457,13 @@ func (cfg *Config) Validate() error {
 		return ErrUnsetAdvertiseClientURLsFlag
 	}
 
+	switch cfg.AutoCompactionMode {
+	case "":
+	case compactor.ModeRevision, compactor.ModePeriodic:
+	default:
+		return fmt.Errorf("unknown auto-compaction-mode %q", cfg.AutoCompactionMode)
+	}
+
 	return nil
 }
 

embed/config_test.go

@@ -148,3 +148,22 @@ func mustCreateCfgFile(t *testing.T, b []byte) *os.File {
 	}
 	return tmpfile
 }
+
+func TestAutoCompactionModeInvalid(t *testing.T) {
+	cfg := NewConfig()
+	cfg.AutoCompactionMode = "period"
+	err := cfg.Validate()
+	if err == nil {
+		t.Errorf("expected non-nil error, got %v", err)
+	}
+}
+
+func TestAutoCompactionModeParse(t *testing.T) {
+	dur, err := parseCompactionRetention("revision", "1")
+	if err != nil {
+		t.Error(err)
+	}
+	if dur != 1 {
+		t.Fatalf("AutoCompactionRetention expected 1, got %d", dur)
+	}
+}

embed/etcd.go

@@ -27,6 +27,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/coreos/etcd/compactor"
 	"github.com/coreos/etcd/etcdserver"
 	"github.com/coreos/etcd/etcdserver/api/etcdhttp"
 	"github.com/coreos/etcd/etcdserver/api/v2http"
@@ -134,22 +135,13 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		}
 	}
 
-	var (
-		autoCompactionRetention time.Duration
-		h                       int
-	)
 	// AutoCompactionRetention defaults to "0" if not set.
 	if len(cfg.AutoCompactionRetention) == 0 {
 		cfg.AutoCompactionRetention = "0"
 	}
-	h, err = strconv.Atoi(cfg.AutoCompactionRetention)
-	if err == nil {
-		autoCompactionRetention = time.Duration(int64(h)) * time.Hour
-	} else {
-		autoCompactionRetention, err = time.ParseDuration(cfg.AutoCompactionRetention)
-		if err != nil {
-			return nil, fmt.Errorf("error parsing AutoCompactionRetention: %v", err)
-		}
+	autoCompactionRetention, err := parseCompactionRetention(cfg.AutoCompactionMode, cfg.AutoCompactionRetention)
+	if err != nil {
+		return e, err
 	}
 
 	srvcfg := etcdserver.ServerConfig{
@@ -562,3 +554,22 @@ func (e *Etcd) errHandler(err error) {
 	case e.errc <- err:
 	}
 }
+
+func parseCompactionRetention(mode, retention string) (ret time.Duration, err error) {
+	h, err := strconv.Atoi(retention)
+	if err == nil {
+		switch mode {
+		case compactor.ModeRevision:
+			ret = time.Duration(int64(h))
+		case compactor.ModePeriodic:
+			ret = time.Duration(int64(h)) * time.Hour
+		}
+	} else {
+		// periodic compaction
+		ret, err = time.ParseDuration(retention)
+		if err != nil {
+			return 0, fmt.Errorf("error parsing CompactionRetention: %v", err)
+		}
+	}
+	return ret, nil
+}

etcdctl/README.md

@@ -876,10 +876,11 @@ If NOSPACE alarm is present:
 
 ### DEFRAG [options]
 
-DEFRAG defragments the backend database file for a set of given endpoints while etcd is running, or directly defragments an
-etcd data directory while etcd is not running. When an etcd member reclaims storage space from deleted and compacted keys, the
-space is kept in a free list and the database file remains the same size. By defragmenting the database, the etcd member
-releases this free space back to the file system.
+DEFRAG defragments the backend database file for a set of given endpoints while etcd is running, or directly defragments an etcd data directory while etcd is not running. When an etcd member reclaims storage space from deleted and compacted keys, the space is kept in a free list and the database file remains the same size. By defragmenting the database, the etcd member releases this free space back to the file system.
+
+**Note that defragmentation to a live member blocks the system from reading and writing data while rebuilding its states.**
+
+**Note that defragmentation request does not get replicated over cluster. That is, the request is only applied to the local node. Specify all members in `--endpoints` flag.**
 
 #### Options
 

etcdmain/grpc_proxy.go

@@ -53,6 +53,8 @@ var (
 	grpcProxyDNSCluster        string
 	grpcProxyInsecureDiscovery bool
 	grpcProxyDataDir           string
+	grpcMaxCallSendMsgSize     int
+	grpcMaxCallRecvMsgSize     int
 
 	// tls for connecting to etcd
 
@@ -82,6 +84,8 @@ var (
 	grpcProxyDebug bool
 )
 
+const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024
+
 func init() {
 	rootCmd.AddCommand(newGRPCProxyCommand())
 }
@@ -115,6 +119,8 @@ func newGRPCProxyStartCommand() *cobra.Command {
 	cmd.Flags().StringVar(&grpcProxyNamespace, "namespace", "", "string to prefix to all keys for namespacing requests")
 	cmd.Flags().BoolVar(&grpcProxyEnablePprof, "enable-pprof", false, `Enable runtime profiling data via HTTP server. Address is at client URL + "/debug/pprof/"`)
 	cmd.Flags().StringVar(&grpcProxyDataDir, "data-dir", "default.proxy", "Data directory for persistent data")
+	cmd.Flags().IntVar(&grpcMaxCallSendMsgSize, "max-send-bytes", defaultGRPCMaxCallSendMsgSize, "message send limits in bytes (default value is 1.5 MiB)")
+	cmd.Flags().IntVar(&grpcMaxCallRecvMsgSize, "max-recv-bytes", math.MaxInt32, "message receive limits in bytes (default value is math.MaxInt32)")
 
 	// client TLS for connecting to server
 	cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file")
@@ -241,6 +247,14 @@ func newClientCfg(eps []string) (*clientv3.Config, error) {
 		Endpoints:   eps,
 		DialTimeout: 5 * time.Second,
 	}
+
+	if grpcMaxCallSendMsgSize > 0 {
+		cfg.MaxCallSendMsgSize = grpcMaxCallSendMsgSize
+	}
+	if grpcMaxCallRecvMsgSize > 0 {
+		cfg.MaxCallRecvMsgSize = grpcMaxCallRecvMsgSize
+	}
+
 	tls := newTLS(grpcProxyCA, grpcProxyCert, grpcProxyKey)
 	if tls == nil && grpcProxyInsecureSkipTLSVerify {
 		tls = &transport.TLSInfo{}

etcdserver/api/v3election/election.go

@@ -16,12 +16,17 @@ package v3election
 
 import (
 	"context"
+	"errors"
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/clientv3/concurrency"
 	epb "github.com/coreos/etcd/etcdserver/api/v3election/v3electionpb"
 )
 
+// ErrMissingLeaderKey is returned when election API request
+// is missing the "leader" field.
+var ErrMissingLeaderKey = errors.New(`"leader" field must be provided`)
+
 type electionServer struct {
 	c *clientv3.Client
 }
@@ -51,6 +56,9 @@ func (es *electionServer) Campaign(ctx context.Context, req *epb.CampaignRequest
 }
 
 func (es *electionServer) Proclaim(ctx context.Context, req *epb.ProclaimRequest) (*epb.ProclaimResponse, error) {
+	if req.Leader == nil {
+		return nil, ErrMissingLeaderKey
+	}
 	s, err := es.session(ctx, req.Leader.Lease)
 	if err != nil {
 		return nil, err
@@ -98,6 +106,9 @@ func (es *electionServer) Leader(ctx context.Context, req *epb.LeaderRequest) (*
 }
 
 func (es *electionServer) Resign(ctx context.Context, req *epb.ResignRequest) (*epb.ResignResponse, error) {
+	if req.Leader == nil {
+		return nil, ErrMissingLeaderKey
+	}
 	s, err := es.session(ctx, req.Leader.Lease)
 	if err != nil {
 		return nil, err

etcdserver/api/v3rpc/rpctypes/error.go

@@ -31,8 +31,9 @@ var (
 	ErrGRPCFutureRev     = status.New(codes.OutOfRange, "etcdserver: mvcc: required revision is a future revision").Err()
 	ErrGRPCNoSpace       = status.New(codes.ResourceExhausted, "etcdserver: mvcc: database space exceeded").Err()
 
-	ErrGRPCLeaseNotFound = status.New(codes.NotFound, "etcdserver: requested lease not found").Err()
-	ErrGRPCLeaseExist    = status.New(codes.FailedPrecondition, "etcdserver: lease already exists").Err()
+	ErrGRPCLeaseNotFound    = status.New(codes.NotFound, "etcdserver: requested lease not found").Err()
+	ErrGRPCLeaseExist       = status.New(codes.FailedPrecondition, "etcdserver: lease already exists").Err()
+	ErrGRPCLeaseTTLTooLarge = status.New(codes.OutOfRange, "etcdserver: too large lease TTL").Err()
 
 	ErrGRPCMemberExist            = status.New(codes.FailedPrecondition, "etcdserver: member ID already exist").Err()
 	ErrGRPCPeerURLExist           = status.New(codes.FailedPrecondition, "etcdserver: Peer URLs already exists").Err()
@@ -80,8 +81,9 @@ var (
 		ErrorDesc(ErrGRPCFutureRev):    ErrGRPCFutureRev,
 		ErrorDesc(ErrGRPCNoSpace):      ErrGRPCNoSpace,
 
-		ErrorDesc(ErrGRPCLeaseNotFound): ErrGRPCLeaseNotFound,
-		ErrorDesc(ErrGRPCLeaseExist):    ErrGRPCLeaseExist,
+		ErrorDesc(ErrGRPCLeaseNotFound):    ErrGRPCLeaseNotFound,
+		ErrorDesc(ErrGRPCLeaseExist):       ErrGRPCLeaseExist,
+		ErrorDesc(ErrGRPCLeaseTTLTooLarge): ErrGRPCLeaseTTLTooLarge,
 
 		ErrorDesc(ErrGRPCMemberExist):            ErrGRPCMemberExist,
 		ErrorDesc(ErrGRPCPeerURLExist):           ErrGRPCPeerURLExist,
@@ -131,8 +133,9 @@ var (
 	ErrFutureRev     = Error(ErrGRPCFutureRev)
 	ErrNoSpace       = Error(ErrGRPCNoSpace)
 
-	ErrLeaseNotFound = Error(ErrGRPCLeaseNotFound)
-	ErrLeaseExist    = Error(ErrGRPCLeaseExist)
+	ErrLeaseNotFound    = Error(ErrGRPCLeaseNotFound)
+	ErrLeaseExist       = Error(ErrGRPCLeaseExist)
+	ErrLeaseTTLTooLarge = Error(ErrGRPCLeaseTTLTooLarge)
 
 	ErrMemberExist            = Error(ErrGRPCMemberExist)
 	ErrPeerURLExist           = Error(ErrGRPCPeerURLExist)

etcdserver/api/v3rpc/util.go

@@ -52,8 +52,9 @@ var toGRPCErrorMap = map[error]error{
 	etcdserver.ErrKeyNotFound:                rpctypes.ErrGRPCKeyNotFound,
 	etcdserver.ErrCorrupt:                    rpctypes.ErrGRPCCorrupt,
 
-	lease.ErrLeaseNotFound: rpctypes.ErrGRPCLeaseNotFound,
-	lease.ErrLeaseExists:   rpctypes.ErrGRPCLeaseExist,
+	lease.ErrLeaseNotFound:    rpctypes.ErrGRPCLeaseNotFound,
+	lease.ErrLeaseExists:      rpctypes.ErrGRPCLeaseExist,
+	lease.ErrLeaseTTLTooLarge: rpctypes.ErrGRPCLeaseTTLTooLarge,
 
 	auth.ErrRootUserNotExist:     rpctypes.ErrGRPCRootUserNotExist,
 	auth.ErrRootRoleNotExist:     rpctypes.ErrGRPCRootRoleNotExist,

etcdserver/apply.go

@@ -107,6 +107,8 @@ func (s *EtcdServer) newApplierV3() applierV3 {
 }
 
 func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
+	defer warnOfExpensiveRequest(time.Now(), r)
+
 	ar := &applyResult{}
 
 	// call into a.s.applyV3.F instead of a.F so upper appliers can check individual calls

etcdserver/apply_v2.go

@@ -107,6 +107,8 @@ func (a *applierV2store) Sync(r *RequestV2) Response {
 // applyV2Request interprets r as a call to store.X and returns a Response interpreted
 // from store.Event
 func (s *EtcdServer) applyV2Request(r *RequestV2) Response {
+	defer warnOfExpensiveRequest(time.Now(), r)
+
 	switch r.Method {
 	case "POST":
 		return s.applyV2.Post(r)

etcdserver/backend.go

@@ -58,8 +58,8 @@ func openBackend(cfg ServerConfig) backend.Backend {
 	select {
 	case be := <-beOpened:
 		return be
-	case <-time.After(time.Second):
-		plog.Warningf("another etcd process is using %q and holds the file lock.", fn)
+	case <-time.After(10 * time.Second):
+		plog.Warningf("another etcd process is using %q and holds the file lock, or loading backend file is taking >10 seconds", fn)
 		plog.Warningf("waiting for it to exit before starting...")
 	}
 	return <-beOpened

etcdserver/config.go

@@ -124,7 +124,8 @@ func (c *ServerConfig) advertiseMatchesCluster() error {
 	sort.Strings(apurls)
 	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
 	defer cancel()
-	if netutil.URLStringsEqual(ctx, apurls, urls.StringSlice()) {
+	ok, err := netutil.URLStringsEqual(ctx, apurls, urls.StringSlice())
+	if ok {
 		return nil
 	}
 
@@ -148,7 +149,7 @@ func (c *ServerConfig) advertiseMatchesCluster() error {
 		}
 		mstr := strings.Join(missing, ",")
 		apStr := strings.Join(apurls, ",")
-		return fmt.Errorf("--initial-cluster has %s but missing from --initial-advertise-peer-urls=%s ", mstr, apStr)
+		return fmt.Errorf("--initial-cluster has %s but missing from --initial-advertise-peer-urls=%s (%v)", mstr, apStr, err)
 	}
 
 	for url := range apMap {
@@ -156,9 +157,16 @@ func (c *ServerConfig) advertiseMatchesCluster() error {
 			missing = append(missing, url)
 		}
 	}
-	mstr := strings.Join(missing, ",")
+	if len(missing) > 0 {
+		mstr := strings.Join(missing, ",")
+		umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs})
+		return fmt.Errorf("--initial-advertise-peer-urls has %s but missing from --initial-cluster=%s", mstr, umap.String())
+	}
+
+	// resolved URLs from "--initial-advertise-peer-urls" and "--initial-cluster" did not match or failed
+	apStr := strings.Join(apurls, ",")
 	umap := types.URLsMap(map[string]types.URLs{c.Name: c.PeerURLs})
-	return fmt.Errorf("--initial-advertise-peer-urls has %s but missing from --initial-cluster=%s", mstr, umap.String())
+	return fmt.Errorf("failed to resolve %s to match --initial-cluster=%s (%v)", apStr, umap.String(), err)
 }
 
 func (c *ServerConfig) MemberDir() string { return filepath.Join(c.DataDir, "member") }

etcdserver/membership/cluster.go

@@ -490,8 +490,8 @@ func ValidateClusterAndAssignIDs(local *RaftCluster, existing *RaftCluster) erro
 	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
 	defer cancel()
 	for i := range ems {
-		if !netutil.URLStringsEqual(ctx, ems[i].PeerURLs, lms[i].PeerURLs) {
-			return fmt.Errorf("unmatched member while checking PeerURLs")
+		if ok, err := netutil.URLStringsEqual(ctx, ems[i].PeerURLs, lms[i].PeerURLs); !ok {
+			return fmt.Errorf("unmatched member while checking PeerURLs (%v)", err)
 		}
 		lms[i].ID = ems[i].ID
 	}

etcdserver/raft.go

@@ -95,6 +95,7 @@ type raftNode struct {
 	term  uint64
 	lead  uint64
 
+	tickMu *sync.Mutex
 	raftNodeConfig
 
 	// a chan to send/receive snapshot
@@ -131,6 +132,7 @@ type raftNodeConfig struct {
 
 func newRaftNode(cfg raftNodeConfig) *raftNode {
 	r := &raftNode{
+		tickMu:         new(sync.Mutex),
 		raftNodeConfig: cfg,
 		// set up contention detectors for raft heartbeat message.
 		// expect to send a heartbeat within 2 heartbeat intervals.
@@ -149,6 +151,13 @@ func newRaftNode(cfg raftNodeConfig) *raftNode {
 	return r
 }
 
+// raft.Node does not have locks in Raft package
+func (r *raftNode) tick() {
+	r.tickMu.Lock()
+	r.Tick()
+	r.tickMu.Unlock()
+}
+
 // start prepares and starts raftNode in a new goroutine. It is no longer safe
 // to modify the fields after it has been started.
 func (r *raftNode) start(rh *raftReadyHandler) {
@@ -161,7 +170,7 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 		for {
 			select {
 			case <-r.ticker.C:
-				r.Tick()
+				r.tick()
 			case rd := <-r.Ready():
 				if rd.SoftState != nil {
 					newLeader := rd.SoftState.Lead != raft.None && atomic.LoadUint64(&r.lead) != rd.SoftState.Lead
@@ -368,13 +377,13 @@ func (r *raftNode) resumeSending() {
 	p.Resume()
 }
 
-// advanceTicksForElection advances ticks to the node for fast election.
-// This reduces the time to wait for first leader election if bootstrapping the whole
-// cluster, while leaving at least 1 heartbeat for possible existing leader
-// to contact it.
-func advanceTicksForElection(n raft.Node, electionTicks int) {
-	for i := 0; i < electionTicks-1; i++ {
-		n.Tick()
+// advanceTicks advances ticks of Raft node.
+// This can be used for fast-forwarding election
+// ticks in multi data-center deployments, thus
+// speeding up election process.
+func (r *raftNode) advanceTicks(ticks int) {
+	for i := 0; i < ticks; i++ {
+		r.tick()
 	}
 }
 
@@ -415,7 +424,6 @@ func startNode(cfg ServerConfig, cl *membership.RaftCluster, ids []types.ID) (id
 	raftStatusMu.Lock()
 	raftStatus = n.Status
 	raftStatusMu.Unlock()
-	advanceTicksForElection(n, c.ElectionTick)
 	return id, n, s, w
 }
 
@@ -449,7 +457,6 @@ func restartNode(cfg ServerConfig, snapshot *raftpb.Snapshot) (types.ID, *member
 	raftStatusMu.Lock()
 	raftStatus = n.Status
 	raftStatusMu.Unlock()
-	advanceTicksForElection(n, c.ElectionTick)
 	return id, cl, n, s, w
 }
 
@@ -498,6 +505,7 @@ func restartAsStandaloneNode(cfg ServerConfig, snapshot *raftpb.Snapshot) (types
 		Storage:         s,
 		MaxSizePerMsg:   maxSizePerMsg,
 		MaxInflightMsgs: maxInflightMsgs,
+		CheckQuorum:     true,
 	}
 	n := raft.RestartNode(c)
 	raftStatus = n.Status

etcdserver/server.go

@@ -521,12 +521,51 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 	return srv, nil
 }
 
+func (s *EtcdServer) adjustTicks() {
+	clusterN := len(s.cluster.Members())
+
+	// single-node fresh start, or single-node recovers from snapshot
+	if clusterN == 1 {
+		ticks := s.Cfg.ElectionTicks - 1
+		plog.Infof("%s as single-node; fast-forwarding %d ticks (election ticks %d)", s.ID(), ticks, s.Cfg.ElectionTicks)
+		s.r.advanceTicks(ticks)
+		return
+	}
+
+	// retry up to "rafthttp.ConnReadTimeout", which is 5-sec
+	// until peer connection reports; otherwise:
+	// 1. all connections failed, or
+	// 2. no active peers, or
+	// 3. restarted single-node with no snapshot
+	// then, do nothing, because advancing ticks would have no effect
+	waitTime := rafthttp.ConnReadTimeout
+	itv := 50 * time.Millisecond
+	for i := int64(0); i < int64(waitTime/itv); i++ {
+		select {
+		case <-time.After(itv):
+		case <-s.stopping:
+			return
+		}
+
+		peerN := s.r.transport.ActivePeers()
+		if peerN > 1 {
+			// multi-node received peer connection reports
+			// adjust ticks, in case slow leader message receive
+			ticks := s.Cfg.ElectionTicks - 2
+			plog.Infof("%s initialzed peer connection; fast-forwarding %d ticks (election ticks %d) with %d active peer(s)", s.ID(), ticks, s.Cfg.ElectionTicks, peerN)
+			s.r.advanceTicks(ticks)
+			return
+		}
+	}
+}
+
 // Start performs any initialization of the Server necessary for it to
 // begin serving requests. It must be called before Do or Process.
 // Start must be non-blocking; any long-running server functionality
 // should be implemented in goroutines.
 func (s *EtcdServer) Start() {
 	s.start()
+	s.goAttach(func() { s.adjustTicks() })
 	s.goAttach(func() { s.publish(s.Cfg.ReqTimeout()) })
 	s.goAttach(s.purgeFile)
 	s.goAttach(func() { monitorFileDescriptor(s.stopping) })
@@ -794,14 +833,8 @@ func (s *EtcdServer) run() {
 
 func (s *EtcdServer) applyAll(ep *etcdProgress, apply *apply) {
 	s.applySnapshot(ep, apply)
-	st := time.Now()
 	s.applyEntries(ep, apply)
-	d := time.Since(st)
-	entriesNum := len(apply.entries)
-	if entriesNum != 0 && d > time.Duration(entriesNum)*warnApplyDuration {
-		plog.Warningf("apply entries took too long [%v for %d entries]", d, len(apply.entries))
-		plog.Warningf("avoid queries with large range/delete range!")
-	}
+
 	proposalsApplied.Set(float64(ep.appliedi))
 	s.applyWait.Trigger(ep.appliedi)
 	// wait for the raft routine to finish the disk writes before triggering a

etcdserver/util.go

@@ -15,6 +15,7 @@
 package etcdserver
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/coreos/etcd/etcdserver/membership"
@@ -95,3 +96,19 @@ func (nc *notifier) notify(err error) {
 	nc.err = err
 	close(nc.c)
 }
+
+func warnOfExpensiveRequest(now time.Time, stringer fmt.Stringer) {
+	warnOfExpensiveGenericRequest(now, stringer, "")
+}
+
+func warnOfExpensiveReadOnlyRangeRequest(now time.Time, stringer fmt.Stringer) {
+	warnOfExpensiveGenericRequest(now, stringer, "read-only range ")
+}
+
+func warnOfExpensiveGenericRequest(now time.Time, stringer fmt.Stringer, prefix string) {
+	// TODO: add metrics
+	d := time.Since(now)
+	if d > warnApplyDuration {
+		plog.Warningf("%srequest %q took too long (%v) to execute", prefix, stringer.String(), d)
+	}
+}

etcdserver/util_test.go

@@ -83,6 +83,7 @@ func (s *nopTransporterWithActiveTime) RemovePeer(id types.ID)              {}
 func (s *nopTransporterWithActiveTime) RemoveAllPeers()                     {}
 func (s *nopTransporterWithActiveTime) UpdatePeer(id types.ID, us []string) {}
 func (s *nopTransporterWithActiveTime) ActiveSince(id types.ID) time.Time   { return s.activeMap[id] }
+func (s *nopTransporterWithActiveTime) ActivePeers() int                    { return 0 }
 func (s *nopTransporterWithActiveTime) Stop()                               {}
 func (s *nopTransporterWithActiveTime) Pause()                              {}
 func (s *nopTransporterWithActiveTime) Resume()                             {}

etcdserver/v2_server.go

@@ -158,3 +158,8 @@ func (r *RequestV2) Handle(ctx context.Context, v2api RequestV2Handler) (Respons
 	}
 	return Response{}, ErrUnknownMethod
 }
+
+func (r *RequestV2) String() string {
+	rpb := pb.Request(*r)
+	return rpb.String()
+}

etcdserver/v3_server.go

@@ -84,6 +84,8 @@ type Authenticator interface {
 }
 
 func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) {
+	defer warnOfExpensiveReadOnlyRangeRequest(time.Now(), r)
+
 	if !r.Serializable {
 		err := s.linearizableReadNotify(ctx)
 		if err != nil {
@@ -95,6 +97,7 @@ func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeRe
 	chk := func(ai *auth.AuthInfo) error {
 		return s.authStore.IsRangePermitted(ai, r.Key, r.RangeEnd)
 	}
+
 	get := func() { resp, err = s.applyV3Base.Range(nil, r) }
 	if serr := s.doSerialize(ctx, chk, get); serr != nil {
 		return nil, serr
@@ -131,12 +134,16 @@ func (s *EtcdServer) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse
 		chk := func(ai *auth.AuthInfo) error {
 			return checkTxnAuth(s.authStore, ai, r)
 		}
+
+		defer warnOfExpensiveReadOnlyRangeRequest(time.Now(), r)
+
 		get := func() { resp, err = s.applyV3Base.Txn(r) }
 		if serr := s.doSerialize(ctx, chk, get); serr != nil {
 			return nil, serr
 		}
 		return resp, err
 	}
+
 	resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{Txn: r})
 	if err != nil {
 		return nil, err

hack/scripts-dev/Makefile

@@ -1,8 +1,13 @@
 # run from repository root
-#
+
+
+
 # Example:
-#   make clean -f ./hack/scripts-dev/Makefile
 #   make build -f ./hack/scripts-dev/Makefile
+#   make clean -f ./hack/scripts-dev/Makefile
+#   make clean-docker -f ./hack/scripts-dev/Makefile
+#   make restart-docker -f ./hack/scripts-dev/Makefile
+#   make delete-docker-images -f ./hack/scripts-dev/Makefile
 
 .PHONY: build
 build:
@@ -23,53 +28,89 @@ clean:
 	rm -f ./clientv3/integration/127.0.0.1:* ./clientv3/integration/localhost:*
 	rm -f ./clientv3/ordering/127.0.0.1:* ./clientv3/ordering/localhost:*
 
-_GO_VERSION = 1.9.2
-ifdef GO_VERSION
-	_GO_VERSION = $(GO_VERSION)
+clean-docker:
+	docker images
+	docker image prune --force
+
+restart-docker:
+	service docker restart
+
+delete-docker-images:
+	docker rm --force $(docker ps -a -q) || true
+	docker rmi --force $(docker images -q) || true
+
+
+
+GO_VERSION ?= 1.10
+ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound")
+
+TEST_SUFFIX = $(shell date +%s | base64 | head -c 15)
+TEST_OPTS ?= PASSES='unit'
+
+TMP_DIR_MOUNT_FLAG = --mount type=tmpfs,destination=/tmp
+ifdef HOST_TMP_DIR
+	TMP_DIR_MOUNT_FLAG = --mount type=bind,source=$(HOST_TMP_DIR),destination=/tmp
 endif
 
+
+
 # Example:
-#   GO_VERSION=1.8.5 make build-docker-test -f ./hack/scripts-dev/Makefile
+#   GO_VERSION=1.8.7 make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
-#   GO_VERSION=1.8.5 make push-docker-test -f ./hack/scripts-dev/Makefile
+#   GO_VERSION=1.8.7 make push-docker-test -f ./hack/scripts-dev/Makefile
 #   make push-docker-test -f ./hack/scripts-dev/Makefile
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
-#   GO_VERSION=1.8.5 make pull-docker-test -f ./hack/scripts-dev/Makefile
+#   GO_VERSION=1.8.7 make pull-docker-test -f ./hack/scripts-dev/Makefile
 #   make pull-docker-test -f ./hack/scripts-dev/Makefile
 
 build-docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	@cat ./Dockerfile-test | sed s/REPLACE_ME_GO_VERSION/$(_GO_VERSION)/ \
-	  > ./.Dockerfile-test
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./Dockerfile-test
 	docker build \
-	  --tag gcr.io/etcd-development/etcd-test:go$(_GO_VERSION) \
-	  --file ./.Dockerfile-test .
+	  --tag gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  --file ./Dockerfile-test .
+	@mv ./Dockerfile-test.bak ./Dockerfile-test
 
 push-docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-test:go$(GO_VERSION)
 
 pull-docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	docker pull gcr.io/etcd-development/etcd-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-test:go$(GO_VERSION)
+
+
 
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make compile-with-docker-test -f ./hack/scripts-dev/Makefile
+#   make compile-setup-gopath-with-docker-test -f ./hack/scripts-dev/Makefile
 
 compile-with-docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
+	docker run \
+	  --rm \
+	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version"
+
+compile-setup-gopath-with-docker-test:
+	$(info GO_VERSION: $(GO_VERSION))
 	docker run \
 	  --rm \
 	  --mount type=bind,source=`pwd`,destination=/etcd \
-	  gcr.io/etcd-development/etcd-test:go$(_GO_VERSION) \
-	  /bin/bash -c "cd /etcd && GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version"
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && ETCD_SETUP_GOPATH=1 GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version && rm -rf ./gopath"
 
+
+
+# Example:
+#
 # Local machine:
 #   TEST_OPTS="PASSES='fmt'" make test -f ./hack/scripts-dev/Makefile
 #   TEST_OPTS="PASSES='fmt bom dep compile build unit'" make test -f ./hack/scripts-dev/Makefile
-#   TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional'" make test -f ./hack/scripts-dev/Makefile
+#   TEST_OPTS="PASSES='build unit release integration_e2e functional'" make test -f ./hack/scripts-dev/Makefile
 #   TEST_OPTS="PASSES='build grpcproxy'" make test -f ./hack/scripts-dev/Makefile
 #
 # Example (test with docker):
@@ -81,88 +122,77 @@ compile-with-docker-test:
 #   TEST_OPTS="PASSES='fmt bom dep compile build unit'" make docker-test -f ./hack/scripts-dev/Makefile
 #
 # Semaphore CI (test with docker):
-#   TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional'" make docker-test -f ./hack/scripts-dev/Makefile
-#   HOST_TMP_DI=/tmp TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional'" make docker-test -f ./hack/scripts-dev/Makefile
+#   TEST_OPTS="PASSES='build unit release integration_e2e functional'" make docker-test -f ./hack/scripts-dev/Makefile
+#   HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build unit release integration_e2e functional'" make docker-test -f ./hack/scripts-dev/Makefile
 #   TEST_OPTS="GOARCH=386 PASSES='build unit integration_e2e'" make docker-test -f ./hack/scripts-dev/Makefile
 #
 # grpc-proxy tests (test with docker):
 #   TEST_OPTS="PASSES='build grpcproxy'" make docker-test -f ./hack/scripts-dev/Makefile
-#   HOST_TMP_DI=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test -f ./hack/scripts-dev/Makefile
-
-TEST_SUFFIX = $(shell date +%s | base64 | head -c 15)
-
-_TEST_OPTS = PASSES='unit'
-ifdef TEST_OPTS
-	_TEST_OPTS = $(TEST_OPTS)
-endif
-
-_TMP_DIR_MOUNT_FLAG = --mount type=tmpfs,destination=/tmp
-ifdef HOST_TMP_DIR
-	_TMP_DIR_MOUNT_FLAG = --mount type=bind,source=$(HOST_TMP_DIR),destination=/tmp
-endif
+#   HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test -f ./hack/scripts-dev/Makefile
 
 .PHONY: test
 test:
-	$(info TEST_OPTS: $(_TEST_OPTS))
+	$(info TEST_OPTS: $(TEST_OPTS))
 	$(info log-file: test-$(TEST_SUFFIX).log)
-	$(_TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log
+	$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log
 	! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log
 
 docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	$(info TEST_OPTS: $(_TEST_OPTS))
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	$(info TEST_OPTS: $(TEST_OPTS))
 	$(info log-file: test-$(TEST_SUFFIX).log)
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
-	  gcr.io/etcd-development/etcd-test:go$(_GO_VERSION) \
-	  /bin/bash -c "$(_TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log"
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log"
 	! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log
 
 docker-test-coverage:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
 	$(info log-file: docker-test-coverage-$(TEST_SUFFIX).log)
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
-	  gcr.io/etcd-development/etcd-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
 	  /bin/bash -c "COVERDIR=covdir PASSES='build build_cov cov' ./test 2>&1 | tee docker-test-coverage-$(TEST_SUFFIX).log && /codecov -t 6040de41-c073-4d6f-bbf8-d89256ef31e1"
 	! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 docker-test-coverage-$(TEST_SUFFIX).log
 
-# build release container image with Linux
-_ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound")
-ifdef ETCD_VERSION
-	_ETCD_VERSION = $(ETCD_VERSION)
-endif
+
 
 # Example:
-#   ETCD_VERSION=v3.3.0-test.0 make build-docker-release-master -f ./hack/scripts-dev/Makefile
-#   ETCD_VERSION=v3.3.0-test.0 make push-docker-release-master -f ./hack/scripts-dev/Makefile
+#   make compile-with-docker-test -f ./hack/scripts-dev/Makefile
+#   ETCD_VERSION=v3-test make build-docker-release-master -f ./hack/scripts-dev/Makefile
+#   ETCD_VERSION=v3-test make push-docker-release-master -f ./hack/scripts-dev/Makefile
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
 
-build-docker-release-master: compile-with-docker-test
-	$(info ETCD_VERSION: $(_ETCD_VERSION))
+build-docker-release-master:
+	$(info ETCD_VERSION: $(ETCD_VERSION))
 	cp ./Dockerfile-release ./bin/Dockerfile-release
 	docker build \
-	  --tag gcr.io/etcd-development/etcd:$(_ETCD_VERSION) \
+	  --tag gcr.io/etcd-development/etcd:$(ETCD_VERSION) \
 	  --file ./bin/Dockerfile-release \
 	  ./bin
 	rm -f ./bin/Dockerfile-release
 
 	docker run \
 	  --rm \
-	  gcr.io/etcd-development/etcd:$(_ETCD_VERSION) \
+	  gcr.io/etcd-development/etcd:$(ETCD_VERSION) \
 	  /bin/sh -c "/usr/local/bin/etcd --version && ETCDCTL_API=3 /usr/local/bin/etcdctl version"
 
 push-docker-release-master:
-	$(info ETCD_VERSION: $(_ETCD_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd:$(_ETCD_VERSION)
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd:$(ETCD_VERSION)
+
+
 
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
@@ -176,49 +206,50 @@ push-docker-release-master:
 #   make docker-static-ip-test-certs-metrics-proxy-run -f ./hack/scripts-dev/Makefile
 
 build-docker-static-ip-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	@cat ./hack/scripts-dev/docker-static-ip/Dockerfile | sed s/REPLACE_ME_GO_VERSION/$(_GO_VERSION)/ \
-	  > ./hack/scripts-dev/docker-static-ip/.Dockerfile
-
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./hack/scripts-dev/docker-static-ip/Dockerfile
 	docker build \
-	  --tag gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION) \
-	  --file ./hack/scripts-dev/docker-static-ip/.Dockerfile \
+	  --tag gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
+	  --file ./hack/scripts-dev/docker-static-ip/Dockerfile \
 	  ./hack/scripts-dev/docker-static-ip
+	@mv ./hack/scripts-dev/docker-static-ip/Dockerfile.bak ./hack/scripts-dev/docker-static-ip/Dockerfile
 
 push-docker-static-ip-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION)
 
 pull-docker-static-ip-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	docker pull gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION)
 
 docker-static-ip-test-certs-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-static-ip/certs,destination=/certs \
-	  gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
 
 docker-static-ip-test-certs-metrics-proxy-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-static-ip/certs-metrics-proxy,destination=/certs-metrics-proxy \
-	  gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-metrics-proxy/run.sh && rm -rf m*.etcd"
 
+
+
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make compile-with-docker-test -f ./hack/scripts-dev/Makefile
@@ -227,76 +258,122 @@ docker-static-ip-test-certs-metrics-proxy-run:
 #   make push-docker-dns-test -f ./hack/scripts-dev/Makefile
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
 #   make pull-docker-dns-test -f ./hack/scripts-dev/Makefile
+#   make docker-dns-test-insecure-run -f ./hack/scripts-dev/Makefile
 #   make docker-dns-test-certs-run -f ./hack/scripts-dev/Makefile
 #   make docker-dns-test-certs-gateway-run -f ./hack/scripts-dev/Makefile
 #   make docker-dns-test-certs-wildcard-run -f ./hack/scripts-dev/Makefile
+#   make docker-dns-test-certs-common-name-auth-run -f ./hack/scripts-dev/Makefile
+#   make docker-dns-test-certs-common-name-multi-run -f ./hack/scripts-dev/Makefile
 
 build-docker-dns-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	@cat ./hack/scripts-dev/docker-dns/Dockerfile | sed s/REPLACE_ME_GO_VERSION/$(_GO_VERSION)/ \
-	  > ./hack/scripts-dev/docker-dns/.Dockerfile
-
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./hack/scripts-dev/docker-dns/Dockerfile
 	docker build \
-	  --tag gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
-	  --file ./hack/scripts-dev/docker-dns/.Dockerfile \
+	  --tag gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  --file ./hack/scripts-dev/docker-dns/Dockerfile \
 	  ./hack/scripts-dev/docker-dns
+	@mv ./hack/scripts-dev/docker-dns/Dockerfile.bak ./hack/scripts-dev/docker-dns/Dockerfile
 
 	docker run \
 	  --rm \
 	  --dns 127.0.0.1 \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "/etc/init.d/bind9 start && cat /dev/null >/etc/hosts && dig etcd.local"
 
 push-docker-dns-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION)
 
 pull-docker-dns-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	docker pull gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION)
 
-docker-dns-test-certs-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+docker-dns-test-insecure-run:
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/insecure,destination=/insecure \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /insecure/run.sh && rm -rf m*.etcd"
+
+docker-dns-test-certs-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs,destination=/certs \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
 
 docker-dns-test-certs-gateway-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-gateway,destination=/certs-gateway \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-gateway/run.sh && rm -rf m*.etcd"
 
 docker-dns-test-certs-wildcard-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-wildcard,destination=/certs-wildcard \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-wildcard/run.sh && rm -rf m*.etcd"
 
+docker-dns-test-certs-common-name-auth-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-common-name-auth,destination=/certs-common-name-auth \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-common-name-auth/run.sh && rm -rf m*.etcd"
+
+docker-dns-test-certs-common-name-multi-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-common-name-multi,destination=/certs-common-name-multi \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-common-name-multi/run.sh && rm -rf m*.etcd"
+
+
+
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make compile-with-docker-test -f ./hack/scripts-dev/Makefile
@@ -310,84 +387,113 @@ docker-dns-test-certs-wildcard-run:
 #   make docker-dns-srv-test-certs-wildcard-run -f ./hack/scripts-dev/Makefile
 
 build-docker-dns-srv-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	@cat ./hack/scripts-dev/docker-dns-srv/Dockerfile | sed s/REPLACE_ME_GO_VERSION/$(_GO_VERSION)/ \
-	  > ./hack/scripts-dev/docker-dns-srv/.Dockerfile
-
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./hack/scripts-dev/docker-dns-srv/Dockerfile
 	docker build \
-	  --tag gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
-	  --file ./hack/scripts-dev/docker-dns-srv/.Dockerfile \
+	  --tag gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
+	  --file ./hack/scripts-dev/docker-dns-srv/Dockerfile \
 	  ./hack/scripts-dev/docker-dns-srv
+	@mv ./hack/scripts-dev/docker-dns-srv/Dockerfile.bak ./hack/scripts-dev/docker-dns-srv/Dockerfile
 
 	docker run \
 	  --rm \
 	  --dns 127.0.0.1 \
-	  gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
 	  /bin/bash -c "/etc/init.d/bind9 start && cat /dev/null >/etc/hosts && dig +noall +answer SRV _etcd-client-ssl._tcp.etcd.local && dig +noall +answer SRV _etcd-server-ssl._tcp.etcd.local && dig +noall +answer m1.etcd.local m2.etcd.local m3.etcd.local"
 
 push-docker-dns-srv-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION)
 
 pull-docker-dns-srv-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	docker pull gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION)
 
 docker-dns-srv-test-certs-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns-srv/certs,destination=/certs \
-	  gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
 
 docker-dns-srv-test-certs-gateway-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns-srv/certs-gateway,destination=/certs-gateway \
-	  gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-gateway/run.sh && rm -rf m*.etcd"
 
 docker-dns-srv-test-certs-wildcard-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns-srv/certs-wildcard,destination=/certs-wildcard \
-	  gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-wildcard/run.sh && rm -rf m*.etcd"
 
-# example workflow for common name + auth
-# TODO: make this as tests
-# make docker-dns-example-certs-common-name-run -f ./hack/scripts-dev/Makefile
-docker-dns-example-certs-common-name-run:
-	$(info GO_VERSION: $(_GO_VERSION))
-	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+
+
+# Example:
+#   make build-etcd-test-proxy -f ./hack/scripts-dev/Makefile
+
+build-etcd-test-proxy:
+	go build -v -o ./bin/etcd-test-proxy ./tools/etcd-test-proxy
+
+
+
+# Example:
+#   make build-docker-functional-tester -f ./hack/scripts-dev/Makefile
+#   make push-docker-functional-tester -f ./hack/scripts-dev/Makefile
+#   make pull-docker-functional-tester -f ./hack/scripts-dev/Makefile
+
+build-docker-functional-tester:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./Dockerfile-functional-tester
+	docker build \
+	  --tag gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) \
+	  --file ./Dockerfile-functional-tester \
+	  .
+	@mv ./Dockerfile-functional-tester.bak ./Dockerfile-functional-tester
+
 	docker run \
 	  --rm \
-	  --tty \
-	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
-	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
-	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-common-name,destination=/certs-common-name \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
-	  /bin/bash -c "cd /etcd && /certs-common-name/run.sh && rm -rf m*.etcd"
+	  gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) \
+	  /bin/bash -c "/etcd --version && \
+	   /etcd-failpoints --version && \
+	   ETCDCTL_API=3 /etcdctl version && \
+	   /etcd-agent -help || true && \
+	   /etcd-tester -help || true && \
+	   /etcd-runner --help || true && \
+	   /benchmark --help || true && \
+	   /etcd-test-proxy -help || true"
+
+push-docker-functional-tester:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION)
+
+pull-docker-functional-tester:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	docker pull gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION)

hack/scripts-dev/docker-dns-srv/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.10
+FROM ubuntu:17.10
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

hack/scripts-dev/docker-dns/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.10
+FROM ubuntu:17.10
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

hack/scripts-dev/docker-dns/certs-common-name-auth/Procfile

@@ -0,0 +1,6 @@
+# Use goreman to run `go get github.com/mattn/goreman`
+etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls https://127.0.0.1:2379 --advertise-client-urls https://m1.etcd.local:2379 --listen-peer-urls https://127.0.0.1:2380 --initial-advertise-peer-urls=https://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-auth/server.crt --peer-key-file=/certs-common-name-auth/server.key.insecure --peer-trusted-ca-file=/certs-common-name-auth/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name-auth/server.crt --key-file=/certs-common-name-auth/server.key.insecure --trusted-ca-file=/certs-common-name-auth/ca.crt --client-cert-auth
+
+etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls https://127.0.0.1:22379 --advertise-client-urls https://m2.etcd.local:22379 --listen-peer-urls https://127.0.0.1:22380 --initial-advertise-peer-urls=https://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-auth/server.crt --peer-key-file=/certs-common-name-auth/server.key.insecure --peer-trusted-ca-file=/certs-common-name-auth/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name-auth/server.crt --key-file=/certs-common-name-auth/server.key.insecure --trusted-ca-file=/certs-common-name-auth/ca.crt --client-cert-auth
+
+etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls https://127.0.0.1:32379 --advertise-client-urls https://m3.etcd.local:32379 --listen-peer-urls https://127.0.0.1:32380 --initial-advertise-peer-urls=https://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-auth/server.crt --peer-key-file=/certs-common-name-auth/server.key.insecure --peer-trusted-ca-file=/certs-common-name-auth/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name-auth/server.crt --key-file=/certs-common-name-auth/server.key.insecure --trusted-ca-file=/certs-common-name-auth/ca.crt --client-cert-auth

hack/scripts-dev/docker-dns/certs-common-name-auth/run.sh

@@ -6,65 +6,65 @@ rm -rf /tmp/m1.data /tmp/m2.data /tmp/m3.data
 # get rid of hosts so go lookup won't resolve 127.0.0.1 to localhost
 cat /dev/null >/etc/hosts
 
-goreman -f /certs-common-name/Procfile start &
+goreman -f /certs-common-name-auth/Procfile start &
 
 # TODO: remove random sleeps
 sleep 7s
 
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379 \
   endpoint health --cluster
 
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   put abc def
 
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   get abc
 
 sleep 1s && printf "\n"
 echo "Step 1. creating root role"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   role add root
 
 sleep 1s && printf "\n"
 echo "Step 2. granting readwrite 'foo' permission to role 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   role grant-permission root readwrite foo
 
 sleep 1s && printf "\n"
 echo "Step 3. getting role 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   role get root
 
 sleep 1s && printf "\n"
 echo "Step 4. creating user 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --interactive=false \
   user add root:123
@@ -72,36 +72,36 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 5. granting role 'root' to user 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   user grant-role root root
 
 sleep 1s && printf "\n"
 echo "Step 6. getting user 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   user get root
 
 sleep 1s && printf "\n"
 echo "Step 7. enabling auth"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   auth enable
 
 sleep 1s && printf "\n"
 echo "Step 8. writing 'foo' with 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   put foo bar
@@ -109,9 +109,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 9. writing 'aaa' with 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   put aaa bbb
@@ -119,18 +119,18 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 10. writing 'foo' without 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   put foo bar
 
 sleep 1s && printf "\n"
 echo "Step 11. reading 'foo' with 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   get foo
@@ -138,9 +138,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 12. reading 'aaa' with 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   get aaa
@@ -148,9 +148,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 13. creating a new user 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   --interactive=false \
@@ -159,9 +159,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 14. creating a role 'test-role'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   role add test-role
@@ -169,9 +169,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 15. granting readwrite 'aaa' --prefix permission to role 'test-role'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   role grant-permission test-role readwrite aaa --prefix
@@ -179,9 +179,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 16. getting role 'test-role'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   role get test-role
@@ -189,9 +189,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 17. granting role 'test-role' to user 'test-common-name'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   user grant-role test-common-name test-role
@@ -199,9 +199,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 18. writing 'aaa' with 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=test-common-name:test-pass \
   put aaa bbb
@@ -209,9 +209,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 19. writing 'bbb' with 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=test-common-name:test-pass \
   put bbb bbb
@@ -219,9 +219,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 20. reading 'aaa' with 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=test-common-name:test-pass \
   get aaa
@@ -229,9 +229,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 21. reading 'bbb' with 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=test-common-name:test-pass \
   get bbb
@@ -239,17 +239,17 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 22. writing 'aaa' with CommonName 'test-common-name'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   put aaa ccc
 
 sleep 1s && printf "\n"
 echo "Step 23. reading 'aaa' with CommonName 'test-common-name'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   get aaa

hack/scripts-dev/docker-dns/certs-common-name-multi/Procfile

@@ -0,0 +1,6 @@
+# Use goreman to run `go get github.com/mattn/goreman`
+etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls https://127.0.0.1:2379 --advertise-client-urls https://m1.etcd.local:2379 --listen-peer-urls https://127.0.0.1:2380 --initial-advertise-peer-urls=https://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-multi/server-1.crt --peer-key-file=/certs-common-name-multi/server-1.key.insecure --peer-trusted-ca-file=/certs-common-name-multi/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-common-name-multi/server-1.crt --key-file=/certs-common-name-multi/server-1.key.insecure --trusted-ca-file=/certs-common-name-multi/ca.crt --client-cert-auth
+
+etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls https://127.0.0.1:22379 --advertise-client-urls https://m2.etcd.local:22379 --listen-peer-urls https://127.0.0.1:22380 --initial-advertise-peer-urls=https://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-multi/server-2.crt --peer-key-file=/certs-common-name-multi/server-2.key.insecure --peer-trusted-ca-file=/certs-common-name-multi/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-common-name-multi/server-2.crt --key-file=/certs-common-name-multi/server-2.key.insecure --trusted-ca-file=/certs-common-name-multi/ca.crt --client-cert-auth
+
+etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls https://127.0.0.1:32379 --advertise-client-urls https://m3.etcd.local:32379 --listen-peer-urls https://127.0.0.1:32380 --initial-advertise-peer-urls=https://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-multi/server-3.crt --peer-key-file=/certs-common-name-multi/server-3.key.insecure --peer-trusted-ca-file=/certs-common-name-multi/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-common-name-multi/server-3.crt --key-file=/certs-common-name-multi/server-3.key.insecure --trusted-ca-file=/certs-common-name-multi/ca.crt --client-cert-auth

hack/scripts-dev/docker-dns/certs-common-name-multi/ca-csr.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "ca",
+  "ca": {
+    "expiry": "87600h"
+  }
+}

hack/scripts-dev/docker-dns/certs-common-name-multi/ca.crt

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID0jCCArqgAwIBAgIUd3UZnVmZFo8x9MWWhUrYQvZHLrQwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMjAwNjAwMDBaFw0yODAxMTgwNjAw
+MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCqgFTgSFl+ugXkZuiN5PXp84Zv05crwI5x2ePMnc2/3u1s7cQBvXQGCJcq
+OwWD7tjcy4K2PDC0DLRa4Mkd8JpwADmf6ojbMH/3a1pXY2B3BJQwmNPFnxRJbDZL
+Iti6syWKwyfLVb1KFCU08G+ZrWmGIXPWDiE+rTn/ArD/6WbQI1LYBFJm25NLpttM
+mA3HnWoErNGY4Z/AR54ROdQSPL7RSUZBa0Kn1riXeOJ40/05qosR2O/hBSAGkD+m
+5Rj+A6oek44zZqVzCSEncLsRJAKqgZIqsBrErAho72irEgTwv4OM0MyOCsY/9erf
+hNYRSoQeX+zUvEvgToalfWGt6kT3AgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIBBjAS
+BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBRDePNja5CK4zUfO5x1vzGvdmUF
+CzAfBgNVHSMEGDAWgBRDePNja5CK4zUfO5x1vzGvdmUFCzANBgkqhkiG9w0BAQsF
+AAOCAQEAZu0a3B7Ef/z5Ct99xgzPy4z9RwglqPuxk446hBWR5TYT9fzm+voHCAwb
+MJEaQK3hvAz47qAjyR9/b+nBw4LRTMxg0WqB+UEEVwBGJxtfcOHx4mJHc3lgVJnR
+LiEWtIND7lu5Ql0eOjSehQzkJZhUb4SnXD7yk64zukQQv9zlZYZCHPDAQ9LzR2vI
+ii4yhwdWl7iiZ0lOyR4xqPB3Cx/2kjtuRiSkbpHGwWBJLng2ZqgO4K+gL3naNgqN
+TRtdOSK3j/E5WtAeFUUT68Gjsg7yXxqyjUFq+piunFfQHhPB+6sPPy56OtIogOk4
+dFCfFAygYNrFKz366KY+7CbpB+4WKA==
+-----END CERTIFICATE-----

hack/scripts-dev/docker-dns/certs-common-name-multi/gencert.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "87600h"
+    }
+  }
+}

hack/scripts-dev/docker-dns/certs-common-name-multi/gencerts.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if ! [[ "$0" =~ "./gencerts.sh" ]]; then
+	echo "must be run from 'fixtures'"
+	exit 255
+fi
+
+if ! which cfssl; then
+	echo "cfssl is not installed"
+	exit 255
+fi
+
+cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca
+mv ca.pem ca.crt
+openssl x509 -in ca.crt -noout -text
+
+# generate wildcard certificates DNS: m1/m2/m3.etcd.local
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-1.json | cfssljson --bare ./server-1
+mv server-1.pem server-1.crt
+mv server-1-key.pem server-1.key.insecure
+
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-2.json | cfssljson --bare ./server-2
+mv server-2.pem server-2.crt
+mv server-2-key.pem server-2.key.insecure
+
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-3.json | cfssljson --bare ./server-3
+mv server-3.pem server-3.crt
+mv server-3-key.pem server-3.key.insecure
+
+rm -f *.csr *.pem *.stderr *.txt

hack/scripts-dev/docker-dns/certs-common-name-multi/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+rm -rf /tmp/m1.data /tmp/m2.data /tmp/m3.data
+
+/etc/init.d/bind9 start
+
+# get rid of hosts so go lookup won't resolve 127.0.0.1 to localhost
+cat /dev/null >/etc/hosts
+
+goreman -f /certs-common-name-multi/Procfile start &
+
+# TODO: remove random sleeps
+sleep 7s
+
+ETCDCTL_API=3 ./etcdctl \
+  --cacert=/certs-common-name-multi/ca.crt \
+  --cert=/certs-common-name-multi/server-1.crt \
+  --key=/certs-common-name-multi/server-1.key.insecure \
+  --endpoints=https://m1.etcd.local:2379 \
+  endpoint health --cluster
+
+ETCDCTL_API=3 ./etcdctl \
+  --cacert=/certs-common-name-multi/ca.crt \
+  --cert=/certs-common-name-multi/server-2.crt \
+  --key=/certs-common-name-multi/server-2.key.insecure \
+  --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
+  put abc def
+
+ETCDCTL_API=3 ./etcdctl \
+  --cacert=/certs-common-name-multi/ca.crt \
+  --cert=/certs-common-name-multi/server-3.crt \
+  --key=/certs-common-name-multi/server-3.key.insecure \
+  --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
+  get abc

hack/scripts-dev/docker-dns/certs-common-name-multi/server-1.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEIDCCAwigAwIBAgIUaDLXBmJpHrElwENdnVk9hvAvlKcwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMjAwNjAwMDBaFw0yODAxMTgwNjAw
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAOb5CdovL9QCdgsxnCBikTbJko6r5mrF+eA47gDLcVbWrRW5
+d8eZYV1Fyn5qe80O6LB6LKPrRftxyAGABKqIBCHR57E97UsICC4lGycBWaav6cJ+
+7Spkpf8cSSDjjgb4KC6VVPf9MCsHxBYSTfme8JEFE+6KjlG8Mqt2yv/5aIyRYITN
+WzXvV7wxS9aOgDdXLbojW9FJQCuzttOPfvINTyhtvUvCM8S61La5ymCdAdPpx1U9
+m5KC23k6ZbkAC8/jcOV+68adTUuMWLefPf9Ww3qMT8382k86gJgQjZuJDGUl3Xi5
+GXmO0GfrMh+v91yiaiqjsJCDp3uVcUSeH7qSkb0CAwEAAaOBqzCBqDAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFEwLLCuIHilzynJ7DlTrikyhy2TAMB8GA1UdIwQYMBaA
+FEN482NrkIrjNR87nHW/Ma92ZQULMCkGA1UdEQQiMCCCDW0xLmV0Y2QubG9jYWyC
+CWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAkERnrIIvkZHWsyih
+mFNf/JmFHC+0/UAG9Ti9msRlr9j1fh+vBIid3FAIShX0zFXf+AtN/+Bz5SVvQHUT
+tm71AK/vER1Ue059SIty+Uz5mNAjwtXy0WaUgSuF4uju7MkYD5yUnSGv1iBfm88a
+q+q1Vd5m6PkOCfuyNQQm5RKUiJiO4OS+2F9/JOpyr0qqdQthOWr266CqXuvVhd+Z
+oZZn5TLq5GHCaTxfngSqS3TXl55QEGl65SUgYdGqpIfaQt3QKq2dqVg/syLPkTJt
+GNJVLxJuUIu0PLrfuWynUm+1mOOfwXd8NZVZITUxC7Tl5ecFbTaOzU/4a7Cyssny
+Wr3dUg==
+-----END CERTIFICATE-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-1.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA5vkJ2i8v1AJ2CzGcIGKRNsmSjqvmasX54DjuAMtxVtatFbl3
+x5lhXUXKfmp7zQ7osHoso+tF+3HIAYAEqogEIdHnsT3tSwgILiUbJwFZpq/pwn7t
+KmSl/xxJIOOOBvgoLpVU9/0wKwfEFhJN+Z7wkQUT7oqOUbwyq3bK//lojJFghM1b
+Ne9XvDFL1o6AN1ctuiNb0UlAK7O2049+8g1PKG29S8IzxLrUtrnKYJ0B0+nHVT2b
+koLbeTpluQALz+Nw5X7rxp1NS4xYt589/1bDeoxPzfzaTzqAmBCNm4kMZSXdeLkZ
+eY7QZ+syH6/3XKJqKqOwkIOne5VxRJ4fupKRvQIDAQABAoIBAQCYQsXm6kJqTbEJ
+kgutIa0+48TUfqen7Zja4kyrg3HU4DI75wb6MreHqFFj4sh4FoL4i6HP8XIx3wEN
+VBo/XOj0bo6BPiSm2MWjvdxXa0Fxa/f6uneYAb+YHEps/vWKzJ6YjuLzlBnj0/vE
+3Q5AJzHJOAK6tuY5JYp1lBsggYcVWiQSW6wGQRReU/B/GdFgglL1chqL33Dt11Uv
+Y6+oJz/PyqzPLPHcPbhqyQRMOZXnhx+8/+ooq5IojqOHfpa9JQURcHY7isBnpI/G
+ZAa8tZctgTqtL4hB1rxDhdq1fS2YC12lxkBZse4jszcm0tYzy2gWmNTH480uo/0J
+GOxX7eP1AoGBAO7O+aLhQWrspWQ//8YFbPWNhyscQub+t6WYjc0wn9j0dz8vkhMw
+rh5O8uMcZBMDQdq185BcB3aHInw9COWZEcWNIen4ZyNJa5VCN4FY0a2GtFSSGG3f
+ilKmQ7cjB950q2jl1AR3t2H7yah+i1ZChzPx+GEe+51LcJZX8mMjGvwjAoGBAPeZ
+qJ2W4O2dOyupAfnKpZZclrEBqlyg7Xj85u20eBMUqtaIEcI/u2kaotQPeuaekUH0
+b1ybr3sJBTp3qzHUaNV3iMfgrnbWEOkIV2TCReWQb1Fk93o3gilMIkhGLIhxwWpM
+UpQy3JTjGG/Y6gIOs7YnOBGVMA0o+RvouwooU6ifAoGAH6D6H0CGUYsWPLjdP3To
+gX1FMciEc+O4nw4dede+1BVM1emPB0ujRBBgywOvnXUI+9atc6k8s84iGyJaU056
+tBeFLl/gCSRoQ1SJ1W/WFY2JxMm0wpig0WGEBnV1TVlWeoY2FoFkoG2gv9hCzCHz
+lkWuB+76lFKxjrgHOmoj4NECgYB+COmbzkGQsoh8IPuwe0bu0xKh54cgv4oiHBow
+xbyZedu8eGcRyf9L8RMRfw/AdNbcC+Dj8xvQNTdEG8Y5BzaV8tLda7FjLHRPKr/R
+ulJ6GJuRgyO2Qqsu+mI5B/+DNOSPh2pBpeJCp5a42GHFylYQUsZnrNlY2ZJ0cnND
+KGPtYQKBgQDL30+BB95FtRUvFoJIWwASCp7TIqW7N7RGWgqmsXU0EZ0Mya4dquqG
+rJ1QuXQIJ+xV060ehwJR+iDUAY2xUg3/LCoDD0rwBzSdh+NEKjOmRNFRtn7WT03Q
+264E80r6VTRSN4sWQwAAbd1VF1uGO5tkzZdJGWGhQhvTUZ498dE+9Q==
+-----END RSA PRIVATE KEY-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-2.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEIDCCAwigAwIBAgIUHXDUS+Vry/Tquc6S6OoaeuGozrEwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMjAwNjAwMDBaFw0yODAxMTgwNjAw
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAOO+FsO+6pwpv+5K+VQTYQb0lT0BjnM7Y2qSZIiTGCDp/M0P
+yHSed4oTzxBeA9hEytczH/oddAUuSZNgag5sGFVgjFNdiZli4wQqJaMQRodivuUl
+ZscqnWwtP3GYVAfg+t/4YdGB+dQRDQvHBl9BRYmUh2ixOA98OXKfNMr+u+3sh5Gy
+dwx5ZEBRvgBcRrgCaIMsvVeIzHQBMHrNySAD1bGgm3xGdLeVPhAp24yUKZ5IbN6/
++5hyCRARtGwLH/1Q/h10Sr5jxQi00eEXH+CNOvcerH6b2II/BxHIcqKd0u36pUfG
+0KsY+ia0fvYi510V6Q0FAn45luEjHEk5ITN/LnMCAwEAAaOBqzCBqDAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFE69SZun6mXZe6cd3Cb2HWrK281MMB8GA1UdIwQYMBaA
+FEN482NrkIrjNR87nHW/Ma92ZQULMCkGA1UdEQQiMCCCDW0yLmV0Y2QubG9jYWyC
+CWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAI5nHHULV7eUJMsvv
+zk1shv826kOwXbMX10iRaf49/r7TWBq0pbPapvf5VXRsZ5wlDrDzjaNstpsaow/j
+fhZ1zpU0h1bdifxE+omFSWZjpVM8kQD/yzT34VdyA+P2HuxG8ZTa8r7wTGrooD60
+TjBBM5gFV4nGVe+KbApQ26KWr+P8biKaWe6MM/jAv6TNeXiWReHqyM5v404PZQXK
+cIN+fBb8bQfuaKaN1dkOUI3uSHmVmeYc5OGNJ2QKL9Uzm1VGbbM+1BOLhmF53QSm
+5m2B64lPKy+vpTcRLN7oW1FHZOKts+1OEaLMCyjWFKFbdcrmJI+AP2IB+V6ODECn
+RwJDtA==
+-----END CERTIFICATE-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-2.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA474Ww77qnCm/7kr5VBNhBvSVPQGOcztjapJkiJMYIOn8zQ/I
+dJ53ihPPEF4D2ETK1zMf+h10BS5Jk2BqDmwYVWCMU12JmWLjBColoxBGh2K+5SVm
+xyqdbC0/cZhUB+D63/hh0YH51BENC8cGX0FFiZSHaLE4D3w5cp80yv677eyHkbJ3
+DHlkQFG+AFxGuAJogyy9V4jMdAEwes3JIAPVsaCbfEZ0t5U+ECnbjJQpnkhs3r/7
+mHIJEBG0bAsf/VD+HXRKvmPFCLTR4Rcf4I069x6sfpvYgj8HEchyop3S7fqlR8bQ
+qxj6JrR+9iLnXRXpDQUCfjmW4SMcSTkhM38ucwIDAQABAoIBAQCHYF6N2zYAwDyL
+/Ns65A4gIVF5Iyy3SM0u83h5St7j6dNRXhltYSlz1ZSXiRtF+paM16IhflKSJdKs
+nXpNumm4jpy7jXWWzRZfSmJ3DNyv673H3rS6nZVYUYlOEBubV1wpuK8E5/tG2R/l
+KVibVORuBPF9BSNq6RAJF6Q9KrExmvH4MmG/3Y+iYbZgn0OK1WHxzbeMzdI8OO4z
+eg4gTKuMoRFt5B4rZmC5QiXGHdnUXRWfy+yPLTH3hfTek4JT98akFNS01Q4UAi9p
+5cC3TOqDNiZdAkN83UKhW9TNAc/vJlq6d5oXW5R+yPt+d8yMvEch4KfpYo33j0oz
+qB40pdJRAoGBAP8ZXnWXxhzLhZ4o+aKefnsUUJjaiVhhSRH/kGAAg65lc4IEnt+N
+nzyNIwz/2vPv2Gq2BpStrTsTNKVSZCKgZhoBTavP60FaszDSM0bKHTWHW7zaQwc0
+bQG6YvvCiP0iwEzXw7S4BhdAl+x/5C30dUZgKMSDFzuBI187h6dQQNZpAoGBAOSL
+/MBuRYBgrHIL9V1v9JGDBeawGc3j2D5c56TeDtGGv8WGeCuE/y9tn+LcKQ+bCGyi
+qkW+hobro/iaXODwUZqSKaAVbxC7uBLBTRB716weMzrnD8zSTOiMWg/gh+FOnr/4
+ZfcBco2Pmm5qQ3ZKwVk2jsfLhz6ZKwMrjSaO1Zp7AoGBAJZsajPjRHI0XN0vgkyv
+Mxv2lbQcoYKZE1JmpcbGZt/OePdBLEHcq/ozq2h98qmHU9FQ9r5zT0QXhiK6W8vD
+U5GgFSHsH+hQyHtQZ+YlRmYLJEBPX9j+xAyR0M5uHwNNm6F0VbXaEdViRHOz0mR6
+0zClgUSnnGp9MtN0MgCqJSGJAoGAJYba3Jn+rYKyLhPKmSoN5Wq3KFbYFdeIpUzJ
++GdB1aOjj4Jx7utqn1YHv89YqqhRLM1U2hjbrAG7LdHi2Eh9jbzcOt3qG7xHEEVP
+Kxq6ohdfYBean44UdMa+7wZ2KUeoh2r5CyLgtV/UArdOFnlV4Bk2PpYrwdqSlnWr
+Op6PcksCgYEA6HmIHLRTGyOUzS82BEcs5an2mzhQ8XCNdYS6sDaYSiDu2qlPukyZ
+jons6P4qpOxlP9Cr6DW7px2fUZrEuPUV8fRJOc+a5AtZ5TmV6N1uH/G1rKmmAMCc
+jGAmTJW87QguauTpuUto5u6IhyO2CRsYEy8K1A/1HUQKl721faZBIMA=
+-----END RSA PRIVATE KEY-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-3.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEIDCCAwigAwIBAgIURfpNMXGb1/oZVwEWyc0Ofn7IItQwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMjAwNjAwMDBaFw0yODAxMTgwNjAw
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBALgCDkDM4qayF6CFt1ZScKR8B+/7qrn1iQ/qYnzRHQ1hlkuS
+b3TkQtt7amGAuoD42d8jLYYvHn2Pbmdhn0mtgYZpFfLFCg4O67ZbX54lBHi+yDEh
+QhneM9Ovsc42A0EVvabINYtKR6B2YRN00QRXS5R1t+QmclpshFgY0+ITsxlJeygs
+wojXthPEfjTQK04JUi5LTHP15rLVzDEd7MguCWdEWRnOu/mSfPHlyz2noUcKuy0M
+awsnSMwf+KBwQMLbJhTXtA4MG2FYsm/2en3/oAc8/0Z8sMOX05F+b0MgHl+a31aQ
+UHM5ykfDNm3hGQfzjQCx4y4hjDoFxbuXvsey6GMCAwEAAaOBqzCBqDAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFDMydqyg/s43/dJTMt25zJubI/CUMB8GA1UdIwQYMBaA
+FEN482NrkIrjNR87nHW/Ma92ZQULMCkGA1UdEQQiMCCCDW0zLmV0Y2QubG9jYWyC
+CWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAVs3VQjgx9CycaWKS
+P6EvMtlqOkanJEe3zr69sI66cc2ZhfJ5xK38ox4oYpMOA131WRvwq0hjKhhZoVQ8
+aQ4yALi1XBltuIyEyrTX9GWAMeDzY95MdWKhyI8ps6/OOoXN596g9ZdOdIbZAMT4
+XAXm43WccM2W2jiKCEKcE4afIF8RiMIaFwG8YU8oHtnnNvxTVa0wrpcObtEtIzC5
+RJxzX9bkHCTHTgJog4OPChU4zffn18U/AVJ7MZ8gweVwhc4gGe0kwOJE+mLHcC5G
+uoFSuVmAhYrH/OPpZhSDOaCED4dsF5jN25CbR3NufEBFRXBH20ZHNkNvbbBnYCBU
+4+Rx5w==
+-----END CERTIFICATE-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-3.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAuAIOQMziprIXoIW3VlJwpHwH7/uqufWJD+pifNEdDWGWS5Jv
+dORC23tqYYC6gPjZ3yMthi8efY9uZ2GfSa2BhmkV8sUKDg7rtltfniUEeL7IMSFC
+Gd4z06+xzjYDQRW9psg1i0pHoHZhE3TRBFdLlHW35CZyWmyEWBjT4hOzGUl7KCzC
+iNe2E8R+NNArTglSLktMc/XmstXMMR3syC4JZ0RZGc67+ZJ88eXLPaehRwq7LQxr
+CydIzB/4oHBAwtsmFNe0DgwbYViyb/Z6ff+gBzz/Rnyww5fTkX5vQyAeX5rfVpBQ
+cznKR8M2beEZB/ONALHjLiGMOgXFu5e+x7LoYwIDAQABAoIBAQCY54RmjprNAHKn
+vlXCEpFt7W8/GXcePg2ePxuGMtKcevpEZDPgA4oXDnAxA6J3Z9LMHFRJC8Cff9+z
+YqjVtatLQOmvKdMYKYfvqfBD3ujfWVHLmaJvEnkor/flrnZ30BQfkoED9T6d9aDn
+ZQwHOm8gt82OdfBSeZhkCIWReOM73622qJhmLWUUY3xEucRAFF6XffOLvJAT87Vu
+pXKtCnQxhzxkUsCYNIOeH/pTX+XoLkysFBKxnrlbTeM0cEgWpYMICt/vsUrp6DHs
+jygxR1EnT2/4ufe81aFSO4SzUZKJrz8zj4yIyDOR0Mp6FW+xMp8S0fDOywHhLlXn
+xQOevmGBAoGBAOMQaWWs2FcxWvLfX95RyWPtkQ+XvmWlL5FR427TlLhtU6EPs0xZ
+eeanMtQqSRHlDkatwc0XQk+s30/UJ+5i1iz3shLwtnZort/pbnyWrxkE9pcR0fgr
+IklujJ8e8kQHpY75gOLmEiADrUITqvfbvSMsaG3h1VydPNU3JYTUuYmjAoGBAM91
+Atnri0PH3UKonAcMPSdwQ5NexqAD1JUk6KUoX2poXBXO3zXBFLgbMeJaWthbe+dG
+Raw/zjBET/oRfDOssh+QTD8TutI9LA2+EN7TG7Kr6NFciz4Q2pioaimv9KUhJx+8
+HH2wCANYgkv69IWUFskF0uDCW9FQVvpepcctCJJBAoGAMlWxB5kJXErUnoJl/iKj
+QkOnpI0+58l2ggBlKmw8y6VwpIOWe5ZaL4dg/Sdii1T7lS9vhsdhK8hmuIuPToka
+cV13XDuANz99hKV6mKPOrP0srNCGez0UnLKk+aEik3IegVNN/v6BhhdKkRtLCybr
+BqERhUpKwf0ZPyq6ZnfBqYECgYEAsiD2YcctvPVPtnyv/B02JTbvzwoB4kNntOgM
+GkOgKe2Ro+gNIEq5T5uKKaELf9qNePeNu2jN0gPV6BI7YuNVzmRIE6ENOJfty573
+PVxm2/Nf5ORhatlt2MZC4aiDl4Xv4f/TNth/COBmgHbqngeZyOGHQBWiYQdqp2+9
+SFgSlAECgYEA1zLhxj6f+psM5Gpx56JJIEraHfyuyR1Oxii5mo7I3PLsbF/s6YDR
+q9E64GoR5PdgCQlMm09f6wfT61NVwsYrbLlLET6tAiG0eNxXe71k1hUb6aa4DpNQ
+IcS3E3hb5KREXUH5d+PKeD2qrf52mtakjn9b2aH2rQw2e2YNkIDV+XA=
+-----END RSA PRIVATE KEY-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-ca-csr-1.json

@@ -0,0 +1,21 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "etcd.local",
+  "hosts": [
+    "m1.etcd.local",
+    "127.0.0.1",
+    "localhost"
+  ]
+}

hack/scripts-dev/docker-dns/certs-common-name-multi/server-ca-csr-2.json

@@ -0,0 +1,21 @@
+{
+    "key": {
+      "algo": "rsa",
+      "size": 2048
+    },
+    "names": [
+      {
+        "O": "etcd",
+        "OU": "etcd Security",
+        "L": "San Francisco",
+        "ST": "California",
+        "C": "USA"
+      }
+    ],
+    "CN": "etcd.local",
+    "hosts": [
+      "m2.etcd.local",
+      "127.0.0.1",
+      "localhost"
+    ]
+  }

hack/scripts-dev/docker-dns/certs-common-name-multi/server-ca-csr-3.json

@@ -0,0 +1,21 @@
+{
+    "key": {
+      "algo": "rsa",
+      "size": 2048
+    },
+    "names": [
+      {
+        "O": "etcd",
+        "OU": "etcd Security",
+        "L": "San Francisco",
+        "ST": "California",
+        "C": "USA"
+      }
+    ],
+    "CN": "etcd.local",
+    "hosts": [
+      "m3.etcd.local",
+      "127.0.0.1",
+      "localhost"
+    ]
+  }

hack/scripts-dev/docker-dns/certs-common-name/Procfile

@@ -1,6 +0,0 @@
-# Use goreman to run `go get github.com/mattn/goreman`
-etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls https://127.0.0.1:2379 --advertise-client-urls https://m1.etcd.local:2379 --listen-peer-urls https://127.0.0.1:2380 --initial-advertise-peer-urls=https://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name/server.crt --peer-key-file=/certs-common-name/server.key.insecure --peer-trusted-ca-file=/certs-common-name/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name/server.crt --key-file=/certs-common-name/server.key.insecure --trusted-ca-file=/certs-common-name/ca.crt --client-cert-auth
-
-etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls https://127.0.0.1:22379 --advertise-client-urls https://m2.etcd.local:22379 --listen-peer-urls https://127.0.0.1:22380 --initial-advertise-peer-urls=https://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name/server.crt --peer-key-file=/certs-common-name/server.key.insecure --peer-trusted-ca-file=/certs-common-name/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name/server.crt --key-file=/certs-common-name/server.key.insecure --trusted-ca-file=/certs-common-name/ca.crt --client-cert-auth
-
-etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls https://127.0.0.1:32379 --advertise-client-urls https://m3.etcd.local:32379 --listen-peer-urls https://127.0.0.1:32380 --initial-advertise-peer-urls=https://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name/server.crt --peer-key-file=/certs-common-name/server.key.insecure --peer-trusted-ca-file=/certs-common-name/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name/server.crt --key-file=/certs-common-name/server.key.insecure --trusted-ca-file=/certs-common-name/ca.crt --client-cert-auth

hack/scripts-dev/docker-dns/certs/run.sh

@@ -31,3 +31,52 @@ ETCDCTL_API=3 ./etcdctl \
   --key=/certs/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   get abc
+
+printf "\nWriting v2 key...\n"
+curl -L https://127.0.0.1:2379/v2/keys/queue \
+  --cacert /certs/ca.crt \
+  --cert /certs/server.crt \
+  --key /certs/server.key.insecure \
+  -X POST \
+  -d value=data
+
+printf "\nWriting v2 key...\n"
+curl -L https://m1.etcd.local:2379/v2/keys/queue \
+  --cacert /certs/ca.crt \
+  --cert /certs/server.crt \
+  --key /certs/server.key.insecure \
+  -X POST \
+  -d value=data
+
+printf "\nWriting v3 key...\n"
+curl -L https://127.0.0.1:2379/v3/kv/put \
+  --cacert /certs/ca.crt \
+  --cert /certs/server.crt \
+  --key /certs/server.key.insecure \
+  -X POST \
+  -d '{"key": "Zm9v", "value": "YmFy"}'
+
+printf "\n\nWriting v3 key...\n"
+curl -L https://m1.etcd.local:2379/v3/kv/put \
+  --cacert /certs/ca.crt \
+  --cert /certs/server.crt \
+  --key /certs/server.key.insecure \
+	-X POST \
+  -d '{"key": "Zm9v", "value": "YmFy"}'
+
+printf "\n\nReading v3 key...\n"
+curl -L https://m1.etcd.local:2379/v3/kv/range \
+  --cacert /certs/ca.crt \
+  --cert /certs/server.crt \
+  --key /certs/server.key.insecure \
+	-X POST \
+  -d '{"key": "Zm9v"}'
+
+printf "\n\nFetching 'curl https://m1.etcd.local:2379/metrics'...\n"
+curl \
+  --cacert /certs/ca.crt \
+  --cert /certs/server.crt \
+  --key /certs/server.key.insecure \
+  -L https://m1.etcd.local:2379/metrics | grep Put | tail -3
+
+printf "\n\nDone!!!\n\n"

hack/scripts-dev/docker-dns/insecure/Procfile

@@ -0,0 +1,6 @@
+# Use goreman to run `go get github.com/mattn/goreman`
+etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls http://127.0.0.1:2379 --advertise-client-urls http://m1.etcd.local:2379 --listen-peer-urls http://127.0.0.1:2380 --initial-advertise-peer-urls=http://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=http://m1.etcd.local:2380,m2=http://m2.etcd.local:22380,m3=http://m3.etcd.local:32380 --host-whitelist "localhost,127.0.0.1,m1.etcd.local"
+
+etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls http://127.0.0.1:22379 --advertise-client-urls http://m2.etcd.local:22379 --listen-peer-urls http://127.0.0.1:22380 --initial-advertise-peer-urls=http://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=http://m1.etcd.local:2380,m2=http://m2.etcd.local:22380,m3=http://m3.etcd.local:32380 --host-whitelist "localhost,127.0.0.1,m1.etcd.local"
+
+etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls http://127.0.0.1:32379 --advertise-client-urls http://m3.etcd.local:32379 --listen-peer-urls http://127.0.0.1:32380 --initial-advertise-peer-urls=http://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=http://m1.etcd.local:2380,m2=http://m2.etcd.local:22380,m3=http://m3.etcd.local:32380 --host-whitelist "localhost,127.0.0.1,m1.etcd.local"

hack/scripts-dev/docker-dns/insecure/run.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+rm -rf /tmp/m1.data /tmp/m2.data /tmp/m3.data
+
+/etc/init.d/bind9 start
+
+# get rid of hosts so go lookup won't resolve 127.0.0.1 to localhost
+cat /dev/null >/etc/hosts
+
+goreman -f /insecure/Procfile start &
+
+# TODO: remove random sleeps
+sleep 7s
+
+ETCDCTL_API=3 ./etcdctl \
+  --endpoints=http://m1.etcd.local:2379 \
+  endpoint health --cluster
+
+ETCDCTL_API=3 ./etcdctl \
+  --endpoints=http://m1.etcd.local:2379,http://m2.etcd.local:22379,http://m3.etcd.local:32379 \
+  put abc def
+
+ETCDCTL_API=3 ./etcdctl \
+  --endpoints=http://m1.etcd.local:2379,http://m2.etcd.local:22379,http://m3.etcd.local:32379 \
+  get abc
+
+printf "\nWriting v2 key...\n"
+curl \
+  -L http://127.0.0.1:2379/v2/keys/queue \
+  -X POST \
+  -d value=data
+
+printf "\nWriting v2 key...\n"
+curl \
+  -L http://m1.etcd.local:2379/v2/keys/queue \
+  -X POST \
+  -d value=data
+
+printf "\nWriting v3 key...\n"
+curl \
+  -L http://127.0.0.1:2379/v3/kv/put \
+  -X POST \
+  -d '{"key": "Zm9v", "value": "YmFy"}'
+
+printf "\n\nWriting v3 key...\n"
+curl \
+  -L http://m1.etcd.local:2379/v3/kv/put \
+  -X POST \
+  -d '{"key": "Zm9v", "value": "YmFy"}'
+
+printf "\n\nReading v3 key...\n"
+curl \
+  -L http://m1.etcd.local:2379/v3/kv/range \
+  -X POST \
+  -d '{"key": "Zm9v"}'
+
+printf "\n\nFetching 'curl http://m1.etcd.local:2379/metrics'...\n"
+curl \
+  -L http://m1.etcd.local:2379/metrics | grep Put | tail -3
+
+name1=$(base64 <<< "/election-prefix")
+val1=$(base64 <<< "v1")
+data1="{\"name\":\"${name1}\", \"value\":\"${val1}\"}"
+
+printf "\n\nCampaign: ${data1}\n"
+result1=$(curl -L http://m1.etcd.local:2379/v3/election/campaign -X POST -d "${data1}")
+echo ${result1}
+
+# should not panic servers
+val2=$(base64 <<< "v2")
+data2="{\"value\": \"${val2}\"}"
+printf "\n\nProclaim (wrong-format): ${data2}\n"
+curl \
+  -L http://m1.etcd.local:2379/v3/election/proclaim \
+  -X POST \
+  -d "${data2}"
+
+printf "\n\nProclaim (wrong-format)...\n"
+curl \
+  -L http://m1.etcd.local:2379/v3/election/proclaim \
+  -X POST \
+  -d '}'
+
+printf "\n\nProclaim (wrong-format)...\n"
+curl \
+  -L http://m1.etcd.local:2379/v3/election/proclaim \
+  -X POST \
+  -d '{"value": "Zm9v"}'
+
+printf "\n\nDone!!!\n\n"

hack/scripts-dev/docker-static-ip/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.10
+FROM ubuntu:17.10
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

lease/lessor.go

@@ -29,6 +29,9 @@ import (
 // NoLease is a special LeaseID representing the absence of a lease.
 const NoLease = LeaseID(0)
 
+// MaxLeaseTTL is the maximum lease TTL value
+const MaxLeaseTTL = 9000000000
+
 var (
 	forever = time.Time{}
 
@@ -37,9 +40,10 @@ var (
 	// maximum number of leases to revoke per second; configurable for tests
 	leaseRevokeRate = 1000
 
-	ErrNotPrimary    = errors.New("not a primary lessor")
-	ErrLeaseNotFound = errors.New("lease not found")
-	ErrLeaseExists   = errors.New("lease already exists")
+	ErrNotPrimary       = errors.New("not a primary lessor")
+	ErrLeaseNotFound    = errors.New("lease not found")
+	ErrLeaseExists      = errors.New("lease already exists")
+	ErrLeaseTTLTooLarge = errors.New("too large lease TTL")
 )
 
 // TxnDelete is a TxnWrite that only permits deletes. Defined here
@@ -198,6 +202,10 @@ func (le *lessor) Grant(id LeaseID, ttl int64) (*Lease, error) {
 		return nil, ErrLeaseNotFound
 	}
 
+	if ttl > MaxLeaseTTL {
+		return nil, ErrLeaseTTLTooLarge
+	}
+
 	// TODO: when lessor is under high load, it should give out lease
 	// with longer TTL to reduce renew load.
 	l := &Lease{

lease/lessor_test.go

@@ -451,6 +451,20 @@ func TestLessorExpireAndDemote(t *testing.T) {
 	}
 }
 
+func TestLessorMaxTTL(t *testing.T) {
+	dir, be := NewTestBackend(t)
+	defer os.RemoveAll(dir)
+	defer be.Close()
+
+	le := newLessor(be, minLeaseTTL)
+	defer le.Stop()
+
+	_, err := le.Grant(1, MaxLeaseTTL+1)
+	if err != ErrLeaseTTLTooLarge {
+		t.Fatalf("grant unexpectedly succeeded")
+	}
+}
+
 type fakeDeleter struct {
 	deleted []string
 	tx      backend.BatchTx

mvcc/watchable_store.go

@@ -192,7 +192,7 @@ func (s *watchableStore) Restore(b backend.Backend) error {
 	}
 
 	for wa := range s.synced.watchers {
-		s.unsynced.watchers.add(wa)
+		s.unsynced.add(wa)
 	}
 	s.synced = newWatcherGroup()
 	return nil

mvcc/watchable_store_test.go

@@ -297,36 +297,45 @@ func TestWatchFutureRev(t *testing.T) {
 }
 
 func TestWatchRestore(t *testing.T) {
-	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
-	defer cleanup(s, b, tmpPath)
+	test := func(delay time.Duration) func(t *testing.T) {
+		return func(t *testing.T) {
+			b, tmpPath := backend.NewDefaultTmpBackend()
+			s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+			defer cleanup(s, b, tmpPath)
 
-	testKey := []byte("foo")
-	testValue := []byte("bar")
-	rev := s.Put(testKey, testValue, lease.NoLease)
+			testKey := []byte("foo")
+			testValue := []byte("bar")
+			rev := s.Put(testKey, testValue, lease.NoLease)
 
-	newBackend, newPath := backend.NewDefaultTmpBackend()
-	newStore := newWatchableStore(newBackend, &lease.FakeLessor{}, nil)
-	defer cleanup(newStore, newBackend, newPath)
+			newBackend, newPath := backend.NewDefaultTmpBackend()
+			newStore := newWatchableStore(newBackend, &lease.FakeLessor{}, nil)
+			defer cleanup(newStore, newBackend, newPath)
 
-	w := newStore.NewWatchStream()
-	w.Watch(testKey, nil, rev-1)
+			w := newStore.NewWatchStream()
+			w.Watch(testKey, nil, rev-1)
 
-	newStore.Restore(b)
-	select {
-	case resp := <-w.Chan():
-		if resp.Revision != rev {
-			t.Fatalf("rev = %d, want %d", resp.Revision, rev)
+			time.Sleep(delay)
+
+			newStore.Restore(b)
+			select {
+			case resp := <-w.Chan():
+				if resp.Revision != rev {
+					t.Fatalf("rev = %d, want %d", resp.Revision, rev)
+				}
+				if len(resp.Events) != 1 {
+					t.Fatalf("failed to get events from the response")
+				}
+				if resp.Events[0].Kv.ModRevision != rev {
+					t.Fatalf("kv.rev = %d, want %d", resp.Events[0].Kv.ModRevision, rev)
+				}
+			case <-time.After(time.Second):
+				t.Fatal("failed to receive event in 1 second.")
+			}
 		}
-		if len(resp.Events) != 1 {
-			t.Fatalf("failed to get events from the response")
-		}
-		if resp.Events[0].Kv.ModRevision != rev {
-			t.Fatalf("kv.rev = %d, want %d", resp.Events[0].Kv.ModRevision, rev)
-		}
-	case <-time.After(time.Second):
-		t.Fatal("failed to receive event in 1 second.")
 	}
+
+	t.Run("Normal", test(0))
+	t.Run("RunSyncWatchLoopBeforeRestore", test(time.Millisecond*120)) // longer than default waitDuration
 }
 
 // TestWatchBatchUnsynced tests batching on unsynced watchers

pkg/netutil/netutil.go

@@ -17,6 +17,7 @@ package netutil
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/url"
 	"reflect"
@@ -73,14 +74,14 @@ func resolveTCPAddrs(ctx context.Context, urls [][]url.URL) ([][]url.URL, error)
 		for i, u := range us {
 			nu, err := url.Parse(u.String())
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("failed to parse %q (%v)", u.String(), err)
 			}
 			nus[i] = *nu
 		}
 		for i, u := range nus {
 			h, err := resolveURL(ctx, u)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("failed to resolve %q (%v)", u.String(), err)
 			}
 			if h != "" {
 				nus[i].Host = h
@@ -123,35 +124,41 @@ func resolveURL(ctx context.Context, u url.URL) (string, error) {
 
 // urlsEqual checks equality of url.URLS between two arrays.
 // This check pass even if an URL is in hostname and opposite is in IP address.
-func urlsEqual(ctx context.Context, a []url.URL, b []url.URL) bool {
+func urlsEqual(ctx context.Context, a []url.URL, b []url.URL) (bool, error) {
 	if len(a) != len(b) {
-		return false
+		return false, fmt.Errorf("len(%q) != len(%q)", urlsToStrings(a), urlsToStrings(b))
 	}
 	urls, err := resolveTCPAddrs(ctx, [][]url.URL{a, b})
 	if err != nil {
-		return false
+		return false, err
 	}
+	preva, prevb := a, b
 	a, b = urls[0], urls[1]
 	sort.Sort(types.URLs(a))
 	sort.Sort(types.URLs(b))
 	for i := range a {
 		if !reflect.DeepEqual(a[i], b[i]) {
-			return false
+			return false, fmt.Errorf("%q(resolved from %q) != %q(resolved from %q)",
+				a[i].String(), preva[i].String(),
+				b[i].String(), prevb[i].String(),
+			)
 		}
 	}
-
-	return true
+	return true, nil
 }
 
-func URLStringsEqual(ctx context.Context, a []string, b []string) bool {
+// URLStringsEqual returns "true" if given URLs are valid
+// and resolved to same IP addresses. Otherwise, return "false"
+// and error, if any.
+func URLStringsEqual(ctx context.Context, a []string, b []string) (bool, error) {
 	if len(a) != len(b) {
-		return false
+		return false, fmt.Errorf("len(%q) != len(%q)", a, b)
 	}
 	urlsA := make([]url.URL, 0)
 	for _, str := range a {
 		u, err := url.Parse(str)
 		if err != nil {
-			return false
+			return false, fmt.Errorf("failed to parse %q", str)
 		}
 		urlsA = append(urlsA, *u)
 	}
@@ -159,14 +166,21 @@ func URLStringsEqual(ctx context.Context, a []string, b []string) bool {
 	for _, str := range b {
 		u, err := url.Parse(str)
 		if err != nil {
-			return false
+			return false, fmt.Errorf("failed to parse %q", str)
 		}
 		urlsB = append(urlsB, *u)
 	}
-
 	return urlsEqual(ctx, urlsA, urlsB)
 }
 
+func urlsToStrings(us []url.URL) []string {
+	rs := make([]string, len(us))
+	for i := range us {
+		rs[i] = us[i].String()
+	}
+	return rs
+}
+
 func IsNetworkTimeoutError(err error) bool {
 	nerr, ok := err.(net.Error)
 	return ok && nerr.Timeout()

pkg/netutil/netutil_test.go

@@ -167,6 +167,7 @@ func TestURLsEqual(t *testing.T) {
 		a      []url.URL
 		b      []url.URL
 		expect bool
+		err    error
 	}{
 		{
 			a:      []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}},
@@ -182,11 +183,13 @@ func TestURLsEqual(t *testing.T) {
 			a:      []url.URL{{Scheme: "http", Host: "example.com:2379"}},
 			b:      []url.URL{{Scheme: "https", Host: "10.0.10.1:2379"}},
 			expect: false,
+			err:    errors.New(`"http://10.0.10.1:2379"(resolved from "http://example.com:2379") != "https://10.0.10.1:2379"(resolved from "https://10.0.10.1:2379")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "https", Host: "example.com:2379"}},
 			b:      []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}},
 			expect: false,
+			err:    errors.New(`"https://10.0.10.1:2379"(resolved from "https://example.com:2379") != "http://10.0.10.1:2379"(resolved from "http://10.0.10.1:2379")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "unix", Host: "abc:2379"}},
@@ -212,46 +215,55 @@ func TestURLsEqual(t *testing.T) {
 			a:      []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}},
 			b:      []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}},
 			expect: false,
+			err:    errors.New(`"http://127.0.0.1:2379"(resolved from "http://127.0.0.1:2379") != "http://127.0.0.1:2380"(resolved from "http://127.0.0.1:2380")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "example.com:2380"}},
 			b:      []url.URL{{Scheme: "http", Host: "10.0.10.1:2379"}},
 			expect: false,
+			err:    errors.New(`"http://10.0.10.1:2380"(resolved from "http://example.com:2380") != "http://10.0.10.1:2379"(resolved from "http://10.0.10.1:2379")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}},
 			b:      []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}},
 			expect: false,
+			err:    errors.New(`"http://127.0.0.1:2379"(resolved from "http://127.0.0.1:2379") != "http://10.0.0.1:2379"(resolved from "http://10.0.0.1:2379")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "example.com:2379"}},
 			b:      []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}},
 			expect: false,
+			err:    errors.New(`"http://10.0.10.1:2379"(resolved from "http://example.com:2379") != "http://10.0.0.1:2379"(resolved from "http://10.0.0.1:2379")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			b:      []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			expect: false,
+			err:    errors.New(`"http://127.0.0.1:2379"(resolved from "http://127.0.0.1:2379") != "http://127.0.0.1:2380"(resolved from "http://127.0.0.1:2380")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			b:      []url.URL{{Scheme: "http", Host: "127.0.0.1:2380"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			expect: false,
+			err:    errors.New(`"http://10.0.10.1:2379"(resolved from "http://example.com:2379") != "http://127.0.0.1:2380"(resolved from "http://127.0.0.1:2380")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "127.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			b:      []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			expect: false,
+			err:    errors.New(`"http://127.0.0.1:2379"(resolved from "http://127.0.0.1:2379") != "http://10.0.0.1:2379"(resolved from "http://10.0.0.1:2379")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "example.com:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			b:      []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			expect: false,
+			err:    errors.New(`"http://10.0.10.1:2379"(resolved from "http://example.com:2379") != "http://10.0.0.1:2379"(resolved from "http://10.0.0.1:2379")`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}},
 			b:      []url.URL{{Scheme: "http", Host: "10.0.0.1:2379"}, {Scheme: "http", Host: "127.0.0.1:2380"}},
 			expect: false,
+			err:    errors.New(`len(["http://10.0.0.1:2379"]) != len(["http://10.0.0.1:2379" "http://127.0.0.1:2380"])`),
 		},
 		{
 			a:      []url.URL{{Scheme: "http", Host: "first.com:2379"}, {Scheme: "http", Host: "second.com:2380"}},
@@ -265,16 +277,24 @@ func TestURLsEqual(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
-		result := urlsEqual(context.TODO(), test.a, test.b)
+	for i, test := range tests {
+		result, err := urlsEqual(context.TODO(), test.a, test.b)
 		if result != test.expect {
-			t.Errorf("a:%v b:%v, expected %v but %v", test.a, test.b, test.expect, result)
+			t.Errorf("#%d: a:%v b:%v, expected %v but %v", i, test.a, test.b, test.expect, result)
+		}
+		if test.err != nil {
+			if err.Error() != test.err.Error() {
+				t.Errorf("#%d: err expected %v but %v", i, test.err, err)
+			}
 		}
 	}
 }
 func TestURLStringsEqual(t *testing.T) {
-	result := URLStringsEqual(context.TODO(), []string{"http://127.0.0.1:8080"}, []string{"http://127.0.0.1:8080"})
+	result, err := URLStringsEqual(context.TODO(), []string{"http://127.0.0.1:8080"}, []string{"http://127.0.0.1:8080"})
 	if !result {
 		t.Errorf("unexpected result %v", result)
 	}
+	if err != nil {
+		t.Errorf("unexpected error %v", err)
+	}
 }

pkg/transport/fixtures/ca-csr.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "ca",
+  "ca": {
+    "expiry": "87600h"
+  }
+}

pkg/transport/fixtures/ca.crt

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDsTCCApmgAwIBAgIUZzOo4zcHY/nEXY1PD8A7povXlWUwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMDIxNjQxMDBaFw0yNzEyMzExNjQx
+MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDD4Ys48LDWGyojj3Rcr6fnESY+UycaaGoTXADWLPmm+sQR3KcsJxF4054S
+d2G+NBfJHZvTHhVqOeqZxNtoqgje4paY2A5TbWBdV+xoGfbakwwngiX1yeF1I54k
+KH19zb8rBKAm7xixO60hE2CIYzMuw9lDkwoHpI6/PJdy7jwtytbo2Oac512JiO9Y
+dHp9dr3mrCzoKEBRtL1asRKfzp6gBC5rIw5T4jrq37feerV4pDEJX7fvexxVocVm
+tT4bmMq3Ap6OFFAzmE/ITI8pXvFaOd9lyebNXQmrreKJLUfEIZa6JulLCYxfkJ8z
++CcNLyn6ZXNMaIZ8G9Hm6VRdRi8/AgMBAAGjRTBDMA4GA1UdDwEB/wQEAwIBBjAS
+BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBRDLNYEX8XI7nM53k1rUR+mpTjQ
+NTANBgkqhkiG9w0BAQsFAAOCAQEACDe3Fa1KE/rvVtyCLW/IBfKV01NShFTsb6x8
+GrPEQ6NJLZQ2MzdyJgAF2a/nZ9KVgrhGXoyoZBCKP9Dd/JDzSSZcBztfNK8dRv2A
+XHBBF6tZ19I+XY9c7/CfhJ2CEYJpeN9r3GKSqV+njkmg8n/On2BTlFsij88plK8H
+ORyemc1nQI+ARPSu2r3rJbYa4yI2U6w4L4BTCVImg3bX50GImmXGlwvnJMFik1FX
++0hdfetRxxMZ1pm2Uy6099KkULnSKabZGwRiBUHQJYh0EeuAOQ4a6MG5DRkURWNs
+dInjPOLY9/7S5DQKwz/NtqXA8EEymZosHxpiRp+zzKB4XaV9Ig==
+-----END CERTIFICATE-----

pkg/transport/fixtures/gencert.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "87600h"
+    }
+  }
+}

pkg/transport/fixtures/gencerts.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if ! [[ "$0" =~ "./gencerts.sh" ]]; then
+	echo "must be run from 'fixtures'"
+	exit 255
+fi
+
+if ! which cfssl; then
+	echo "cfssl is not installed"
+	exit 255
+fi
+
+cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca
+mv ca.pem ca.crt
+openssl x509 -in ca.crt -noout -text
+
+# generate DNS: localhost, IP: 127.0.0.1, CN: example.com certificates
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr.json | cfssljson --bare ./server
+mv server.pem server.crt
+mv server-key.pem server.key.insecure
+
+rm -f *.csr *.pem *.stderr *.txt

pkg/transport/fixtures/server-ca-csr.json

@@ -0,0 +1,20 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "example.com",
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

pkg/transport/fixtures/server.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEEjCCAvqgAwIBAgIUIYc+vmysep1pDc2ua/VQEeMFQVAwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMDIxNjQxMDBaFw0yNzEyMzExNjQx
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDEq7aT2BQZfmJ2xpUm8xWJlN0c3cOLVZRH9mIrEutIHmip
+BYq3ZIq3q52w+T3sMcaJNMGjCteE8Lu+G9YSmtfZMAWnkaM02KOjVMkkQcK7Z4vM
+lOUjlO+dsvhfmw3CPghqSs6M1K2CTqhuEiXdOBofuEMmwKNRgkV/jT92PUs0h8kq
+loc/I3/H+hx/ZJ1i0S0xkZKpaImc0oZ9ZDo07biMrsUIzjwbN69mEs+CtVkah4sy
+k6UyRoU2k21lyRTK0LxNjWc9ylzDNUuf6DwduU7lPZsqTaJrFNAAPpOlI4k2EcjL
+3zD8amKkJGDm+PQz97PbTA381ec4ZAtB8volxCebAgMBAAGjgZwwgZkwDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBTTZQnMn5tuUgVE+8c9W0hmbghGoDAfBgNVHSMEGDAW
+gBRDLNYEX8XI7nM53k1rUR+mpTjQNTAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAKUQVj0YDuxg4tinlOZhp4ge7tCA+gL7vV+Q
+iDrkWfOlGjDgwYqWMYDXMHWKIW9ea8LzyI/bVEcaHlnBmNOYuS7g47EWNiU7WUA5
+iTkm3CKA5zHFFPcXHW0GQeCQrX9y3SepKS3cP8TAyZFfC/FvV24Kn1oQhJbEe0ZV
+In/vPHssW7jlVe0FGVUn7FutRQgiA1pTAtS6AP4LeZ9O41DTWkPqV4nBgcxlvkgD
+KjEoXXSb5C0LoR5zwAo9zB3RtmqnmvkHAOv3G92YctdS2VbCmd8CNLj9H7gMmQiH
+ThsStVOhb2uo6Ni4PgzUIYKGTd4ZjUXCYxFKck//ajDyCHlL8v4=
+-----END CERTIFICATE-----

pkg/transport/fixtures/server.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAxKu2k9gUGX5idsaVJvMViZTdHN3Di1WUR/ZiKxLrSB5oqQWK
+t2SKt6udsPk97DHGiTTBowrXhPC7vhvWEprX2TAFp5GjNNijo1TJJEHCu2eLzJTl
+I5TvnbL4X5sNwj4IakrOjNStgk6obhIl3TgaH7hDJsCjUYJFf40/dj1LNIfJKpaH
+PyN/x/ocf2SdYtEtMZGSqWiJnNKGfWQ6NO24jK7FCM48GzevZhLPgrVZGoeLMpOl
+MkaFNpNtZckUytC8TY1nPcpcwzVLn+g8HblO5T2bKk2iaxTQAD6TpSOJNhHIy98w
+/GpipCRg5vj0M/ez20wN/NXnOGQLQfL6JcQnmwIDAQABAoIBAGTx1eaQk9B6BEP+
+rXOudTGGzO8SDFop9M/y8HQ3Y7hCk2mdxJNY8bJQTcIWS+g9rC+kencbC3/aqCJt
+2zT1cTCy61QU9nYbc/JThGIttqvF/AVnryzSNyL0R3Oa/Dbk7CDSgK3cQ6qMgPru
+Ka0gLJh3VVBAtBMUEGPltdsUntM4sHTh5FAabP0ioBJ1QLG6Aak7LOQikjBEFJoc
+Tea4uRsE7IreP5Mn7UW92nkt1ey5UGzBtNNtpHbVaHmfQojwlwkLtnV35sumbvK6
+6KTMNREZv6xSIMwkYxm1zRE3Cus/1jGIc8MZF0BxgcCR+G37l+BKwL8CSymHPxhH
+dvGxoPECgYEA3STp52CbI/KyVfvjxK2OIex/NV1jKh85wQsLtkaRv3/a/EEg7MV7
+54dEvo5KKOZXfeOd9r9G9h1RffjSD9MhxfPhyGwuOcqa8IE1zNwlY/v7KL7HtDIf
+2mrXWF5Klafh8aXYcaRH0ZSLnl/nXUXYht4/0NRGiXnttUgqs6hvY70CgYEA46tO
+J5QkgF3YVY0gx10wRCAnnKLkAaHdtxtteXOJh79xsGXQ4LLngc+mz1hLt+TNJza+
+BZhoWwY/ZgyiTH0pebGr/U0QUMoUHlGgjgj3Aa/XFpOhtyLU+IU/PYl0BUz9dqsN
+TDtv6p/HQhfd98vUNsbACQda+YAo+oRdO5kLQjcCgYB3OAZNcXxRte5EgoY5KqN8
+UGYH2++w7qKRGqZWvtamGYRyB557Zr+0gu0hmc4LHJrASGyJcHcOCaI8Ol7snxMP
+B7qJ9SA6kapTzCS361rQ+zBct/UrhPY9JuovPq4Q3i/luVXldf4t01otqGAvnY7s
+rnZS242nYa8v0tcKgdyDNQKBgB3Z60BzQyn1pBTrkT2ysU5tbOQz03OHVrvYg80l
+4gWDi5OWdgHQU1yI7pVHPX5aKLAYlGfFaQFuW0e1Jl6jFpoXOrbWsOn25RZom4Wk
+FUcKWEhkiRKrJYOEbRtTd3vucVlq6i5xqKX51zWKTZddCXE5NBq69Sm7rSPT0Sms
+UnaXAoGAXYAE5slvjcylJpMV4lxTBmNtA9+pw1T7I379mIyqZ0OS25nmpskHU7FR
+SQDSRHw7hHuyjEHyhMoHEGLfUMIltQoi+pcrieVQelJdSuX7VInzHPAR5RppUVFl
+jOZZKlIiqs+UfCoOgsIblXuw7a/ATnAnXakutSFgHU1lN1gN02U=
+-----END RSA PRIVATE KEY-----

pkg/transport/proxy.go

@@ -0,0 +1,801 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transport
+
+import (
+	"fmt"
+	"io"
+	mrand "math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	humanize "github.com/dustin/go-humanize"
+	"google.golang.org/grpc/grpclog"
+)
+
+// Proxy defines proxy layer that simulates common network faults,
+// such as latency spikes, packet drop/corruption, etc..
+type Proxy interface {
+	// From returns proxy source address in "scheme://host:port" format.
+	From() string
+	// To returns proxy destination address in "scheme://host:port" format.
+	To() string
+
+	// Ready returns when proxy is ready to serve.
+	Ready() <-chan struct{}
+	// Done returns when proxy has been closed.
+	Done() <-chan struct{}
+	// Error sends errors while serving proxy.
+	Error() <-chan error
+	// Close closes listener and transport.
+	Close() error
+
+	// DelayAccept adds latency ± random variable to accepting new incoming connections.
+	DelayAccept(latency, rv time.Duration)
+	// UndelayAccept removes sending latencies.
+	UndelayAccept()
+	// LatencyAccept returns current latency on accepting new incoming connections.
+	LatencyAccept() time.Duration
+	// DelayTx adds latency ± random variable to "sending" layer.
+	DelayTx(latency, rv time.Duration)
+	// UndelayTx removes sending latencies.
+	UndelayTx()
+	// LatencyTx returns current send latency.
+	LatencyTx() time.Duration
+	// DelayRx adds latency ± random variable to "receiving" layer.
+	DelayRx(latency, rv time.Duration)
+	// UndelayRx removes "receiving" latencies.
+	UndelayRx()
+	// LatencyRx returns current receive latency.
+	LatencyRx() time.Duration
+
+	// PauseAccept stops accepting new connections.
+	PauseAccept()
+	// UnpauseAccept removes pause operation on accepting new connections.
+	UnpauseAccept()
+	// PauseTx stops "forwarding" packets.
+	PauseTx()
+	// UnpauseTx removes "forwarding" pause operation.
+	UnpauseTx()
+	// PauseRx stops "receiving" packets to client.
+	PauseRx()
+	// UnpauseRx removes "receiving" pause operation.
+	UnpauseRx()
+
+	// BlackholeTx drops all incoming packets before "forwarding".
+	BlackholeTx()
+	// UnblackholeTx removes blackhole operation on "sending".
+	UnblackholeTx()
+	// BlackholeRx drops all incoming packets to client.
+	BlackholeRx()
+	// UnblackholeRx removes blackhole operation on "receiving".
+	UnblackholeRx()
+
+	// CorruptTx corrupts incoming packets from the listener.
+	CorruptTx(f func(data []byte) []byte)
+	// UncorruptTx removes corrupt operation on "forwarding".
+	UncorruptTx()
+	// CorruptRx corrupts incoming packets to client.
+	CorruptRx(f func(data []byte) []byte)
+	// UncorruptRx removes corrupt operation on "receiving".
+	UncorruptRx()
+
+	// ResetListener closes and restarts listener.
+	ResetListener() error
+}
+
+type proxy struct {
+	from, to      url.URL
+	tlsInfo       TLSInfo
+	dialTimeout   time.Duration
+	bufferSize    int
+	retryInterval time.Duration
+	logger        grpclog.LoggerV2
+
+	readyc chan struct{}
+	donec  chan struct{}
+	errc   chan error
+
+	closeOnce sync.Once
+	closeWg   sync.WaitGroup
+
+	listenerMu sync.RWMutex
+	listener   net.Listener
+
+	latencyAcceptMu sync.RWMutex
+	latencyAccept   time.Duration
+	latencyTxMu     sync.RWMutex
+	latencyTx       time.Duration
+	latencyRxMu     sync.RWMutex
+	latencyRx       time.Duration
+
+	corruptTxMu sync.RWMutex
+	corruptTx   func(data []byte) []byte
+	corruptRxMu sync.RWMutex
+	corruptRx   func(data []byte) []byte
+
+	acceptMu     sync.Mutex
+	pauseAcceptc chan struct{}
+	txMu         sync.Mutex
+	pauseTxc     chan struct{}
+	blackholeTxc chan struct{}
+	rxMu         sync.Mutex
+	pauseRxc     chan struct{}
+	blackholeRxc chan struct{}
+}
+
+// ProxyConfig defines proxy configuration.
+type ProxyConfig struct {
+	From          url.URL
+	To            url.URL
+	TLSInfo       TLSInfo
+	DialTimeout   time.Duration
+	BufferSize    int
+	RetryInterval time.Duration
+	Logger        grpclog.LoggerV2
+}
+
+var (
+	defaultDialTimeout   = 3 * time.Second
+	defaultBufferSize    = 48 * 1024
+	defaultRetryInterval = 10 * time.Millisecond
+	defaultLogger        = grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 0)
+)
+
+// NewProxy returns a proxy implementation with no iptables/tc dependencies.
+// The proxy layer overhead is <1ms.
+func NewProxy(cfg ProxyConfig) Proxy {
+	p := &proxy{
+		from:          cfg.From,
+		to:            cfg.To,
+		tlsInfo:       cfg.TLSInfo,
+		dialTimeout:   cfg.DialTimeout,
+		bufferSize:    cfg.BufferSize,
+		retryInterval: cfg.RetryInterval,
+		logger:        cfg.Logger,
+
+		readyc: make(chan struct{}),
+		donec:  make(chan struct{}),
+		errc:   make(chan error, 16),
+
+		pauseAcceptc: make(chan struct{}),
+		pauseTxc:     make(chan struct{}),
+		blackholeTxc: make(chan struct{}),
+		pauseRxc:     make(chan struct{}),
+		blackholeRxc: make(chan struct{}),
+	}
+	if p.dialTimeout == 0 {
+		p.dialTimeout = defaultDialTimeout
+	}
+	if p.bufferSize == 0 {
+		p.bufferSize = defaultBufferSize
+	}
+	if p.retryInterval == 0 {
+		p.retryInterval = defaultRetryInterval
+	}
+	if p.logger == nil {
+		p.logger = defaultLogger
+	}
+	close(p.pauseAcceptc)
+	close(p.pauseTxc)
+	close(p.pauseRxc)
+
+	if strings.HasPrefix(p.from.Scheme, "http") {
+		p.from.Scheme = "tcp"
+	}
+	if strings.HasPrefix(p.to.Scheme, "http") {
+		p.to.Scheme = "tcp"
+	}
+
+	var ln net.Listener
+	var err error
+	if !p.tlsInfo.Empty() {
+		ln, err = NewListener(p.from.Host, p.from.Scheme, &p.tlsInfo)
+	} else {
+		ln, err = net.Listen(p.from.Scheme, p.from.Host)
+	}
+	if err != nil {
+		p.errc <- err
+		p.Close()
+		return p
+	}
+	p.listener = ln
+
+	p.closeWg.Add(1)
+	go p.listenAndServe()
+	p.logger.Infof("started proxying [%s -> %s]", p.From(), p.To())
+	return p
+}
+
+func (p *proxy) From() string {
+	return fmt.Sprintf("%s://%s", p.from.Scheme, p.from.Host)
+}
+
+func (p *proxy) To() string {
+	return fmt.Sprintf("%s://%s", p.to.Scheme, p.to.Host)
+}
+
+// TODO: implement packet reordering from multiple TCP connections
+// buffer packets per connection for awhile, reorder before transmit
+// - https://github.com/coreos/etcd/issues/5614
+// - https://github.com/coreos/etcd/pull/6918#issuecomment-264093034
+
+func (p *proxy) listenAndServe() {
+	defer p.closeWg.Done()
+
+	p.logger.Infof("listen %q", p.From())
+	close(p.readyc)
+
+	for {
+		p.acceptMu.Lock()
+		pausec := p.pauseAcceptc
+		p.acceptMu.Unlock()
+		select {
+		case <-pausec:
+		case <-p.donec:
+			return
+		}
+
+		p.latencyAcceptMu.RLock()
+		lat := p.latencyAccept
+		p.latencyAcceptMu.RUnlock()
+		if lat > 0 {
+			select {
+			case <-time.After(lat):
+			case <-p.donec:
+				return
+			}
+		}
+
+		p.listenerMu.RLock()
+		ln := p.listener
+		p.listenerMu.RUnlock()
+
+		in, err := ln.Accept()
+		if err != nil {
+			select {
+			case p.errc <- err:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if p.logger.V(5) {
+				p.logger.Errorf("listener accept error %q", err.Error())
+			}
+
+			if strings.HasSuffix(err.Error(), "use of closed network connection") {
+				select {
+				case <-time.After(p.retryInterval):
+				case <-p.donec:
+					return
+				}
+				if p.logger.V(5) {
+					p.logger.Errorf("listener is closed; retry listen %q", p.From())
+				}
+
+				if err = p.ResetListener(); err != nil {
+					select {
+					case p.errc <- err:
+						select {
+						case <-p.donec:
+							return
+						default:
+						}
+					case <-p.donec:
+						return
+					}
+					p.logger.Errorf("failed to reset listener %q", err.Error())
+				}
+			}
+
+			continue
+		}
+
+		var out net.Conn
+		if !p.tlsInfo.Empty() {
+			var tp *http.Transport
+			tp, err = NewTransport(p.tlsInfo, p.dialTimeout)
+			if err != nil {
+				select {
+				case p.errc <- err:
+					select {
+					case <-p.donec:
+						return
+					default:
+					}
+				case <-p.donec:
+					return
+				}
+				continue
+			}
+			out, err = tp.Dial(p.to.Scheme, p.to.Host)
+		} else {
+			out, err = net.Dial(p.to.Scheme, p.to.Host)
+		}
+		if err != nil {
+			select {
+			case p.errc <- err:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if p.logger.V(5) {
+				p.logger.Errorf("dial error %q", err.Error())
+			}
+			continue
+		}
+
+		go func() {
+			// read incoming bytes from listener, dispatch to outgoing connection
+			p.transmit(out, in)
+			out.Close()
+			in.Close()
+		}()
+		go func() {
+			// read response from outgoing connection, write back to listener
+			p.receive(in, out)
+			in.Close()
+			out.Close()
+		}()
+	}
+}
+
+func (p *proxy) transmit(dst io.Writer, src io.Reader) { p.ioCopy(dst, src, true) }
+func (p *proxy) receive(dst io.Writer, src io.Reader)  { p.ioCopy(dst, src, false) }
+func (p *proxy) ioCopy(dst io.Writer, src io.Reader, proxySend bool) {
+	buf := make([]byte, p.bufferSize)
+	for {
+		nr, err := src.Read(buf)
+		if err != nil {
+			if err == io.EOF {
+				return
+			}
+			// connection already closed
+			if strings.HasSuffix(err.Error(), "read: connection reset by peer") {
+				return
+			}
+			if strings.HasSuffix(err.Error(), "use of closed network connection") {
+				return
+			}
+			select {
+			case p.errc <- err:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if p.logger.V(5) {
+				p.logger.Errorf("read error %q", err.Error())
+			}
+			return
+		}
+		if nr == 0 {
+			return
+		}
+		data := buf[:nr]
+
+		var pausec chan struct{}
+		var blackholec chan struct{}
+		if proxySend {
+			p.txMu.Lock()
+			pausec = p.pauseTxc
+			blackholec = p.blackholeTxc
+			p.txMu.Unlock()
+		} else {
+			p.rxMu.Lock()
+			pausec = p.pauseRxc
+			blackholec = p.blackholeRxc
+			p.rxMu.Unlock()
+		}
+		select {
+		case <-pausec:
+		case <-p.donec:
+			return
+		}
+		blackholed := false
+		select {
+		case <-blackholec:
+			blackholed = true
+		case <-p.donec:
+			return
+		default:
+		}
+		if blackholed {
+			if p.logger.V(5) {
+				if proxySend {
+					p.logger.Infof("dropped %s [%s -> %s]", humanize.Bytes(uint64(nr)), p.From(), p.To())
+				} else {
+					p.logger.Infof("dropped %s [%s <- %s]", humanize.Bytes(uint64(nr)), p.From(), p.To())
+				}
+			}
+			continue
+		}
+
+		var lat time.Duration
+		if proxySend {
+			p.latencyTxMu.RLock()
+			lat = p.latencyTx
+			p.latencyTxMu.RUnlock()
+		} else {
+			p.latencyRxMu.RLock()
+			lat = p.latencyRx
+			p.latencyRxMu.RUnlock()
+		}
+		if lat > 0 {
+			select {
+			case <-time.After(lat):
+			case <-p.donec:
+				return
+			}
+		}
+
+		if proxySend {
+			p.corruptTxMu.RLock()
+			if p.corruptTx != nil {
+				data = p.corruptTx(data)
+			}
+			p.corruptTxMu.RUnlock()
+		} else {
+			p.corruptRxMu.RLock()
+			if p.corruptRx != nil {
+				data = p.corruptRx(data)
+			}
+			p.corruptRxMu.RUnlock()
+		}
+
+		var nw int
+		nw, err = dst.Write(data)
+		if err != nil {
+			if err == io.EOF {
+				return
+			}
+			select {
+			case p.errc <- err:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if p.logger.V(5) {
+				if proxySend {
+					p.logger.Errorf("write error while sending (%q)", err.Error())
+				} else {
+					p.logger.Errorf("write error while receiving (%q)", err.Error())
+				}
+			}
+			return
+		}
+
+		if nr != nw {
+			select {
+			case p.errc <- io.ErrShortWrite:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if proxySend {
+				p.logger.Errorf("write error while sending (%q); read %d bytes != wrote %d bytes", io.ErrShortWrite.Error(), nr, nw)
+			} else {
+				p.logger.Errorf("write error while receiving (%q); read %d bytes != wrote %d bytes", io.ErrShortWrite.Error(), nr, nw)
+			}
+			return
+		}
+
+		if p.logger.V(5) {
+			if proxySend {
+				p.logger.Infof("transmitted %s [%s -> %s]", humanize.Bytes(uint64(nr)), p.From(), p.To())
+			} else {
+				p.logger.Infof("received %s [%s <- %s]", humanize.Bytes(uint64(nr)), p.From(), p.To())
+			}
+		}
+	}
+}
+
+func (p *proxy) Ready() <-chan struct{} { return p.readyc }
+func (p *proxy) Done() <-chan struct{}  { return p.donec }
+func (p *proxy) Error() <-chan error    { return p.errc }
+func (p *proxy) Close() (err error) {
+	p.closeOnce.Do(func() {
+		close(p.donec)
+		p.listenerMu.Lock()
+		if p.listener != nil {
+			err = p.listener.Close()
+			p.logger.Infof("closed proxy listener on %q", p.From())
+		}
+		p.listenerMu.Unlock()
+	})
+	p.closeWg.Wait()
+	return err
+}
+
+func (p *proxy) DelayAccept(latency, rv time.Duration) {
+	if latency <= 0 {
+		return
+	}
+	d := computeLatency(latency, rv)
+	p.latencyAcceptMu.Lock()
+	p.latencyAccept = d
+	p.latencyAcceptMu.Unlock()
+	p.logger.Infof("set accept latency %v(%v±%v) [%s -> %s]", d, latency, rv, p.From(), p.To())
+}
+
+func (p *proxy) UndelayAccept() {
+	p.latencyAcceptMu.Lock()
+	d := p.latencyAccept
+	p.latencyAccept = 0
+	p.latencyAcceptMu.Unlock()
+	p.logger.Infof("removed accept latency %v [%s -> %s]", d, p.From(), p.To())
+}
+
+func (p *proxy) LatencyAccept() time.Duration {
+	p.latencyAcceptMu.RLock()
+	d := p.latencyAccept
+	p.latencyAcceptMu.RUnlock()
+	return d
+}
+
+func (p *proxy) DelayTx(latency, rv time.Duration) {
+	if latency <= 0 {
+		return
+	}
+	d := computeLatency(latency, rv)
+	p.latencyTxMu.Lock()
+	p.latencyTx = d
+	p.latencyTxMu.Unlock()
+	p.logger.Infof("set transmit latency %v(%v±%v) [%s -> %s]", d, latency, rv, p.From(), p.To())
+}
+
+func (p *proxy) UndelayTx() {
+	p.latencyTxMu.Lock()
+	d := p.latencyTx
+	p.latencyTx = 0
+	p.latencyTxMu.Unlock()
+	p.logger.Infof("removed transmit latency %v [%s -> %s]", d, p.From(), p.To())
+}
+
+func (p *proxy) LatencyTx() time.Duration {
+	p.latencyTxMu.RLock()
+	d := p.latencyTx
+	p.latencyTxMu.RUnlock()
+	return d
+}
+
+func (p *proxy) DelayRx(latency, rv time.Duration) {
+	if latency <= 0 {
+		return
+	}
+	d := computeLatency(latency, rv)
+	p.latencyRxMu.Lock()
+	p.latencyRx = d
+	p.latencyRxMu.Unlock()
+	p.logger.Infof("set receive latency %v(%v±%v) [%s <- %s]", d, latency, rv, p.From(), p.To())
+}
+
+func (p *proxy) UndelayRx() {
+	p.latencyRxMu.Lock()
+	d := p.latencyRx
+	p.latencyRx = 0
+	p.latencyRxMu.Unlock()
+	p.logger.Infof("removed receive latency %v [%s <- %s]", d, p.From(), p.To())
+}
+
+func (p *proxy) LatencyRx() time.Duration {
+	p.latencyRxMu.RLock()
+	d := p.latencyRx
+	p.latencyRxMu.RUnlock()
+	return d
+}
+
+func computeLatency(lat, rv time.Duration) time.Duration {
+	if rv == 0 {
+		return lat
+	}
+	if rv < 0 {
+		rv *= -1
+	}
+	if rv > lat {
+		rv = lat / 10
+	}
+	now := time.Now()
+	mrand.Seed(int64(now.Nanosecond()))
+	sign := 1
+	if now.Second()%2 == 0 {
+		sign = -1
+	}
+	return lat + time.Duration(int64(sign)*mrand.Int63n(rv.Nanoseconds()))
+}
+
+func (p *proxy) PauseAccept() {
+	p.acceptMu.Lock()
+	p.pauseAcceptc = make(chan struct{})
+	p.acceptMu.Unlock()
+	p.logger.Infof("paused accepting new connections [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnpauseAccept() {
+	p.acceptMu.Lock()
+	select {
+	case <-p.pauseAcceptc: // already unpaused
+	case <-p.donec:
+		p.acceptMu.Unlock()
+		return
+	default:
+		close(p.pauseAcceptc)
+	}
+	p.acceptMu.Unlock()
+	p.logger.Infof("unpaused accepting new connections [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) PauseTx() {
+	p.txMu.Lock()
+	p.pauseTxc = make(chan struct{})
+	p.txMu.Unlock()
+	p.logger.Infof("paused transmit listen [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnpauseTx() {
+	p.txMu.Lock()
+	select {
+	case <-p.pauseTxc: // already unpaused
+	case <-p.donec:
+		p.txMu.Unlock()
+		return
+	default:
+		close(p.pauseTxc)
+	}
+	p.txMu.Unlock()
+	p.logger.Infof("unpaused transmit listen [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) PauseRx() {
+	p.rxMu.Lock()
+	p.pauseRxc = make(chan struct{})
+	p.rxMu.Unlock()
+	p.logger.Infof("paused receive listen [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnpauseRx() {
+	p.rxMu.Lock()
+	select {
+	case <-p.pauseRxc: // already unpaused
+	case <-p.donec:
+		p.rxMu.Unlock()
+		return
+	default:
+		close(p.pauseRxc)
+	}
+	p.rxMu.Unlock()
+	p.logger.Infof("unpaused receive listen [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) BlackholeTx() {
+	p.txMu.Lock()
+	select {
+	case <-p.blackholeTxc: // already blackholed
+	case <-p.donec:
+		p.txMu.Unlock()
+		return
+	default:
+		close(p.blackholeTxc)
+	}
+	p.txMu.Unlock()
+	p.logger.Infof("blackholed transmit [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnblackholeTx() {
+	p.txMu.Lock()
+	p.blackholeTxc = make(chan struct{})
+	p.txMu.Unlock()
+	p.logger.Infof("unblackholed transmit [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) BlackholeRx() {
+	p.rxMu.Lock()
+	select {
+	case <-p.blackholeRxc: // already blackholed
+	case <-p.donec:
+		p.rxMu.Unlock()
+		return
+	default:
+		close(p.blackholeRxc)
+	}
+	p.rxMu.Unlock()
+	p.logger.Infof("blackholed receive [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnblackholeRx() {
+	p.rxMu.Lock()
+	p.blackholeRxc = make(chan struct{})
+	p.rxMu.Unlock()
+	p.logger.Infof("unblackholed receive [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) CorruptTx(f func([]byte) []byte) {
+	p.corruptTxMu.Lock()
+	p.corruptTx = f
+	p.corruptTxMu.Unlock()
+	p.logger.Infof("corrupting transmit [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) UncorruptTx() {
+	p.corruptTxMu.Lock()
+	p.corruptTx = nil
+	p.corruptTxMu.Unlock()
+	p.logger.Infof("stopped corrupting transmit [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) CorruptRx(f func([]byte) []byte) {
+	p.corruptRxMu.Lock()
+	p.corruptRx = f
+	p.corruptRxMu.Unlock()
+	p.logger.Infof("corrupting receive [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) UncorruptRx() {
+	p.corruptRxMu.Lock()
+	p.corruptRx = nil
+	p.corruptRxMu.Unlock()
+	p.logger.Infof("stopped corrupting receive [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) ResetListener() error {
+	p.listenerMu.Lock()
+	defer p.listenerMu.Unlock()
+
+	if err := p.listener.Close(); err != nil {
+		// already closed
+		if !strings.HasSuffix(err.Error(), "use of closed network connection") {
+			return err
+		}
+	}
+
+	var ln net.Listener
+	var err error
+	if !p.tlsInfo.Empty() {
+		ln, err = NewListener(p.from.Host, p.from.Scheme, &p.tlsInfo)
+	} else {
+		ln, err = net.Listen(p.from.Scheme, p.from.Host)
+	}
+	if err != nil {
+		return err
+	}
+	p.listener = ln
+
+	p.logger.Infof("reset listener %q", p.From())
+	return nil
+}

pkg/transport/proxy_test.go

@@ -0,0 +1,606 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transport
+
+import (
+	"bytes"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/grpclog"
+)
+
+var testTLSInfo = TLSInfo{
+	KeyFile:        "./fixtures/server.key.insecure",
+	CertFile:       "./fixtures/server.crt",
+	TrustedCAFile:  "./fixtures/ca.crt",
+	ClientCertAuth: true,
+}
+
+func TestProxy_Unix_Insecure(t *testing.T)         { testProxy(t, "unix", false, false) }
+func TestProxy_TCP_Insecure(t *testing.T)          { testProxy(t, "tcp", false, false) }
+func TestProxy_Unix_Secure(t *testing.T)           { testProxy(t, "unix", true, false) }
+func TestProxy_TCP_Secure(t *testing.T)            { testProxy(t, "tcp", true, false) }
+func TestProxy_Unix_Insecure_DelayTx(t *testing.T) { testProxy(t, "unix", false, true) }
+func TestProxy_TCP_Insecure_DelayTx(t *testing.T)  { testProxy(t, "tcp", false, true) }
+func TestProxy_Unix_Secure_DelayTx(t *testing.T)   { testProxy(t, "unix", true, true) }
+func TestProxy_TCP_Secure_DelayTx(t *testing.T)    { testProxy(t, "tcp", true, true) }
+func testProxy(t *testing.T, scheme string, secure bool, delayTx bool) {
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	if scheme == "tcp" {
+		ln1, ln2 := listen(t, "tcp", "localhost:0", TLSInfo{}), listen(t, "tcp", "localhost:0", TLSInfo{})
+		srcAddr, dstAddr = ln1.Addr().String(), ln2.Addr().String()
+		ln1.Close()
+		ln2.Close()
+	} else {
+		defer func() {
+			os.RemoveAll(srcAddr)
+			os.RemoveAll(dstAddr)
+		}()
+	}
+	tlsInfo := testTLSInfo
+	if !secure {
+		tlsInfo = TLSInfo{}
+	}
+	ln := listen(t, scheme, dstAddr, tlsInfo)
+	defer ln.Close()
+
+	cfg := ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	}
+	if secure {
+		cfg.TLSInfo = testTLSInfo
+	}
+	p := NewProxy(cfg)
+	<-p.Ready()
+	defer p.Close()
+
+	data1 := []byte("Hello World!")
+	donec, writec := make(chan struct{}), make(chan []byte)
+
+	go func() {
+		defer close(donec)
+		for data := range writec {
+			send(t, data, scheme, srcAddr, tlsInfo)
+		}
+	}()
+
+	recvc := make(chan []byte)
+	go func() {
+		for i := 0; i < 2; i++ {
+			recvc <- receive(t, ln)
+		}
+	}()
+
+	writec <- data1
+	now := time.Now()
+	if d := <-recvc; !bytes.Equal(data1, d) {
+		t.Fatalf("expected %q, got %q", string(data1), string(d))
+	}
+	took1 := time.Since(now)
+	t.Logf("took %v with no latency", took1)
+
+	lat, rv := 50*time.Millisecond, 5*time.Millisecond
+	if delayTx {
+		p.DelayTx(lat, rv)
+	}
+
+	data2 := []byte("new data")
+	writec <- data2
+	now = time.Now()
+	if d := <-recvc; !bytes.Equal(data2, d) {
+		t.Fatalf("expected %q, got %q", string(data2), string(d))
+	}
+	took2 := time.Since(now)
+	if delayTx {
+		t.Logf("took %v with latency %v±%v", took2, lat, rv)
+	} else {
+		t.Logf("took %v with no latency", took2)
+	}
+
+	if delayTx {
+		p.UndelayTx()
+		if took1 >= took2 {
+			t.Fatalf("expected took1 %v < took2 %v (with latency)", took1, took2)
+		}
+	}
+
+	close(writec)
+	select {
+	case <-donec:
+	case <-time.After(3 * time.Second):
+		t.Fatal("took too long to write")
+	}
+
+	select {
+	case <-p.Done():
+		t.Fatal("unexpected done")
+	case err := <-p.Error():
+		t.Fatal(err)
+	default:
+	}
+
+	if err := p.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case <-p.Done():
+	case err := <-p.Error():
+		if !strings.HasPrefix(err.Error(), "accept ") &&
+			!strings.HasSuffix(err.Error(), "use of closed network connection") {
+			t.Fatal(err)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("took too long to close")
+	}
+}
+
+func TestProxy_Unix_Insecure_DelayAccept(t *testing.T) { testProxyDelayAccept(t, false) }
+func TestProxy_Unix_Secure_DelayAccept(t *testing.T)   { testProxyDelayAccept(t, true) }
+func testProxyDelayAccept(t *testing.T, secure bool) {
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	tlsInfo := testTLSInfo
+	if !secure {
+		tlsInfo = TLSInfo{}
+	}
+	scheme := "unix"
+	ln := listen(t, scheme, dstAddr, tlsInfo)
+	defer ln.Close()
+
+	cfg := ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	}
+	if secure {
+		cfg.TLSInfo = testTLSInfo
+	}
+	p := NewProxy(cfg)
+	<-p.Ready()
+	defer p.Close()
+
+	data := []byte("Hello World!")
+
+	now := time.Now()
+	send(t, data, scheme, srcAddr, tlsInfo)
+	if d := receive(t, ln); !bytes.Equal(data, d) {
+		t.Fatalf("expected %q, got %q", string(data), string(d))
+	}
+	took1 := time.Since(now)
+	t.Logf("took %v with no latency", took1)
+
+	lat, rv := 700*time.Millisecond, 10*time.Millisecond
+	p.DelayAccept(lat, rv)
+	defer p.UndelayAccept()
+	if err := p.ResetListener(); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(200 * time.Millisecond)
+
+	now = time.Now()
+	send(t, data, scheme, srcAddr, tlsInfo)
+	if d := receive(t, ln); !bytes.Equal(data, d) {
+		t.Fatalf("expected %q, got %q", string(data), string(d))
+	}
+	took2 := time.Since(now)
+	t.Logf("took %v with latency %v±%v", took2, lat, rv)
+
+	if took1 >= took2 {
+		t.Fatalf("expected took1 %v < took2 %v", took1, took2)
+	}
+}
+
+func TestProxy_PauseTx(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	p.PauseTx()
+
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+
+	recvc := make(chan []byte)
+	go func() {
+		recvc <- receive(t, ln)
+	}()
+
+	select {
+	case d := <-recvc:
+		t.Fatalf("received unexpected data %q during pause", string(d))
+	case <-time.After(200 * time.Millisecond):
+	}
+
+	p.UnpauseTx()
+
+	select {
+	case d := <-recvc:
+		if !bytes.Equal(data, d) {
+			t.Fatalf("expected %q, got %q", string(data), string(d))
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("took too long to receive after unpause")
+	}
+}
+
+func TestProxy_BlackholeTx(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	p.BlackholeTx()
+
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+
+	recvc := make(chan []byte)
+	go func() {
+		recvc <- receive(t, ln)
+	}()
+
+	select {
+	case d := <-recvc:
+		t.Fatalf("unexpected data receive %q during blackhole", string(d))
+	case <-time.After(200 * time.Millisecond):
+	}
+
+	p.UnblackholeTx()
+
+	// expect different data, old data dropped
+	data[0]++
+	send(t, data, scheme, srcAddr, TLSInfo{})
+
+	select {
+	case d := <-recvc:
+		if !bytes.Equal(data, d) {
+			t.Fatalf("expected %q, got %q", string(data), string(d))
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("took too long to receive after unblackhole")
+	}
+}
+
+func TestProxy_CorruptTx(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	p.CorruptTx(func(d []byte) []byte {
+		d[len(d)/2]++
+		return d
+	})
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+	if d := receive(t, ln); bytes.Equal(d, data) {
+		t.Fatalf("expected corrupted data, got %q", string(d))
+	}
+
+	p.UncorruptTx()
+	send(t, data, scheme, srcAddr, TLSInfo{})
+	if d := receive(t, ln); !bytes.Equal(d, data) {
+		t.Fatalf("expected uncorrupted data, got %q", string(d))
+	}
+}
+
+func TestProxy_Shutdown(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	px, _ := p.(*proxy)
+	px.listener.Close()
+	time.Sleep(200 * time.Millisecond)
+
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+	if d := receive(t, ln); !bytes.Equal(d, data) {
+		t.Fatalf("expected %q, got %q", string(data), string(d))
+	}
+}
+
+func TestProxy_ShutdownListener(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	// shut down destination
+	ln.Close()
+	time.Sleep(200 * time.Millisecond)
+
+	ln = listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+	if d := receive(t, ln); !bytes.Equal(d, data) {
+		t.Fatalf("expected %q, got %q", string(data), string(d))
+	}
+}
+
+func TestProxyHTTP_Insecure_DelayTx(t *testing.T) { testProxyHTTP(t, false, true) }
+func TestProxyHTTP_Secure_DelayTx(t *testing.T)   { testProxyHTTP(t, true, true) }
+func TestProxyHTTP_Insecure_DelayRx(t *testing.T) { testProxyHTTP(t, false, false) }
+func TestProxyHTTP_Secure_DelayRx(t *testing.T)   { testProxyHTTP(t, true, false) }
+func testProxyHTTP(t *testing.T, secure, delayTx bool) {
+	scheme := "tcp"
+	ln1, ln2 := listen(t, scheme, "localhost:0", TLSInfo{}), listen(t, scheme, "localhost:0", TLSInfo{})
+	srcAddr, dstAddr := ln1.Addr().String(), ln2.Addr().String()
+	ln1.Close()
+	ln2.Close()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/hello", func(w http.ResponseWriter, req *http.Request) {
+		d, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err = w.Write([]byte(fmt.Sprintf("%q(confirmed)", string(d)))); err != nil {
+			t.Fatal(err)
+		}
+	})
+	var tlsConfig *tls.Config
+	var err error
+	if secure {
+		tlsConfig, err = testTLSInfo.ServerConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	srv := &http.Server{
+		Addr:      dstAddr,
+		Handler:   mux,
+		TLSConfig: tlsConfig,
+	}
+
+	donec := make(chan struct{})
+	defer func() {
+		srv.Close()
+		<-donec
+	}()
+	go func() {
+		defer close(donec)
+		if !secure {
+			srv.ListenAndServe()
+		} else {
+			srv.ListenAndServeTLS(testTLSInfo.CertFile, testTLSInfo.KeyFile)
+		}
+	}()
+	time.Sleep(200 * time.Millisecond)
+
+	cfg := ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	}
+	if secure {
+		cfg.TLSInfo = testTLSInfo
+	}
+	p := NewProxy(cfg)
+	<-p.Ready()
+	defer p.Close()
+
+	data := "Hello World!"
+
+	now := time.Now()
+	var resp *http.Response
+	if secure {
+		tp, terr := NewTransport(testTLSInfo, 3*time.Second)
+		if terr != nil {
+			t.Fatal(terr)
+		}
+		cli := &http.Client{Transport: tp}
+		resp, err = cli.Post("https://"+srcAddr+"/hello", "", strings.NewReader(data))
+	} else {
+		resp, err = http.Post("http://"+srcAddr+"/hello", "", strings.NewReader(data))
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	took1 := time.Since(now)
+	t.Logf("took %v with no latency", took1)
+
+	rs1 := string(d)
+	exp := fmt.Sprintf("%q(confirmed)", data)
+	if rs1 != exp {
+		t.Fatalf("got %q, expected %q", rs1, exp)
+	}
+
+	lat, rv := 100*time.Millisecond, 10*time.Millisecond
+	if delayTx {
+		p.DelayTx(lat, rv)
+		defer p.UndelayTx()
+	} else {
+		p.DelayRx(lat, rv)
+		defer p.UndelayRx()
+	}
+
+	now = time.Now()
+	if secure {
+		tp, terr := NewTransport(testTLSInfo, 3*time.Second)
+		if terr != nil {
+			t.Fatal(terr)
+		}
+		cli := &http.Client{Transport: tp}
+		resp, err = cli.Post("https://"+srcAddr+"/hello", "", strings.NewReader(data))
+	} else {
+		resp, err = http.Post("http://"+srcAddr+"/hello", "", strings.NewReader(data))
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	took2 := time.Since(now)
+	t.Logf("took %v with latency %v±%v", took2, lat, rv)
+
+	rs2 := string(d)
+	if rs2 != exp {
+		t.Fatalf("got %q, expected %q", rs2, exp)
+	}
+	if took1 > took2 {
+		t.Fatalf("expected took1 %v < took2 %v", took1, took2)
+	}
+}
+
+func newUnixAddr() string {
+	now := time.Now().UnixNano()
+	rand.Seed(now)
+	addr := fmt.Sprintf("%X%X.unix-conn", now, rand.Intn(35000))
+	os.RemoveAll(addr)
+	return addr
+}
+
+func listen(t *testing.T, scheme, addr string, tlsInfo TLSInfo) (ln net.Listener) {
+	var err error
+	if !tlsInfo.Empty() {
+		ln, err = NewListener(addr, scheme, &tlsInfo)
+	} else {
+		ln, err = net.Listen(scheme, addr)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ln
+}
+
+func send(t *testing.T, data []byte, scheme, addr string, tlsInfo TLSInfo) {
+	var out net.Conn
+	var err error
+	if !tlsInfo.Empty() {
+		tp, terr := NewTransport(tlsInfo, 3*time.Second)
+		if terr != nil {
+			t.Fatal(terr)
+		}
+		out, err = tp.Dial(scheme, addr)
+	} else {
+		out, err = net.Dial(scheme, addr)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err = out.Write(data); err != nil {
+		t.Fatal(err)
+	}
+	if err = out.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func receive(t *testing.T, ln net.Listener) (data []byte) {
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+	for {
+		in, err := ln.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+		var n int64
+		n, err = buf.ReadFrom(in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n > 0 {
+			break
+		}
+	}
+	return buf.Bytes()
+}

proxy/httpproxy/reverse.go

@@ -119,6 +119,7 @@ func (p *reverseProxy) ServeHTTP(rw http.ResponseWriter, clientreq *http.Request
 			case <-closeCh:
 				atomic.StoreInt32(&requestClosed, 1)
 				plog.Printf("client %v closed request prematurely", clientreq.RemoteAddr)
+				cancel()
 			case <-completeCh:
 			}
 		}()

rafthttp/peer.go

@@ -230,6 +230,7 @@ func (p *peer) send(m raftpb.Message) {
 			plog.MergeWarningf("dropped internal raft message to %s since %s's sending buffer is full (bad/overloaded network)", p.id, name)
 		}
 		plog.Debugf("dropped %s to %s since %s's sending buffer is full", m.Type, p.id, name)
+		sentFailures.WithLabelValues(types.ID(m.To).String()).Inc()
 	}
 }
 

rafthttp/remote.go

@@ -53,6 +53,7 @@ func (g *remote) send(m raftpb.Message) {
 			plog.MergeWarningf("dropped internal raft message to %s since sending buffer is full (bad/overloaded network)", g.id)
 		}
 		plog.Debugf("dropped %s to %s since sending buffer is full", m.Type, g.id)
+		sentFailures.WithLabelValues(types.ID(m.To).String()).Inc()
 	}
 }
 

rafthttp/transport.go

@@ -85,6 +85,8 @@ type Transporter interface {
 	// If the connection is active since peer was added, it returns the adding time.
 	// If the connection is currently inactive, it returns zero time.
 	ActiveSince(id types.ID) time.Time
+	// ActivePeers returns the number of active peers.
+	ActivePeers() int
 	// Stop closes the connections and stops the transporter.
 	Stop()
 }
@@ -375,6 +377,20 @@ func (t *Transport) Resume() {
 	}
 }
 
+// ActivePeers returns a channel that closes when an initial
+// peer connection has been established. Use this to wait until the
+// first peer connection becomes active.
+func (t *Transport) ActivePeers() (cnt int) {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+	for _, p := range t.peers {
+		if !p.activeSince().IsZero() {
+			cnt++
+		}
+	}
+	return cnt
+}
+
 type nopTransporter struct{}
 
 func NewNopTransporter() Transporter {
@@ -391,6 +407,7 @@ func (s *nopTransporter) RemovePeer(id types.ID)              {}
 func (s *nopTransporter) RemoveAllPeers()                     {}
 func (s *nopTransporter) UpdatePeer(id types.ID, us []string) {}
 func (s *nopTransporter) ActiveSince(id types.ID) time.Time   { return time.Time{} }
+func (s *nopTransporter) ActivePeers() int                    { return 0 }
 func (s *nopTransporter) Stop()                               {}
 func (s *nopTransporter) Pause()                              {}
 func (s *nopTransporter) Resume()                             {}

test

@@ -114,7 +114,7 @@ function functional_pass {
 
 	for a in 1 2 3; do
 		mkdir -p ./agent-$a
-		./bin/etcd-agent -etcd-path ./bin/etcd -etcd-log-dir "./agent-$a" -port ":${a}9027" -use-root=false &
+		./bin/etcd-agent -etcd-path ./bin/etcd -etcd-log-dir "./agent-$a" -port ":${a}9027" &
 		pid="$!"
 		agent_pids="${agent_pids} $pid"
 	done
@@ -129,10 +129,12 @@ function functional_pass {
 	echo "Starting 'etcd-tester'"
 	./bin/etcd-tester \
 		-agent-endpoints "127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027" \
-		-client-ports 12379,22379,32379 \
-		-peer-ports 12380,22380,32380 \
+		-client-ports 1379,2379,3379 \
+		-advertise-client-ports 13790,23790,33790 \
+		-peer-ports 1380,2380,3380 \
+		-advertise-peer-ports 13800,23800,33800 \
 		-limit 1 \
-		-schedule-cases "0 1 2 3 4 5" \
+		-schedule-cases "0 1 2 3 4 5 6 7 8 9" \
 		-stress-qps 1000 \
 		-stress-key-txn-count 100 \
 		-stress-key-txn-ops 10 \