version: 3.3.1

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

.gitignore

@@ -7,7 +7,6 @@
 /release
 /machine*
 /bin
-.Dockerfile-test
 .vagrant
 *.etcd
 *.log
@@ -15,8 +14,6 @@
 *.swp
 /hack/insta-discovery/.env
 *.test
-tools/functional-tester/docker/bin
-hack/scripts-dev/docker-dns/.Dockerfile
-hack/scripts-dev/docker-dns-srv/.Dockerfile
 hack/tls-setup/certs
 .idea
+*.bak

.semaphore.sh

@@ -2,7 +2,7 @@
 
 TEST_SUFFIX=$(date +%s | base64 | head -c 15)
 
-TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional' MANUAL_VER=v3.3.0-rc.0"
+TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional' MANUAL_VER=v3.3.0"
 if [ "$TEST_ARCH" == "386" ]; then
 	TEST_OPTS="GOARCH=386 PASSES='build unit integration_e2e'"
 fi
@@ -10,7 +10,7 @@ fi
 docker run \
 	--rm \
 	--volume=`pwd`:/go/src/github.com/coreos/etcd \
-	gcr.io/etcd-development/etcd-test:go1.9.3 \
+	gcr.io/etcd-development/etcd-test:go1.9.4 \
 	/bin/bash -c "${TEST_OPTS} ./test 2>&1 | tee test-${TEST_SUFFIX}.log"
 
 ! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-${TEST_SUFFIX}.log

.travis.yml

@@ -6,7 +6,7 @@ sudo: required
 services: docker
 
 go:
-- 1.9.3
+- 1.9.4
 - tip
 
 notifications:
@@ -30,7 +30,7 @@ matrix:
   - go: tip
     env: TARGET=amd64-go-tip
   exclude:
-  - go: 1.9.3
+  - go: 1.9.4
     env: TARGET=amd64-go-tip
   - go: tip
     env: TARGET=amd64
@@ -48,17 +48,18 @@ matrix:
     env: TARGET=ppc64le
 
 before_install:
-- docker pull gcr.io/etcd-development/etcd-test:go1.9.3
+- if [[ $TRAVIS_GO_VERSION == 1.* ]]; then docker pull gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION}; fi
 
 install:
 - pushd cmd/etcd && go get -t -v ./... && popd
 
 script:
+ - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
  - >
     case "${TARGET}" in
       amd64)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.3 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GOARCH=amd64 ./test"
         ;;
       amd64-go-tip)
@@ -66,23 +67,23 @@ script:
         ;;
       darwin-amd64)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.3 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOOS=darwin GOARCH=amd64 ./build"
         ;;
       windows-amd64)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.3 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOOS=windows GOARCH=amd64 ./build"
         ;;
       386)
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.3 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GOARCH=386 PASSES='build unit' ./test"
         ;;
       *)
         # test building out of gopath
         docker run --rm \
-          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go1.9.3 \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
           /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOARCH='${TARGET}' ./build"
         ;;
     esac

Dockerfile-functional-tester

@@ -0,0 +1,53 @@
+FROM ubuntu:17.10
+
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
+RUN apt-get -y update \
+  && apt-get -y install \
+  build-essential \
+  gcc \
+  apt-utils \
+  pkg-config \
+  software-properties-common \
+  apt-transport-https \
+  libssl-dev \
+  sudo \
+  bash \
+  curl \
+  wget \
+  tar \
+  git \
+  && apt-get -y update \
+  && apt-get -y upgrade \
+  && apt-get -y autoremove \
+  && apt-get -y autoclean
+
+ENV GOROOT /usr/local/go
+ENV GOPATH /go
+ENV PATH ${GOPATH}/bin:${GOROOT}/bin:${PATH}
+ENV GO_VERSION REPLACE_ME_GO_VERSION
+ENV GO_DOWNLOAD_URL https://storage.googleapis.com/golang
+RUN rm -rf ${GOROOT} \
+  && curl -s ${GO_DOWNLOAD_URL}/go${GO_VERSION}.linux-amd64.tar.gz | tar -v -C /usr/local/ -xz \
+  && mkdir -p ${GOPATH}/src ${GOPATH}/bin \
+  && go version
+
+RUN mkdir -p ${GOPATH}/src/github.com/coreos/etcd
+ADD . ${GOPATH}/src/github.com/coreos/etcd
+
+RUN go get -v github.com/coreos/gofail \
+  && pushd ${GOPATH}/src/github.com/coreos/etcd \
+  && GO_BUILD_FLAGS="-v" ./build \
+  && cp ./bin/etcd /etcd \
+  && cp ./bin/etcdctl /etcdctl \
+  && GO_BUILD_FLAGS="-v" FAILPOINTS=1 ./build \
+  && cp ./bin/etcd /etcd-failpoints \
+  && ./tools/functional-tester/build \
+  && cp ./bin/etcd-agent /etcd-agent \
+  && cp ./bin/etcd-tester /etcd-tester \
+  && cp ./bin/etcd-runner /etcd-runner \
+  && go build -v -o /benchmark ./cmd/tools/benchmark \
+  && go build -v -o /etcd-test-proxy ./cmd/tools/etcd-test-proxy \
+  && popd \
+  && rm -rf ${GOPATH}/src/github.com/coreos/etcd

Documentation/dev-guide/limit.md

@@ -6,5 +6,4 @@ etcd is designed to handle small key value pairs typical for metadata. Larger re
 
 ## Storage size limit
 
-The default storage size limit is 2GB, configurable with `--quota-backend-bytes` flag; supports up to 8GB.
-
+The default storage size limit is 2GB, configurable with `--quota-backend-bytes` flag. 8GB is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.

Documentation/faq.md

@@ -22,7 +22,7 @@ A member's advertised peer URLs come from `--initial-advertise-peer-urls` on ini
 
 ### System requirements
 
-Since etcd writes data to disk, SSD is highly recommended. To prevent performance degradation or unintentionally overloading the key-value store, etcd enforces a 2GB default storage size quota, configurable up to 8GB. To avoid swapping or running out of memory, the machine should have at least as much RAM to cover the quota. At CoreOS, an etcd cluster is usually deployed on dedicated CoreOS Container Linux machines with dual-core processors, 2GB of RAM, and 80GB of SSD *at the very least*. **Note that performance is intrinsically workload dependent; please test before production deployment**. See [hardware][hardware-setup] for more recommendations.
+Since etcd writes data to disk, SSD is highly recommended. To prevent performance degradation or unintentionally overloading the key-value store, etcd enforces a configurable storage size quota set to 2GB by default. To avoid swapping or running out of memory, the machine should have at least as much RAM to cover the quota. 8GB is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it. At CoreOS, an etcd cluster is usually deployed on dedicated CoreOS Container Linux machines with dual-core processors, 2GB of RAM, and 80GB of SSD *at the very least*. **Note that performance is intrinsically workload dependent; please test before production deployment**. See [hardware][hardware-setup] for more recommendations.
 
 Most stable production environment is Linux operating system with amd64 architecture; see [supported platform][supported-platform] for more.
 

etcdserver/apply.go

@@ -107,6 +107,8 @@ func (s *EtcdServer) newApplierV3() applierV3 {
 }
 
 func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
+	defer warnOfExpensiveRequest(time.Now(), r)
+
 	ar := &applyResult{}
 
 	// call into a.s.applyV3.F instead of a.F so upper appliers can check individual calls

etcdserver/apply_v2.go

@@ -107,6 +107,8 @@ func (a *applierV2store) Sync(r *RequestV2) Response {
 // applyV2Request interprets r as a call to store.X and returns a Response interpreted
 // from store.Event
 func (s *EtcdServer) applyV2Request(r *RequestV2) Response {
+	defer warnOfExpensiveRequest(time.Now(), r)
+
 	switch r.Method {
 	case "POST":
 		return s.applyV2.Post(r)

etcdserver/server.go

@@ -794,14 +794,8 @@ func (s *EtcdServer) run() {
 
 func (s *EtcdServer) applyAll(ep *etcdProgress, apply *apply) {
 	s.applySnapshot(ep, apply)
-	st := time.Now()
 	s.applyEntries(ep, apply)
-	d := time.Since(st)
-	entriesNum := len(apply.entries)
-	if entriesNum != 0 && d > time.Duration(entriesNum)*warnApplyDuration {
-		plog.Warningf("apply entries took too long [%v for %d entries]", d, len(apply.entries))
-		plog.Warningf("avoid queries with large range/delete range!")
-	}
+
 	proposalsApplied.Set(float64(ep.appliedi))
 	s.applyWait.Trigger(ep.appliedi)
 	// wait for the raft routine to finish the disk writes before triggering a

etcdserver/util.go

@@ -15,6 +15,7 @@
 package etcdserver
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/coreos/etcd/etcdserver/membership"
@@ -95,3 +96,19 @@ func (nc *notifier) notify(err error) {
 	nc.err = err
 	close(nc.c)
 }
+
+func warnOfExpensiveRequest(now time.Time, stringer fmt.Stringer) {
+	warnOfExpensiveGenericRequest(now, stringer, "")
+}
+
+func warnOfExpensiveReadOnlyRangeRequest(now time.Time, stringer fmt.Stringer) {
+	warnOfExpensiveGenericRequest(now, stringer, "read-only range ")
+}
+
+func warnOfExpensiveGenericRequest(now time.Time, stringer fmt.Stringer, prefix string) {
+	// TODO: add metrics
+	d := time.Since(now)
+	if d > warnApplyDuration {
+		plog.Warningf("%srequest %q took too long (%v) to execute", prefix, stringer.String(), d)
+	}
+}

etcdserver/v2_server.go

@@ -158,3 +158,8 @@ func (r *RequestV2) Handle(ctx context.Context, v2api RequestV2Handler) (Respons
 	}
 	return Response{}, ErrUnknownMethod
 }
+
+func (r *RequestV2) String() string {
+	rpb := pb.Request(*r)
+	return rpb.String()
+}

etcdserver/v3_server.go

@@ -84,6 +84,8 @@ type Authenticator interface {
 }
 
 func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) {
+	defer warnOfExpensiveReadOnlyRangeRequest(time.Now(), r)
+
 	if !r.Serializable {
 		err := s.linearizableReadNotify(ctx)
 		if err != nil {
@@ -95,6 +97,7 @@ func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeRe
 	chk := func(ai *auth.AuthInfo) error {
 		return s.authStore.IsRangePermitted(ai, r.Key, r.RangeEnd)
 	}
+
 	get := func() { resp, err = s.applyV3Base.Range(nil, r) }
 	if serr := s.doSerialize(ctx, chk, get); serr != nil {
 		return nil, serr
@@ -131,12 +134,16 @@ func (s *EtcdServer) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse
 		chk := func(ai *auth.AuthInfo) error {
 			return checkTxnAuth(s.authStore, ai, r)
 		}
+
+		defer warnOfExpensiveReadOnlyRangeRequest(time.Now(), r)
+
 		get := func() { resp, err = s.applyV3Base.Txn(r) }
 		if serr := s.doSerialize(ctx, chk, get); serr != nil {
 			return nil, serr
 		}
 		return resp, err
 	}
+
 	resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{Txn: r})
 	if err != nil {
 		return nil, err

hack/scripts-dev/Makefile

@@ -1,5 +1,7 @@
 # run from repository root
-#
+
+
+
 # Example:
 #   make clean -f ./hack/scripts-dev/Makefile
 #   make build -f ./hack/scripts-dev/Makefile
@@ -23,49 +25,65 @@ clean:
 	rm -f ./clientv3/integration/127.0.0.1:* ./clientv3/integration/localhost:*
 	rm -f ./clientv3/ordering/127.0.0.1:* ./clientv3/ordering/localhost:*
 
-_GO_VERSION = 1.9.2
-ifdef GO_VERSION
-	_GO_VERSION = $(GO_VERSION)
+
+
+GO_VERSION ?= 1.9.4
+ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound")
+
+TEST_SUFFIX = $(shell date +%s | base64 | head -c 15)
+TEST_OPTS ?= PASSES='unit'
+
+TMP_DIR_MOUNT_FLAG = --mount type=tmpfs,destination=/tmp
+ifdef HOST_TMP_DIR
+	TMP_DIR_MOUNT_FLAG = --mount type=bind,source=$(HOST_TMP_DIR),destination=/tmp
 endif
 
+
+
 # Example:
-#   GO_VERSION=1.8.5 make build-docker-test -f ./hack/scripts-dev/Makefile
+#   GO_VERSION=1.8.7 make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
-#   GO_VERSION=1.8.5 make push-docker-test -f ./hack/scripts-dev/Makefile
+#   GO_VERSION=1.8.7 make push-docker-test -f ./hack/scripts-dev/Makefile
 #   make push-docker-test -f ./hack/scripts-dev/Makefile
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
-#   GO_VERSION=1.8.5 make pull-docker-test -f ./hack/scripts-dev/Makefile
+#   GO_VERSION=1.8.7 make pull-docker-test -f ./hack/scripts-dev/Makefile
 #   make pull-docker-test -f ./hack/scripts-dev/Makefile
 
 build-docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	@cat ./Dockerfile-test | sed s/REPLACE_ME_GO_VERSION/$(_GO_VERSION)/ \
-	  > ./.Dockerfile-test
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./Dockerfile-test
 	docker build \
-	  --tag gcr.io/etcd-development/etcd-test:go$(_GO_VERSION) \
-	  --file ./.Dockerfile-test .
+	  --tag gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  --file ./Dockerfile-test .
+	@mv ./Dockerfile-test.bak ./Dockerfile-test
 
 push-docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-test:go$(GO_VERSION)
 
 pull-docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	docker pull gcr.io/etcd-development/etcd-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-test:go$(GO_VERSION)
+
+
 
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make compile-with-docker-test -f ./hack/scripts-dev/Makefile
 
 compile-with-docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	docker run \
 	  --rm \
 	  --mount type=bind,source=`pwd`,destination=/etcd \
-	  gcr.io/etcd-development/etcd-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && GO_BUILD_FLAGS=-v ./build && ./bin/etcd --version"
 
+
+
+# Example:
+#
 # Local machine:
 #   TEST_OPTS="PASSES='fmt'" make test -f ./hack/scripts-dev/Makefile
 #   TEST_OPTS="PASSES='fmt bom dep compile build unit'" make test -f ./hack/scripts-dev/Makefile
@@ -82,64 +100,50 @@ compile-with-docker-test:
 #
 # Semaphore CI (test with docker):
 #   TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional'" make docker-test -f ./hack/scripts-dev/Makefile
-#   HOST_TMP_DI=/tmp TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional'" make docker-test -f ./hack/scripts-dev/Makefile
+#   HOST_TMP_DIR=/tmp TEST_OPTS="RELEASE_TEST=y INTEGRATION=y PASSES='build unit release integration_e2e functional'" make docker-test -f ./hack/scripts-dev/Makefile
 #   TEST_OPTS="GOARCH=386 PASSES='build unit integration_e2e'" make docker-test -f ./hack/scripts-dev/Makefile
 #
 # grpc-proxy tests (test with docker):
 #   TEST_OPTS="PASSES='build grpcproxy'" make docker-test -f ./hack/scripts-dev/Makefile
-#   HOST_TMP_DI=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test -f ./hack/scripts-dev/Makefile
-
-TEST_SUFFIX = $(shell date +%s | base64 | head -c 15)
-
-_TEST_OPTS = PASSES='unit'
-ifdef TEST_OPTS
-	_TEST_OPTS = $(TEST_OPTS)
-endif
-
-_TMP_DIR_MOUNT_FLAG = --mount type=tmpfs,destination=/tmp
-ifdef HOST_TMP_DIR
-	_TMP_DIR_MOUNT_FLAG = --mount type=bind,source=$(HOST_TMP_DIR),destination=/tmp
-endif
+#   HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test -f ./hack/scripts-dev/Makefile
 
 .PHONY: test
 test:
-	$(info TEST_OPTS: $(_TEST_OPTS))
+	$(info TEST_OPTS: $(TEST_OPTS))
 	$(info log-file: test-$(TEST_SUFFIX).log)
-	$(_TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log
+	$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log
 	! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log
 
 docker-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	$(info TEST_OPTS: $(_TEST_OPTS))
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	$(info TEST_OPTS: $(TEST_OPTS))
 	$(info log-file: test-$(TEST_SUFFIX).log)
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
-	  gcr.io/etcd-development/etcd-test:go$(_GO_VERSION) \
-	  /bin/bash -c "$(_TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log"
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
+	  /bin/bash -c "$(TEST_OPTS) ./test 2>&1 | tee test-$(TEST_SUFFIX).log"
 	! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-$(TEST_SUFFIX).log
 
 docker-test-coverage:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
 	$(info log-file: docker-test-coverage-$(TEST_SUFFIX).log)
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`,destination=/go/src/github.com/coreos/etcd \
-	  gcr.io/etcd-development/etcd-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-test:go$(GO_VERSION) \
 	  /bin/bash -c "COVERDIR=covdir PASSES='build build_cov cov' ./test 2>&1 | tee docker-test-coverage-$(TEST_SUFFIX).log && /codecov -t 6040de41-c073-4d6f-bbf8-d89256ef31e1"
 	! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 docker-test-coverage-$(TEST_SUFFIX).log
 
-# build release container image with Linux
-_ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound")
-ifdef ETCD_VERSION
-	_ETCD_VERSION = $(ETCD_VERSION)
-endif
+
 
 # Example:
 #   ETCD_VERSION=v3.3.0-test.0 make build-docker-release-master -f ./hack/scripts-dev/Makefile
@@ -147,22 +151,24 @@ endif
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
 
 build-docker-release-master: compile-with-docker-test
-	$(info ETCD_VERSION: $(_ETCD_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
 	cp ./Dockerfile-release ./bin/Dockerfile-release
 	docker build \
-	  --tag gcr.io/etcd-development/etcd:$(_ETCD_VERSION) \
+	  --tag gcr.io/etcd-development/etcd:$(ETCD_VERSION) \
 	  --file ./bin/Dockerfile-release \
 	  ./bin
 	rm -f ./bin/Dockerfile-release
 
 	docker run \
 	  --rm \
-	  gcr.io/etcd-development/etcd:$(_ETCD_VERSION) \
+	  gcr.io/etcd-development/etcd:$(ETCD_VERSION) \
 	  /bin/sh -c "/usr/local/bin/etcd --version && ETCDCTL_API=3 /usr/local/bin/etcdctl version"
 
 push-docker-release-master:
-	$(info ETCD_VERSION: $(_ETCD_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd:$(_ETCD_VERSION)
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd:$(ETCD_VERSION)
+
+
 
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
@@ -176,49 +182,50 @@ push-docker-release-master:
 #   make docker-static-ip-test-certs-metrics-proxy-run -f ./hack/scripts-dev/Makefile
 
 build-docker-static-ip-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	@cat ./hack/scripts-dev/docker-static-ip/Dockerfile | sed s/REPLACE_ME_GO_VERSION/$(_GO_VERSION)/ \
-	  > ./hack/scripts-dev/docker-static-ip/.Dockerfile
-
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./hack/scripts-dev/docker-static-ip/Dockerfile
 	docker build \
-	  --tag gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION) \
-	  --file ./hack/scripts-dev/docker-static-ip/.Dockerfile \
+	  --tag gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
+	  --file ./hack/scripts-dev/docker-static-ip/Dockerfile \
 	  ./hack/scripts-dev/docker-static-ip
+	@mv ./hack/scripts-dev/docker-static-ip/Dockerfile.bak ./hack/scripts-dev/docker-static-ip/Dockerfile
 
 push-docker-static-ip-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION)
 
 pull-docker-static-ip-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	docker pull gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION)
 
 docker-static-ip-test-certs-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-static-ip/certs,destination=/certs \
-	  gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
 
 docker-static-ip-test-certs-metrics-proxy-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-static-ip/certs-metrics-proxy,destination=/certs-metrics-proxy \
-	  gcr.io/etcd-development/etcd-static-ip-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-static-ip-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-metrics-proxy/run.sh && rm -rf m*.etcd"
 
+
+
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make compile-with-docker-test -f ./hack/scripts-dev/Makefile
@@ -230,73 +237,104 @@ docker-static-ip-test-certs-metrics-proxy-run:
 #   make docker-dns-test-certs-run -f ./hack/scripts-dev/Makefile
 #   make docker-dns-test-certs-gateway-run -f ./hack/scripts-dev/Makefile
 #   make docker-dns-test-certs-wildcard-run -f ./hack/scripts-dev/Makefile
+#   make docker-dns-test-certs-common-name-auth-run -f ./hack/scripts-dev/Makefile
+#   make docker-dns-test-certs-common-name-multi-run -f ./hack/scripts-dev/Makefile
 
 build-docker-dns-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	@cat ./hack/scripts-dev/docker-dns/Dockerfile | sed s/REPLACE_ME_GO_VERSION/$(_GO_VERSION)/ \
-	  > ./hack/scripts-dev/docker-dns/.Dockerfile
-
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./hack/scripts-dev/docker-dns/Dockerfile
 	docker build \
-	  --tag gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
-	  --file ./hack/scripts-dev/docker-dns/.Dockerfile \
+	  --tag gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  --file ./hack/scripts-dev/docker-dns/Dockerfile \
 	  ./hack/scripts-dev/docker-dns
+	@mv ./hack/scripts-dev/docker-dns/Dockerfile.bak ./hack/scripts-dev/docker-dns/Dockerfile
 
 	docker run \
 	  --rm \
 	  --dns 127.0.0.1 \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "/etc/init.d/bind9 start && cat /dev/null >/etc/hosts && dig etcd.local"
 
 push-docker-dns-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION)
 
 pull-docker-dns-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	docker pull gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION)
 
 docker-dns-test-certs-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs,destination=/certs \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
 
 docker-dns-test-certs-gateway-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-gateway,destination=/certs-gateway \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-gateway/run.sh && rm -rf m*.etcd"
 
 docker-dns-test-certs-wildcard-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-wildcard,destination=/certs-wildcard \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-wildcard/run.sh && rm -rf m*.etcd"
 
+docker-dns-test-certs-common-name-auth-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-common-name-auth,destination=/certs-common-name-auth \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-common-name-auth/run.sh && rm -rf m*.etcd"
+
+docker-dns-test-certs-common-name-multi-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-common-name-multi,destination=/certs-common-name-multi \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-common-name-multi/run.sh && rm -rf m*.etcd"
+
+
+
 # Example:
 #   make build-docker-test -f ./hack/scripts-dev/Makefile
 #   make compile-with-docker-test -f ./hack/scripts-dev/Makefile
@@ -310,84 +348,113 @@ docker-dns-test-certs-wildcard-run:
 #   make docker-dns-srv-test-certs-wildcard-run -f ./hack/scripts-dev/Makefile
 
 build-docker-dns-srv-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	@cat ./hack/scripts-dev/docker-dns-srv/Dockerfile | sed s/REPLACE_ME_GO_VERSION/$(_GO_VERSION)/ \
-	  > ./hack/scripts-dev/docker-dns-srv/.Dockerfile
-
+	$(info GO_VERSION: $(GO_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./hack/scripts-dev/docker-dns-srv/Dockerfile
 	docker build \
-	  --tag gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
-	  --file ./hack/scripts-dev/docker-dns-srv/.Dockerfile \
+	  --tag gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
+	  --file ./hack/scripts-dev/docker-dns-srv/Dockerfile \
 	  ./hack/scripts-dev/docker-dns-srv
+	@mv ./hack/scripts-dev/docker-dns-srv/Dockerfile.bak ./hack/scripts-dev/docker-dns-srv/Dockerfile
 
 	docker run \
 	  --rm \
 	  --dns 127.0.0.1 \
-	  gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
 	  /bin/bash -c "/etc/init.d/bind9 start && cat /dev/null >/etc/hosts && dig +noall +answer SRV _etcd-client-ssl._tcp.etcd.local && dig +noall +answer SRV _etcd-server-ssl._tcp.etcd.local && dig +noall +answer m1.etcd.local m2.etcd.local m3.etcd.local"
 
 push-docker-dns-srv-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	gcloud docker -- push gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION)
 
 pull-docker-dns-srv-test:
-	$(info GO_VERSION: $(_GO_VERSION))
-	docker pull gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION)
+	$(info GO_VERSION: $(GO_VERSION))
+	docker pull gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION)
 
 docker-dns-srv-test-certs-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns-srv/certs,destination=/certs \
-	  gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs/run.sh && rm -rf m*.etcd"
 
 docker-dns-srv-test-certs-gateway-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns-srv/certs-gateway,destination=/certs-gateway \
-	  gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-gateway/run.sh && rm -rf m*.etcd"
 
 docker-dns-srv-test-certs-wildcard-run:
-	$(info GO_VERSION: $(_GO_VERSION))
+	$(info GO_VERSION: $(GO_VERSION))
 	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
 	docker run \
 	  --rm \
 	  --tty \
 	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
+	  $(TMP_DIR_MOUNT_FLAG) \
 	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
 	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns-srv/certs-wildcard,destination=/certs-wildcard \
-	  gcr.io/etcd-development/etcd-dns-srv-test:go$(_GO_VERSION) \
+	  gcr.io/etcd-development/etcd-dns-srv-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-wildcard/run.sh && rm -rf m*.etcd"
 
-# example workflow for common name + auth
-# TODO: make this as tests
-# make docker-dns-example-certs-common-name-run -f ./hack/scripts-dev/Makefile
-docker-dns-example-certs-common-name-run:
-	$(info GO_VERSION: $(_GO_VERSION))
-	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
-	$(info TMP_DIR_MOUNT_FLAG: $(_TMP_DIR_MOUNT_FLAG))
+
+
+# Example:
+#   make build-etcd-test-proxy -f ./hack/scripts-dev/Makefile
+
+build-etcd-test-proxy:
+	go build -v -o ./bin/etcd-test-proxy ./cmd/tools/etcd-test-proxy
+
+
+
+# Example:
+#   make build-docker-functional-tester -f ./hack/scripts-dev/Makefile
+#   make push-docker-functional-tester -f ./hack/scripts-dev/Makefile
+#   make pull-docker-functional-tester -f ./hack/scripts-dev/Makefile
+
+build-docker-functional-tester:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	@sed -i.bak 's|REPLACE_ME_GO_VERSION|$(GO_VERSION)|g' ./Dockerfile-functional-tester
+	docker build \
+	  --tag gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) \
+	  --file ./Dockerfile-functional-tester \
+	  .
+	@mv ./Dockerfile-functional-tester.bak ./Dockerfile-functional-tester
+
 	docker run \
 	  --rm \
-	  --tty \
-	  --dns 127.0.0.1 \
-	  $(_TMP_DIR_MOUNT_FLAG) \
-	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
-	  --mount type=bind,source=`pwd`/hack/scripts-dev/docker-dns/certs-common-name,destination=/certs-common-name \
-	  gcr.io/etcd-development/etcd-dns-test:go$(_GO_VERSION) \
-	  /bin/bash -c "cd /etcd && /certs-common-name/run.sh && rm -rf m*.etcd"
+	  gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION) \
+	  /bin/bash -c "/etcd --version && \
+	   /etcd-failpoints --version && \
+	   ETCDCTL_API=3 /etcdctl version && \
+	   /etcd-agent -help || true && \
+	   /etcd-tester -help || true && \
+	   /etcd-runner --help || true && \
+	   /benchmark --help || true && \
+	   /etcd-test-proxy -help || true"
+
+push-docker-functional-tester:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	gcloud docker -- push gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION)
+
+pull-docker-functional-tester:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info ETCD_VERSION: $(ETCD_VERSION))
+	docker pull gcr.io/etcd-development/etcd-functional-tester:go$(GO_VERSION)

hack/scripts-dev/docker-dns-srv/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.10
+FROM ubuntu:17.10
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

hack/scripts-dev/docker-dns/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.10
+FROM ubuntu:17.10
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

hack/scripts-dev/docker-dns/certs-common-name-auth/Procfile

@@ -0,0 +1,6 @@
+# Use goreman to run `go get github.com/mattn/goreman`
+etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls https://127.0.0.1:2379 --advertise-client-urls https://m1.etcd.local:2379 --listen-peer-urls https://127.0.0.1:2380 --initial-advertise-peer-urls=https://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-auth/server.crt --peer-key-file=/certs-common-name-auth/server.key.insecure --peer-trusted-ca-file=/certs-common-name-auth/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name-auth/server.crt --key-file=/certs-common-name-auth/server.key.insecure --trusted-ca-file=/certs-common-name-auth/ca.crt --client-cert-auth
+
+etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls https://127.0.0.1:22379 --advertise-client-urls https://m2.etcd.local:22379 --listen-peer-urls https://127.0.0.1:22380 --initial-advertise-peer-urls=https://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-auth/server.crt --peer-key-file=/certs-common-name-auth/server.key.insecure --peer-trusted-ca-file=/certs-common-name-auth/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name-auth/server.crt --key-file=/certs-common-name-auth/server.key.insecure --trusted-ca-file=/certs-common-name-auth/ca.crt --client-cert-auth
+
+etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls https://127.0.0.1:32379 --advertise-client-urls https://m3.etcd.local:32379 --listen-peer-urls https://127.0.0.1:32380 --initial-advertise-peer-urls=https://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-auth/server.crt --peer-key-file=/certs-common-name-auth/server.key.insecure --peer-trusted-ca-file=/certs-common-name-auth/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name-auth/server.crt --key-file=/certs-common-name-auth/server.key.insecure --trusted-ca-file=/certs-common-name-auth/ca.crt --client-cert-auth

hack/scripts-dev/docker-dns/certs-common-name-auth/run.sh

@@ -6,65 +6,65 @@ rm -rf /tmp/m1.data /tmp/m2.data /tmp/m3.data
 # get rid of hosts so go lookup won't resolve 127.0.0.1 to localhost
 cat /dev/null >/etc/hosts
 
-goreman -f /certs-common-name/Procfile start &
+goreman -f /certs-common-name-auth/Procfile start &
 
 # TODO: remove random sleeps
 sleep 7s
 
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379 \
   endpoint health --cluster
 
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   put abc def
 
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   get abc
 
 sleep 1s && printf "\n"
 echo "Step 1. creating root role"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   role add root
 
 sleep 1s && printf "\n"
 echo "Step 2. granting readwrite 'foo' permission to role 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   role grant-permission root readwrite foo
 
 sleep 1s && printf "\n"
 echo "Step 3. getting role 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   role get root
 
 sleep 1s && printf "\n"
 echo "Step 4. creating user 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --interactive=false \
   user add root:123
@@ -72,36 +72,36 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 5. granting role 'root' to user 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   user grant-role root root
 
 sleep 1s && printf "\n"
 echo "Step 6. getting user 'root'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   user get root
 
 sleep 1s && printf "\n"
 echo "Step 7. enabling auth"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   auth enable
 
 sleep 1s && printf "\n"
 echo "Step 8. writing 'foo' with 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   put foo bar
@@ -109,9 +109,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 9. writing 'aaa' with 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   put aaa bbb
@@ -119,18 +119,18 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 10. writing 'foo' without 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   put foo bar
 
 sleep 1s && printf "\n"
 echo "Step 11. reading 'foo' with 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   get foo
@@ -138,9 +138,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 12. reading 'aaa' with 'root:123'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   get aaa
@@ -148,9 +148,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 13. creating a new user 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   --interactive=false \
@@ -159,9 +159,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 14. creating a role 'test-role'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   role add test-role
@@ -169,9 +169,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 15. granting readwrite 'aaa' --prefix permission to role 'test-role'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   role grant-permission test-role readwrite aaa --prefix
@@ -179,9 +179,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 16. getting role 'test-role'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   role get test-role
@@ -189,9 +189,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 17. granting role 'test-role' to user 'test-common-name'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=root:123 \
   user grant-role test-common-name test-role
@@ -199,9 +199,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 18. writing 'aaa' with 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=test-common-name:test-pass \
   put aaa bbb
@@ -209,9 +209,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 19. writing 'bbb' with 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=test-common-name:test-pass \
   put bbb bbb
@@ -219,9 +219,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 20. reading 'aaa' with 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=test-common-name:test-pass \
   get aaa
@@ -229,9 +229,9 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 21. reading 'bbb' with 'test-common-name:test-pass'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   --user=test-common-name:test-pass \
   get bbb
@@ -239,17 +239,17 @@ ETCDCTL_API=3 ./etcdctl \
 sleep 1s && printf "\n"
 echo "Step 22. writing 'aaa' with CommonName 'test-common-name'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   put aaa ccc
 
 sleep 1s && printf "\n"
 echo "Step 23. reading 'aaa' with CommonName 'test-common-name'"
 ETCDCTL_API=3 ./etcdctl \
-  --cacert=/certs-common-name/ca.crt \
-  --cert=/certs-common-name/server.crt \
-  --key=/certs-common-name/server.key.insecure \
+  --cacert=/certs-common-name-auth/ca.crt \
+  --cert=/certs-common-name-auth/server.crt \
+  --key=/certs-common-name-auth/server.key.insecure \
   --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
   get aaa

hack/scripts-dev/docker-dns/certs-common-name-multi/Procfile

@@ -0,0 +1,6 @@
+# Use goreman to run `go get github.com/mattn/goreman`
+etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls https://127.0.0.1:2379 --advertise-client-urls https://m1.etcd.local:2379 --listen-peer-urls https://127.0.0.1:2380 --initial-advertise-peer-urls=https://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-multi/server-1.crt --peer-key-file=/certs-common-name-multi/server-1.key.insecure --peer-trusted-ca-file=/certs-common-name-multi/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-common-name-multi/server-1.crt --key-file=/certs-common-name-multi/server-1.key.insecure --trusted-ca-file=/certs-common-name-multi/ca.crt --client-cert-auth
+
+etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls https://127.0.0.1:22379 --advertise-client-urls https://m2.etcd.local:22379 --listen-peer-urls https://127.0.0.1:22380 --initial-advertise-peer-urls=https://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-multi/server-2.crt --peer-key-file=/certs-common-name-multi/server-2.key.insecure --peer-trusted-ca-file=/certs-common-name-multi/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-common-name-multi/server-2.crt --key-file=/certs-common-name-multi/server-2.key.insecure --trusted-ca-file=/certs-common-name-multi/ca.crt --client-cert-auth
+
+etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls https://127.0.0.1:32379 --advertise-client-urls https://m3.etcd.local:32379 --listen-peer-urls https://127.0.0.1:32380 --initial-advertise-peer-urls=https://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name-multi/server-3.crt --peer-key-file=/certs-common-name-multi/server-3.key.insecure --peer-trusted-ca-file=/certs-common-name-multi/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-common-name-multi/server-3.crt --key-file=/certs-common-name-multi/server-3.key.insecure --trusted-ca-file=/certs-common-name-multi/ca.crt --client-cert-auth

hack/scripts-dev/docker-dns/certs-common-name-multi/ca-csr.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "ca",
+  "ca": {
+    "expiry": "87600h"
+  }
+}

hack/scripts-dev/docker-dns/certs-common-name-multi/ca.crt

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID0jCCArqgAwIBAgIUd3UZnVmZFo8x9MWWhUrYQvZHLrQwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMjAwNjAwMDBaFw0yODAxMTgwNjAw
+MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCqgFTgSFl+ugXkZuiN5PXp84Zv05crwI5x2ePMnc2/3u1s7cQBvXQGCJcq
+OwWD7tjcy4K2PDC0DLRa4Mkd8JpwADmf6ojbMH/3a1pXY2B3BJQwmNPFnxRJbDZL
+Iti6syWKwyfLVb1KFCU08G+ZrWmGIXPWDiE+rTn/ArD/6WbQI1LYBFJm25NLpttM
+mA3HnWoErNGY4Z/AR54ROdQSPL7RSUZBa0Kn1riXeOJ40/05qosR2O/hBSAGkD+m
+5Rj+A6oek44zZqVzCSEncLsRJAKqgZIqsBrErAho72irEgTwv4OM0MyOCsY/9erf
+hNYRSoQeX+zUvEvgToalfWGt6kT3AgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIBBjAS
+BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBRDePNja5CK4zUfO5x1vzGvdmUF
+CzAfBgNVHSMEGDAWgBRDePNja5CK4zUfO5x1vzGvdmUFCzANBgkqhkiG9w0BAQsF
+AAOCAQEAZu0a3B7Ef/z5Ct99xgzPy4z9RwglqPuxk446hBWR5TYT9fzm+voHCAwb
+MJEaQK3hvAz47qAjyR9/b+nBw4LRTMxg0WqB+UEEVwBGJxtfcOHx4mJHc3lgVJnR
+LiEWtIND7lu5Ql0eOjSehQzkJZhUb4SnXD7yk64zukQQv9zlZYZCHPDAQ9LzR2vI
+ii4yhwdWl7iiZ0lOyR4xqPB3Cx/2kjtuRiSkbpHGwWBJLng2ZqgO4K+gL3naNgqN
+TRtdOSK3j/E5WtAeFUUT68Gjsg7yXxqyjUFq+piunFfQHhPB+6sPPy56OtIogOk4
+dFCfFAygYNrFKz366KY+7CbpB+4WKA==
+-----END CERTIFICATE-----

hack/scripts-dev/docker-dns/certs-common-name-multi/gencert.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "87600h"
+    }
+  }
+}

hack/scripts-dev/docker-dns/certs-common-name-multi/gencerts.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if ! [[ "$0" =~ "./gencerts.sh" ]]; then
+	echo "must be run from 'fixtures'"
+	exit 255
+fi
+
+if ! which cfssl; then
+	echo "cfssl is not installed"
+	exit 255
+fi
+
+cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca
+mv ca.pem ca.crt
+openssl x509 -in ca.crt -noout -text
+
+# generate wildcard certificates DNS: m1/m2/m3.etcd.local
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-1.json | cfssljson --bare ./server-1
+mv server-1.pem server-1.crt
+mv server-1-key.pem server-1.key.insecure
+
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-2.json | cfssljson --bare ./server-2
+mv server-2.pem server-2.crt
+mv server-2-key.pem server-2.key.insecure
+
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-3.json | cfssljson --bare ./server-3
+mv server-3.pem server-3.crt
+mv server-3-key.pem server-3.key.insecure
+
+rm -f *.csr *.pem *.stderr *.txt

hack/scripts-dev/docker-dns/certs-common-name-multi/run.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+rm -rf /tmp/m1.data /tmp/m2.data /tmp/m3.data
+
+/etc/init.d/bind9 start
+
+# get rid of hosts so go lookup won't resolve 127.0.0.1 to localhost
+cat /dev/null >/etc/hosts
+
+goreman -f /certs-common-name-multi/Procfile start &
+
+# TODO: remove random sleeps
+sleep 7s
+
+ETCDCTL_API=3 ./etcdctl \
+  --cacert=/certs-common-name-multi/ca.crt \
+  --cert=/certs-common-name-multi/server-1.crt \
+  --key=/certs-common-name-multi/server-1.key.insecure \
+  --endpoints=https://m1.etcd.local:2379 \
+  endpoint health --cluster
+
+ETCDCTL_API=3 ./etcdctl \
+  --cacert=/certs-common-name-multi/ca.crt \
+  --cert=/certs-common-name-multi/server-2.crt \
+  --key=/certs-common-name-multi/server-2.key.insecure \
+  --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
+  put abc def
+
+ETCDCTL_API=3 ./etcdctl \
+  --cacert=/certs-common-name-multi/ca.crt \
+  --cert=/certs-common-name-multi/server-3.crt \
+  --key=/certs-common-name-multi/server-3.key.insecure \
+  --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
+  get abc

hack/scripts-dev/docker-dns/certs-common-name-multi/server-1.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEIDCCAwigAwIBAgIUaDLXBmJpHrElwENdnVk9hvAvlKcwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMjAwNjAwMDBaFw0yODAxMTgwNjAw
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAOb5CdovL9QCdgsxnCBikTbJko6r5mrF+eA47gDLcVbWrRW5
+d8eZYV1Fyn5qe80O6LB6LKPrRftxyAGABKqIBCHR57E97UsICC4lGycBWaav6cJ+
+7Spkpf8cSSDjjgb4KC6VVPf9MCsHxBYSTfme8JEFE+6KjlG8Mqt2yv/5aIyRYITN
+WzXvV7wxS9aOgDdXLbojW9FJQCuzttOPfvINTyhtvUvCM8S61La5ymCdAdPpx1U9
+m5KC23k6ZbkAC8/jcOV+68adTUuMWLefPf9Ww3qMT8382k86gJgQjZuJDGUl3Xi5
+GXmO0GfrMh+v91yiaiqjsJCDp3uVcUSeH7qSkb0CAwEAAaOBqzCBqDAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFEwLLCuIHilzynJ7DlTrikyhy2TAMB8GA1UdIwQYMBaA
+FEN482NrkIrjNR87nHW/Ma92ZQULMCkGA1UdEQQiMCCCDW0xLmV0Y2QubG9jYWyC
+CWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAkERnrIIvkZHWsyih
+mFNf/JmFHC+0/UAG9Ti9msRlr9j1fh+vBIid3FAIShX0zFXf+AtN/+Bz5SVvQHUT
+tm71AK/vER1Ue059SIty+Uz5mNAjwtXy0WaUgSuF4uju7MkYD5yUnSGv1iBfm88a
+q+q1Vd5m6PkOCfuyNQQm5RKUiJiO4OS+2F9/JOpyr0qqdQthOWr266CqXuvVhd+Z
+oZZn5TLq5GHCaTxfngSqS3TXl55QEGl65SUgYdGqpIfaQt3QKq2dqVg/syLPkTJt
+GNJVLxJuUIu0PLrfuWynUm+1mOOfwXd8NZVZITUxC7Tl5ecFbTaOzU/4a7Cyssny
+Wr3dUg==
+-----END CERTIFICATE-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-1.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA5vkJ2i8v1AJ2CzGcIGKRNsmSjqvmasX54DjuAMtxVtatFbl3
+x5lhXUXKfmp7zQ7osHoso+tF+3HIAYAEqogEIdHnsT3tSwgILiUbJwFZpq/pwn7t
+KmSl/xxJIOOOBvgoLpVU9/0wKwfEFhJN+Z7wkQUT7oqOUbwyq3bK//lojJFghM1b
+Ne9XvDFL1o6AN1ctuiNb0UlAK7O2049+8g1PKG29S8IzxLrUtrnKYJ0B0+nHVT2b
+koLbeTpluQALz+Nw5X7rxp1NS4xYt589/1bDeoxPzfzaTzqAmBCNm4kMZSXdeLkZ
+eY7QZ+syH6/3XKJqKqOwkIOne5VxRJ4fupKRvQIDAQABAoIBAQCYQsXm6kJqTbEJ
+kgutIa0+48TUfqen7Zja4kyrg3HU4DI75wb6MreHqFFj4sh4FoL4i6HP8XIx3wEN
+VBo/XOj0bo6BPiSm2MWjvdxXa0Fxa/f6uneYAb+YHEps/vWKzJ6YjuLzlBnj0/vE
+3Q5AJzHJOAK6tuY5JYp1lBsggYcVWiQSW6wGQRReU/B/GdFgglL1chqL33Dt11Uv
+Y6+oJz/PyqzPLPHcPbhqyQRMOZXnhx+8/+ooq5IojqOHfpa9JQURcHY7isBnpI/G
+ZAa8tZctgTqtL4hB1rxDhdq1fS2YC12lxkBZse4jszcm0tYzy2gWmNTH480uo/0J
+GOxX7eP1AoGBAO7O+aLhQWrspWQ//8YFbPWNhyscQub+t6WYjc0wn9j0dz8vkhMw
+rh5O8uMcZBMDQdq185BcB3aHInw9COWZEcWNIen4ZyNJa5VCN4FY0a2GtFSSGG3f
+ilKmQ7cjB950q2jl1AR3t2H7yah+i1ZChzPx+GEe+51LcJZX8mMjGvwjAoGBAPeZ
+qJ2W4O2dOyupAfnKpZZclrEBqlyg7Xj85u20eBMUqtaIEcI/u2kaotQPeuaekUH0
+b1ybr3sJBTp3qzHUaNV3iMfgrnbWEOkIV2TCReWQb1Fk93o3gilMIkhGLIhxwWpM
+UpQy3JTjGG/Y6gIOs7YnOBGVMA0o+RvouwooU6ifAoGAH6D6H0CGUYsWPLjdP3To
+gX1FMciEc+O4nw4dede+1BVM1emPB0ujRBBgywOvnXUI+9atc6k8s84iGyJaU056
+tBeFLl/gCSRoQ1SJ1W/WFY2JxMm0wpig0WGEBnV1TVlWeoY2FoFkoG2gv9hCzCHz
+lkWuB+76lFKxjrgHOmoj4NECgYB+COmbzkGQsoh8IPuwe0bu0xKh54cgv4oiHBow
+xbyZedu8eGcRyf9L8RMRfw/AdNbcC+Dj8xvQNTdEG8Y5BzaV8tLda7FjLHRPKr/R
+ulJ6GJuRgyO2Qqsu+mI5B/+DNOSPh2pBpeJCp5a42GHFylYQUsZnrNlY2ZJ0cnND
+KGPtYQKBgQDL30+BB95FtRUvFoJIWwASCp7TIqW7N7RGWgqmsXU0EZ0Mya4dquqG
+rJ1QuXQIJ+xV060ehwJR+iDUAY2xUg3/LCoDD0rwBzSdh+NEKjOmRNFRtn7WT03Q
+264E80r6VTRSN4sWQwAAbd1VF1uGO5tkzZdJGWGhQhvTUZ498dE+9Q==
+-----END RSA PRIVATE KEY-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-2.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEIDCCAwigAwIBAgIUHXDUS+Vry/Tquc6S6OoaeuGozrEwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMjAwNjAwMDBaFw0yODAxMTgwNjAw
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAOO+FsO+6pwpv+5K+VQTYQb0lT0BjnM7Y2qSZIiTGCDp/M0P
+yHSed4oTzxBeA9hEytczH/oddAUuSZNgag5sGFVgjFNdiZli4wQqJaMQRodivuUl
+ZscqnWwtP3GYVAfg+t/4YdGB+dQRDQvHBl9BRYmUh2ixOA98OXKfNMr+u+3sh5Gy
+dwx5ZEBRvgBcRrgCaIMsvVeIzHQBMHrNySAD1bGgm3xGdLeVPhAp24yUKZ5IbN6/
++5hyCRARtGwLH/1Q/h10Sr5jxQi00eEXH+CNOvcerH6b2II/BxHIcqKd0u36pUfG
+0KsY+ia0fvYi510V6Q0FAn45luEjHEk5ITN/LnMCAwEAAaOBqzCBqDAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFE69SZun6mXZe6cd3Cb2HWrK281MMB8GA1UdIwQYMBaA
+FEN482NrkIrjNR87nHW/Ma92ZQULMCkGA1UdEQQiMCCCDW0yLmV0Y2QubG9jYWyC
+CWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAI5nHHULV7eUJMsvv
+zk1shv826kOwXbMX10iRaf49/r7TWBq0pbPapvf5VXRsZ5wlDrDzjaNstpsaow/j
+fhZ1zpU0h1bdifxE+omFSWZjpVM8kQD/yzT34VdyA+P2HuxG8ZTa8r7wTGrooD60
+TjBBM5gFV4nGVe+KbApQ26KWr+P8biKaWe6MM/jAv6TNeXiWReHqyM5v404PZQXK
+cIN+fBb8bQfuaKaN1dkOUI3uSHmVmeYc5OGNJ2QKL9Uzm1VGbbM+1BOLhmF53QSm
+5m2B64lPKy+vpTcRLN7oW1FHZOKts+1OEaLMCyjWFKFbdcrmJI+AP2IB+V6ODECn
+RwJDtA==
+-----END CERTIFICATE-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-2.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEA474Ww77qnCm/7kr5VBNhBvSVPQGOcztjapJkiJMYIOn8zQ/I
+dJ53ihPPEF4D2ETK1zMf+h10BS5Jk2BqDmwYVWCMU12JmWLjBColoxBGh2K+5SVm
+xyqdbC0/cZhUB+D63/hh0YH51BENC8cGX0FFiZSHaLE4D3w5cp80yv677eyHkbJ3
+DHlkQFG+AFxGuAJogyy9V4jMdAEwes3JIAPVsaCbfEZ0t5U+ECnbjJQpnkhs3r/7
+mHIJEBG0bAsf/VD+HXRKvmPFCLTR4Rcf4I069x6sfpvYgj8HEchyop3S7fqlR8bQ
+qxj6JrR+9iLnXRXpDQUCfjmW4SMcSTkhM38ucwIDAQABAoIBAQCHYF6N2zYAwDyL
+/Ns65A4gIVF5Iyy3SM0u83h5St7j6dNRXhltYSlz1ZSXiRtF+paM16IhflKSJdKs
+nXpNumm4jpy7jXWWzRZfSmJ3DNyv673H3rS6nZVYUYlOEBubV1wpuK8E5/tG2R/l
+KVibVORuBPF9BSNq6RAJF6Q9KrExmvH4MmG/3Y+iYbZgn0OK1WHxzbeMzdI8OO4z
+eg4gTKuMoRFt5B4rZmC5QiXGHdnUXRWfy+yPLTH3hfTek4JT98akFNS01Q4UAi9p
+5cC3TOqDNiZdAkN83UKhW9TNAc/vJlq6d5oXW5R+yPt+d8yMvEch4KfpYo33j0oz
+qB40pdJRAoGBAP8ZXnWXxhzLhZ4o+aKefnsUUJjaiVhhSRH/kGAAg65lc4IEnt+N
+nzyNIwz/2vPv2Gq2BpStrTsTNKVSZCKgZhoBTavP60FaszDSM0bKHTWHW7zaQwc0
+bQG6YvvCiP0iwEzXw7S4BhdAl+x/5C30dUZgKMSDFzuBI187h6dQQNZpAoGBAOSL
+/MBuRYBgrHIL9V1v9JGDBeawGc3j2D5c56TeDtGGv8WGeCuE/y9tn+LcKQ+bCGyi
+qkW+hobro/iaXODwUZqSKaAVbxC7uBLBTRB716weMzrnD8zSTOiMWg/gh+FOnr/4
+ZfcBco2Pmm5qQ3ZKwVk2jsfLhz6ZKwMrjSaO1Zp7AoGBAJZsajPjRHI0XN0vgkyv
+Mxv2lbQcoYKZE1JmpcbGZt/OePdBLEHcq/ozq2h98qmHU9FQ9r5zT0QXhiK6W8vD
+U5GgFSHsH+hQyHtQZ+YlRmYLJEBPX9j+xAyR0M5uHwNNm6F0VbXaEdViRHOz0mR6
+0zClgUSnnGp9MtN0MgCqJSGJAoGAJYba3Jn+rYKyLhPKmSoN5Wq3KFbYFdeIpUzJ
++GdB1aOjj4Jx7utqn1YHv89YqqhRLM1U2hjbrAG7LdHi2Eh9jbzcOt3qG7xHEEVP
+Kxq6ohdfYBean44UdMa+7wZ2KUeoh2r5CyLgtV/UArdOFnlV4Bk2PpYrwdqSlnWr
+Op6PcksCgYEA6HmIHLRTGyOUzS82BEcs5an2mzhQ8XCNdYS6sDaYSiDu2qlPukyZ
+jons6P4qpOxlP9Cr6DW7px2fUZrEuPUV8fRJOc+a5AtZ5TmV6N1uH/G1rKmmAMCc
+jGAmTJW87QguauTpuUto5u6IhyO2CRsYEy8K1A/1HUQKl721faZBIMA=
+-----END RSA PRIVATE KEY-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-3.crt

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEIDCCAwigAwIBAgIURfpNMXGb1/oZVwEWyc0Ofn7IItQwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMjAwNjAwMDBaFw0yODAxMTgwNjAw
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBALgCDkDM4qayF6CFt1ZScKR8B+/7qrn1iQ/qYnzRHQ1hlkuS
+b3TkQtt7amGAuoD42d8jLYYvHn2Pbmdhn0mtgYZpFfLFCg4O67ZbX54lBHi+yDEh
+QhneM9Ovsc42A0EVvabINYtKR6B2YRN00QRXS5R1t+QmclpshFgY0+ITsxlJeygs
+wojXthPEfjTQK04JUi5LTHP15rLVzDEd7MguCWdEWRnOu/mSfPHlyz2noUcKuy0M
+awsnSMwf+KBwQMLbJhTXtA4MG2FYsm/2en3/oAc8/0Z8sMOX05F+b0MgHl+a31aQ
+UHM5ykfDNm3hGQfzjQCx4y4hjDoFxbuXvsey6GMCAwEAAaOBqzCBqDAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFDMydqyg/s43/dJTMt25zJubI/CUMB8GA1UdIwQYMBaA
+FEN482NrkIrjNR87nHW/Ma92ZQULMCkGA1UdEQQiMCCCDW0zLmV0Y2QubG9jYWyC
+CWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEAVs3VQjgx9CycaWKS
+P6EvMtlqOkanJEe3zr69sI66cc2ZhfJ5xK38ox4oYpMOA131WRvwq0hjKhhZoVQ8
+aQ4yALi1XBltuIyEyrTX9GWAMeDzY95MdWKhyI8ps6/OOoXN596g9ZdOdIbZAMT4
+XAXm43WccM2W2jiKCEKcE4afIF8RiMIaFwG8YU8oHtnnNvxTVa0wrpcObtEtIzC5
+RJxzX9bkHCTHTgJog4OPChU4zffn18U/AVJ7MZ8gweVwhc4gGe0kwOJE+mLHcC5G
+uoFSuVmAhYrH/OPpZhSDOaCED4dsF5jN25CbR3NufEBFRXBH20ZHNkNvbbBnYCBU
+4+Rx5w==
+-----END CERTIFICATE-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-3.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAuAIOQMziprIXoIW3VlJwpHwH7/uqufWJD+pifNEdDWGWS5Jv
+dORC23tqYYC6gPjZ3yMthi8efY9uZ2GfSa2BhmkV8sUKDg7rtltfniUEeL7IMSFC
+Gd4z06+xzjYDQRW9psg1i0pHoHZhE3TRBFdLlHW35CZyWmyEWBjT4hOzGUl7KCzC
+iNe2E8R+NNArTglSLktMc/XmstXMMR3syC4JZ0RZGc67+ZJ88eXLPaehRwq7LQxr
+CydIzB/4oHBAwtsmFNe0DgwbYViyb/Z6ff+gBzz/Rnyww5fTkX5vQyAeX5rfVpBQ
+cznKR8M2beEZB/ONALHjLiGMOgXFu5e+x7LoYwIDAQABAoIBAQCY54RmjprNAHKn
+vlXCEpFt7W8/GXcePg2ePxuGMtKcevpEZDPgA4oXDnAxA6J3Z9LMHFRJC8Cff9+z
+YqjVtatLQOmvKdMYKYfvqfBD3ujfWVHLmaJvEnkor/flrnZ30BQfkoED9T6d9aDn
+ZQwHOm8gt82OdfBSeZhkCIWReOM73622qJhmLWUUY3xEucRAFF6XffOLvJAT87Vu
+pXKtCnQxhzxkUsCYNIOeH/pTX+XoLkysFBKxnrlbTeM0cEgWpYMICt/vsUrp6DHs
+jygxR1EnT2/4ufe81aFSO4SzUZKJrz8zj4yIyDOR0Mp6FW+xMp8S0fDOywHhLlXn
+xQOevmGBAoGBAOMQaWWs2FcxWvLfX95RyWPtkQ+XvmWlL5FR427TlLhtU6EPs0xZ
+eeanMtQqSRHlDkatwc0XQk+s30/UJ+5i1iz3shLwtnZort/pbnyWrxkE9pcR0fgr
+IklujJ8e8kQHpY75gOLmEiADrUITqvfbvSMsaG3h1VydPNU3JYTUuYmjAoGBAM91
+Atnri0PH3UKonAcMPSdwQ5NexqAD1JUk6KUoX2poXBXO3zXBFLgbMeJaWthbe+dG
+Raw/zjBET/oRfDOssh+QTD8TutI9LA2+EN7TG7Kr6NFciz4Q2pioaimv9KUhJx+8
+HH2wCANYgkv69IWUFskF0uDCW9FQVvpepcctCJJBAoGAMlWxB5kJXErUnoJl/iKj
+QkOnpI0+58l2ggBlKmw8y6VwpIOWe5ZaL4dg/Sdii1T7lS9vhsdhK8hmuIuPToka
+cV13XDuANz99hKV6mKPOrP0srNCGez0UnLKk+aEik3IegVNN/v6BhhdKkRtLCybr
+BqERhUpKwf0ZPyq6ZnfBqYECgYEAsiD2YcctvPVPtnyv/B02JTbvzwoB4kNntOgM
+GkOgKe2Ro+gNIEq5T5uKKaELf9qNePeNu2jN0gPV6BI7YuNVzmRIE6ENOJfty573
+PVxm2/Nf5ORhatlt2MZC4aiDl4Xv4f/TNth/COBmgHbqngeZyOGHQBWiYQdqp2+9
+SFgSlAECgYEA1zLhxj6f+psM5Gpx56JJIEraHfyuyR1Oxii5mo7I3PLsbF/s6YDR
+q9E64GoR5PdgCQlMm09f6wfT61NVwsYrbLlLET6tAiG0eNxXe71k1hUb6aa4DpNQ
+IcS3E3hb5KREXUH5d+PKeD2qrf52mtakjn9b2aH2rQw2e2YNkIDV+XA=
+-----END RSA PRIVATE KEY-----

hack/scripts-dev/docker-dns/certs-common-name-multi/server-ca-csr-1.json

@@ -0,0 +1,21 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "etcd.local",
+  "hosts": [
+    "m1.etcd.local",
+    "127.0.0.1",
+    "localhost"
+  ]
+}

hack/scripts-dev/docker-dns/certs-common-name-multi/server-ca-csr-2.json

@@ -0,0 +1,21 @@
+{
+    "key": {
+      "algo": "rsa",
+      "size": 2048
+    },
+    "names": [
+      {
+        "O": "etcd",
+        "OU": "etcd Security",
+        "L": "San Francisco",
+        "ST": "California",
+        "C": "USA"
+      }
+    ],
+    "CN": "etcd.local",
+    "hosts": [
+      "m2.etcd.local",
+      "127.0.0.1",
+      "localhost"
+    ]
+  }

hack/scripts-dev/docker-dns/certs-common-name-multi/server-ca-csr-3.json

@@ -0,0 +1,21 @@
+{
+    "key": {
+      "algo": "rsa",
+      "size": 2048
+    },
+    "names": [
+      {
+        "O": "etcd",
+        "OU": "etcd Security",
+        "L": "San Francisco",
+        "ST": "California",
+        "C": "USA"
+      }
+    ],
+    "CN": "etcd.local",
+    "hosts": [
+      "m3.etcd.local",
+      "127.0.0.1",
+      "localhost"
+    ]
+  }

hack/scripts-dev/docker-dns/certs-common-name/Procfile

@@ -1,6 +0,0 @@
-# Use goreman to run `go get github.com/mattn/goreman`
-etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls https://127.0.0.1:2379 --advertise-client-urls https://m1.etcd.local:2379 --listen-peer-urls https://127.0.0.1:2380 --initial-advertise-peer-urls=https://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name/server.crt --peer-key-file=/certs-common-name/server.key.insecure --peer-trusted-ca-file=/certs-common-name/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name/server.crt --key-file=/certs-common-name/server.key.insecure --trusted-ca-file=/certs-common-name/ca.crt --client-cert-auth
-
-etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls https://127.0.0.1:22379 --advertise-client-urls https://m2.etcd.local:22379 --listen-peer-urls https://127.0.0.1:22380 --initial-advertise-peer-urls=https://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name/server.crt --peer-key-file=/certs-common-name/server.key.insecure --peer-trusted-ca-file=/certs-common-name/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name/server.crt --key-file=/certs-common-name/server.key.insecure --trusted-ca-file=/certs-common-name/ca.crt --client-cert-auth
-
-etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls https://127.0.0.1:32379 --advertise-client-urls https://m3.etcd.local:32379 --listen-peer-urls https://127.0.0.1:32380 --initial-advertise-peer-urls=https://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-common-name/server.crt --peer-key-file=/certs-common-name/server.key.insecure --peer-trusted-ca-file=/certs-common-name/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn test-common-name --cert-file=/certs-common-name/server.crt --key-file=/certs-common-name/server.key.insecure --trusted-ca-file=/certs-common-name/ca.crt --client-cert-auth

hack/scripts-dev/docker-static-ip/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.10
+FROM ubuntu:17.10
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

mvcc/watchable_store.go

@@ -192,7 +192,7 @@ func (s *watchableStore) Restore(b backend.Backend) error {
 	}
 
 	for wa := range s.synced.watchers {
-		s.unsynced.watchers.add(wa)
+		s.unsynced.add(wa)
 	}
 	s.synced = newWatcherGroup()
 	return nil

mvcc/watchable_store_test.go

@@ -297,36 +297,45 @@ func TestWatchFutureRev(t *testing.T) {
 }
 
 func TestWatchRestore(t *testing.T) {
-	b, tmpPath := backend.NewDefaultTmpBackend()
-	s := newWatchableStore(b, &lease.FakeLessor{}, nil)
-	defer cleanup(s, b, tmpPath)
+	test := func(delay time.Duration) func(t *testing.T) {
+		return func(t *testing.T) {
+			b, tmpPath := backend.NewDefaultTmpBackend()
+			s := newWatchableStore(b, &lease.FakeLessor{}, nil)
+			defer cleanup(s, b, tmpPath)
 
-	testKey := []byte("foo")
-	testValue := []byte("bar")
-	rev := s.Put(testKey, testValue, lease.NoLease)
+			testKey := []byte("foo")
+			testValue := []byte("bar")
+			rev := s.Put(testKey, testValue, lease.NoLease)
 
-	newBackend, newPath := backend.NewDefaultTmpBackend()
-	newStore := newWatchableStore(newBackend, &lease.FakeLessor{}, nil)
-	defer cleanup(newStore, newBackend, newPath)
+			newBackend, newPath := backend.NewDefaultTmpBackend()
+			newStore := newWatchableStore(newBackend, &lease.FakeLessor{}, nil)
+			defer cleanup(newStore, newBackend, newPath)
 
-	w := newStore.NewWatchStream()
-	w.Watch(testKey, nil, rev-1)
+			w := newStore.NewWatchStream()
+			w.Watch(testKey, nil, rev-1)
 
-	newStore.Restore(b)
-	select {
-	case resp := <-w.Chan():
-		if resp.Revision != rev {
-			t.Fatalf("rev = %d, want %d", resp.Revision, rev)
+			time.Sleep(delay)
+
+			newStore.Restore(b)
+			select {
+			case resp := <-w.Chan():
+				if resp.Revision != rev {
+					t.Fatalf("rev = %d, want %d", resp.Revision, rev)
+				}
+				if len(resp.Events) != 1 {
+					t.Fatalf("failed to get events from the response")
+				}
+				if resp.Events[0].Kv.ModRevision != rev {
+					t.Fatalf("kv.rev = %d, want %d", resp.Events[0].Kv.ModRevision, rev)
+				}
+			case <-time.After(time.Second):
+				t.Fatal("failed to receive event in 1 second.")
+			}
 		}
-		if len(resp.Events) != 1 {
-			t.Fatalf("failed to get events from the response")
-		}
-		if resp.Events[0].Kv.ModRevision != rev {
-			t.Fatalf("kv.rev = %d, want %d", resp.Events[0].Kv.ModRevision, rev)
-		}
-	case <-time.After(time.Second):
-		t.Fatal("failed to receive event in 1 second.")
 	}
+
+	t.Run("Normal", test(0))
+	t.Run("RunSyncWatchLoopBeforeRestore", test(time.Millisecond*120)) // longer than default waitDuration
 }
 
 // TestWatchBatchUnsynced tests batching on unsynced watchers

pkg/transport/fixtures/ca-csr.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "ca",
+  "ca": {
+    "expiry": "87600h"
+  }
+}

pkg/transport/fixtures/ca.crt

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDsTCCApmgAwIBAgIUZzOo4zcHY/nEXY1PD8A7povXlWUwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMDIxNjQxMDBaFw0yNzEyMzExNjQx
+MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDD4Ys48LDWGyojj3Rcr6fnESY+UycaaGoTXADWLPmm+sQR3KcsJxF4054S
+d2G+NBfJHZvTHhVqOeqZxNtoqgje4paY2A5TbWBdV+xoGfbakwwngiX1yeF1I54k
+KH19zb8rBKAm7xixO60hE2CIYzMuw9lDkwoHpI6/PJdy7jwtytbo2Oac512JiO9Y
+dHp9dr3mrCzoKEBRtL1asRKfzp6gBC5rIw5T4jrq37feerV4pDEJX7fvexxVocVm
+tT4bmMq3Ap6OFFAzmE/ITI8pXvFaOd9lyebNXQmrreKJLUfEIZa6JulLCYxfkJ8z
++CcNLyn6ZXNMaIZ8G9Hm6VRdRi8/AgMBAAGjRTBDMA4GA1UdDwEB/wQEAwIBBjAS
+BgNVHRMBAf8ECDAGAQH/AgECMB0GA1UdDgQWBBRDLNYEX8XI7nM53k1rUR+mpTjQ
+NTANBgkqhkiG9w0BAQsFAAOCAQEACDe3Fa1KE/rvVtyCLW/IBfKV01NShFTsb6x8
+GrPEQ6NJLZQ2MzdyJgAF2a/nZ9KVgrhGXoyoZBCKP9Dd/JDzSSZcBztfNK8dRv2A
+XHBBF6tZ19I+XY9c7/CfhJ2CEYJpeN9r3GKSqV+njkmg8n/On2BTlFsij88plK8H
+ORyemc1nQI+ARPSu2r3rJbYa4yI2U6w4L4BTCVImg3bX50GImmXGlwvnJMFik1FX
++0hdfetRxxMZ1pm2Uy6099KkULnSKabZGwRiBUHQJYh0EeuAOQ4a6MG5DRkURWNs
+dInjPOLY9/7S5DQKwz/NtqXA8EEymZosHxpiRp+zzKB4XaV9Ig==
+-----END CERTIFICATE-----

pkg/transport/fixtures/gencert.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "87600h"
+    }
+  }
+}

pkg/transport/fixtures/gencerts.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if ! [[ "$0" =~ "./gencerts.sh" ]]; then
+	echo "must be run from 'fixtures'"
+	exit 255
+fi
+
+if ! which cfssl; then
+	echo "cfssl is not installed"
+	exit 255
+fi
+
+cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca
+mv ca.pem ca.crt
+openssl x509 -in ca.crt -noout -text
+
+# generate DNS: localhost, IP: 127.0.0.1, CN: example.com certificates
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr.json | cfssljson --bare ./server
+mv server.pem server.crt
+mv server-key.pem server.key.insecure
+
+rm -f *.csr *.pem *.stderr *.txt

pkg/transport/fixtures/server-ca-csr.json

@@ -0,0 +1,20 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "example.com",
+  "hosts": [
+    "127.0.0.1",
+    "localhost"
+  ]
+}

pkg/transport/fixtures/server.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEEjCCAvqgAwIBAgIUIYc+vmysep1pDc2ua/VQEeMFQVAwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xODAxMDIxNjQxMDBaFw0yNzEyMzExNjQx
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDEq7aT2BQZfmJ2xpUm8xWJlN0c3cOLVZRH9mIrEutIHmip
+BYq3ZIq3q52w+T3sMcaJNMGjCteE8Lu+G9YSmtfZMAWnkaM02KOjVMkkQcK7Z4vM
+lOUjlO+dsvhfmw3CPghqSs6M1K2CTqhuEiXdOBofuEMmwKNRgkV/jT92PUs0h8kq
+loc/I3/H+hx/ZJ1i0S0xkZKpaImc0oZ9ZDo07biMrsUIzjwbN69mEs+CtVkah4sy
+k6UyRoU2k21lyRTK0LxNjWc9ylzDNUuf6DwduU7lPZsqTaJrFNAAPpOlI4k2EcjL
+3zD8amKkJGDm+PQz97PbTA381ec4ZAtB8volxCebAgMBAAGjgZwwgZkwDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBTTZQnMn5tuUgVE+8c9W0hmbghGoDAfBgNVHSMEGDAW
+gBRDLNYEX8XI7nM53k1rUR+mpTjQNTAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAKUQVj0YDuxg4tinlOZhp4ge7tCA+gL7vV+Q
+iDrkWfOlGjDgwYqWMYDXMHWKIW9ea8LzyI/bVEcaHlnBmNOYuS7g47EWNiU7WUA5
+iTkm3CKA5zHFFPcXHW0GQeCQrX9y3SepKS3cP8TAyZFfC/FvV24Kn1oQhJbEe0ZV
+In/vPHssW7jlVe0FGVUn7FutRQgiA1pTAtS6AP4LeZ9O41DTWkPqV4nBgcxlvkgD
+KjEoXXSb5C0LoR5zwAo9zB3RtmqnmvkHAOv3G92YctdS2VbCmd8CNLj9H7gMmQiH
+ThsStVOhb2uo6Ni4PgzUIYKGTd4ZjUXCYxFKck//ajDyCHlL8v4=
+-----END CERTIFICATE-----

pkg/transport/fixtures/server.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAxKu2k9gUGX5idsaVJvMViZTdHN3Di1WUR/ZiKxLrSB5oqQWK
+t2SKt6udsPk97DHGiTTBowrXhPC7vhvWEprX2TAFp5GjNNijo1TJJEHCu2eLzJTl
+I5TvnbL4X5sNwj4IakrOjNStgk6obhIl3TgaH7hDJsCjUYJFf40/dj1LNIfJKpaH
+PyN/x/ocf2SdYtEtMZGSqWiJnNKGfWQ6NO24jK7FCM48GzevZhLPgrVZGoeLMpOl
+MkaFNpNtZckUytC8TY1nPcpcwzVLn+g8HblO5T2bKk2iaxTQAD6TpSOJNhHIy98w
+/GpipCRg5vj0M/ez20wN/NXnOGQLQfL6JcQnmwIDAQABAoIBAGTx1eaQk9B6BEP+
+rXOudTGGzO8SDFop9M/y8HQ3Y7hCk2mdxJNY8bJQTcIWS+g9rC+kencbC3/aqCJt
+2zT1cTCy61QU9nYbc/JThGIttqvF/AVnryzSNyL0R3Oa/Dbk7CDSgK3cQ6qMgPru
+Ka0gLJh3VVBAtBMUEGPltdsUntM4sHTh5FAabP0ioBJ1QLG6Aak7LOQikjBEFJoc
+Tea4uRsE7IreP5Mn7UW92nkt1ey5UGzBtNNtpHbVaHmfQojwlwkLtnV35sumbvK6
+6KTMNREZv6xSIMwkYxm1zRE3Cus/1jGIc8MZF0BxgcCR+G37l+BKwL8CSymHPxhH
+dvGxoPECgYEA3STp52CbI/KyVfvjxK2OIex/NV1jKh85wQsLtkaRv3/a/EEg7MV7
+54dEvo5KKOZXfeOd9r9G9h1RffjSD9MhxfPhyGwuOcqa8IE1zNwlY/v7KL7HtDIf
+2mrXWF5Klafh8aXYcaRH0ZSLnl/nXUXYht4/0NRGiXnttUgqs6hvY70CgYEA46tO
+J5QkgF3YVY0gx10wRCAnnKLkAaHdtxtteXOJh79xsGXQ4LLngc+mz1hLt+TNJza+
+BZhoWwY/ZgyiTH0pebGr/U0QUMoUHlGgjgj3Aa/XFpOhtyLU+IU/PYl0BUz9dqsN
+TDtv6p/HQhfd98vUNsbACQda+YAo+oRdO5kLQjcCgYB3OAZNcXxRte5EgoY5KqN8
+UGYH2++w7qKRGqZWvtamGYRyB557Zr+0gu0hmc4LHJrASGyJcHcOCaI8Ol7snxMP
+B7qJ9SA6kapTzCS361rQ+zBct/UrhPY9JuovPq4Q3i/luVXldf4t01otqGAvnY7s
+rnZS242nYa8v0tcKgdyDNQKBgB3Z60BzQyn1pBTrkT2ysU5tbOQz03OHVrvYg80l
+4gWDi5OWdgHQU1yI7pVHPX5aKLAYlGfFaQFuW0e1Jl6jFpoXOrbWsOn25RZom4Wk
+FUcKWEhkiRKrJYOEbRtTd3vucVlq6i5xqKX51zWKTZddCXE5NBq69Sm7rSPT0Sms
+UnaXAoGAXYAE5slvjcylJpMV4lxTBmNtA9+pw1T7I379mIyqZ0OS25nmpskHU7FR
+SQDSRHw7hHuyjEHyhMoHEGLfUMIltQoi+pcrieVQelJdSuX7VInzHPAR5RppUVFl
+jOZZKlIiqs+UfCoOgsIblXuw7a/ATnAnXakutSFgHU1lN1gN02U=
+-----END RSA PRIVATE KEY-----

pkg/transport/proxy.go

@@ -0,0 +1,801 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transport
+
+import (
+	"fmt"
+	"io"
+	mrand "math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	humanize "github.com/dustin/go-humanize"
+	"google.golang.org/grpc/grpclog"
+)
+
+// Proxy defines proxy layer that simulates common network faults,
+// such as latency spikes, packet drop/corruption, etc..
+type Proxy interface {
+	// From returns proxy source address in "scheme://host:port" format.
+	From() string
+	// To returns proxy destination address in "scheme://host:port" format.
+	To() string
+
+	// Ready returns when proxy is ready to serve.
+	Ready() <-chan struct{}
+	// Done returns when proxy has been closed.
+	Done() <-chan struct{}
+	// Error sends errors while serving proxy.
+	Error() <-chan error
+	// Close closes listener and transport.
+	Close() error
+
+	// DelayAccept adds latency ± random variable to accepting new incoming connections.
+	DelayAccept(latency, rv time.Duration)
+	// UndelayAccept removes sending latencies.
+	UndelayAccept()
+	// LatencyAccept returns current latency on accepting new incoming connections.
+	LatencyAccept() time.Duration
+	// DelayTx adds latency ± random variable to "sending" layer.
+	DelayTx(latency, rv time.Duration)
+	// UndelayTx removes sending latencies.
+	UndelayTx()
+	// LatencyTx returns current send latency.
+	LatencyTx() time.Duration
+	// DelayRx adds latency ± random variable to "receiving" layer.
+	DelayRx(latency, rv time.Duration)
+	// UndelayRx removes "receiving" latencies.
+	UndelayRx()
+	// LatencyRx returns current receive latency.
+	LatencyRx() time.Duration
+
+	// PauseAccept stops accepting new connections.
+	PauseAccept()
+	// UnpauseAccept removes pause operation on accepting new connections.
+	UnpauseAccept()
+	// PauseTx stops "forwarding" packets.
+	PauseTx()
+	// UnpauseTx removes "forwarding" pause operation.
+	UnpauseTx()
+	// PauseRx stops "receiving" packets to client.
+	PauseRx()
+	// UnpauseRx removes "receiving" pause operation.
+	UnpauseRx()
+
+	// BlackholeTx drops all incoming packets before "forwarding".
+	BlackholeTx()
+	// UnblackholeTx removes blackhole operation on "sending".
+	UnblackholeTx()
+	// BlackholeRx drops all incoming packets to client.
+	BlackholeRx()
+	// UnblackholeRx removes blackhole operation on "receiving".
+	UnblackholeRx()
+
+	// CorruptTx corrupts incoming packets from the listener.
+	CorruptTx(f func(data []byte) []byte)
+	// UncorruptTx removes corrupt operation on "forwarding".
+	UncorruptTx()
+	// CorruptRx corrupts incoming packets to client.
+	CorruptRx(f func(data []byte) []byte)
+	// UncorruptRx removes corrupt operation on "receiving".
+	UncorruptRx()
+
+	// ResetListener closes and restarts listener.
+	ResetListener() error
+}
+
+type proxy struct {
+	from, to      url.URL
+	tlsInfo       TLSInfo
+	dialTimeout   time.Duration
+	bufferSize    int
+	retryInterval time.Duration
+	logger        grpclog.LoggerV2
+
+	readyc chan struct{}
+	donec  chan struct{}
+	errc   chan error
+
+	closeOnce sync.Once
+	closeWg   sync.WaitGroup
+
+	listenerMu sync.RWMutex
+	listener   net.Listener
+
+	latencyAcceptMu sync.RWMutex
+	latencyAccept   time.Duration
+	latencyTxMu     sync.RWMutex
+	latencyTx       time.Duration
+	latencyRxMu     sync.RWMutex
+	latencyRx       time.Duration
+
+	corruptTxMu sync.RWMutex
+	corruptTx   func(data []byte) []byte
+	corruptRxMu sync.RWMutex
+	corruptRx   func(data []byte) []byte
+
+	acceptMu     sync.Mutex
+	pauseAcceptc chan struct{}
+	txMu         sync.Mutex
+	pauseTxc     chan struct{}
+	blackholeTxc chan struct{}
+	rxMu         sync.Mutex
+	pauseRxc     chan struct{}
+	blackholeRxc chan struct{}
+}
+
+// ProxyConfig defines proxy configuration.
+type ProxyConfig struct {
+	From          url.URL
+	To            url.URL
+	TLSInfo       TLSInfo
+	DialTimeout   time.Duration
+	BufferSize    int
+	RetryInterval time.Duration
+	Logger        grpclog.LoggerV2
+}
+
+var (
+	defaultDialTimeout   = 3 * time.Second
+	defaultBufferSize    = 48 * 1024
+	defaultRetryInterval = 10 * time.Millisecond
+	defaultLogger        = grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 0)
+)
+
+// NewProxy returns a proxy implementation with no iptables/tc dependencies.
+// The proxy layer overhead is <1ms.
+func NewProxy(cfg ProxyConfig) Proxy {
+	p := &proxy{
+		from:          cfg.From,
+		to:            cfg.To,
+		tlsInfo:       cfg.TLSInfo,
+		dialTimeout:   cfg.DialTimeout,
+		bufferSize:    cfg.BufferSize,
+		retryInterval: cfg.RetryInterval,
+		logger:        cfg.Logger,
+
+		readyc: make(chan struct{}),
+		donec:  make(chan struct{}),
+		errc:   make(chan error, 16),
+
+		pauseAcceptc: make(chan struct{}),
+		pauseTxc:     make(chan struct{}),
+		blackholeTxc: make(chan struct{}),
+		pauseRxc:     make(chan struct{}),
+		blackholeRxc: make(chan struct{}),
+	}
+	if p.dialTimeout == 0 {
+		p.dialTimeout = defaultDialTimeout
+	}
+	if p.bufferSize == 0 {
+		p.bufferSize = defaultBufferSize
+	}
+	if p.retryInterval == 0 {
+		p.retryInterval = defaultRetryInterval
+	}
+	if p.logger == nil {
+		p.logger = defaultLogger
+	}
+	close(p.pauseAcceptc)
+	close(p.pauseTxc)
+	close(p.pauseRxc)
+
+	if strings.HasPrefix(p.from.Scheme, "http") {
+		p.from.Scheme = "tcp"
+	}
+	if strings.HasPrefix(p.to.Scheme, "http") {
+		p.to.Scheme = "tcp"
+	}
+
+	var ln net.Listener
+	var err error
+	if !p.tlsInfo.Empty() {
+		ln, err = NewListener(p.from.Host, p.from.Scheme, &p.tlsInfo)
+	} else {
+		ln, err = net.Listen(p.from.Scheme, p.from.Host)
+	}
+	if err != nil {
+		p.errc <- err
+		p.Close()
+		return p
+	}
+	p.listener = ln
+
+	p.closeWg.Add(1)
+	go p.listenAndServe()
+	p.logger.Infof("started proxying [%s -> %s]", p.From(), p.To())
+	return p
+}
+
+func (p *proxy) From() string {
+	return fmt.Sprintf("%s://%s", p.from.Scheme, p.from.Host)
+}
+
+func (p *proxy) To() string {
+	return fmt.Sprintf("%s://%s", p.to.Scheme, p.to.Host)
+}
+
+// TODO: implement packet reordering from multiple TCP connections
+// buffer packets per connection for awhile, reorder before transmit
+// - https://github.com/coreos/etcd/issues/5614
+// - https://github.com/coreos/etcd/pull/6918#issuecomment-264093034
+
+func (p *proxy) listenAndServe() {
+	defer p.closeWg.Done()
+
+	p.logger.Infof("listen %q", p.From())
+	close(p.readyc)
+
+	for {
+		p.acceptMu.Lock()
+		pausec := p.pauseAcceptc
+		p.acceptMu.Unlock()
+		select {
+		case <-pausec:
+		case <-p.donec:
+			return
+		}
+
+		p.latencyAcceptMu.RLock()
+		lat := p.latencyAccept
+		p.latencyAcceptMu.RUnlock()
+		if lat > 0 {
+			select {
+			case <-time.After(lat):
+			case <-p.donec:
+				return
+			}
+		}
+
+		p.listenerMu.RLock()
+		ln := p.listener
+		p.listenerMu.RUnlock()
+
+		in, err := ln.Accept()
+		if err != nil {
+			select {
+			case p.errc <- err:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if p.logger.V(5) {
+				p.logger.Errorf("listener accept error %q", err.Error())
+			}
+
+			if strings.HasSuffix(err.Error(), "use of closed network connection") {
+				select {
+				case <-time.After(p.retryInterval):
+				case <-p.donec:
+					return
+				}
+				if p.logger.V(5) {
+					p.logger.Errorf("listener is closed; retry listen %q", p.From())
+				}
+
+				if err = p.ResetListener(); err != nil {
+					select {
+					case p.errc <- err:
+						select {
+						case <-p.donec:
+							return
+						default:
+						}
+					case <-p.donec:
+						return
+					}
+					p.logger.Errorf("failed to reset listener %q", err.Error())
+				}
+			}
+
+			continue
+		}
+
+		var out net.Conn
+		if !p.tlsInfo.Empty() {
+			var tp *http.Transport
+			tp, err = NewTransport(p.tlsInfo, p.dialTimeout)
+			if err != nil {
+				select {
+				case p.errc <- err:
+					select {
+					case <-p.donec:
+						return
+					default:
+					}
+				case <-p.donec:
+					return
+				}
+				continue
+			}
+			out, err = tp.Dial(p.to.Scheme, p.to.Host)
+		} else {
+			out, err = net.Dial(p.to.Scheme, p.to.Host)
+		}
+		if err != nil {
+			select {
+			case p.errc <- err:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if p.logger.V(5) {
+				p.logger.Errorf("dial error %q", err.Error())
+			}
+			continue
+		}
+
+		go func() {
+			// read incoming bytes from listener, dispatch to outgoing connection
+			p.transmit(out, in)
+			out.Close()
+			in.Close()
+		}()
+		go func() {
+			// read response from outgoing connection, write back to listener
+			p.receive(in, out)
+			in.Close()
+			out.Close()
+		}()
+	}
+}
+
+func (p *proxy) transmit(dst io.Writer, src io.Reader) { p.ioCopy(dst, src, true) }
+func (p *proxy) receive(dst io.Writer, src io.Reader)  { p.ioCopy(dst, src, false) }
+func (p *proxy) ioCopy(dst io.Writer, src io.Reader, proxySend bool) {
+	buf := make([]byte, p.bufferSize)
+	for {
+		nr, err := src.Read(buf)
+		if err != nil {
+			if err == io.EOF {
+				return
+			}
+			// connection already closed
+			if strings.HasSuffix(err.Error(), "read: connection reset by peer") {
+				return
+			}
+			if strings.HasSuffix(err.Error(), "use of closed network connection") {
+				return
+			}
+			select {
+			case p.errc <- err:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if p.logger.V(5) {
+				p.logger.Errorf("read error %q", err.Error())
+			}
+			return
+		}
+		if nr == 0 {
+			return
+		}
+		data := buf[:nr]
+
+		var pausec chan struct{}
+		var blackholec chan struct{}
+		if proxySend {
+			p.txMu.Lock()
+			pausec = p.pauseTxc
+			blackholec = p.blackholeTxc
+			p.txMu.Unlock()
+		} else {
+			p.rxMu.Lock()
+			pausec = p.pauseRxc
+			blackholec = p.blackholeRxc
+			p.rxMu.Unlock()
+		}
+		select {
+		case <-pausec:
+		case <-p.donec:
+			return
+		}
+		blackholed := false
+		select {
+		case <-blackholec:
+			blackholed = true
+		case <-p.donec:
+			return
+		default:
+		}
+		if blackholed {
+			if p.logger.V(5) {
+				if proxySend {
+					p.logger.Infof("dropped %s [%s -> %s]", humanize.Bytes(uint64(nr)), p.From(), p.To())
+				} else {
+					p.logger.Infof("dropped %s [%s <- %s]", humanize.Bytes(uint64(nr)), p.From(), p.To())
+				}
+			}
+			continue
+		}
+
+		var lat time.Duration
+		if proxySend {
+			p.latencyTxMu.RLock()
+			lat = p.latencyTx
+			p.latencyTxMu.RUnlock()
+		} else {
+			p.latencyRxMu.RLock()
+			lat = p.latencyRx
+			p.latencyRxMu.RUnlock()
+		}
+		if lat > 0 {
+			select {
+			case <-time.After(lat):
+			case <-p.donec:
+				return
+			}
+		}
+
+		if proxySend {
+			p.corruptTxMu.RLock()
+			if p.corruptTx != nil {
+				data = p.corruptTx(data)
+			}
+			p.corruptTxMu.RUnlock()
+		} else {
+			p.corruptRxMu.RLock()
+			if p.corruptRx != nil {
+				data = p.corruptRx(data)
+			}
+			p.corruptRxMu.RUnlock()
+		}
+
+		var nw int
+		nw, err = dst.Write(data)
+		if err != nil {
+			if err == io.EOF {
+				return
+			}
+			select {
+			case p.errc <- err:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if p.logger.V(5) {
+				if proxySend {
+					p.logger.Errorf("write error while sending (%q)", err.Error())
+				} else {
+					p.logger.Errorf("write error while receiving (%q)", err.Error())
+				}
+			}
+			return
+		}
+
+		if nr != nw {
+			select {
+			case p.errc <- io.ErrShortWrite:
+				select {
+				case <-p.donec:
+					return
+				default:
+				}
+			case <-p.donec:
+				return
+			}
+			if proxySend {
+				p.logger.Errorf("write error while sending (%q); read %d bytes != wrote %d bytes", io.ErrShortWrite.Error(), nr, nw)
+			} else {
+				p.logger.Errorf("write error while receiving (%q); read %d bytes != wrote %d bytes", io.ErrShortWrite.Error(), nr, nw)
+			}
+			return
+		}
+
+		if p.logger.V(5) {
+			if proxySend {
+				p.logger.Infof("transmitted %s [%s -> %s]", humanize.Bytes(uint64(nr)), p.From(), p.To())
+			} else {
+				p.logger.Infof("received %s [%s <- %s]", humanize.Bytes(uint64(nr)), p.From(), p.To())
+			}
+		}
+	}
+}
+
+func (p *proxy) Ready() <-chan struct{} { return p.readyc }
+func (p *proxy) Done() <-chan struct{}  { return p.donec }
+func (p *proxy) Error() <-chan error    { return p.errc }
+func (p *proxy) Close() (err error) {
+	p.closeOnce.Do(func() {
+		close(p.donec)
+		p.listenerMu.Lock()
+		if p.listener != nil {
+			err = p.listener.Close()
+			p.logger.Infof("closed proxy listener on %q", p.From())
+		}
+		p.listenerMu.Unlock()
+	})
+	p.closeWg.Wait()
+	return err
+}
+
+func (p *proxy) DelayAccept(latency, rv time.Duration) {
+	if latency <= 0 {
+		return
+	}
+	d := computeLatency(latency, rv)
+	p.latencyAcceptMu.Lock()
+	p.latencyAccept = d
+	p.latencyAcceptMu.Unlock()
+	p.logger.Infof("set accept latency %v(%v±%v) [%s -> %s]", d, latency, rv, p.From(), p.To())
+}
+
+func (p *proxy) UndelayAccept() {
+	p.latencyAcceptMu.Lock()
+	d := p.latencyAccept
+	p.latencyAccept = 0
+	p.latencyAcceptMu.Unlock()
+	p.logger.Infof("removed accept latency %v [%s -> %s]", d, p.From(), p.To())
+}
+
+func (p *proxy) LatencyAccept() time.Duration {
+	p.latencyAcceptMu.RLock()
+	d := p.latencyAccept
+	p.latencyAcceptMu.RUnlock()
+	return d
+}
+
+func (p *proxy) DelayTx(latency, rv time.Duration) {
+	if latency <= 0 {
+		return
+	}
+	d := computeLatency(latency, rv)
+	p.latencyTxMu.Lock()
+	p.latencyTx = d
+	p.latencyTxMu.Unlock()
+	p.logger.Infof("set transmit latency %v(%v±%v) [%s -> %s]", d, latency, rv, p.From(), p.To())
+}
+
+func (p *proxy) UndelayTx() {
+	p.latencyTxMu.Lock()
+	d := p.latencyTx
+	p.latencyTx = 0
+	p.latencyTxMu.Unlock()
+	p.logger.Infof("removed transmit latency %v [%s -> %s]", d, p.From(), p.To())
+}
+
+func (p *proxy) LatencyTx() time.Duration {
+	p.latencyTxMu.RLock()
+	d := p.latencyTx
+	p.latencyTxMu.RUnlock()
+	return d
+}
+
+func (p *proxy) DelayRx(latency, rv time.Duration) {
+	if latency <= 0 {
+		return
+	}
+	d := computeLatency(latency, rv)
+	p.latencyRxMu.Lock()
+	p.latencyRx = d
+	p.latencyRxMu.Unlock()
+	p.logger.Infof("set receive latency %v(%v±%v) [%s <- %s]", d, latency, rv, p.From(), p.To())
+}
+
+func (p *proxy) UndelayRx() {
+	p.latencyRxMu.Lock()
+	d := p.latencyRx
+	p.latencyRx = 0
+	p.latencyRxMu.Unlock()
+	p.logger.Infof("removed receive latency %v [%s <- %s]", d, p.From(), p.To())
+}
+
+func (p *proxy) LatencyRx() time.Duration {
+	p.latencyRxMu.RLock()
+	d := p.latencyRx
+	p.latencyRxMu.RUnlock()
+	return d
+}
+
+func computeLatency(lat, rv time.Duration) time.Duration {
+	if rv == 0 {
+		return lat
+	}
+	if rv < 0 {
+		rv *= -1
+	}
+	if rv > lat {
+		rv = lat / 10
+	}
+	now := time.Now()
+	mrand.Seed(int64(now.Nanosecond()))
+	sign := 1
+	if now.Second()%2 == 0 {
+		sign = -1
+	}
+	return lat + time.Duration(int64(sign)*mrand.Int63n(rv.Nanoseconds()))
+}
+
+func (p *proxy) PauseAccept() {
+	p.acceptMu.Lock()
+	p.pauseAcceptc = make(chan struct{})
+	p.acceptMu.Unlock()
+	p.logger.Infof("paused accepting new connections [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnpauseAccept() {
+	p.acceptMu.Lock()
+	select {
+	case <-p.pauseAcceptc: // already unpaused
+	case <-p.donec:
+		p.acceptMu.Unlock()
+		return
+	default:
+		close(p.pauseAcceptc)
+	}
+	p.acceptMu.Unlock()
+	p.logger.Infof("unpaused accepting new connections [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) PauseTx() {
+	p.txMu.Lock()
+	p.pauseTxc = make(chan struct{})
+	p.txMu.Unlock()
+	p.logger.Infof("paused transmit listen [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnpauseTx() {
+	p.txMu.Lock()
+	select {
+	case <-p.pauseTxc: // already unpaused
+	case <-p.donec:
+		p.txMu.Unlock()
+		return
+	default:
+		close(p.pauseTxc)
+	}
+	p.txMu.Unlock()
+	p.logger.Infof("unpaused transmit listen [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) PauseRx() {
+	p.rxMu.Lock()
+	p.pauseRxc = make(chan struct{})
+	p.rxMu.Unlock()
+	p.logger.Infof("paused receive listen [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnpauseRx() {
+	p.rxMu.Lock()
+	select {
+	case <-p.pauseRxc: // already unpaused
+	case <-p.donec:
+		p.rxMu.Unlock()
+		return
+	default:
+		close(p.pauseRxc)
+	}
+	p.rxMu.Unlock()
+	p.logger.Infof("unpaused receive listen [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) BlackholeTx() {
+	p.txMu.Lock()
+	select {
+	case <-p.blackholeTxc: // already blackholed
+	case <-p.donec:
+		p.txMu.Unlock()
+		return
+	default:
+		close(p.blackholeTxc)
+	}
+	p.txMu.Unlock()
+	p.logger.Infof("blackholed transmit [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnblackholeTx() {
+	p.txMu.Lock()
+	p.blackholeTxc = make(chan struct{})
+	p.txMu.Unlock()
+	p.logger.Infof("unblackholed transmit [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) BlackholeRx() {
+	p.rxMu.Lock()
+	select {
+	case <-p.blackholeRxc: // already blackholed
+	case <-p.donec:
+		p.rxMu.Unlock()
+		return
+	default:
+		close(p.blackholeRxc)
+	}
+	p.rxMu.Unlock()
+	p.logger.Infof("blackholed receive [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) UnblackholeRx() {
+	p.rxMu.Lock()
+	p.blackholeRxc = make(chan struct{})
+	p.rxMu.Unlock()
+	p.logger.Infof("unblackholed receive [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) CorruptTx(f func([]byte) []byte) {
+	p.corruptTxMu.Lock()
+	p.corruptTx = f
+	p.corruptTxMu.Unlock()
+	p.logger.Infof("corrupting transmit [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) UncorruptTx() {
+	p.corruptTxMu.Lock()
+	p.corruptTx = nil
+	p.corruptTxMu.Unlock()
+	p.logger.Infof("stopped corrupting transmit [%s -> %s]", p.From(), p.To())
+}
+
+func (p *proxy) CorruptRx(f func([]byte) []byte) {
+	p.corruptRxMu.Lock()
+	p.corruptRx = f
+	p.corruptRxMu.Unlock()
+	p.logger.Infof("corrupting receive [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) UncorruptRx() {
+	p.corruptRxMu.Lock()
+	p.corruptRx = nil
+	p.corruptRxMu.Unlock()
+	p.logger.Infof("stopped corrupting receive [%s <- %s]", p.From(), p.To())
+}
+
+func (p *proxy) ResetListener() error {
+	p.listenerMu.Lock()
+	defer p.listenerMu.Unlock()
+
+	if err := p.listener.Close(); err != nil {
+		// already closed
+		if !strings.HasSuffix(err.Error(), "use of closed network connection") {
+			return err
+		}
+	}
+
+	var ln net.Listener
+	var err error
+	if !p.tlsInfo.Empty() {
+		ln, err = NewListener(p.from.Host, p.from.Scheme, &p.tlsInfo)
+	} else {
+		ln, err = net.Listen(p.from.Scheme, p.from.Host)
+	}
+	if err != nil {
+		return err
+	}
+	p.listener = ln
+
+	p.logger.Infof("reset listener %q", p.From())
+	return nil
+}

pkg/transport/proxy_test.go

@@ -0,0 +1,606 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transport
+
+import (
+	"bytes"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/grpclog"
+)
+
+var testTLSInfo = TLSInfo{
+	KeyFile:        "./fixtures/server.key.insecure",
+	CertFile:       "./fixtures/server.crt",
+	TrustedCAFile:  "./fixtures/ca.crt",
+	ClientCertAuth: true,
+}
+
+func TestProxy_Unix_Insecure(t *testing.T)         { testProxy(t, "unix", false, false) }
+func TestProxy_TCP_Insecure(t *testing.T)          { testProxy(t, "tcp", false, false) }
+func TestProxy_Unix_Secure(t *testing.T)           { testProxy(t, "unix", true, false) }
+func TestProxy_TCP_Secure(t *testing.T)            { testProxy(t, "tcp", true, false) }
+func TestProxy_Unix_Insecure_DelayTx(t *testing.T) { testProxy(t, "unix", false, true) }
+func TestProxy_TCP_Insecure_DelayTx(t *testing.T)  { testProxy(t, "tcp", false, true) }
+func TestProxy_Unix_Secure_DelayTx(t *testing.T)   { testProxy(t, "unix", true, true) }
+func TestProxy_TCP_Secure_DelayTx(t *testing.T)    { testProxy(t, "tcp", true, true) }
+func testProxy(t *testing.T, scheme string, secure bool, delayTx bool) {
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	if scheme == "tcp" {
+		ln1, ln2 := listen(t, "tcp", "localhost:0", TLSInfo{}), listen(t, "tcp", "localhost:0", TLSInfo{})
+		srcAddr, dstAddr = ln1.Addr().String(), ln2.Addr().String()
+		ln1.Close()
+		ln2.Close()
+	} else {
+		defer func() {
+			os.RemoveAll(srcAddr)
+			os.RemoveAll(dstAddr)
+		}()
+	}
+	tlsInfo := testTLSInfo
+	if !secure {
+		tlsInfo = TLSInfo{}
+	}
+	ln := listen(t, scheme, dstAddr, tlsInfo)
+	defer ln.Close()
+
+	cfg := ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	}
+	if secure {
+		cfg.TLSInfo = testTLSInfo
+	}
+	p := NewProxy(cfg)
+	<-p.Ready()
+	defer p.Close()
+
+	data1 := []byte("Hello World!")
+	donec, writec := make(chan struct{}), make(chan []byte)
+
+	go func() {
+		defer close(donec)
+		for data := range writec {
+			send(t, data, scheme, srcAddr, tlsInfo)
+		}
+	}()
+
+	recvc := make(chan []byte)
+	go func() {
+		for i := 0; i < 2; i++ {
+			recvc <- receive(t, ln)
+		}
+	}()
+
+	writec <- data1
+	now := time.Now()
+	if d := <-recvc; !bytes.Equal(data1, d) {
+		t.Fatalf("expected %q, got %q", string(data1), string(d))
+	}
+	took1 := time.Since(now)
+	t.Logf("took %v with no latency", took1)
+
+	lat, rv := 50*time.Millisecond, 5*time.Millisecond
+	if delayTx {
+		p.DelayTx(lat, rv)
+	}
+
+	data2 := []byte("new data")
+	writec <- data2
+	now = time.Now()
+	if d := <-recvc; !bytes.Equal(data2, d) {
+		t.Fatalf("expected %q, got %q", string(data2), string(d))
+	}
+	took2 := time.Since(now)
+	if delayTx {
+		t.Logf("took %v with latency %v±%v", took2, lat, rv)
+	} else {
+		t.Logf("took %v with no latency", took2)
+	}
+
+	if delayTx {
+		p.UndelayTx()
+		if took1 >= took2 {
+			t.Fatalf("expected took1 %v < took2 %v (with latency)", took1, took2)
+		}
+	}
+
+	close(writec)
+	select {
+	case <-donec:
+	case <-time.After(3 * time.Second):
+		t.Fatal("took too long to write")
+	}
+
+	select {
+	case <-p.Done():
+		t.Fatal("unexpected done")
+	case err := <-p.Error():
+		t.Fatal(err)
+	default:
+	}
+
+	if err := p.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case <-p.Done():
+	case err := <-p.Error():
+		if !strings.HasPrefix(err.Error(), "accept ") &&
+			!strings.HasSuffix(err.Error(), "use of closed network connection") {
+			t.Fatal(err)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("took too long to close")
+	}
+}
+
+func TestProxy_Unix_Insecure_DelayAccept(t *testing.T) { testProxyDelayAccept(t, false) }
+func TestProxy_Unix_Secure_DelayAccept(t *testing.T)   { testProxyDelayAccept(t, true) }
+func testProxyDelayAccept(t *testing.T, secure bool) {
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	tlsInfo := testTLSInfo
+	if !secure {
+		tlsInfo = TLSInfo{}
+	}
+	scheme := "unix"
+	ln := listen(t, scheme, dstAddr, tlsInfo)
+	defer ln.Close()
+
+	cfg := ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	}
+	if secure {
+		cfg.TLSInfo = testTLSInfo
+	}
+	p := NewProxy(cfg)
+	<-p.Ready()
+	defer p.Close()
+
+	data := []byte("Hello World!")
+
+	now := time.Now()
+	send(t, data, scheme, srcAddr, tlsInfo)
+	if d := receive(t, ln); !bytes.Equal(data, d) {
+		t.Fatalf("expected %q, got %q", string(data), string(d))
+	}
+	took1 := time.Since(now)
+	t.Logf("took %v with no latency", took1)
+
+	lat, rv := 700*time.Millisecond, 10*time.Millisecond
+	p.DelayAccept(lat, rv)
+	defer p.UndelayAccept()
+	if err := p.ResetListener(); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(200 * time.Millisecond)
+
+	now = time.Now()
+	send(t, data, scheme, srcAddr, tlsInfo)
+	if d := receive(t, ln); !bytes.Equal(data, d) {
+		t.Fatalf("expected %q, got %q", string(data), string(d))
+	}
+	took2 := time.Since(now)
+	t.Logf("took %v with latency %v±%v", took2, lat, rv)
+
+	if took1 >= took2 {
+		t.Fatalf("expected took1 %v < took2 %v", took1, took2)
+	}
+}
+
+func TestProxy_PauseTx(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	p.PauseTx()
+
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+
+	recvc := make(chan []byte)
+	go func() {
+		recvc <- receive(t, ln)
+	}()
+
+	select {
+	case d := <-recvc:
+		t.Fatalf("received unexpected data %q during pause", string(d))
+	case <-time.After(200 * time.Millisecond):
+	}
+
+	p.UnpauseTx()
+
+	select {
+	case d := <-recvc:
+		if !bytes.Equal(data, d) {
+			t.Fatalf("expected %q, got %q", string(data), string(d))
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("took too long to receive after unpause")
+	}
+}
+
+func TestProxy_BlackholeTx(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	p.BlackholeTx()
+
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+
+	recvc := make(chan []byte)
+	go func() {
+		recvc <- receive(t, ln)
+	}()
+
+	select {
+	case d := <-recvc:
+		t.Fatalf("unexpected data receive %q during blackhole", string(d))
+	case <-time.After(200 * time.Millisecond):
+	}
+
+	p.UnblackholeTx()
+
+	// expect different data, old data dropped
+	data[0]++
+	send(t, data, scheme, srcAddr, TLSInfo{})
+
+	select {
+	case d := <-recvc:
+		if !bytes.Equal(data, d) {
+			t.Fatalf("expected %q, got %q", string(data), string(d))
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("took too long to receive after unblackhole")
+	}
+}
+
+func TestProxy_CorruptTx(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	p.CorruptTx(func(d []byte) []byte {
+		d[len(d)/2]++
+		return d
+	})
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+	if d := receive(t, ln); bytes.Equal(d, data) {
+		t.Fatalf("expected corrupted data, got %q", string(d))
+	}
+
+	p.UncorruptTx()
+	send(t, data, scheme, srcAddr, TLSInfo{})
+	if d := receive(t, ln); !bytes.Equal(d, data) {
+		t.Fatalf("expected uncorrupted data, got %q", string(d))
+	}
+}
+
+func TestProxy_Shutdown(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	px, _ := p.(*proxy)
+	px.listener.Close()
+	time.Sleep(200 * time.Millisecond)
+
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+	if d := receive(t, ln); !bytes.Equal(d, data) {
+		t.Fatalf("expected %q, got %q", string(data), string(d))
+	}
+}
+
+func TestProxy_ShutdownListener(t *testing.T) {
+	scheme := "unix"
+	srcAddr, dstAddr := newUnixAddr(), newUnixAddr()
+	defer func() {
+		os.RemoveAll(srcAddr)
+		os.RemoveAll(dstAddr)
+	}()
+
+	ln := listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	p := NewProxy(ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	})
+	<-p.Ready()
+	defer p.Close()
+
+	// shut down destination
+	ln.Close()
+	time.Sleep(200 * time.Millisecond)
+
+	ln = listen(t, scheme, dstAddr, TLSInfo{})
+	defer ln.Close()
+
+	data := []byte("Hello World!")
+	send(t, data, scheme, srcAddr, TLSInfo{})
+	if d := receive(t, ln); !bytes.Equal(d, data) {
+		t.Fatalf("expected %q, got %q", string(data), string(d))
+	}
+}
+
+func TestProxyHTTP_Insecure_DelayTx(t *testing.T) { testProxyHTTP(t, false, true) }
+func TestProxyHTTP_Secure_DelayTx(t *testing.T)   { testProxyHTTP(t, true, true) }
+func TestProxyHTTP_Insecure_DelayRx(t *testing.T) { testProxyHTTP(t, false, false) }
+func TestProxyHTTP_Secure_DelayRx(t *testing.T)   { testProxyHTTP(t, true, false) }
+func testProxyHTTP(t *testing.T, secure, delayTx bool) {
+	scheme := "tcp"
+	ln1, ln2 := listen(t, scheme, "localhost:0", TLSInfo{}), listen(t, scheme, "localhost:0", TLSInfo{})
+	srcAddr, dstAddr := ln1.Addr().String(), ln2.Addr().String()
+	ln1.Close()
+	ln2.Close()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/hello", func(w http.ResponseWriter, req *http.Request) {
+		d, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err = w.Write([]byte(fmt.Sprintf("%q(confirmed)", string(d)))); err != nil {
+			t.Fatal(err)
+		}
+	})
+	var tlsConfig *tls.Config
+	var err error
+	if secure {
+		tlsConfig, err = testTLSInfo.ServerConfig()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	srv := &http.Server{
+		Addr:      dstAddr,
+		Handler:   mux,
+		TLSConfig: tlsConfig,
+	}
+
+	donec := make(chan struct{})
+	defer func() {
+		srv.Close()
+		<-donec
+	}()
+	go func() {
+		defer close(donec)
+		if !secure {
+			srv.ListenAndServe()
+		} else {
+			srv.ListenAndServeTLS(testTLSInfo.CertFile, testTLSInfo.KeyFile)
+		}
+	}()
+	time.Sleep(200 * time.Millisecond)
+
+	cfg := ProxyConfig{
+		From:   url.URL{Scheme: scheme, Host: srcAddr},
+		To:     url.URL{Scheme: scheme, Host: dstAddr},
+		Logger: grpclog.NewLoggerV2WithVerbosity(os.Stderr, os.Stderr, os.Stderr, 5),
+	}
+	if secure {
+		cfg.TLSInfo = testTLSInfo
+	}
+	p := NewProxy(cfg)
+	<-p.Ready()
+	defer p.Close()
+
+	data := "Hello World!"
+
+	now := time.Now()
+	var resp *http.Response
+	if secure {
+		tp, terr := NewTransport(testTLSInfo, 3*time.Second)
+		if terr != nil {
+			t.Fatal(terr)
+		}
+		cli := &http.Client{Transport: tp}
+		resp, err = cli.Post("https://"+srcAddr+"/hello", "", strings.NewReader(data))
+	} else {
+		resp, err = http.Post("http://"+srcAddr+"/hello", "", strings.NewReader(data))
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	took1 := time.Since(now)
+	t.Logf("took %v with no latency", took1)
+
+	rs1 := string(d)
+	exp := fmt.Sprintf("%q(confirmed)", data)
+	if rs1 != exp {
+		t.Fatalf("got %q, expected %q", rs1, exp)
+	}
+
+	lat, rv := 100*time.Millisecond, 10*time.Millisecond
+	if delayTx {
+		p.DelayTx(lat, rv)
+		defer p.UndelayTx()
+	} else {
+		p.DelayRx(lat, rv)
+		defer p.UndelayRx()
+	}
+
+	now = time.Now()
+	if secure {
+		tp, terr := NewTransport(testTLSInfo, 3*time.Second)
+		if terr != nil {
+			t.Fatal(terr)
+		}
+		cli := &http.Client{Transport: tp}
+		resp, err = cli.Post("https://"+srcAddr+"/hello", "", strings.NewReader(data))
+	} else {
+		resp, err = http.Post("http://"+srcAddr+"/hello", "", strings.NewReader(data))
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	took2 := time.Since(now)
+	t.Logf("took %v with latency %v±%v", took2, lat, rv)
+
+	rs2 := string(d)
+	if rs2 != exp {
+		t.Fatalf("got %q, expected %q", rs2, exp)
+	}
+	if took1 > took2 {
+		t.Fatalf("expected took1 %v < took2 %v", took1, took2)
+	}
+}
+
+func newUnixAddr() string {
+	now := time.Now().UnixNano()
+	rand.Seed(now)
+	addr := fmt.Sprintf("%X%X.unix-conn", now, rand.Intn(35000))
+	os.RemoveAll(addr)
+	return addr
+}
+
+func listen(t *testing.T, scheme, addr string, tlsInfo TLSInfo) (ln net.Listener) {
+	var err error
+	if !tlsInfo.Empty() {
+		ln, err = NewListener(addr, scheme, &tlsInfo)
+	} else {
+		ln, err = net.Listen(scheme, addr)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ln
+}
+
+func send(t *testing.T, data []byte, scheme, addr string, tlsInfo TLSInfo) {
+	var out net.Conn
+	var err error
+	if !tlsInfo.Empty() {
+		tp, terr := NewTransport(tlsInfo, 3*time.Second)
+		if terr != nil {
+			t.Fatal(terr)
+		}
+		out, err = tp.Dial(scheme, addr)
+	} else {
+		out, err = net.Dial(scheme, addr)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err = out.Write(data); err != nil {
+		t.Fatal(err)
+	}
+	if err = out.Close(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func receive(t *testing.T, ln net.Listener) (data []byte) {
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+	for {
+		in, err := ln.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+		var n int64
+		n, err = buf.ReadFrom(in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n > 0 {
+			break
+		}
+	}
+	return buf.Bytes()
+}

test

@@ -114,7 +114,7 @@ function functional_pass {
 
 	for a in 1 2 3; do
 		mkdir -p ./agent-$a
-		./bin/etcd-agent -etcd-path ./bin/etcd -etcd-log-dir "./agent-$a" -port ":${a}9027" -use-root=false &
+		./bin/etcd-agent -etcd-path ./bin/etcd -etcd-log-dir "./agent-$a" -port ":${a}9027" &
 		pid="$!"
 		agent_pids="${agent_pids} $pid"
 	done
@@ -129,10 +129,12 @@ function functional_pass {
 	echo "Starting 'etcd-tester'"
 	./bin/etcd-tester \
 		-agent-endpoints "127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027" \
-		-client-ports 12379,22379,32379 \
-		-peer-ports 12380,22380,32380 \
+		-client-ports 1379,2379,3379 \
+		-advertise-client-ports 13790,23790,33790 \
+		-peer-ports 1380,2380,3380 \
+		-advertise-peer-ports 13800,23800,33800 \
 		-limit 1 \
-		-schedule-cases "0 1 2 3 4 5" \
+		-schedule-cases "0 1 2 3 4 5 6 7 8 9" \
 		-stress-qps 1000 \
 		-stress-key-txn-count 100 \
 		-stress-key-txn-ops 10 \

tools/functional-tester/Procfile

@@ -1,4 +0,0 @@
-agent-1: mkdir -p agent-1 && cd agent-1 && ../bin/etcd-agent -etcd-path ../bin/etcd -port 127.0.0.1:19027 -use-root=false
-agent-2: mkdir -p agent-2 && cd agent-2 && ../bin/etcd-agent -etcd-path ../bin/etcd -port 127.0.0.1:29027 -use-root=false
-agent-3: mkdir -p agent-3 && cd agent-3 && ../bin/etcd-agent -etcd-path ../bin/etcd -port 127.0.0.1:39027 -use-root=false
-stresser: sleep 1s && bin/etcd-tester -agent-endpoints "127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027"  -client-ports 12379,22379,32379 -peer-ports 12380,22380,32380

tools/functional-tester/README.md

@@ -10,42 +10,38 @@ The environment of the cluster must be stable enough, so etcd test suite can ass
 
 ## etcd agent
 
-etcd agent is a daemon on each machines. It can start, stop, restart, isolate and terminate an etcd process. The agent exposes these functionality via HTTP RPC. 
+etcd agent is a daemon on each machines. It can start, stop, restart, isolate and terminate an etcd process. The agent exposes these functionality via HTTP RPC.
 
 ## etcd tester
 
 etcd functional tester control the progress of the functional tests. It calls the RPC of the etcd agent to simulate various test cases. For example, it can start a three members cluster by sending three start RPC calls to three different etcd agents. It can make one of the member failed by sending stop RPC call to one etcd agent.
 
-## with Docker (optionally)
+### Run locally
 
-To run the functional tests using Docker, the provided script can be used to set up an environment using Docker Compose. 
-
-Script (on linux):
-```sh
-./tools/functional-tester/test
+```
+$ PASSES=functional ./test
 ```
 
-Running the script requires:
+### Run with Docker
 
-- Docker 1.9+ (with networking support) - to create isolated network
-- docker-compose - to create etcd cluster and tester
-- A multi-arch Go toolchain (OSX)
+To run locally, first build tester image:
 
-Notes:
-- Docker image is based on Alpine Linux OS running in privileged mode to allow iptables manipulation.
-- To specify testing parameters (etcd-tester arguments) modify tools/functional-tester/docker/docker-compose.yml or start etcd-tester manually
-- (OSX) make sure that etcd binary is built for linux/amd64 (eg. `rm bin/etcd;GOOS=linux GOARCH=amd64 ./tools/functional-tester/test`) otherwise it will return `exec format error`
+```bash
+pushd ../..
 
+GO_VERSION=1.9.3 \
+  make build-docker-functional-tester \
+  -f ./hack/scripts-dev/Makefile
 
-## with Goreman
-
-To run the functional tests on a single machine using Goreman, build with the provided build script and run with the provided Procfile:
-
-```sh
-./tools/functional-tester/build
-goreman -f tools/functional-tester/Procfile start
+popd
 ```
 
-Notes:
-- The etcd-agent will not run with root privileges; iptables manipulation is disabled.
-- To specify testing parameters (etcd-tester arguments) modify tools/functional-tester/Procfile or start etcd-tester manually
+And run [example scripts](./scripts).
+
+```bash
+./scripts/agent-1.sh
+./scripts/agent-2.sh
+./scripts/agent-3.sh
+
+./scripts/tester-limit.sh
+```

tools/functional-tester/docker/Dockerfile

@@ -1,8 +0,0 @@
-FROM alpine
-RUN apk update 
-RUN apk add -v iptables sudo
-ADD bin/etcd-agent /
-ADD bin/etcd /
-ADD bin/etcd-tester /
-RUN mkdir /failure_archive
-CMD ["./etcd-agent", "-etcd-path", "./etcd"]

tools/functional-tester/docker/docker-compose.yml

@@ -1,28 +0,0 @@
-# build according provided Dockerfile
-a1:
-  build: .
-  privileged: true
-  net: etcd-functional
-a2:
-  build: .
-  privileged: true
-  net: etcd-functional
-a3:
-  build: .
-  privileged: true
-  net: etcd-functional
-tester:
-  build: .
-  privileged: true
-  net: etcd-functional
-  command:
-    - /etcd-tester
-    - -agent-endpoints
-    - "172.20.0.2:9027,172.20.0.3:9027,172.20.0.4:9027"
-    - -limit 
-    - "1"
-    - -stress-key-count
-    - "1"
-    - -stress-key-size
-    - "1"
-      

tools/functional-tester/etcd-agent/agent.go

@@ -15,14 +15,19 @@
 package main
 
 import (
+	"fmt"
+	"net"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+	"sync"
 	"syscall"
 	"time"
 
 	"github.com/coreos/etcd/pkg/fileutil"
-	"github.com/coreos/etcd/pkg/netutil"
+	"github.com/coreos/etcd/pkg/transport"
 	"github.com/coreos/etcd/tools/functional-tester/etcd-agent/client"
 )
 
@@ -40,13 +45,15 @@ type Agent struct {
 	logfile *os.File
 
 	cfg AgentConfig
+
+	pmu                  sync.Mutex
+	advertisePortToProxy map[int]transport.Proxy
 }
 
 type AgentConfig struct {
 	EtcdPath      string
 	LogDir        string
 	FailpointAddr string
-	UseRoot       bool
 }
 
 func newAgent(cfg AgentConfig) (*Agent, error) {
@@ -69,7 +76,13 @@ func newAgent(cfg AgentConfig) (*Agent, error) {
 		return nil, err
 	}
 
-	return &Agent{state: stateUninitialized, cmd: c, logfile: f, cfg: cfg}, nil
+	return &Agent{
+		state:                stateUninitialized,
+		cmd:                  c,
+		logfile:              f,
+		cfg:                  cfg,
+		advertisePortToProxy: make(map[int]transport.Proxy),
+	}, nil
 }
 
 // start starts a new etcd process with the given args.
@@ -85,6 +98,85 @@ func (a *Agent) start(args ...string) error {
 	}
 
 	a.state = stateStarted
+
+	a.pmu.Lock()
+	defer a.pmu.Unlock()
+	if len(a.advertisePortToProxy) == 0 {
+		// enough time for etcd start before setting up proxy
+		time.Sleep(time.Second)
+		var (
+			err                    error
+			s                      string
+			listenClientURL        *url.URL
+			advertiseClientURL     *url.URL
+			advertiseClientURLPort int
+			listenPeerURL          *url.URL
+			advertisePeerURL       *url.URL
+			advertisePeerURLPort   int
+		)
+		for i := range args {
+			switch args[i] {
+			case "--listen-client-urls":
+				listenClientURL, err = url.Parse(args[i+1])
+				if err != nil {
+					return err
+				}
+			case "--advertise-client-urls":
+				advertiseClientURL, err = url.Parse(args[i+1])
+				if err != nil {
+					return err
+				}
+				_, s, err = net.SplitHostPort(advertiseClientURL.Host)
+				if err != nil {
+					return err
+				}
+				advertiseClientURLPort, err = strconv.Atoi(s)
+				if err != nil {
+					return err
+				}
+			case "--listen-peer-urls":
+				listenPeerURL, err = url.Parse(args[i+1])
+				if err != nil {
+					return err
+				}
+			case "--initial-advertise-peer-urls":
+				advertisePeerURL, err = url.Parse(args[i+1])
+				if err != nil {
+					return err
+				}
+				_, s, err = net.SplitHostPort(advertisePeerURL.Host)
+				if err != nil {
+					return err
+				}
+				advertisePeerURLPort, err = strconv.Atoi(s)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		clientProxy := transport.NewProxy(transport.ProxyConfig{
+			From: *advertiseClientURL,
+			To:   *listenClientURL,
+		})
+		select {
+		case err = <-clientProxy.Error():
+			return err
+		case <-time.After(time.Second):
+		}
+		a.advertisePortToProxy[advertiseClientURLPort] = clientProxy
+
+		peerProxy := transport.NewProxy(transport.ProxyConfig{
+			From: *advertisePeerURL,
+			To:   *listenPeerURL,
+		})
+		select {
+		case err = <-peerProxy.Error():
+			return err
+		case <-time.After(time.Second):
+		}
+		a.advertisePortToProxy[advertisePeerURLPort] = peerProxy
+	}
 	return nil
 }
 
@@ -94,6 +186,24 @@ func (a *Agent) stopWithSig(sig os.Signal) error {
 		return nil
 	}
 
+	a.pmu.Lock()
+	if len(a.advertisePortToProxy) > 0 {
+		for _, p := range a.advertisePortToProxy {
+			if err := p.Close(); err != nil {
+				a.pmu.Unlock()
+				return err
+			}
+			select {
+			case <-p.Done():
+				// enough time to release port
+				time.Sleep(time.Second)
+			case <-time.After(time.Second):
+			}
+		}
+		a.advertisePortToProxy = make(map[int]transport.Proxy)
+	}
+	a.pmu.Unlock()
+
 	err := stopWithSig(a.cmd, sig)
 	if err != nil {
 		return err
@@ -178,27 +288,46 @@ func (a *Agent) terminate() error {
 }
 
 func (a *Agent) dropPort(port int) error {
-	if !a.cfg.UseRoot {
-		return nil
+	a.pmu.Lock()
+	defer a.pmu.Unlock()
+
+	p, ok := a.advertisePortToProxy[port]
+	if !ok {
+		return fmt.Errorf("%d does not have proxy", port)
 	}
-	return netutil.DropPort(port)
+	p.BlackholeTx()
+	p.BlackholeRx()
+	return nil
 }
 
 func (a *Agent) recoverPort(port int) error {
-	if !a.cfg.UseRoot {
-		return nil
+	a.pmu.Lock()
+	defer a.pmu.Unlock()
+
+	p, ok := a.advertisePortToProxy[port]
+	if !ok {
+		return fmt.Errorf("%d does not have proxy", port)
 	}
-	return netutil.RecoverPort(port)
+	p.UnblackholeTx()
+	p.UnblackholeRx()
+	return nil
 }
 
 func (a *Agent) setLatency(ms, rv int) error {
-	if !a.cfg.UseRoot {
-		return nil
-	}
+	a.pmu.Lock()
+	defer a.pmu.Unlock()
+
 	if ms == 0 {
-		return netutil.RemoveLatency()
+		for _, p := range a.advertisePortToProxy {
+			p.UndelayTx()
+			p.UndelayRx()
+		}
 	}
-	return netutil.SetLatency(ms, rv)
+	for _, p := range a.advertisePortToProxy {
+		p.DelayTx(time.Duration(ms)*time.Millisecond, time.Duration(rv)*time.Millisecond)
+		p.DelayRx(time.Duration(ms)*time.Millisecond, time.Duration(rv)*time.Millisecond)
+	}
+	return nil
 }
 
 func (a *Agent) status() client.Status {

tools/functional-tester/etcd-agent/main.go

@@ -16,7 +16,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
 	"os"
 	"path/filepath"
 
@@ -29,7 +28,6 @@ func main() {
 	etcdPath := flag.String("etcd-path", filepath.Join(os.Getenv("GOPATH"), "bin/etcd"), "the path to etcd binary")
 	etcdLogDir := flag.String("etcd-log-dir", "etcd-log", "directory to store etcd logs, data directories, failure archive")
 	port := flag.String("port", ":9027", "port to serve agent server")
-	useRoot := flag.Bool("use-root", true, "use root permissions")
 	failpointAddr := flag.String("failpoint-addr", ":2381", "interface for gofail's HTTP server")
 	flag.Parse()
 
@@ -37,17 +35,7 @@ func main() {
 		EtcdPath:      *etcdPath,
 		LogDir:        *etcdLogDir,
 		FailpointAddr: *failpointAddr,
-		UseRoot:       *useRoot,
 	}
-
-	if *useRoot && os.Getuid() != 0 {
-		fmt.Println("got --use-root=true but not root user")
-		os.Exit(1)
-	}
-	if !*useRoot {
-		fmt.Println("root permissions disabled, agent will not modify network")
-	}
-
 	a, err := newAgent(cfg)
 	if err != nil {
 		plog.Fatal(err)

tools/functional-tester/etcd-tester/cluster.go

@@ -30,10 +30,12 @@ import (
 
 // agentConfig holds information needed to interact/configure an agent and its etcd process
 type agentConfig struct {
-	endpoint      string
-	clientPort    int
-	peerPort      int
-	failpointPort int
+	endpoint            string
+	clientPort          int
+	advertiseClientPort int
+	peerPort            int
+	advertisePeerPort   int
+	failpointPort       int
 }
 
 type cluster struct {
@@ -61,12 +63,14 @@ func (c *cluster) bootstrap() error {
 			return err
 		}
 		members[i] = &member{
-			Agent:        agent,
-			Endpoint:     a.endpoint,
-			Name:         fmt.Sprintf("etcd-%d", i),
-			ClientURL:    fmt.Sprintf("http://%s:%d", host, a.clientPort),
-			PeerURL:      fmt.Sprintf("http://%s:%d", host, a.peerPort),
-			FailpointURL: fmt.Sprintf("http://%s:%d", host, a.failpointPort),
+			Agent:              agent,
+			Endpoint:           a.endpoint,
+			Name:               fmt.Sprintf("etcd-%d", i),
+			ClientURL:          fmt.Sprintf("http://%s:%d", host, a.clientPort),
+			AdvertiseClientURL: fmt.Sprintf("http://%s:%d", host, a.advertiseClientPort),
+			PeerURL:            fmt.Sprintf("http://%s:%d", host, a.peerPort),
+			AdvertisePeerURL:   fmt.Sprintf("http://%s:%d", host, a.advertisePeerPort),
+			FailpointURL:       fmt.Sprintf("http://%s:%d", host, a.failpointPort),
 		}
 		memberNameURLs[i] = members[i].ClusterEntry()
 	}

tools/functional-tester/etcd-tester/failure.go

@@ -128,7 +128,10 @@ func (f *failureDelay) Inject(c *cluster, round int) error {
 	if err := f.failure.Inject(c, round); err != nil {
 		return err
 	}
-	time.Sleep(f.delayDuration)
+	if f.delayDuration > 0 {
+		plog.Infof("sleeping delay duration %v for %q", f.delayDuration, f.failure.Desc())
+		time.Sleep(f.delayDuration)
+	}
 	return nil
 }
 

tools/functional-tester/etcd-tester/failure_agent.go

@@ -24,6 +24,9 @@ const (
 	slowNetworkLatency = 500 // 500 millisecond
 	randomVariation    = 50
 
+	// delay duration to trigger leader election (default election timeout 1s)
+	triggerElectionDur = 5 * time.Second
+
 	// Wait more when it recovers from slow network, because network layer
 	// needs extra time to propagate traffic control (tc command) change.
 	// Otherwise, we get different hash values from the previous revision.
@@ -82,19 +85,27 @@ func injectDropPort(m *member) error  { return m.Agent.DropPort(m.peerPort()) }
 func recoverDropPort(m *member) error { return m.Agent.RecoverPort(m.peerPort()) }
 
 func newFailureIsolate() failure {
-	return &failureOne{
+	f := &failureOne{
 		description:   "isolate one member",
 		injectMember:  injectDropPort,
 		recoverMember: recoverDropPort,
 	}
+	return &failureDelay{
+		failure:       f,
+		delayDuration: triggerElectionDur,
+	}
 }
 
 func newFailureIsolateAll() failure {
-	return &failureAll{
+	f := &failureAll{
 		description:   "isolate all members",
 		injectMember:  injectDropPort,
 		recoverMember: recoverDropPort,
 	}
+	return &failureDelay{
+		failure:       f,
+		delayDuration: triggerElectionDur,
+	}
 }
 
 func injectLatency(m *member) error {
@@ -115,11 +126,15 @@ func recoverLatency(m *member) error {
 
 func newFailureSlowNetworkOneMember() failure {
 	desc := fmt.Sprintf("slow down one member's network by adding %d ms latency", slowNetworkLatency)
-	return &failureOne{
+	f := &failureOne{
 		description:   description(desc),
 		injectMember:  injectLatency,
 		recoverMember: recoverLatency,
 	}
+	return &failureDelay{
+		failure:       f,
+		delayDuration: triggerElectionDur,
+	}
 }
 
 func newFailureSlowNetworkLeader() failure {
@@ -129,15 +144,23 @@ func newFailureSlowNetworkLeader() failure {
 		injectMember:  injectLatency,
 		recoverMember: recoverLatency,
 	}
-	return &failureLeader{ff, 0}
+	f := &failureLeader{ff, 0}
+	return &failureDelay{
+		failure:       f,
+		delayDuration: triggerElectionDur,
+	}
 }
 
 func newFailureSlowNetworkAll() failure {
-	return &failureAll{
+	f := &failureAll{
 		description:   "slow down all members' network",
 		injectMember:  injectLatency,
 		recoverMember: recoverLatency,
 	}
+	return &failureDelay{
+		failure:       f,
+		delayDuration: triggerElectionDur,
+	}
 }
 
 func newFailureNop() failure {

tools/functional-tester/etcd-tester/main.go

@@ -41,7 +41,9 @@ const (
 func main() {
 	endpointStr := flag.String("agent-endpoints", "localhost:9027", "HTTP RPC endpoints of agents. Do not specify the schema.")
 	clientPorts := flag.String("client-ports", "", "etcd client port for each agent endpoint")
+	advertiseClientPorts := flag.String("advertise-client-ports", "", "etcd advertise client port for each agent endpoint")
 	peerPorts := flag.String("peer-ports", "", "etcd peer port for each agent endpoint")
+	advertisePeerPorts := flag.String("advertise-peer-ports", "", "etcd advertise peer port for each agent endpoint")
 	failpointPorts := flag.String("failpoint-ports", "", "etcd failpoint port for each agent endpoint")
 
 	stressKeyLargeSize := flag.Uint("stress-key-large-size", 32*1024+1, "the size of each large key written into etcd.")
@@ -67,14 +69,18 @@ func main() {
 
 	eps := strings.Split(*endpointStr, ",")
 	cports := portsFromArg(*clientPorts, len(eps), defaultClientPort)
+	acports := portsFromArg(*advertiseClientPorts, len(eps), defaultClientPort)
 	pports := portsFromArg(*peerPorts, len(eps), defaultPeerPort)
+	apports := portsFromArg(*advertisePeerPorts, len(eps), defaultPeerPort)
 	fports := portsFromArg(*failpointPorts, len(eps), defaultFailpointPort)
 	agents := make([]agentConfig, len(eps))
 
 	for i := range eps {
 		agents[i].endpoint = eps[i]
 		agents[i].clientPort = cports[i]
+		agents[i].advertiseClientPort = acports[i]
 		agents[i].peerPort = pports[i]
+		agents[i].advertisePeerPort = apports[i]
 		agents[i].failpointPort = fports[i]
 	}
 

tools/functional-tester/etcd-tester/member.go

@@ -29,23 +29,25 @@ import (
 )
 
 type member struct {
-	Agent        client.Agent
-	Endpoint     string
-	Name         string
-	ClientURL    string
-	PeerURL      string
-	FailpointURL string
+	Agent              client.Agent
+	Endpoint           string
+	Name               string
+	ClientURL          string
+	AdvertiseClientURL string
+	PeerURL            string
+	AdvertisePeerURL   string
+	FailpointURL       string
 }
 
-func (m *member) ClusterEntry() string { return m.Name + "=" + m.PeerURL }
+func (m *member) ClusterEntry() string { return m.Name + "=" + m.AdvertisePeerURL }
 
 func (m *member) Flags() []string {
 	return []string{
 		"--name", m.Name,
 		"--listen-client-urls", m.ClientURL,
-		"--advertise-client-urls", m.ClientURL,
+		"--advertise-client-urls", m.AdvertiseClientURL,
 		"--listen-peer-urls", m.PeerURL,
-		"--initial-advertise-peer-urls", m.PeerURL,
+		"--initial-advertise-peer-urls", m.AdvertisePeerURL,
 		"--initial-cluster-state", "new",
 		"--experimental-initial-corrupt-check",
 	}
@@ -54,7 +56,7 @@ func (m *member) Flags() []string {
 func (m *member) CheckCompact(rev int64) error {
 	cli, err := m.newClientV3()
 	if err != nil {
-		return fmt.Errorf("%v (endpoint %s)", err, m.ClientURL)
+		return fmt.Errorf("%v (endpoint %s)", err, m.AdvertiseClientURL)
 	}
 	defer cli.Close()
 
@@ -64,29 +66,29 @@ func (m *member) CheckCompact(rev int64) error {
 	cancel()
 
 	if !ok {
-		return fmt.Errorf("watch channel terminated (endpoint %s)", m.ClientURL)
+		return fmt.Errorf("watch channel terminated (endpoint %s)", m.AdvertiseClientURL)
 	}
 	if wr.CompactRevision != rev {
-		return fmt.Errorf("got compact revision %v, wanted %v (endpoint %s)", wr.CompactRevision, rev, m.ClientURL)
+		return fmt.Errorf("got compact revision %v, wanted %v (endpoint %s)", wr.CompactRevision, rev, m.AdvertiseClientURL)
 	}
 
 	return nil
 }
 
 func (m *member) Defrag() error {
-	plog.Printf("defragmenting %s\n", m.ClientURL)
+	plog.Printf("defragmenting %s\n", m.AdvertiseClientURL)
 	cli, err := m.newClientV3()
 	if err != nil {
 		return err
 	}
 	defer cli.Close()
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	_, err = cli.Defragment(ctx, m.ClientURL)
+	_, err = cli.Defragment(ctx, m.AdvertiseClientURL)
 	cancel()
 	if err != nil {
 		return err
 	}
-	plog.Printf("defragmented %s\n", m.ClientURL)
+	plog.Printf("defragmented %s\n", m.AdvertiseClientURL)
 	return nil
 }
 
@@ -114,7 +116,7 @@ func (m *member) Rev(ctx context.Context) (int64, error) {
 		return 0, err
 	}
 	defer cli.Close()
-	resp, err := cli.Status(ctx, m.ClientURL)
+	resp, err := cli.Status(ctx, m.AdvertiseClientURL)
 	if err != nil {
 		return 0, err
 	}
@@ -127,7 +129,7 @@ func (m *member) IsLeader() (bool, error) {
 		return false, err
 	}
 	defer cli.Close()
-	resp, err := cli.Status(context.Background(), m.ClientURL)
+	resp, err := cli.Status(context.Background(), m.AdvertiseClientURL)
 	if err != nil {
 		return false, err
 	}
@@ -137,7 +139,7 @@ func (m *member) IsLeader() (bool, error) {
 func (m *member) SetHealthKeyV3() error {
 	cli, err := m.newClientV3()
 	if err != nil {
-		return fmt.Errorf("%v (%s)", err, m.ClientURL)
+		return fmt.Errorf("%v (%s)", err, m.AdvertiseClientURL)
 	}
 	defer cli.Close()
 	// give enough time-out in case expensive requests (range/delete) are pending
@@ -145,14 +147,14 @@ func (m *member) SetHealthKeyV3() error {
 	_, err = cli.Put(ctx, "health", "good")
 	cancel()
 	if err != nil {
-		return fmt.Errorf("%v (%s)", err, m.ClientURL)
+		return fmt.Errorf("%v (%s)", err, m.AdvertiseClientURL)
 	}
 	return nil
 }
 
 func (m *member) newClientV3() (*clientv3.Client, error) {
 	return clientv3.New(clientv3.Config{
-		Endpoints:   []string{m.ClientURL},
+		Endpoints:   []string{m.AdvertiseClientURL},
 		DialTimeout: 5 * time.Second,
 	})
 }
@@ -163,7 +165,7 @@ func (m *member) dialGRPC() (*grpc.ClientConn, error) {
 
 // grpcAddr gets the host from clientURL so it works with grpc.Dial()
 func (m *member) grpcAddr() string {
-	u, err := url.Parse(m.ClientURL)
+	u, err := url.Parse(m.AdvertiseClientURL)
 	if err != nil {
 		panic(err)
 	}
@@ -171,7 +173,7 @@ func (m *member) grpcAddr() string {
 }
 
 func (m *member) peerPort() (port int) {
-	u, err := url.Parse(m.PeerURL)
+	u, err := url.Parse(m.AdvertisePeerURL)
 	if err != nil {
 		panic(err)
 	}

tools/functional-tester/scripts/agent-1.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+<<COMMENT
+# to run agent
+./scripts/agent-1.sh
+
+# to run with failpoints
+ETCD_EXEC_PATH=/etcd-failpoints ./scripts/agent-1.sh
+COMMENT
+
+if ! [[ "$0" =~ "scripts/agent-1.sh" ]]; then
+  echo "must be run from tools/functional-tester"
+  exit 255
+fi
+
+if [ -z "${ETCD_EXEC_PATH}" ]; then
+  ETCD_EXEC_PATH=/etcd
+	echo "Running agent without failpoints:" ${ETCD_EXEC_PATH}
+elif [[ "${ETCD_EXEC_PATH}" == "/etcd-failpoints" ]]; then
+	echo "Running agent with failpoints:" ${ETCD_EXEC_PATH}
+else
+  echo "Cannot find executable:" ${ETCD_EXEC_PATH}
+  exit 255
+fi
+
+rm -rf `pwd`/agent-1 && mkdir -p `pwd`/agent-1
+docker run \
+  --rm \
+  --net=host \
+  --name agent-1 \
+  --mount type=bind,source=`pwd`/agent-1,destination=/agent-1 \
+  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
+  /bin/bash -c "/etcd-agent \
+    --etcd-path ${ETCD_EXEC_PATH} \
+    --etcd-log-dir /agent-1 \
+    --port :19027 \
+    --failpoint-addr :7381"

tools/functional-tester/scripts/agent-2.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+<<COMMENT
+# to run agent
+./scripts/agent-2.sh
+
+# to run with failpoints
+ETCD_EXEC_PATH=/etcd-failpoints ./scripts/agent-2.sh
+COMMENT
+
+if ! [[ "$0" =~ "scripts/agent-2.sh" ]]; then
+  echo "must be run from tools/functional-tester"
+  exit 255
+fi
+
+if [ -z "${ETCD_EXEC_PATH}" ]; then
+  ETCD_EXEC_PATH=/etcd
+	echo "Running agent without failpoints:" ${ETCD_EXEC_PATH}
+elif [[ "${ETCD_EXEC_PATH}" == "/etcd-failpoints" ]]; then
+	echo "Running agent with failpoints:" ${ETCD_EXEC_PATH}
+else
+  echo "Cannot find executable:" ${ETCD_EXEC_PATH}
+  exit 255
+fi
+
+rm -rf `pwd`/agent-2 && mkdir -p `pwd`/agent-2
+docker run \
+  --rm \
+  --net=host \
+  --name agent-2 \
+  --mount type=bind,source=`pwd`/agent-2,destination=/agent-2 \
+  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
+  /bin/bash -c "/etcd-agent \
+    --etcd-path ${ETCD_EXEC_PATH} \
+    --etcd-log-dir /agent-2 \
+    --port :29027 \
+    --failpoint-addr :7382"

tools/functional-tester/scripts/agent-3.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+<<COMMENT
+# to run agent
+./scripts/agent-3.sh
+
+# to run with failpoints
+ETCD_EXEC_PATH=/etcd-failpoints ./scripts/agent-3.sh
+COMMENT
+
+if ! [[ "$0" =~ "scripts/agent-3.sh" ]]; then
+  echo "must be run from tools/functional-tester"
+  exit 255
+fi
+
+if [ -z "${ETCD_EXEC_PATH}" ]; then
+  ETCD_EXEC_PATH=/etcd
+	echo "Running agent without failpoints:" ${ETCD_EXEC_PATH}
+elif [[ "${ETCD_EXEC_PATH}" == "/etcd-failpoints" ]]; then
+	echo "Running agent with failpoints:" ${ETCD_EXEC_PATH}
+else
+  echo "Cannot find executable:" ${ETCD_EXEC_PATH}
+  exit 255
+fi
+
+rm -rf `pwd`/agent-3 && mkdir -p `pwd`/agent-3
+docker run \
+  --rm \
+  --net=host \
+  --name agent-3 \
+  --mount type=bind,source=`pwd`/agent-3,destination=/agent-3 \
+  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
+  /bin/bash -c "/etcd-agent \
+    --etcd-path ${ETCD_EXEC_PATH} \
+    --etcd-log-dir /agent-3 \
+    --port :39027 \
+    --failpoint-addr :7383"

tools/functional-tester/scripts/tester-limit.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+if ! [[ "$0" =~ "scripts/tester-limit.sh" ]]; then
+  echo "must be run from tools/functional-tester"
+  exit 255
+fi
+
+# to run only 1 test round
+docker run \
+  --rm \
+  --net=host \
+  --name tester \
+  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
+  /bin/bash -c "/etcd-tester \
+    --agent-endpoints '127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027' \
+    --client-ports 1379,2379,3379 \
+    --advertise-client-ports 13790,23790,33790 \
+    --peer-ports 1380,2380,3380 \
+    --advertise-peer-ports 13800,23800,33800 \
+    --limit 1 \
+    --stress-qps=2500 \
+    --stress-key-txn-count 100 \
+    --stress-key-txn-ops 10 \
+    --exit-on-failure"

tools/functional-tester/scripts/tester-runner.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+if ! [[ "$0" =~ "scripts/tester-runner.sh" ]]; then
+  echo "must be run from tools/functional-tester"
+  exit 255
+fi
+
+# to run with etcd-runner
+docker run \
+  --rm \
+  --net=host \
+  --name tester \
+  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
+  /bin/bash -c "/etcd-tester \
+    --agent-endpoints '127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027' \
+    --client-ports 1379,2379,3379 \
+    --advertise-client-ports 13790,23790,33790 \
+    --peer-ports 1380,2380,3380 \
+    --advertise-peer-ports 13800,23800,33800 \
+    --stress-qps=2500 \
+    --stress-key-txn-count 100 \
+    --stress-key-txn-ops 10 \
+    --etcd-runner /etcd-runner \
+    --stresser=keys,lease,election-runner,watch-runner,lock-racer-runner,lease-runner \
+    --exit-on-failure"

tools/functional-tester/scripts/tester.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+if ! [[ "$0" =~ "scripts/tester.sh" ]]; then
+  echo "must be run from tools/functional-tester"
+  exit 255
+fi
+
+docker run \
+  --rm \
+  --net=host \
+  --name tester \
+  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
+  /bin/bash -c "/etcd-tester \
+    --agent-endpoints '127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027' \
+    --client-ports 1379,2379,3379 \
+    --advertise-client-ports 13790,23790,33790 \
+    --peer-ports 1380,2380,3380 \
+    --advertise-peer-ports 13800,23800,33800 \
+    --stress-qps=2500 \
+    --stress-key-txn-count 100 \
+    --stress-key-txn-ops 10 \
+    --exit-on-failure"

tools/functional-tester/test

@@ -1,23 +0,0 @@
-#!/bin/sh -e
-set -x
-set -e
-
-# 1. build etcd binaries
-[ -f bin/etcd ] || ./build
-
-# 2. build agent & tester
-[ -f bin/etcd-agent -a -f bin/etcd-tester ] || ./tools/functional-tester/build
-
-# 3. build docker image (alpine based)
-mkdir -p ./tools/functional-tester/docker/bin
-cp -v bin/etcd-agent bin/etcd-tester bin/etcd ./tools/functional-tester/docker/bin
-docker-compose -f tools/functional-tester/docker/docker-compose.yml build
-
-# 4. create network (assumption - no overlaps)
-docker network ls | grep etcd-functional || docker network create --subnet 172.20.0.0/16 etcd-functional
-
-# 5. run cluster and tester (assumption - agents'll get first ip addresses)
-docker-compose -f tools/functional-tester/docker/docker-compose.yml up -d a1 a2 a3
-
-# 6. run tester
-docker-compose -f tools/functional-tester/docker/docker-compose.yml run tester

version/version.go

@@ -26,7 +26,7 @@ import (
 var (
 	// MinClusterVersion is the min cluster version this etcd binary is compatible with.
 	MinClusterVersion = "3.0.0"
-	Version           = "3.3.0"
+	Version           = "3.3.1"
 	APIVersion        = "unknown"
 
 	// Git SHA Value will be set during build