Merge pull request #16020 from tjungblu/putauthshort_3.4

[3.4] Early exit auth check on lease puts

.github/workflows/release.yaml

@@ -0,0 +1,32 @@
+name: Release
+on: [push, pull_request]
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - id: goversion
+      run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ${{ steps.goversion.outputs.goversion }}
+    - name: release
+      run: |
+        set -euo pipefail
+
+        git config --global user.email "github-action@etcd.io"
+        git config --global user.name "Github Action"
+        gpg --batch --gen-key <<EOF
+        %no-protection
+        Key-Type: 1
+        Key-Length: 2048
+        Subkey-Type: 1
+        Subkey-Length: 2048
+        Name-Real: Github Action
+        Name-Email: github-action@etcd.io
+        Expire-Date: 0
+        EOF
+        DRY_RUN=true ./scripts/release.sh --no-upload --no-docker-push --in-place 3.4.99
+    - name: test-image
+      run: |
+        VERSION=3.4.99 ./scripts/test_images.sh

.github/workflows/tests.yaml

@@ -0,0 +1,78 @@
+name: Tests
+on: [push, pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+        - linux-amd64-fmt
+        - linux-amd64-integration-1-cpu
+        - linux-amd64-integration-2-cpu
+        - linux-amd64-integration-4-cpu
+        - linux-amd64-functional
+        - linux-amd64-unit-4-cpu-race
+        - all-build
+        - linux-amd64-grpcproxy
+        - linux-amd64-e2e
+        - linux-386-unit
+    steps:
+    - uses: actions/checkout@v2
+    - id: goversion
+      run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
+    - uses: actions/setup-go@v2
+      with:
+        go-version: ${{ steps.goversion.outputs.goversion }}
+    - run: date
+    - env:
+        TARGET: ${{ matrix.target }}
+      run: |
+        set -euo pipefail
+
+        go version
+        echo ${GOROOT}
+        echo "${TARGET}"
+        case "${TARGET}" in
+          linux-amd64-fmt)
+            GOARCH=amd64 PASSES='fmt bom dep' ./test
+            ;;
+          linux-amd64-integration-1-cpu)
+            GOARCH=amd64 CPU=1 PASSES='integration' RACE='false' ./test
+            ;;
+          linux-amd64-integration-2-cpu)
+            GOARCH=amd64 CPU=2 PASSES='integration' RACE='false' ./test
+            ;;
+          linux-amd64-integration-4-cpu)
+            GOARCH=amd64 CPU=4 PASSES='integration' RACE='false' ./test
+            ;;
+          linux-amd64-functional)
+            ./build && GOARCH=amd64 PASSES='functional' ./test
+            ;;
+          linux-amd64-unit-4-cpu-race)
+            GOARCH=amd64 PASSES='unit' RACE='true' CPU='4' ./test -p=2
+            ;;
+          all-build)
+            GOARCH=amd64 PASSES='build' ./test
+            GOARCH=386 PASSES='build' ./test
+            GO_BUILD_FLAGS='-v' GOOS=darwin GOARCH=amd64 ./build
+            GO_BUILD_FLAGS='-v' GOOS=windows GOARCH=amd64 ./build
+            GO_BUILD_FLAGS='-v' GOARCH=arm ./build
+            GO_BUILD_FLAGS='-v' GOARCH=arm64 ./build
+            GO_BUILD_FLAGS='-v' GOARCH=ppc64le ./build
+            GO_BUILD_FLAGS='-v' GOARCH=s390x ./build
+            ;;
+          linux-amd64-grpcproxy)
+            PASSES='build grpcproxy' CPU='4' RACE='true' ./test
+            ;;
+          linux-amd64-e2e)
+            GOARCH=amd64 PASSES='build release e2e' ./test
+            ;;
+          linux-386-unit)
+            GOARCH=386 PASSES='unit' ./test
+            ;;
+          *)
+            echo "Failed to find target"
+            exit 1
+            ;;
+        esac

.github/workflows/trivy-nightly-scan.yaml

@@ -0,0 +1,37 @@
+name: Trivy Nightly Scan
+on:
+  schedule:
+    - cron: '0 2 * * *' # run at 2 AM UTC
+
+permissions: read-all 
+jobs:
+  nightly-scan:
+    name: Trivy Scan nightly
+    strategy:
+      fail-fast: false
+      matrix:
+        # maintain the versions of etcd that need to be actively
+        # security scanned
+        versions: [v3.4.22]
+    permissions:
+      security-events: write  # for github/codeql-action/upload-sarif to upload SARIF results
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          ref: release-3.4
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@9ab158e8597f3b310480b9a69402b419bc03dbd5 # master
+        with:
+          image-ref: 'gcr.io/etcd-development/etcd:${{ matrix.versions }}'
+          severity: 'CRITICAL,HIGH'
+          format: 'template'
+          template: '@/contrib/sarif.tpl'
+          output: 'trivy-results-3-4.sarif'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@a669cc5936cc5e1b6a362ec1ff9e410dc570d190 # v2.1.36
+        with:
+          sarif_file: 'trivy-results-3-4.sarif'

.gitignore

@@ -31,6 +31,7 @@ vendor/**/*
 !vendor/**/License*
 !vendor/**/LICENCE*
 !vendor/**/LICENSE*
+!vendor/modules.txt
 vendor/**/*_test.go
 
 *.bak

.go-version

@@ -0,0 +1 @@
+1.19.10

.travis.yml

@@ -1,94 +0,0 @@
-language: go
-go_import_path: go.etcd.io/etcd
-
-sudo: required
-
-services: docker
-
-go:
-- 1.12.12
-
-notifications:
-  on_success: never
-  on_failure: never
-
-env:
-  matrix:
-  - TARGET=linux-amd64-fmt
-  - TARGET=linux-amd64-integration-1-cpu
-  - TARGET=linux-amd64-integration-2-cpu
-  - TARGET=linux-amd64-integration-4-cpu
-  - TARGET=linux-amd64-functional
-  - TARGET=linux-amd64-unit
-  - TARGET=all-build
-  - TARGET=linux-amd64-grpcproxy
-  - TARGET=linux-386-unit
-
-matrix:
-  fast_finish: true
-  allow_failures:
-  - go: 1.12.12
-    env: TARGET=linux-amd64-grpcproxy
-  - go: 1.12.12
-    env: TARGET=linux-386-unit
-
-before_install:
-- if [[ $TRAVIS_GO_VERSION == 1.* ]]; then docker pull gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION}; fi
-
-install:
-- go get -t -v -d ./...
-
-script:
- - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
- - >
-    case "${TARGET}" in
-      linux-amd64-fmt)
-        docker run --rm \
-          --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 PASSES='fmt bom dep' ./test"
-        ;;
-      linux-amd64-integration-1-cpu)
-        docker run --rm \
-          --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' ./test"
-        ;;
-      linux-amd64-integration-2-cpu)
-        docker run --rm \
-          --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' ./test"
-        ;;
-      linux-amd64-integration-4-cpu)
-        docker run --rm \
-          --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' ./test"
-        ;;
-      linux-amd64-functional)
-        docker run --rm \
-          --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "./build && GOARCH=amd64 PASSES='functional' ./test"
-        ;;
-      linux-amd64-unit)
-        docker run --rm \
-          --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 PASSES='unit' ./test"
-        ;;
-      all-build)
-        docker run --rm \
-          --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 PASSES='build' ./test \
-            && GOARCH=386 PASSES='build' ./test \
-            && GO_BUILD_FLAGS='-v' GOOS=darwin GOARCH=amd64 ./build \
-            && GO_BUILD_FLAGS='-v' GOOS=windows GOARCH=amd64 ./build \
-            && GO_BUILD_FLAGS='-v' GOARCH=arm ./build \
-            && GO_BUILD_FLAGS='-v' GOARCH=arm64 ./build \
-            && GO_BUILD_FLAGS='-v' GOARCH=ppc64le ./build"
-        ;;
-      linux-amd64-grpcproxy)
-        sudo HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test
-        ;;
-      linux-386-unit)
-        docker run --rm \
-          --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=386 PASSES='unit' ./test"
-        ;;
-    esac

Dockerfile-release

@@ -1,15 +1,10 @@
-FROM k8s.gcr.io/debian-base:v1.0.0
+FROM --platform=linux/amd64 gcr.io/distroless/static-debian11
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/
-RUN mkdir -p /var/etcd/
-RUN mkdir -p /var/lib/etcd/
 
-# Alpine Linux doesn't use pam, which means that there is no /etc/nsswitch.conf,
-# but Golang relies on /etc/nsswitch.conf to check the order of DNS resolving
-# (see https://github.com/golang/go/commit/9dee7771f561cf6aee081c0af6658cc81fac3918)
-# To fix this we just create /etc/nsswitch.conf and add the following line:
-RUN echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
+WORKDIR /var/etcd/
+WORKDIR /var/lib/etcd/
 
 EXPOSE 2379 2380
 

Dockerfile-release.arm64

@@ -1,9 +1,10 @@
-FROM k8s.gcr.io/debian-base-arm64:v1.0.0
+FROM --platform=linux/arm64 gcr.io/distroless/static-debian11
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/
-ADD var/etcd /var/etcd
-ADD var/lib/etcd /var/lib/etcd
+
+WORKDIR /var/etcd/
+WORKDIR /var/lib/etcd/
 
 EXPOSE 2379 2380
 

Dockerfile-release.ppc64le

@@ -1,9 +1,10 @@
-FROM k8s.gcr.io/debian-base-ppc64le:v1.0.0
+FROM --platform=linux/ppc64le gcr.io/distroless/static-debian11
 
 ADD etcd /usr/local/bin/
 ADD etcdctl /usr/local/bin/
-ADD var/etcd /var/etcd
-ADD var/lib/etcd /var/lib/etcd
+
+WORKDIR /var/etcd/
+WORKDIR /var/lib/etcd/
 
 EXPOSE 2379 2380
 

Documentation/dev-internal/release.md

@@ -128,7 +128,7 @@ for TARGET_ARCH in "amd64" "arm64" "ppc64le"; do
   TAG=quay.io/coreos/etcd GOARCH=${TARGET_ARCH} \
     BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} \
     BUILDDIR=release \
-    ./scripts/build-docker ${VERSION}
+    ./scripts/build-docker.sh ${VERSION}
 done
 ```
 

Makefile

@@ -51,7 +51,7 @@ docker-remove:
 
 
 
-GO_VERSION ?= 1.12.12
+GO_VERSION ?= 1.19.9
 ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound")
 
 TEST_SUFFIX = $(shell date +%s | base64 | head -c 15)
@@ -65,11 +65,11 @@ endif
 
 
 # Example:
-#   GO_VERSION=1.12.12 make build-docker-test
+#   GO_VERSION=1.12.17 make build-docker-test
 #   make build-docker-test
 #
 #   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
-#   GO_VERSION=1.12.12 make push-docker-test
+#   GO_VERSION=1.12.17 make push-docker-test
 #   make push-docker-test
 #
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com

auth/authpb/auth.pb.go

@@ -2,16 +2,18 @@
 // source: auth.proto
 
 /*
-	Package authpb is a generated protocol buffer package.
+Package authpb is a generated protocol buffer package.
 
-	It is generated from these files:
-		auth.proto
+It is generated from these files:
 
-	It has these top-level messages:
-		UserAddOptions
-		User
-		Permission
-		Role
+	auth.proto
+
+It has these top-level messages:
+
+	UserAddOptions
+	User
+	Permission
+	Role
 */
 package authpb
 

auth/jwt.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/golang-jwt/jwt"
 	"go.uber.org/zap"
 )
 
@@ -42,7 +42,7 @@ func (t *tokenJWT) info(ctx context.Context, token string, rev uint64) (*AuthInf
 	// rev isn't used in JWT, it is only used in simple token
 	var (
 		username string
-		revision uint64
+		revision float64
 	)
 
 	parsed, err := jwt.Parse(token, func(token *jwt.Token) (interface{}, error) {
@@ -82,10 +82,19 @@ func (t *tokenJWT) info(ctx context.Context, token string, rev uint64) (*AuthInf
 		return nil, false
 	}
 
-	username = claims["username"].(string)
-	revision = uint64(claims["revision"].(float64))
+	username, ok = claims["username"].(string)
+	if !ok {
+		t.lg.Warn("failed to obtain user claims from jwt token")
+		return nil, false
+	}
 
-	return &AuthInfo{Username: username, Revision: revision}, true
+	revision, ok = claims["revision"].(float64)
+	if !ok {
+		t.lg.Warn("failed to obtain revision claims from jwt token")
+		return nil, false
+	}
+
+	return &AuthInfo{Username: username, Revision: uint64(revision)}, true
 }
 
 func (t *tokenJWT) assign(ctx context.Context, username string, revision uint64) (string, error) {

auth/jwt_test.go

@@ -18,7 +18,10 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
+	"github.com/golang-jwt/jwt"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 )
 
@@ -202,3 +205,75 @@ func TestJWTBad(t *testing.T) {
 func testJWTOpts() string {
 	return fmt.Sprintf("%s,pub-key=%s,priv-key=%s,sign-method=RS256", tokenTypeJWT, jwtRSAPubKey, jwtRSAPrivKey)
 }
+
+func TestJWTTokenWithMissingFields(t *testing.T) {
+	testCases := []struct {
+		name        string
+		username    string // An empty string means not present
+		revision    uint64 // 0 means not present
+		expectValid bool
+	}{
+		{
+			name:        "valid token",
+			username:    "hello",
+			revision:    100,
+			expectValid: true,
+		},
+		{
+			name:        "no username",
+			username:    "",
+			revision:    100,
+			expectValid: false,
+		},
+		{
+			name:        "no revision",
+			username:    "hello",
+			revision:    0,
+			expectValid: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		optsMap := map[string]string{
+			"priv-key":    jwtRSAPrivKey,
+			"sign-method": "RS256",
+			"ttl":         "1h",
+		}
+
+		t.Run(tc.name, func(t *testing.T) {
+			// prepare claims
+			claims := jwt.MapClaims{
+				"exp": time.Now().Add(time.Hour).Unix(),
+			}
+			if tc.username != "" {
+				claims["username"] = tc.username
+			}
+			if tc.revision != 0 {
+				claims["revision"] = tc.revision
+			}
+
+			// generate a JWT token with the given claims
+			var opts jwtOptions
+			err := opts.ParseWithDefaults(optsMap)
+			require.NoError(t, err)
+			key, err := opts.Key()
+			require.NoError(t, err)
+
+			tk := jwt.NewWithClaims(opts.SignMethod, claims)
+			token, err := tk.SignedString(key)
+			require.NoError(t, err)
+
+			// verify the token
+			jwtProvider, err := newTokenProviderJWT(zap.NewNop(), optsMap)
+			require.NoError(t, err)
+			ai, ok := jwtProvider.info(context.TODO(), token, 123)
+
+			require.Equal(t, tc.expectValid, ok)
+			if ok {
+				require.Equal(t, tc.username, ai.Username)
+				require.Equal(t, tc.revision, ai.Revision)
+			}
+		})
+	}
+}

auth/options.go

@@ -21,7 +21,7 @@ import (
 	"io/ioutil"
 	"time"
 
-	jwt "github.com/dgrijalva/jwt-go"
+	"github.com/golang-jwt/jwt"
 )
 
 const (

auth/range_perm_cache.go

@@ -76,8 +76,10 @@ func checkKeyInterval(
 	cachedPerms *unifiedRangePermissions,
 	key, rangeEnd []byte,
 	permtyp authpb.Permission_Type) bool {
-	if len(rangeEnd) == 1 && rangeEnd[0] == 0 {
+	if isOpenEnded(rangeEnd) {
 		rangeEnd = nil
+		// nil rangeEnd will be converetd to []byte{}, the largest element of BytesAffineComparable,
+		// in NewBytesAffineInterval().
 	}
 
 	ivl := adt.NewBytesAffineInterval(key, rangeEnd)
@@ -113,41 +115,99 @@ func checkKeyPoint(lg *zap.Logger, cachedPerms *unifiedRangePermissions, key []b
 	return false
 }
 
-func (as *authStore) isRangeOpPermitted(tx backend.BatchTx, userName string, key, rangeEnd []byte, permtyp authpb.Permission_Type) bool {
-	// assumption: tx is Lock()ed
-	_, ok := as.rangePermCache[userName]
+func (as *authStore) isRangeOpPermitted(userName string, key, rangeEnd []byte, permtyp authpb.Permission_Type) bool {
+	as.rangePermCacheMu.RLock()
+	defer as.rangePermCacheMu.RUnlock()
+
+	rangePerm, ok := as.rangePermCache[userName]
 	if !ok {
-		perms := getMergedPerms(as.lg, tx, userName)
-		if perms == nil {
-			if as.lg != nil {
-				as.lg.Warn(
-					"failed to create a merged permission",
-					zap.String("user-name", userName),
-				)
-			} else {
-				plog.Errorf("failed to create a unified permission of user %s", userName)
-			}
-			return false
-		}
-		as.rangePermCache[userName] = perms
+		as.lg.Error(
+			"user doesn't exist",
+			zap.String("user-name", userName),
+		)
+		return false
 	}
 
 	if len(rangeEnd) == 0 {
-		return checkKeyPoint(as.lg, as.rangePermCache[userName], key, permtyp)
+		return checkKeyPoint(as.lg, rangePerm, key, permtyp)
 	}
 
-	return checkKeyInterval(as.lg, as.rangePermCache[userName], key, rangeEnd, permtyp)
+	return checkKeyInterval(as.lg, rangePerm, key, rangeEnd, permtyp)
 }
 
-func (as *authStore) clearCachedPerm() {
+func (as *authStore) refreshRangePermCache(tx backend.BatchTx) {
+	// Note that every authentication configuration update calls this method and it invalidates the entire
+	// rangePermCache and reconstruct it based on information of users and roles stored in the backend.
+	// This can be a costly operation.
+	as.rangePermCacheMu.Lock()
+	defer as.rangePermCacheMu.Unlock()
+
 	as.rangePermCache = make(map[string]*unifiedRangePermissions)
-}
 
-func (as *authStore) invalidateCachedPerm(userName string) {
-	delete(as.rangePermCache, userName)
+	users := getAllUsers(as.lg, tx)
+	for _, user := range users {
+		userName := string(user.Name)
+		perms := getMergedPerms(as.lg, tx, userName)
+		if perms == nil {
+			as.lg.Error(
+				"failed to create a merged permission",
+				zap.String("user-name", userName),
+			)
+			continue
+		}
+		as.rangePermCache[userName] = perms
+	}
 }
 
 type unifiedRangePermissions struct {
 	readPerms  adt.IntervalTree
 	writePerms adt.IntervalTree
 }
+
+// Constraints related to key range
+// Assumptions:
+// a1. key must be non-nil
+// a2. []byte{} (in the case of string, "") is not a valid key of etcd
+// For representing an open-ended range, BytesAffineComparable uses []byte{} as the largest element.
+// a3. []byte{0x00} is the minimum valid etcd key
+//
+// Based on the above assumptions, key and rangeEnd must follow below rules:
+// b1. for representing a single key point, rangeEnd should be nil or zero length byte array (in the case of string, "")
+// Rule a2 guarantees that (X, []byte{}) for any X is not a valid range. So such ranges can be used for representing
+// a single key permission.
+//
+// b2. key range with upper limit, like (X, Y), larger or equal to X and smaller than Y
+//
+// b3. key range with open-ended, like (X, <open ended>), is represented like (X, []byte{0x00})
+// Because of rule a3, if we have (X, []byte{0x00}), such a range represents an empty range and makes no sense to have
+// such a permission. So we use []byte{0x00} for representing an open-ended permission.
+// Note that rangeEnd with []byte{0x00} will be converted into []byte{} before inserted into the interval tree
+// (rule a2 ensures that this is the largest element).
+// Special range like key = []byte{0x00} and rangeEnd = []byte{0x00} is treated as a range which matches with all keys.
+//
+// Treating a range whose rangeEnd with []byte{0x00} as an open-ended comes from the rules of Range() and Watch() API.
+
+func isOpenEnded(rangeEnd []byte) bool { // check rule b3
+	return len(rangeEnd) == 1 && rangeEnd[0] == 0
+}
+
+func isValidPermissionRange(key, rangeEnd []byte) bool {
+	if len(key) == 0 {
+		return false
+	}
+	if rangeEnd == nil || len(rangeEnd) == 0 { // ensure rule b1
+		return true
+	}
+
+	begin := adt.BytesAffineComparable(key)
+	end := adt.BytesAffineComparable(rangeEnd)
+	if begin.Compare(end) == -1 { // rule b2
+		return true
+	}
+
+	if isOpenEnded(rangeEnd) {
+		return true
+	}
+
+	return false
+}

auth/range_perm_cache_test.go

@@ -45,6 +45,26 @@ func TestRangePermission(t *testing.T) {
 			[]byte("a"), []byte("f"),
 			true,
 		},
+		{
+			[]adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte("f"))},
+			[]byte("a"), []byte{},
+			false,
+		},
+		{
+			[]adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte{})},
+			[]byte("a"), []byte{},
+			true,
+		},
+		{
+			[]adt.Interval{adt.NewBytesAffineInterval([]byte{0x00}, []byte{})},
+			[]byte("a"), []byte{},
+			true,
+		},
+		{
+			[]adt.Interval{adt.NewBytesAffineInterval([]byte{0x00}, []byte{})},
+			[]byte{0x00}, []byte{},
+			true,
+		},
 	}
 
 	for i, tt := range tests {
@@ -86,6 +106,16 @@ func TestKeyPermission(t *testing.T) {
 			[]byte("f"),
 			false,
 		},
+		{
+			[]adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte("c"), []byte{})},
+			[]byte("f"),
+			true,
+		},
+		{
+			[]adt.Interval{adt.NewBytesAffineInterval([]byte("a"), []byte("d")), adt.NewBytesAffineInterval([]byte("a"), []byte("b")), adt.NewBytesAffineInterval([]byte{0x00}, []byte{})},
+			[]byte("f"),
+			true,
+		},
 	}
 
 	for i, tt := range tests {
@@ -100,3 +130,88 @@ func TestKeyPermission(t *testing.T) {
 		}
 	}
 }
+
+func TestRangeCheck(t *testing.T) {
+	tests := []struct {
+		name     string
+		key      []byte
+		rangeEnd []byte
+		want     bool
+	}{
+		{
+			name:     "valid single key",
+			key:      []byte("a"),
+			rangeEnd: []byte(""),
+			want:     true,
+		},
+		{
+			name:     "valid single key",
+			key:      []byte("a"),
+			rangeEnd: nil,
+			want:     true,
+		},
+		{
+			name:     "valid key range, key < rangeEnd",
+			key:      []byte("a"),
+			rangeEnd: []byte("b"),
+			want:     true,
+		},
+		{
+			name:     "invalid empty key range, key == rangeEnd",
+			key:      []byte("a"),
+			rangeEnd: []byte("a"),
+			want:     false,
+		},
+		{
+			name:     "invalid empty key range, key > rangeEnd",
+			key:      []byte("b"),
+			rangeEnd: []byte("a"),
+			want:     false,
+		},
+		{
+			name:     "invalid key, key must not be \"\"",
+			key:      []byte(""),
+			rangeEnd: []byte("a"),
+			want:     false,
+		},
+		{
+			name:     "invalid key range, key must not be \"\"",
+			key:      []byte(""),
+			rangeEnd: []byte(""),
+			want:     false,
+		},
+		{
+			name:     "invalid key range, key must not be \"\"",
+			key:      []byte(""),
+			rangeEnd: []byte("\x00"),
+			want:     false,
+		},
+		{
+			name:     "valid single key (not useful in practice)",
+			key:      []byte("\x00"),
+			rangeEnd: []byte(""),
+			want:     true,
+		},
+		{
+			name:     "valid key range, larger or equals to \"a\"",
+			key:      []byte("a"),
+			rangeEnd: []byte("\x00"),
+			want:     true,
+		},
+		{
+			name:     "valid key range, which includes all keys",
+			key:      []byte("\x00"),
+			rangeEnd: []byte("\x00"),
+			want:     true,
+		},
+	}
+
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := isValidPermissionRange(tt.key, tt.rangeEnd)
+			if result != tt.want {
+				t.Errorf("#%d: result=%t, want=%t", i, result, tt.want)
+			}
+		})
+	}
+}

auth/simple_token.go

@@ -159,6 +159,11 @@ func (t *tokenSimple) invalidateUser(username string) {
 }
 
 func (t *tokenSimple) enable() {
+	t.simpleTokensMu.Lock()
+	defer t.simpleTokensMu.Unlock()
+	if t.simpleTokenKeeper != nil { // already enabled
+		return
+	}
 	if t.simpleTokenTTL <= 0 {
 		t.simpleTokenTTL = simpleTokenTTLDefault
 	}

auth/store.go

@@ -59,6 +59,7 @@ var (
 	ErrRoleAlreadyExist     = errors.New("auth: role already exists")
 	ErrRoleNotFound         = errors.New("auth: role not found")
 	ErrRoleEmpty            = errors.New("auth: role name is empty")
+	ErrPermissionNotGiven   = errors.New("auth: permission not given")
 	ErrAuthFailed           = errors.New("auth: authentication failed, invalid user ID or password")
 	ErrNoPasswordUser       = errors.New("auth: authentication failed, password was given for no password user")
 	ErrPermissionDenied     = errors.New("auth: permission denied")
@@ -215,7 +216,14 @@ type authStore struct {
 	enabled   bool
 	enabledMu sync.RWMutex
 
-	rangePermCache map[string]*unifiedRangePermissions // username -> unifiedRangePermissions
+	// rangePermCache needs to be protected by rangePermCacheMu
+	// rangePermCacheMu needs to be write locked only in initialization phase or configuration changes
+	// Hot paths like Range(), needs to acquire read lock for improving performance
+	//
+	// Note that BatchTx and ReadTx cannot be a mutex for rangePermCache because they are independent resources
+	// see also: https://github.com/etcd-io/etcd/pull/13920#discussion_r849114855
+	rangePermCache   map[string]*unifiedRangePermissions // username -> unifiedRangePermissions
+	rangePermCacheMu sync.RWMutex
 
 	tokenProvider       TokenProvider
 	syncConsistentIndex saveConsistentIndexFunc
@@ -258,7 +266,7 @@ func (as *authStore) AuthEnable() error {
 	as.enabled = true
 	as.tokenProvider.enable()
 
-	as.rangePermCache = make(map[string]*unifiedRangePermissions)
+	as.refreshRangePermCache(tx)
 
 	as.setRevision(getRevision(tx))
 
@@ -395,11 +403,15 @@ func (as *authStore) Recover(be backend.Backend) {
 	}
 
 	as.setRevision(getRevision(tx))
+	as.refreshRangePermCache(tx)
 
 	tx.Unlock()
 
 	as.enabledMu.Lock()
 	as.enabled = enabled
+	if enabled {
+		as.tokenProvider.enable()
+	}
 	as.enabledMu.Unlock()
 }
 
@@ -454,6 +466,7 @@ func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse,
 
 	as.commitRevision(tx)
 	as.saveConsistentIndex(tx)
+	as.refreshRangePermCache(tx)
 
 	if as.lg != nil {
 		as.lg.Info("added a user", zap.String("user-name", r.Name))
@@ -486,8 +499,8 @@ func (as *authStore) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDelete
 
 	as.commitRevision(tx)
 	as.saveConsistentIndex(tx)
+	as.refreshRangePermCache(tx)
 
-	as.invalidateCachedPerm(r.Name)
 	as.tokenProvider.invalidateUser(r.Name)
 
 	if as.lg != nil {
@@ -539,8 +552,8 @@ func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*p
 
 	as.commitRevision(tx)
 	as.saveConsistentIndex(tx)
+	as.refreshRangePermCache(tx)
 
-	as.invalidateCachedPerm(r.Name)
 	as.tokenProvider.invalidateUser(r.Name)
 
 	if as.lg != nil {
@@ -592,10 +605,9 @@ func (as *authStore) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUser
 
 	putUser(as.lg, tx, user)
 
-	as.invalidateCachedPerm(r.User)
-
 	as.commitRevision(tx)
 	as.saveConsistentIndex(tx)
+	as.refreshRangePermCache(tx)
 
 	if as.lg != nil {
 		as.lg.Info(
@@ -679,10 +691,9 @@ func (as *authStore) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUs
 
 	putUser(as.lg, tx, updatedUser)
 
-	as.invalidateCachedPerm(r.Name)
-
 	as.commitRevision(tx)
 	as.saveConsistentIndex(tx)
+	as.refreshRangePermCache(tx)
 
 	if as.lg != nil {
 		as.lg.Info(
@@ -752,12 +763,9 @@ func (as *authStore) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest)
 
 	putRole(as.lg, tx, updatedRole)
 
-	// TODO(mitake): currently single role update invalidates every cache
-	// It should be optimized.
-	as.clearCachedPerm()
-
 	as.commitRevision(tx)
 	as.saveConsistentIndex(tx)
+	as.refreshRangePermCache(tx)
 
 	if as.lg != nil {
 		as.lg.Info(
@@ -813,11 +821,11 @@ func (as *authStore) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDelete
 
 		putUser(as.lg, tx, updatedUser)
 
-		as.invalidateCachedPerm(string(user.Name))
 	}
 
 	as.commitRevision(tx)
 	as.saveConsistentIndex(tx)
+	as.refreshRangePermCache(tx)
 
 	if as.lg != nil {
 		as.lg.Info("deleted a role", zap.String("role-name", r.Role))
@@ -877,6 +885,13 @@ func (perms permSlice) Swap(i, j int) {
 }
 
 func (as *authStore) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (*pb.AuthRoleGrantPermissionResponse, error) {
+	if r.Perm == nil {
+		return nil, ErrPermissionNotGiven
+	}
+	if !isValidPermissionRange(r.Perm.Key, r.Perm.RangeEnd) {
+		return nil, ErrInvalidAuthMgmt
+	}
+
 	tx := as.be.BatchTx()
 	tx.Lock()
 	defer tx.Unlock()
@@ -907,12 +922,9 @@ func (as *authStore) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (
 
 	putRole(as.lg, tx, role)
 
-	// TODO(mitake): currently single role update invalidates every cache
-	// It should be optimized.
-	as.clearCachedPerm()
-
 	as.commitRevision(tx)
 	as.saveConsistentIndex(tx)
+	as.refreshRangePermCache(tx)
 
 	if as.lg != nil {
 		as.lg.Info(
@@ -973,7 +985,7 @@ func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeE
 		return nil
 	}
 
-	if as.isRangeOpPermitted(tx, userName, key, rangeEnd, permTyp) {
+	if as.isRangeOpPermitted(userName, key, rangeEnd, permTyp) {
 		return nil
 	}
 
@@ -1039,7 +1051,15 @@ func getUser(lg *zap.Logger, tx backend.BatchTx, username string) *authpb.User {
 }
 
 func getAllUsers(lg *zap.Logger, tx backend.BatchTx) []*authpb.User {
-	_, vs := tx.UnsafeRange(authUsersBucketName, []byte{0}, []byte{0xff}, -1)
+	var vs [][]byte
+	err := tx.UnsafeForEach(authUsersBucketName, func(k []byte, v []byte) error {
+		vs = append(vs, v)
+		return nil
+	})
+	if err != nil {
+		lg.Panic("failed to get users",
+			zap.Error(err))
+	}
 	if len(vs) == 0 {
 		return nil
 	}
@@ -1192,6 +1212,8 @@ func NewAuthStore(lg *zap.Logger, be backend.Backend, tp TokenProvider, bcryptCo
 
 	as.setupMetricsReporter()
 
+	as.refreshRangePermCache(tx)
+
 	tx.Unlock()
 	be.ForceCommit()
 

auth/store_test.go

@@ -16,6 +16,7 @@ package auth
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"reflect"
@@ -28,6 +29,7 @@ import (
 	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
 	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
 	"go.etcd.io/etcd/mvcc/backend"
+	"go.etcd.io/etcd/pkg/adt"
 
 	"go.uber.org/zap"
 	"golang.org/x/crypto/bcrypt"
@@ -151,7 +153,8 @@ func TestUserAdd(t *testing.T) {
 	as, tearDown := setupAuthStore(t)
 	defer tearDown(t)
 
-	ua := &pb.AuthUserAddRequest{Name: "foo", Options: &authpb.UserAddOptions{NoPassword: false}}
+	const userName = "foo"
+	ua := &pb.AuthUserAddRequest{Name: userName, Options: &authpb.UserAddOptions{NoPassword: false}}
 	_, err := as.UserAdd(ua) // add an existing user
 	if err == nil {
 		t.Fatalf("expected %v, got %v", ErrUserAlreadyExist, err)
@@ -165,6 +168,11 @@ func TestUserAdd(t *testing.T) {
 	if err != ErrUserEmpty {
 		t.Fatal(err)
 	}
+
+	if _, ok := as.rangePermCache[userName]; !ok {
+		t.Fatalf("user %s should be added but it doesn't exist in rangePermCache", userName)
+
+	}
 }
 
 func TestRecover(t *testing.T) {
@@ -179,6 +187,30 @@ func TestRecover(t *testing.T) {
 	}
 }
 
+func TestRecoverWithEmptyRangePermCache(t *testing.T) {
+	as, tearDown := setupAuthStore(t)
+	defer as.Close()
+	defer tearDown(t)
+
+	as.enabled = false
+	as.rangePermCache = map[string]*unifiedRangePermissions{}
+	as.Recover(as.be)
+
+	if !as.IsAuthEnabled() {
+		t.Fatalf("expected auth enabled got disabled")
+	}
+
+	if len(as.rangePermCache) != 2 {
+		t.Fatalf("rangePermCache should have permission information for 2 users (\"root\" and \"foo\"), but has %d information", len(as.rangePermCache))
+	}
+	if _, ok := as.rangePermCache["root"]; !ok {
+		t.Fatal("user \"root\" should be created by setupAuthStore() but doesn't exist in rangePermCache")
+	}
+	if _, ok := as.rangePermCache["foo"]; !ok {
+		t.Fatal("user \"foo\" should be created by setupAuthStore() but doesn't exist in rangePermCache")
+	}
+}
+
 func TestCheckPassword(t *testing.T) {
 	as, tearDown := setupAuthStore(t)
 	defer tearDown(t)
@@ -213,7 +245,8 @@ func TestUserDelete(t *testing.T) {
 	defer tearDown(t)
 
 	// delete an existing user
-	ud := &pb.AuthUserDeleteRequest{Name: "foo"}
+	const userName = "foo"
+	ud := &pb.AuthUserDeleteRequest{Name: userName}
 	_, err := as.UserDelete(ud)
 	if err != nil {
 		t.Fatal(err)
@@ -227,6 +260,47 @@ func TestUserDelete(t *testing.T) {
 	if err != ErrUserNotFound {
 		t.Fatalf("expected %v, got %v", ErrUserNotFound, err)
 	}
+
+	if _, ok := as.rangePermCache[userName]; ok {
+		t.Fatalf("user %s should be deleted but it exists in rangePermCache", userName)
+
+	}
+}
+
+func TestUserDeleteAndPermCache(t *testing.T) {
+	as, tearDown := setupAuthStore(t)
+	defer tearDown(t)
+
+	// delete an existing user
+	const deletedUserName = "foo"
+	ud := &pb.AuthUserDeleteRequest{Name: deletedUserName}
+	_, err := as.UserDelete(ud)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// delete a non-existing user
+	_, err = as.UserDelete(ud)
+	if err != ErrUserNotFound {
+		t.Fatalf("expected %v, got %v", ErrUserNotFound, err)
+	}
+
+	if _, ok := as.rangePermCache[deletedUserName]; ok {
+		t.Fatalf("user %s should be deleted but it exists in rangePermCache", deletedUserName)
+	}
+
+	// add a new user
+	const newUser = "bar"
+	ua := &pb.AuthUserAddRequest{Name: newUser, Options: &authpb.UserAddOptions{NoPassword: false}}
+	_, err = as.UserAdd(ua)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := as.rangePermCache[newUser]; !ok {
+		t.Fatalf("user %s should exist but it doesn't exist in rangePermCache", deletedUserName)
+
+	}
 }
 
 func TestUserChangePassword(t *testing.T) {
@@ -445,6 +519,162 @@ func TestRoleGrantPermission(t *testing.T) {
 	if !reflect.DeepEqual(perm, r.Perm[0]) {
 		t.Errorf("expected %v, got %v", perm, r.Perm[0])
 	}
+
+	// trying to grant nil permissions returns an error (and doesn't change the actual permissions!)
+	_, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{
+		Name: "role-test-1",
+	})
+
+	if err != ErrPermissionNotGiven {
+		t.Error(err)
+	}
+
+	r, err = as.RoleGet(&pb.AuthRoleGetRequest{Role: "role-test-1"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(perm, r.Perm[0]) {
+		t.Errorf("expected %v, got %v", perm, r.Perm[0])
+	}
+}
+
+func TestRoleGrantInvalidPermission(t *testing.T) {
+	as, tearDown := setupAuthStore(t)
+	defer tearDown(t)
+
+	_, err := as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test-1"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name string
+		perm *authpb.Permission
+		want error
+	}{
+		{
+			name: "valid range",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte("Keys"),
+				RangeEnd: []byte("RangeEnd"),
+			},
+			want: nil,
+		},
+		{
+			name: "invalid range: nil key",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      nil,
+				RangeEnd: []byte("RangeEnd"),
+			},
+			want: ErrInvalidAuthMgmt,
+		},
+		{
+			name: "valid range: single key",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte("Keys"),
+				RangeEnd: nil,
+			},
+			want: nil,
+		},
+		{
+			name: "valid range: single key",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte("Keys"),
+				RangeEnd: []byte{},
+			},
+			want: nil,
+		},
+		{
+			name: "invalid range: empty (Key == RangeEnd)",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte("a"),
+				RangeEnd: []byte("a"),
+			},
+			want: ErrInvalidAuthMgmt,
+		},
+		{
+			name: "invalid range: empty (Key > RangeEnd)",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte("b"),
+				RangeEnd: []byte("a"),
+			},
+			want: ErrInvalidAuthMgmt,
+		},
+		{
+			name: "invalid range: length of key is 0",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte(""),
+				RangeEnd: []byte("a"),
+			},
+			want: ErrInvalidAuthMgmt,
+		},
+		{
+			name: "invalid range: length of key is 0",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte(""),
+				RangeEnd: []byte(""),
+			},
+			want: ErrInvalidAuthMgmt,
+		},
+		{
+			name: "invalid range: length of key is 0",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte(""),
+				RangeEnd: []byte{0x00},
+			},
+			want: ErrInvalidAuthMgmt,
+		},
+		{
+			name: "valid range: single key permission for []byte{0x00}",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte{0x00},
+				RangeEnd: []byte(""),
+			},
+			want: nil,
+		},
+		{
+			name: "valid range: \"a\" or larger keys",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte("a"),
+				RangeEnd: []byte{0x00},
+			},
+			want: nil,
+		},
+		{
+			name: "valid range: the entire keys",
+			perm: &authpb.Permission{
+				PermType: authpb.WRITE,
+				Key:      []byte{0x00},
+				RangeEnd: []byte{0x00},
+			},
+			want: nil,
+		},
+	}
+
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{
+				Name: "role-test-1",
+				Perm: tt.perm,
+			})
+
+			if !errors.Is(err, tt.want) {
+				t.Errorf("#%d: result=%t, want=%t", i, err, tt.want)
+			}
+		})
+	}
 }
 
 func TestRoleRevokePermission(t *testing.T) {
@@ -503,17 +733,44 @@ func TestUserRevokePermission(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "role-test"})
+	const userName = "foo"
+	_, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userName, Role: "role-test"})
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	_, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "foo", Role: "role-test-1"})
+	_, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: userName, Role: "role-test-1"})
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	u, err := as.UserGet(&pb.AuthUserGetRequest{Name: "foo"})
+	perm := &authpb.Permission{
+		PermType: authpb.WRITE,
+		Key:      []byte("WriteKeyBegin"),
+		RangeEnd: []byte("WriteKeyEnd"),
+	}
+	_, err = as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{
+		Name: "role-test-1",
+		Perm: perm,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := as.rangePermCache[userName]; !ok {
+		t.Fatalf("User %s should have its entry in rangePermCache", userName)
+	}
+	unifiedPerm := as.rangePermCache[userName]
+	pt1 := adt.NewBytesAffinePoint([]byte("WriteKeyBegin"))
+	if !unifiedPerm.writePerms.Contains(pt1) {
+		t.Fatal("rangePermCache should contain WriteKeyBegin")
+	}
+	pt2 := adt.NewBytesAffinePoint([]byte("OutOfRange"))
+	if unifiedPerm.writePerms.Contains(pt2) {
+		t.Fatal("rangePermCache should not contain OutOfRange")
+	}
+
+	u, err := as.UserGet(&pb.AuthUserGetRequest{Name: userName})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -523,12 +780,12 @@ func TestUserRevokePermission(t *testing.T) {
 		t.Fatalf("expected %v, got %v", expected, u.Roles)
 	}
 
-	_, err = as.UserRevokeRole(&pb.AuthUserRevokeRoleRequest{Name: "foo", Role: "role-test-1"})
+	_, err = as.UserRevokeRole(&pb.AuthUserRevokeRoleRequest{Name: userName, Role: "role-test-1"})
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	u, err = as.UserGet(&pb.AuthUserGetRequest{Name: "foo"})
+	u, err = as.UserGet(&pb.AuthUserGetRequest{Name: userName})
 	if err != nil {
 		t.Fatal(err)
 	}

bill-of-materials.json

@@ -44,15 +44,6 @@
 			}
 		]
 	},
-	{
-		"project": "github.com/dgrijalva/jwt-go",
-		"licenses": [
-			{
-				"type": "MIT License",
-				"confidence": 0.9891304347826086
-			}
-		]
-	},
 	{
 		"project": "github.com/dustin/go-humanize",
 		"licenses": [
@@ -71,6 +62,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/golang-jwt/jwt",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
 	{
 		"project": "github.com/golang/groupcache/lru",
 		"licenses": [
@@ -378,7 +378,7 @@
 		]
 	},
 	{
-		"project": "golang.org/x/sys/unix",
+		"project": "golang.org/x/sys",
 		"licenses": [
 			{
 				"type": "BSD 3-clause \"New\" or \"Revised\" License",

build

@@ -1,22 +1,24 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 # set some environment variables
 ORG_PATH="go.etcd.io"
 REPO_PATH="${ORG_PATH}/etcd"
 
 GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
-if [[ -n "$FAILPOINTS" ]]; then
+if [[ -n "${FAILPOINTS:-}" ]]; then
 	GIT_SHA="$GIT_SHA"-FAILPOINTS
 fi
 
 # Set GO_LDFLAGS="-s" for building without symbols for debugging.
-GO_LDFLAGS="$GO_LDFLAGS -X ${REPO_PATH}/version.GitSHA=${GIT_SHA}"
+GO_LDFLAGS="${GO_LDFLAGS:-} -X ${REPO_PATH}/version.GitSHA=${GIT_SHA}"
 
 # enable/disable failpoints
 toggle_failpoints() {
 	mode="$1"
 	if command -v gofail >/dev/null 2>&1; then
-		gofail "$mode" etcdserver/ mvcc/backend/
+		gofail "$mode" etcdserver/ mvcc/backend/ wal/
 	elif [[ "$mode" != "disable" ]]; then
 		echo "FAILPOINTS set but gofail not found"
 		exit 1
@@ -30,7 +32,7 @@ etcd_setup_gopath() {
 	cd "$CDIR" || return
 	etcdGOPATH="${CDIR}/gopath"
 	# preserve old gopath to support building with unvendored tooling deps (e.g., gofail)
-	if [[ -n "$GOPATH" ]]; then
+	if [[ -n "${GOPATH:-}" ]]; then
 		GOPATH=":$GOPATH"
 	fi
 	rm -rf "${etcdGOPATH:?}/"
@@ -42,23 +44,23 @@ etcd_setup_gopath() {
 
 toggle_failpoints_default() {
 	mode="disable"
-	if [[ -n "$FAILPOINTS" ]]; then mode="enable"; fi
+	if [[ -n "${FAILPOINTS:-}" ]]; then mode="enable"; fi
 	toggle_failpoints "$mode"
 }
 
 etcd_build() {
 	out="bin"
-	if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+	if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi
 	toggle_failpoints_default
 
 	# Static compilation is useful when etcd is run in a container. $GO_BUILD_FLAGS is OK
 	# shellcheck disable=SC2086
-	CGO_ENABLED=0 go build $GO_BUILD_FLAGS \
+	CGO_ENABLED=0 go build ${GO_BUILD_FLAGS:-} \
 		-installsuffix cgo \
 		-ldflags "$GO_LDFLAGS" \
 		-o "${out}/etcd" ${REPO_PATH} || return
 	# shellcheck disable=SC2086
-	CGO_ENABLED=0 go build $GO_BUILD_FLAGS \
+	CGO_ENABLED=0 go build ${GO_BUILD_FLAGS:-} \
 		-installsuffix cgo \
 		-ldflags "$GO_LDFLAGS" \
 		-o "${out}/etcdctl" ${REPO_PATH}/etcdctl || return
@@ -66,7 +68,7 @@ etcd_build() {
 
 tools_build() {
 	out="bin"
-	if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+	if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi
 	tools_path="tools/benchmark
 	tools/etcd-dump-db
 	tools/etcd-dump-logs
@@ -88,7 +90,7 @@ tools_build() {
 
 toggle_failpoints_default
 
-if [[ "${ETCD_SETUP_GOPATH}" == "1" ]]; then
+if [[ "${ETCD_SETUP_GOPATH:-}" == "1" ]]; then
 	etcd_setup_gopath
 fi
 

client/doc.go

@@ -68,6 +68,5 @@ Use a custom context to set timeouts on your operations:
 			// handle error
 		}
 	}
-
 */
 package client

clientv3/balancer/balancer_test.go

@@ -19,7 +19,6 @@ import (
 	"fmt"
 	"strings"
 	"testing"
-	"time"
 
 	"go.etcd.io/etcd/clientv3/balancer/picker"
 	"go.etcd.io/etcd/clientv3/balancer/resolver/endpoint"
@@ -92,24 +91,25 @@ func TestRoundRobinBalancedResolvableNoFailover(t *testing.T) {
 				return picked, err
 			}
 
-			prev, switches := "", 0
+			_, picked, err := warmupConnections(reqFunc, tc.serverCount, "")
+			if err != nil {
+				t.Fatalf("Unexpected failure %v", err)
+			}
+
+			// verify that we round robin
+			prev, switches := picked, 0
 			for i := 0; i < tc.reqN; i++ {
-				picked, err := reqFunc(context.Background())
+				picked, err = reqFunc(context.Background())
 				if err != nil {
 					t.Fatalf("#%d: unexpected failure %v", i, err)
 				}
-				if prev == "" {
-					prev = picked
-					continue
-				}
 				if prev != picked {
 					switches++
 				}
 				prev = picked
 			}
-			if tc.serverCount > 1 && switches < tc.reqN-3 { // -3 for initial resolutions
-				// TODO: FIX ME
-				t.Skipf("expected balanced loads for %d requests, got switches %d", tc.reqN, switches)
+			if tc.serverCount > 1 && switches != tc.reqN {
+				t.Fatalf("expected balanced loads for %d requests, got switches %d", tc.reqN, switches)
 			}
 		})
 	}
@@ -160,26 +160,21 @@ func TestRoundRobinBalancedResolvableFailoverFromServerFail(t *testing.T) {
 	}
 
 	// stop first server, loads should be redistributed
-	// stopped server should never be picked
 	ms.StopAt(0)
-	available := make(map[string]struct{})
-	for i := 1; i < serverCount; i++ {
-		available[eps[i]] = struct{}{}
+	// stopped server will be transitioned into TRANSIENT_FAILURE state
+	// but it doesn't happen instantaneously and it can still be picked for a short period of time
+	// we ignore "transport is closing" in such case
+	available, picked, err := warmupConnections(reqFunc, serverCount-1, "transport is closing")
+	if err != nil {
+		t.Fatalf("Unexpected failure %v", err)
 	}
 
 	reqN := 10
-	prev, switches := "", 0
+	prev, switches := picked, 0
 	for i := 0; i < reqN; i++ {
-		picked, err := reqFunc(context.Background())
-		if err != nil && strings.Contains(err.Error(), "transport is closing") {
-			continue
-		}
-		if prev == "" { // first failover
-			if eps[0] == picked {
-				t.Fatalf("expected failover from %q, picked %q", eps[0], picked)
-			}
-			prev = picked
-			continue
+		picked, err = reqFunc(context.Background())
+		if err != nil {
+			t.Fatalf("#%d: unexpected failure %v", i, err)
 		}
 		if _, ok := available[picked]; !ok {
 			t.Fatalf("picked unavailable address %q (available %v)", picked, available)
@@ -189,18 +184,18 @@ func TestRoundRobinBalancedResolvableFailoverFromServerFail(t *testing.T) {
 		}
 		prev = picked
 	}
-	if switches < reqN-3 { // -3 for initial resolutions + failover
-		// TODO: FIX ME!
-		t.Skipf("expected balanced loads for %d requests, got switches %d", reqN, switches)
+	if switches != reqN {
+		t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches)
 	}
 
 	// now failed server comes back
 	ms.StartAt(0)
+	available, picked, err = warmupConnections(reqFunc, serverCount, "")
+	if err != nil {
+		t.Fatalf("Unexpected failure %v", err)
+	}
 
-	// enough time for reconnecting to recovered server
-	time.Sleep(time.Second)
-
-	prev, switches = "", 0
+	prev, switches = picked, 0
 	recoveredAddr, recovered := eps[0], 0
 	available[recoveredAddr] = struct{}{}
 
@@ -209,10 +204,6 @@ func TestRoundRobinBalancedResolvableFailoverFromServerFail(t *testing.T) {
 		if err != nil {
 			t.Fatalf("#%d: unexpected failure %v", i, err)
 		}
-		if prev == "" {
-			prev = picked
-			continue
-		}
 		if _, ok := available[picked]; !ok {
 			t.Fatalf("#%d: picked unavailable address %q (available %v)", i, picked, available)
 		}
@@ -224,10 +215,10 @@ func TestRoundRobinBalancedResolvableFailoverFromServerFail(t *testing.T) {
 		}
 		prev = picked
 	}
-	if switches < reqN-3 { // -3 for initial resolutions
+	if switches != 2*reqN {
 		t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches)
 	}
-	if recovered < reqN/serverCount {
+	if recovered != 2*reqN/serverCount {
 		t.Fatalf("recovered server %q got only %d requests", recoveredAddr, recovered)
 	}
 }
@@ -242,11 +233,10 @@ func TestRoundRobinBalancedResolvableFailoverFromRequestFail(t *testing.T) {
 	}
 	defer ms.Stop()
 	var eps []string
-	available := make(map[string]struct{})
 	for _, svr := range ms.Servers {
 		eps = append(eps, svr.ResolverAddress().Addr)
-		available[svr.Address] = struct{}{}
 	}
+
 	rsv, err := endpoint.NewResolverGroup("requestfail")
 	if err != nil {
 		t.Fatal(err)
@@ -277,6 +267,11 @@ func TestRoundRobinBalancedResolvableFailoverFromRequestFail(t *testing.T) {
 		return picked, err
 	}
 
+	available, picked, err := warmupConnections(reqFunc, serverCount, "")
+	if err != nil {
+		t.Fatalf("Unexpected failure %v", err)
+	}
+
 	reqN := 20
 	prev, switches := "", 0
 	for i := 0; i < reqN; i++ {
@@ -285,17 +280,13 @@ func TestRoundRobinBalancedResolvableFailoverFromRequestFail(t *testing.T) {
 		if i%2 == 0 {
 			cancel()
 		}
-		picked, err := reqFunc(ctx)
+		picked, err = reqFunc(ctx)
 		if i%2 == 0 {
-			if s, ok := status.FromError(err); ok && s.Code() != codes.Canceled || picked != "" {
+			if s, ok := status.FromError(err); ok && s.Code() != codes.Canceled {
 				t.Fatalf("#%d: expected %v, got %v", i, context.Canceled, err)
 			}
 			continue
 		}
-		if prev == "" && picked != "" {
-			prev = picked
-			continue
-		}
 		if _, ok := available[picked]; !ok {
 			t.Fatalf("#%d: picked unavailable address %q (available %v)", i, picked, available)
 		}
@@ -304,7 +295,29 @@ func TestRoundRobinBalancedResolvableFailoverFromRequestFail(t *testing.T) {
 		}
 		prev = picked
 	}
-	if switches < reqN/2-3 { // -3 for initial resolutions + failover
+	if switches != reqN/2 {
 		t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches)
 	}
 }
+
+type reqFuncT = func(ctx context.Context) (picked string, err error)
+
+func warmupConnections(reqFunc reqFuncT, serverCount int, ignoreErr string) (map[string]struct{}, string, error) {
+	var picked string
+	var err error
+	available := make(map[string]struct{})
+	// cycle through all peers to indirectly verify that balancer subconn list is fully loaded
+	// otherwise we can't reliably count switches between 'picked' peers in the test assert phase
+	for len(available) < serverCount {
+		picked, err = reqFunc(context.Background())
+		if err != nil {
+			if ignoreErr != "" && strings.Contains(err.Error(), ignoreErr) {
+				// skip ignored errors
+				continue
+			}
+			return available, picked, err
+		}
+		available[picked] = struct{}{}
+	}
+	return available, picked, err
+}

clientv3/balancer/connectivity/connectivity.go

@@ -57,9 +57,9 @@ func (rc *recorder) GetCurrentState() (state connectivity.State) {
 // RecordTransition records state change happening in subConn and based on that
 // it evaluates what aggregated state should be.
 //
-//  - If at least one SubConn in Ready, the aggregated state is Ready;
-//  - Else if at least one SubConn in Connecting, the aggregated state is Connecting;
-//  - Else the aggregated state is TransientFailure.
+//   - If at least one SubConn in Ready, the aggregated state is Ready;
+//   - Else if at least one SubConn in Connecting, the aggregated state is Connecting;
+//   - Else the aggregated state is TransientFailure.
 //
 // Idle and Shutdown are not considered.
 //

clientv3/client.go

@@ -174,7 +174,9 @@ func (c *Client) Sync(ctx context.Context) error {
 	}
 	var eps []string
 	for _, m := range mresp.Members {
-		eps = append(eps, m.ClientURLs...)
+		if len(m.Name) != 0 && !m.IsLearner {
+			eps = append(eps, m.ClientURLs...)
+		}
 	}
 	c.SetEndpoints(eps...)
 	return nil

clientv3/client_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
+	"go.etcd.io/etcd/etcdserver/etcdserverpb"
 	"go.etcd.io/etcd/pkg/testutil"
 
 	"google.golang.org/grpc"
@@ -166,3 +167,51 @@ func TestCloseCtxClient(t *testing.T) {
 		t.Errorf("failed to Close the client. %v", err)
 	}
 }
+
+func TestSyncFiltersMembers(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	c, _ := New(Config{Endpoints: []string{"http://254.0.0.1:12345"}})
+	c.Cluster = &mockCluster{
+		[]*etcdserverpb.Member{
+			{ID: 0, Name: "", ClientURLs: []string{"http://254.0.0.1:12345"}, IsLearner: false},
+			{ID: 1, Name: "isStarted", ClientURLs: []string{"http://254.0.0.2:12345"}, IsLearner: true},
+			{ID: 2, Name: "isStartedAndNotLearner", ClientURLs: []string{"http://254.0.0.3:12345"}, IsLearner: false},
+		},
+	}
+	c.Sync(context.Background())
+
+	endpoints := c.Endpoints()
+	if len(endpoints) != 1 || endpoints[0] != "http://254.0.0.3:12345" {
+		t.Error("Client.Sync uses learner and/or non-started member client URLs")
+	}
+	c.Close()
+}
+
+type mockCluster struct {
+	members []*etcdserverpb.Member
+}
+
+func (mc *mockCluster) MemberList(ctx context.Context) (*MemberListResponse, error) {
+	return &MemberListResponse{Members: mc.members}, nil
+}
+
+func (mc *mockCluster) MemberAdd(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) {
+	return nil, nil
+}
+
+func (mc *mockCluster) MemberAddAsLearner(ctx context.Context, peerAddrs []string) (*MemberAddResponse, error) {
+	return nil, nil
+}
+
+func (mc *mockCluster) MemberRemove(ctx context.Context, id uint64) (*MemberRemoveResponse, error) {
+	return nil, nil
+}
+
+func (mc *mockCluster) MemberUpdate(ctx context.Context, id uint64, peerAddrs []string) (*MemberUpdateResponse, error) {
+	return nil, nil
+}
+
+func (mc *mockCluster) MemberPromote(ctx context.Context, id uint64) (*MemberPromoteResponse, error) {
+	return nil, nil
+}

clientv3/concurrency/election_test.go

@@ -65,22 +65,18 @@ func TestResumeElection(t *testing.T) {
 
 	respChan := make(chan *clientv3.GetResponse)
 	go func() {
+		defer close(respChan)
 		o := e.Observe(ctx)
 		respChan <- nil
-		for {
-			select {
-			case resp, ok := <-o:
-				if !ok {
-					t.Fatal("Observe() channel closed prematurely")
-				}
-				// Ignore any observations that candidate1 was elected
-				if string(resp.Kvs[0].Value) == "candidate1" {
-					continue
-				}
-				respChan <- &resp
-				return
+		for resp := range o {
+			// Ignore any observations that candidate1 was elected
+			if string(resp.Kvs[0].Value) == "candidate1" {
+				continue
 			}
+			respChan <- &resp
+			return
 		}
+		t.Error("Observe() channel closed prematurely")
 	}()
 
 	// wait until observe goroutine is running

clientv3/doc.go

@@ -102,5 +102,4 @@
 // The grpc load balancer is registered statically and is shared across etcd clients.
 // To enable detailed load balancer logging, set the ETCD_CLIENT_DEBUG environment
 // variable.  E.g. "ETCD_CLIENT_DEBUG=1".
-//
 package clientv3

clientv3/integration/black_hole_test.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !cluster_proxy
 // +build !cluster_proxy
 
 package integration

clientv3/integration/leasing_test.go

@@ -619,16 +619,28 @@ func TestLeasingTxnOwnerGet(t *testing.T) {
 	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
 	defer clus.Terminate(t)
 
+	client := clus.Client(0)
+
 	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
 	testutil.AssertNil(t, err)
-	defer closeLKV()
+
+	defer func() {
+		// In '--tags cluster_proxy' mode the client need to be closed before
+		// closeLKV(). This interrupts all outstanding watches. Closing by closeLKV()
+		// is not sufficient as (unfortunately) context close does not interrupts Watches.
+		// See ./clientv3/watch.go:
+		// >> Currently, client contexts are overwritten with "valCtx" that never closes. <<
+		clus.TakeClient(0) // avoid double Close() of the client.
+		client.Close()
+		closeLKV()
+	}()
 
 	keyCount := rand.Intn(10) + 1
 	var ops []clientv3.Op
 	presps := make([]*clientv3.PutResponse, keyCount)
 	for i := range presps {
 		k := fmt.Sprintf("k-%d", i)
-		presp, err := clus.Client(0).Put(context.TODO(), k, k+k)
+		presp, err := client.Put(context.TODO(), k, k+k)
 		if err != nil {
 			t.Fatal(err)
 		}

clientv3/integration/network_partition_test.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !cluster_proxy
 // +build !cluster_proxy
 
 package integration

clientv3/integration/user_test.go

@@ -114,7 +114,7 @@ func authSetupRoot(t *testing.T, auth clientv3.Auth) {
 func TestGetTokenWithoutAuth(t *testing.T) {
 	defer testutil.AfterTest(t)
 
-	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 10})
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2})
 	defer clus.Terminate(t)
 
 	authapi := clus.RandClient()
@@ -130,7 +130,7 @@ func TestGetTokenWithoutAuth(t *testing.T) {
 	// "Username" and "Password" must be used
 	cfg := clientv3.Config{
 		Endpoints:   authapi.Endpoints(),
-		DialTimeout: 1 * time.Second, // make sure all connection time of connect all endpoint must be more DialTimeout
+		DialTimeout: 5 * time.Second,
 		Username:    "root",
 		Password:    "123",
 	}
@@ -142,7 +142,7 @@ func TestGetTokenWithoutAuth(t *testing.T) {
 
 	switch err {
 	case nil:
-		t.Log("passes as expected, but may be connection time less than DialTimeout")
+		t.Log("passes as expected")
 	case context.DeadlineExceeded:
 		t.Errorf("not expected result:%v with endpoint:%s", err, authapi.Endpoints())
 	case rpctypes.ErrAuthNotEnabled:
@@ -150,5 +150,4 @@ func TestGetTokenWithoutAuth(t *testing.T) {
 	default:
 		t.Errorf("other errors:%v", err)
 	}
-
 }

clientv3/integration/watch_fragment_test.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !cluster_proxy
 // +build !cluster_proxy
 
 package integration

clientv3/integration/watch_test.go

@@ -338,6 +338,9 @@ func putAndWatch(t *testing.T, wctx *watchctx, key, val string) {
 		if !ok {
 			t.Fatalf("unexpected watch close")
 		}
+		if err := v.Err(); err != nil {
+			t.Fatalf("unexpected watch response error: %v", err)
+		}
 		if string(v.Events[0].Kv.Value) != val {
 			t.Fatalf("bad value got %v, wanted %v", v.Events[0].Kv.Value, val)
 		}
@@ -607,6 +610,9 @@ func TestConfigurableWatchProgressNotifyInterval(t *testing.T) {
 }
 
 func TestWatchRequestProgress(t *testing.T) {
+	if integration.ThroughProxy {
+		t.Skip("grpc-proxy does not support WatchProgress yet")
+	}
 	testCases := []struct {
 		name     string
 		watchers []string
@@ -1010,12 +1016,12 @@ func TestWatchCancelOnServer(t *testing.T) {
 // TestWatchOverlapContextCancel stresses the watcher stream teardown path by
 // creating/canceling watchers to ensure that new watchers are not taken down
 // by a torn down watch stream. The sort of race that's being detected:
-//     1. create w1 using a cancelable ctx with %v as "ctx"
-//     2. cancel ctx
-//     3. watcher client begins tearing down watcher grpc stream since no more watchers
-//     3. start creating watcher w2 using a new "ctx" (not canceled), attaches to old grpc stream
-//     4. watcher client finishes tearing down stream on "ctx"
-//     5. w2 comes back canceled
+//  1. create w1 using a cancelable ctx with %v as "ctx"
+//  2. cancel ctx
+//  3. watcher client begins tearing down watcher grpc stream since no more watchers
+//  3. start creating watcher w2 using a new "ctx" (not canceled), attaches to old grpc stream
+//  4. watcher client finishes tearing down stream on "ctx"
+//  5. w2 comes back canceled
 func TestWatchOverlapContextCancel(t *testing.T) {
 	f := func(clus *integration.ClusterV3) {}
 	testWatchOverlapContextCancel(t, f)

clientv3/leasing/doc.go

@@ -19,28 +19,27 @@
 //
 // First, create a leasing KV from a clientv3.Client 'cli':
 //
-//     lkv, err := leasing.NewKV(cli, "leasing-prefix")
-//     if err != nil {
-//         // handle error
-//     }
+//	lkv, err := leasing.NewKV(cli, "leasing-prefix")
+//	if err != nil {
+//	    // handle error
+//	}
 //
 // A range request for a key "abc" tries to acquire a leasing key so it can cache the range's
 // key locally. On the server, the leasing key is stored to "leasing-prefix/abc":
 //
-//     resp, err := lkv.Get(context.TODO(), "abc")
+//	resp, err := lkv.Get(context.TODO(), "abc")
 //
 // Future linearized read requests using 'lkv' will be served locally for the lease's lifetime:
 //
-//     resp, err = lkv.Get(context.TODO(), "abc")
+//	resp, err = lkv.Get(context.TODO(), "abc")
 //
 // If another leasing client writes to a leased key, then the owner relinquishes its exclusive
 // access, permitting the writer to modify the key:
 //
-//     lkv2, err := leasing.NewKV(cli, "leasing-prefix")
-//     if err != nil {
-//         // handle error
-//     }
-//     lkv2.Put(context.TODO(), "abc", "456")
-//     resp, err = lkv.Get("abc")
-//
+//	lkv2, err := leasing.NewKV(cli, "leasing-prefix")
+//	if err != nil {
+//	    // handle error
+//	}
+//	lkv2.Put(context.TODO(), "abc", "456")
+//	resp, err = lkv.Get("abc")
 package leasing

clientv3/namespace/doc.go

@@ -39,5 +39,4 @@
 //	resp, _ = cli.Get(context.TODO(), "abc")
 //	fmt.Printf("%s\n", resp.Kvs[0].Value)
 //	// Output: 456
-//
 package namespace

clientv3/naming/doc.go

@@ -52,5 +52,4 @@
 //		r := &etcdnaming.GRPCResolver{Client: c}
 //		return r.Update(c.Ctx(), service, naming.Update{Op: naming.Add, Addr: addr}, clientv3.WithLease(lid))
 //	}
-//
 package naming

clientv3/naming/grpc_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package naming
+package naming_test
 
 import (
 	"context"
@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	etcd "go.etcd.io/etcd/clientv3"
+	namingv3 "go.etcd.io/etcd/clientv3/naming"
 	"go.etcd.io/etcd/integration"
 	"go.etcd.io/etcd/pkg/testutil"
 
@@ -33,7 +34,7 @@ func TestGRPCResolver(t *testing.T) {
 	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
 	defer clus.Terminate(t)
 
-	r := GRPCResolver{
+	r := namingv3.GRPCResolver{
 		Client: clus.RandClient(),
 	}
 
@@ -107,7 +108,7 @@ func TestGRPCResolverMulti(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	r := GRPCResolver{c}
+	r := namingv3.GRPCResolver{c}
 
 	w, err := r.Resolve("foo")
 	if err != nil {

clientv3/op.go

@@ -77,6 +77,9 @@ type Op struct {
 	cmps    []Cmp
 	thenOps []Op
 	elseOps []Op
+
+	isOptsWithFromKey bool
+	isOptsWithPrefix  bool
 }
 
 // accessors / mutators
@@ -216,6 +219,10 @@ func (op Op) isWrite() bool {
 	return op.t != tRange
 }
 
+func NewOp() *Op {
+	return &Op{key: []byte("")}
+}
+
 // OpGet returns "get" operation based on given key and operation options.
 func OpGet(key string, opts ...OpOption) Op {
 	// WithPrefix and WithFromKey are not supported together
@@ -387,6 +394,7 @@ func WithPrefix() OpOption {
 			return
 		}
 		op.end = getPrefix(op.key)
+		op.isOptsWithPrefix = true
 	}
 }
 
@@ -406,6 +414,7 @@ func WithFromKey() OpOption {
 			op.key = []byte{0}
 		}
 		op.end = []byte("\x00")
+		op.isOptsWithFromKey = true
 	}
 }
 
@@ -554,7 +563,21 @@ func toLeaseTimeToLiveRequest(id LeaseID, opts ...LeaseOption) *pb.LeaseTimeToLi
 }
 
 // isWithPrefix returns true if WithPrefix is being called in the op
-func isWithPrefix(opts []OpOption) bool { return isOpFuncCalled("WithPrefix", opts) }
+func isWithPrefix(opts []OpOption) bool {
+	ret := NewOp()
+	for _, opt := range opts {
+		opt(ret)
+	}
+
+	return ret.isOptsWithPrefix
+}
 
 // isWithFromKey returns true if WithFromKey is being called in the op
-func isWithFromKey(opts []OpOption) bool { return isOpFuncCalled("WithFromKey", opts) }
+func isWithFromKey(opts []OpOption) bool {
+	ret := NewOp()
+	for _, opt := range opts {
+		opt(ret)
+	}
+
+	return ret.isOptsWithFromKey
+}

clientv3/ordering/doc.go

@@ -38,5 +38,4 @@
 //	cli.KV = ordering.NewKV(cli.KV, vf)
 //
 // Now calls using 'cli' will reject order violations with an error.
-//
 package ordering

clientv3/ordering/util.go

@@ -16,8 +16,7 @@ package ordering
 
 import (
 	"errors"
-	"sync"
-	"time"
+	"sync/atomic"
 
 	"go.etcd.io/etcd/clientv3"
 )
@@ -26,26 +25,18 @@ type OrderViolationFunc func(op clientv3.Op, resp clientv3.OpResponse, prevRev i
 
 var ErrNoGreaterRev = errors.New("etcdclient: no cluster members have a revision higher than the previously received revision")
 
-func NewOrderViolationSwitchEndpointClosure(c clientv3.Client) OrderViolationFunc {
-	var mu sync.Mutex
-	violationCount := 0
-	return func(op clientv3.Op, resp clientv3.OpResponse, prevRev int64) error {
-		if violationCount > len(c.Endpoints()) {
+func NewOrderViolationSwitchEndpointClosure(c *clientv3.Client) OrderViolationFunc {
+	violationCount := int32(0)
+	return func(_ clientv3.Op, _ clientv3.OpResponse, _ int64) error {
+		// Each request is assigned by round-robin load-balancer's picker to a different
+		// endpoints. If we cycled them 5 times (even with some level of concurrency),
+		// with high probability no endpoint points on a member with fresh data.
+		// TODO: Ideally we should track members (resp.opp.Header) that returned
+		// stale result and explicitly temporarily disable them in 'picker'.
+		if atomic.LoadInt32(&violationCount) > int32(5*len(c.Endpoints())) {
 			return ErrNoGreaterRev
 		}
-		mu.Lock()
-		defer mu.Unlock()
-		eps := c.Endpoints()
-		// force client to connect to given endpoint by limiting to a single endpoint
-		c.SetEndpoints(eps[violationCount%len(eps)])
-		// give enough time for operation
-		time.Sleep(1 * time.Second)
-		// set available endpoints back to all endpoints in to ensure
-		// the client has access to all the endpoints.
-		c.SetEndpoints(eps...)
-		// give enough time for operation
-		time.Sleep(1 * time.Second)
-		violationCount++
+		atomic.AddInt32(&violationCount, 1)
 		return nil
 	}
 }

clientv3/ordering/util_test.go

@@ -64,19 +64,19 @@ func TestEndpointSwitchResolvesViolation(t *testing.T) {
 	// NewOrderViolationSwitchEndpointClosure will be able to
 	// access the full list of endpoints.
 	cli.SetEndpoints(eps...)
-	OrderingKv := NewKV(cli.KV, NewOrderViolationSwitchEndpointClosure(*cli))
+	orderingKv := NewKV(cli.KV, NewOrderViolationSwitchEndpointClosure(cli))
 	// set prevRev to the second member's revision of "foo" such that
 	// the revision is higher than the third member's revision of "foo"
-	_, err = OrderingKv.Get(ctx, "foo")
+	_, err = orderingKv.Get(ctx, "foo")
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	t.Logf("Reconfigure client to speak only to the 'partitioned' member")
 	cli.SetEndpoints(clus.Members[2].GRPCAddr())
-	time.Sleep(1 * time.Second) // give enough time for operation
-	_, err = OrderingKv.Get(ctx, "foo", clientv3.WithSerializable())
-	if err != nil {
-		t.Fatalf("failed to resolve order violation %v", err)
+	_, err = orderingKv.Get(ctx, "foo", clientv3.WithSerializable())
+	if err != ErrNoGreaterRev {
+		t.Fatal("While speaking to partitioned leader, we should get ErrNoGreaterRev error")
 	}
 }
 
@@ -123,7 +123,7 @@ func TestUnresolvableOrderViolation(t *testing.T) {
 	// access the full list of endpoints.
 	cli.SetEndpoints(eps...)
 	time.Sleep(1 * time.Second) // give enough time for operation
-	OrderingKv := NewKV(cli.KV, NewOrderViolationSwitchEndpointClosure(*cli))
+	OrderingKv := NewKV(cli.KV, NewOrderViolationSwitchEndpointClosure(cli))
 	// set prevRev to the first member's revision of "foo" such that
 	// the revision is higher than the fourth and fifth members' revision of "foo"
 	_, err = OrderingKv.Get(ctx, "foo")

clientv3/retry_interceptor.go

@@ -73,8 +73,8 @@ func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOpt
 				// its the callCtx deadline or cancellation, in which case try again.
 				continue
 			}
-			if callOpts.retryAuth && rpctypes.Error(lastErr) == rpctypes.ErrInvalidAuthToken {
-				gterr := c.getToken(ctx)
+			if c.shouldRefreshToken(lastErr, callOpts) {
+				gterr := c.refreshToken(ctx)
 				if gterr != nil {
 					logger.Warn(
 						"retrying of unary invoker failed to fetch new auth token",
@@ -142,6 +142,37 @@ func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOp
 	}
 }
 
+// shouldRefreshToken checks whether there's a need to refresh the token based on the error and callOptions,
+// and returns a boolean value.
+func (c *Client) shouldRefreshToken(err error, callOpts *options) bool {
+	if rpctypes.Error(err) == rpctypes.ErrUserEmpty {
+		// refresh the token when username, password is present but the server returns ErrUserEmpty
+		// which is possible when the client token is cleared somehow
+		return c.authTokenBundle != nil // equal to c.Username != "" && c.Password != ""
+	}
+
+	return callOpts.retryAuth &&
+		(rpctypes.Error(err) == rpctypes.ErrInvalidAuthToken || rpctypes.Error(err) == rpctypes.ErrAuthOldRevision)
+}
+
+func (c *Client) refreshToken(ctx context.Context) error {
+	if c.authTokenBundle == nil {
+		// c.authTokenBundle will be initialized only when
+		// c.Username != "" && c.Password != "".
+		//
+		// When users use the TLS CommonName based authentication, the
+		// authTokenBundle is always nil. But it's possible for the clients
+		// to get `rpctypes.ErrAuthOldRevision` response when the clients
+		// concurrently modify auth data (e.g, addUser, deleteUser etc.).
+		// In this case, there is no need to refresh the token; instead the
+		// clients just need to retry the operations (e.g. Put, Delete etc).
+		return nil
+	}
+	// clear auth token before refreshing it.
+	c.authTokenBundle.UpdateAuthToken("")
+	return c.getToken(ctx)
+}
+
 // type serverStreamingRetryingStream is the implementation of grpc.ClientStream that acts as a
 // proxy to the underlying call. If any of the RecvMsg() calls fail, it will try to reestablish
 // a new ClientStream according to the retry policy.
@@ -239,8 +270,8 @@ func (s *serverStreamingRetryingStream) receiveMsgAndIndicateRetry(m interface{}
 		// its the callCtx deadline or cancellation, in which case try again.
 		return true, err
 	}
-	if s.callOpts.retryAuth && rpctypes.Error(err) == rpctypes.ErrInvalidAuthToken {
-		gterr := s.client.getToken(s.ctx)
+	if s.client.shouldRefreshToken(err, s.callOpts) {
+		gterr := s.client.refreshToken(s.ctx)
 		if gterr != nil {
 			s.client.lg.Warn("retry failed to fetch new auth token", zap.Error(gterr))
 			return false, err // return the original error for simplicity

clientv3/retry_interceptor_test.go

@@ -0,0 +1,141 @@
+// Copyright 2022 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Based on github.com/grpc-ecosystem/go-grpc-middleware/retry, but modified to support the more
+// fine grained error checking required by write-at-most-once retry semantics of etcd.
+
+package clientv3
+
+import (
+	"go.etcd.io/etcd/clientv3/credentials"
+	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
+	grpccredentials "google.golang.org/grpc/credentials"
+	"testing"
+)
+
+type dummyAuthTokenBundle struct{}
+
+func (d dummyAuthTokenBundle) TransportCredentials() grpccredentials.TransportCredentials {
+	return nil
+}
+
+func (d dummyAuthTokenBundle) PerRPCCredentials() grpccredentials.PerRPCCredentials {
+	return nil
+}
+
+func (d dummyAuthTokenBundle) NewWithMode(mode string) (grpccredentials.Bundle, error) {
+	return nil, nil
+}
+
+func (d dummyAuthTokenBundle) UpdateAuthToken(token string) {
+}
+
+func TestClientShouldRefreshToken(t *testing.T) {
+	type fields struct {
+		authTokenBundle credentials.Bundle
+	}
+	type args struct {
+		err      error
+		callOpts *options
+	}
+
+	optsWithTrue := &options{
+		retryAuth: true,
+	}
+	optsWithFalse := &options{
+		retryAuth: false,
+	}
+
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   bool
+	}{
+		{
+			name: "ErrUserEmpty and non nil authTokenBundle",
+			fields: fields{
+				authTokenBundle: &dummyAuthTokenBundle{},
+			},
+			args: args{rpctypes.ErrGRPCUserEmpty, optsWithTrue},
+			want: true,
+		},
+		{
+			name: "ErrUserEmpty and nil authTokenBundle",
+			fields: fields{
+				authTokenBundle: nil,
+			},
+			args: args{rpctypes.ErrGRPCUserEmpty, optsWithTrue},
+			want: false,
+		},
+		{
+			name: "ErrGRPCInvalidAuthToken and retryAuth",
+			fields: fields{
+				authTokenBundle: nil,
+			},
+			args: args{rpctypes.ErrGRPCInvalidAuthToken, optsWithTrue},
+			want: true,
+		},
+		{
+			name: "ErrGRPCInvalidAuthToken and !retryAuth",
+			fields: fields{
+				authTokenBundle: nil,
+			},
+			args: args{rpctypes.ErrGRPCInvalidAuthToken, optsWithFalse},
+			want: false,
+		},
+		{
+			name: "ErrGRPCAuthOldRevision and retryAuth",
+			fields: fields{
+				authTokenBundle: nil,
+			},
+			args: args{rpctypes.ErrGRPCAuthOldRevision, optsWithTrue},
+			want: true,
+		},
+		{
+			name: "ErrGRPCAuthOldRevision and !retryAuth",
+			fields: fields{
+				authTokenBundle: nil,
+			},
+			args: args{rpctypes.ErrGRPCAuthOldRevision, optsWithFalse},
+			want: false,
+		},
+		{
+			name: "Other error and retryAuth",
+			fields: fields{
+				authTokenBundle: nil,
+			},
+			args: args{rpctypes.ErrGRPCAuthFailed, optsWithTrue},
+			want: false,
+		},
+		{
+			name: "Other error and !retryAuth",
+			fields: fields{
+				authTokenBundle: nil,
+			},
+			args: args{rpctypes.ErrGRPCAuthFailed, optsWithFalse},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &Client{
+				authTokenBundle: tt.fields.authTokenBundle,
+			}
+			if got := c.shouldRefreshToken(tt.args.err, tt.args.callOpts); got != tt.want {
+				t.Errorf("shouldRefreshToken() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

clientv3/snapshot/member_test.go

@@ -69,8 +69,8 @@ func TestSnapshotV3RestoreMultiMemberAdd(t *testing.T) {
 	cfg.Name = "3"
 	cfg.InitialClusterToken = testClusterTkn
 	cfg.ClusterState = "existing"
-	cfg.LCUrls, cfg.ACUrls = newCURLs, newCURLs
-	cfg.LPUrls, cfg.APUrls = newPURLs, newPURLs
+	cfg.ListenClientUrls, cfg.AdvertiseClientUrls = newCURLs, newCURLs
+	cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = newPURLs, newPURLs
 	cfg.InitialCluster = ""
 	for i := 0; i < clusterN; i++ {
 		cfg.InitialCluster += fmt.Sprintf(",%d=%s", i, pURLs[i].String())

clientv3/snapshot/v3_snapshot.go

@@ -391,7 +391,7 @@ func (s *v3Manager) saveDB() error {
 	be := backend.NewDefaultBackend(dbpath)
 
 	// a lessor never timeouts leases
-	lessor := lease.NewLessor(s.lg, be, lease.LessorConfig{MinLeaseTTL: math.MaxInt64})
+	lessor := lease.NewLessor(s.lg, be, nil, lease.LessorConfig{MinLeaseTTL: math.MaxInt64})
 
 	mvs := mvcc.NewStore(s.lg, be, lessor, (*initIndex)(&commit), mvcc.StoreConfig{CompactionBatchLimit: math.MaxInt32})
 	txn := mvs.Write(traceutil.TODO())

clientv3/snapshot/v3_snapshot_test.go

@@ -51,8 +51,8 @@ func TestSnapshotV3RestoreSingle(t *testing.T) {
 	cfg.Name = "s1"
 	cfg.InitialClusterToken = testClusterTkn
 	cfg.ClusterState = "existing"
-	cfg.LCUrls, cfg.ACUrls = cURLs, cURLs
-	cfg.LPUrls, cfg.APUrls = pURLs, pURLs
+	cfg.ListenClientUrls, cfg.AdvertiseClientUrls = cURLs, cURLs
+	cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = pURLs, pURLs
 	cfg.InitialCluster = fmt.Sprintf("%s=%s", cfg.Name, pURLs[0].String())
 	cfg.Dir = filepath.Join(os.TempDir(), fmt.Sprint(time.Now().Nanosecond()))
 
@@ -87,7 +87,8 @@ func TestSnapshotV3RestoreSingle(t *testing.T) {
 	}
 
 	var cli *clientv3.Client
-	cli, err = clientv3.New(clientv3.Config{Endpoints: []string{cfg.ACUrls[0].String()}})
+	cli, err = clientv3.New(clientv3.Config{Endpoints: []string{cfg.AdvertiseClientUrls[0].String()}})
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -203,8 +204,8 @@ func createSnapshotFile(t *testing.T, kvs []kv) string {
 	cfg.Debug = false
 	cfg.Name = "default"
 	cfg.ClusterState = "new"
-	cfg.LCUrls, cfg.ACUrls = cURLs, cURLs
-	cfg.LPUrls, cfg.APUrls = pURLs, pURLs
+	cfg.ListenClientUrls, cfg.AdvertiseClientUrls = cURLs, cURLs
+	cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = pURLs, pURLs
 	cfg.InitialCluster = fmt.Sprintf("%s=%s", cfg.Name, pURLs[0].String())
 	cfg.Dir = filepath.Join(os.TempDir(), fmt.Sprint(time.Now().Nanosecond()))
 	srv, err := embed.StartEtcd(cfg)
@@ -221,7 +222,7 @@ func createSnapshotFile(t *testing.T, kvs []kv) string {
 		t.Fatalf("failed to start embed.Etcd for creating snapshots")
 	}
 
-	ccfg := clientv3.Config{Endpoints: []string{cfg.ACUrls[0].String()}}
+	ccfg := clientv3.Config{Endpoints: []string{cfg.AdvertiseClientUrls[0].String()}}
 	cli, err := clientv3.New(ccfg)
 	if err != nil {
 		t.Fatal(err)
@@ -271,8 +272,8 @@ func restoreCluster(t *testing.T, clusterN int, dbPath string) (
 		cfg.Name = fmt.Sprintf("%d", i)
 		cfg.InitialClusterToken = testClusterTkn
 		cfg.ClusterState = "existing"
-		cfg.LCUrls, cfg.ACUrls = []url.URL{cURLs[i]}, []url.URL{cURLs[i]}
-		cfg.LPUrls, cfg.APUrls = []url.URL{pURLs[i]}, []url.URL{pURLs[i]}
+		cfg.ListenClientUrls, cfg.AdvertiseClientUrls = []url.URL{cURLs[i]}, []url.URL{cURLs[i]}
+		cfg.ListenPeerUrls, cfg.AdvertisePeerUrls = []url.URL{pURLs[i]}, []url.URL{pURLs[i]}
 		cfg.InitialCluster = ics
 		cfg.Dir = filepath.Join(os.TempDir(), fmt.Sprint(time.Now().Nanosecond()+i))
 

clientv3/txn.go

@@ -25,15 +25,14 @@ import (
 
 // Txn is the interface that wraps mini-transactions.
 //
-//	 Txn(context.TODO()).If(
-//	  Compare(Value(k1), ">", v1),
-//	  Compare(Version(k1), "=", 2)
-//	 ).Then(
-//	  OpPut(k2,v2), OpPut(k3,v3)
-//	 ).Else(
-//	  OpPut(k4,v4), OpPut(k5,v5)
-//	 ).Commit()
-//
+//	Txn(context.TODO()).If(
+//	 Compare(Value(k1), ">", v1),
+//	 Compare(Version(k1), "=", 2)
+//	).Then(
+//	 OpPut(k2,v2), OpPut(k3,v3)
+//	).Else(
+//	 OpPut(k4,v4), OpPut(k5,v5)
+//	).Commit()
 type Txn interface {
 	// If takes a list of comparison. If all comparisons passed in succeed,
 	// the operations passed into Then() will be executed. Or the operations

clientv3/utils.go

@@ -16,9 +16,6 @@ package clientv3
 
 import (
 	"math/rand"
-	"reflect"
-	"runtime"
-	"strings"
 	"time"
 )
 
@@ -32,18 +29,3 @@ func jitterUp(duration time.Duration, jitter float64) time.Duration {
 	multiplier := jitter * (rand.Float64()*2 - 1)
 	return time.Duration(float64(duration) * (1 + multiplier))
 }
-
-// Check if the provided function is being called in the op options.
-func isOpFuncCalled(op string, opts []OpOption) bool {
-	for _, opt := range opts {
-		v := reflect.ValueOf(opt)
-		if v.Kind() == reflect.Func {
-			if opFunc := runtime.FuncForPC(v.Pointer()); opFunc != nil {
-				if strings.Contains(opFunc.Name(), op) {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}

clientv3/watch.go

@@ -37,6 +37,13 @@ const (
 	EventTypePut    = mvccpb.PUT
 
 	closeSendErrTimeout = 250 * time.Millisecond
+
+	// AutoWatchID is the watcher ID passed in WatchStream.Watch when no
+	// user-provided ID is available. If pass, an ID will automatically be assigned.
+	AutoWatchID = 0
+
+	// InvalidWatchID represents an invalid watch ID and prevents duplication with an existing watch.
+	InvalidWatchID = -1
 )
 
 type Event mvccpb.Event
@@ -443,7 +450,7 @@ func (w *watcher) closeStream(wgs *watchGrpcStream) {
 
 func (w *watchGrpcStream) addSubstream(resp *pb.WatchResponse, ws *watcherStream) {
 	// check watch ID for backward compatibility (<= v3.3)
-	if resp.WatchId == -1 || (resp.Canceled && resp.CancelReason != "") {
+	if resp.WatchId == InvalidWatchID || (resp.Canceled && resp.CancelReason != "") {
 		w.closeErr = v3rpc.Error(errors.New(resp.CancelReason))
 		// failed; no channel
 		close(ws.recvc)
@@ -474,7 +481,7 @@ func (w *watchGrpcStream) closeSubstream(ws *watcherStream) {
 	} else if ws.outc != nil {
 		close(ws.outc)
 	}
-	if ws.id != -1 {
+	if ws.id != InvalidWatchID {
 		delete(w.substreams, ws.id)
 		return
 	}
@@ -526,6 +533,7 @@ func (w *watchGrpcStream) run() {
 	cancelSet := make(map[int64]struct{})
 
 	var cur *pb.WatchResponse
+	backoff := time.Millisecond
 	for {
 		select {
 		// Watch() requested
@@ -536,7 +544,7 @@ func (w *watchGrpcStream) run() {
 				// TODO: pass custom watch ID?
 				ws := &watcherStream{
 					initReq: *wreq,
-					id:      -1,
+					id:      InvalidWatchID,
 					outc:    outc,
 					// unbuffered so resumes won't cause repeat events
 					recvc: make(chan *WatchResponse),
@@ -650,6 +658,7 @@ func (w *watchGrpcStream) run() {
 				closeErr = err
 				return
 			}
+			backoff = w.backoffIfUnavailable(backoff, err)
 			if wc, closeErr = w.newWatchClient(); closeErr != nil {
 				return
 			}
@@ -666,7 +675,7 @@ func (w *watchGrpcStream) run() {
 			return
 
 		case ws := <-w.closingc:
-			if ws.id != -1 {
+			if ws.id != InvalidWatchID {
 				// client is closing an established watch; close it on the server proactively instead of waiting
 				// to close when the next message arrives
 				cancelSet[ws.id] = struct{}{}
@@ -723,9 +732,9 @@ func (w *watchGrpcStream) dispatchEvent(pbresp *pb.WatchResponse) bool {
 		cancelReason:    pbresp.CancelReason,
 	}
 
-	// watch IDs are zero indexed, so request notify watch responses are assigned a watch ID of -1 to
+	// watch IDs are zero indexed, so request notify watch responses are assigned a watch ID of InvalidWatchID to
 	// indicate they should be broadcast.
-	if wr.IsProgressNotify() && pbresp.WatchId == -1 {
+	if wr.IsProgressNotify() && pbresp.WatchId == InvalidWatchID {
 		return w.broadcastResponse(wr)
 	}
 
@@ -846,7 +855,7 @@ func (w *watchGrpcStream) serveSubstream(ws *watcherStream, resumec chan struct{
 				}
 			} else {
 				// current progress of watch; <= store revision
-				nextRev = wr.Header.Revision
+				nextRev = wr.Header.Revision + 1
 			}
 
 			if len(wr.Events) > 0 {
@@ -880,7 +889,7 @@ func (w *watchGrpcStream) newWatchClient() (pb.Watch_WatchClient, error) {
 	w.resumec = make(chan struct{})
 	w.joinSubstreams()
 	for _, ws := range w.substreams {
-		ws.id = -1
+		ws.id = InvalidWatchID
 		w.resuming = append(w.resuming, ws)
 	}
 	// strip out nils, if any
@@ -970,6 +979,21 @@ func (w *watchGrpcStream) joinSubstreams() {
 
 var maxBackoff = 100 * time.Millisecond
 
+func (w *watchGrpcStream) backoffIfUnavailable(backoff time.Duration, err error) time.Duration {
+	if isUnavailableErr(w.ctx, err) {
+		// retry, but backoff
+		if backoff < maxBackoff {
+			// 25% backoff factor
+			backoff = backoff + backoff/4
+			if backoff > maxBackoff {
+				backoff = maxBackoff
+			}
+		}
+		time.Sleep(backoff)
+	}
+	return backoff
+}
+
 // openWatchClient retries opening a watch client until success or halt.
 // manually retry in case "ws==nil && err==nil"
 // TODO: remove FailFast=false
@@ -990,17 +1014,7 @@ func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error)
 		if isHaltErr(w.ctx, err) {
 			return nil, v3rpc.Error(err)
 		}
-		if isUnavailableErr(w.ctx, err) {
-			// retry, but backoff
-			if backoff < maxBackoff {
-				// 25% backoff factor
-				backoff = backoff + backoff/4
-				if backoff > maxBackoff {
-					backoff = maxBackoff
-				}
-			}
-			time.Sleep(backoff)
-		}
+		backoff = w.backoffIfUnavailable(backoff, err)
 	}
 	return ws, nil
 }

embed/config.go

@@ -18,6 +18,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"math"
 	"net"
 	"net/http"
 	"net/url"
@@ -53,7 +54,9 @@ const (
 	DefaultMaxSnapshots          = 5
 	DefaultMaxWALs               = 5
 	DefaultMaxTxnOps             = uint(128)
+	DefaultWarningApplyDuration  = 100 * time.Millisecond
 	DefaultMaxRequestBytes       = 1.5 * 1024 * 1024
+	DefaultMaxConcurrentStreams  = math.MaxUint32
 	DefaultGRPCKeepAliveMinTime  = 5 * time.Second
 	DefaultGRPCKeepAliveInterval = 2 * time.Hour
 	DefaultGRPCKeepAliveTimeout  = 20 * time.Second
@@ -176,18 +179,27 @@ type Config struct {
 	MaxTxnOps         uint  `json:"max-txn-ops"`
 	MaxRequestBytes   uint  `json:"max-request-bytes"`
 
-	LPUrls, LCUrls []url.URL
-	APUrls, ACUrls []url.URL
-	ClientTLSInfo  transport.TLSInfo
-	ClientAutoTLS  bool
-	PeerTLSInfo    transport.TLSInfo
-	PeerAutoTLS    bool
+	// MaxConcurrentStreams specifies the maximum number of concurrent
+	// streams that each client can open at a time.
+	MaxConcurrentStreams uint32 `json:"max-concurrent-streams"`
+
+	ListenPeerUrls, ListenClientUrls, ListenClientHttpUrls []url.URL
+	AdvertisePeerUrls, AdvertiseClientUrls                 []url.URL
+	ClientTLSInfo                                          transport.TLSInfo
+	ClientAutoTLS                                          bool
+	PeerTLSInfo                                            transport.TLSInfo
+	PeerAutoTLS                                            bool
 
 	// CipherSuites is a list of supported TLS cipher suites between
 	// client/server and peers. If empty, Go auto-populates the list.
 	// Note that cipher suites are prioritized in the given order.
 	CipherSuites []string `json:"cipher-suites"`
 
+	// TlsMinVersion is the minimum accepted TLS version between client/server and peers.
+	TlsMinVersion string `json:"tls-min-version"`
+	// TlsMaxVersion is the maximum accepted TLS version between client/server and peers.
+	TlsMaxVersion string `json:"tls-max-version"`
+
 	ClusterState          string `json:"initial-cluster-state"`
 	DNSCluster            string `json:"discovery-srv"`
 	DNSClusterServiceName string `json:"discovery-srv-name"`
@@ -273,7 +285,7 @@ type Config struct {
 	AuthToken  string `json:"auth-token"`
 	BcryptCost uint   `json:"bcrypt-cost"`
 
-	//The AuthTokenTTL in seconds of the simple token
+	// AuthTokenTTL specifies the TTL in seconds of the simple token
 	AuthTokenTTL uint `json:"auth-token-ttl"`
 
 	ExperimentalInitialCorruptCheck bool          `json:"experimental-initial-corrupt-check"`
@@ -281,10 +293,18 @@ type Config struct {
 	ExperimentalEnableV2V3          string        `json:"experimental-enable-v2v3"`
 	// ExperimentalBackendFreelistType specifies the type of freelist that boltdb backend uses (array and map are supported types).
 	ExperimentalBackendFreelistType string `json:"experimental-backend-bbolt-freelist-type"`
-	// ExperimentalEnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases.
-	ExperimentalEnableLeaseCheckpoint       bool          `json:"experimental-enable-lease-checkpoint"`
-	ExperimentalCompactionBatchLimit        int           `json:"experimental-compaction-batch-limit"`
-	ExperimentalWatchProgressNotifyInterval time.Duration `json:"experimental-watch-progress-notify-interval"`
+	// ExperimentalEnableLeaseCheckpoint enables leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change.
+	ExperimentalEnableLeaseCheckpoint bool `json:"experimental-enable-lease-checkpoint"`
+	// ExperimentalEnableLeaseCheckpointPersist enables persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled.
+	// Requires experimental-enable-lease-checkpoint to be enabled.
+	// Deprecated in v3.6.
+	// TODO: Delete in v3.7
+	ExperimentalEnableLeaseCheckpointPersist bool          `json:"experimental-enable-lease-checkpoint-persist"`
+	ExperimentalCompactionBatchLimit         int           `json:"experimental-compaction-batch-limit"`
+	ExperimentalWatchProgressNotifyInterval  time.Duration `json:"experimental-watch-progress-notify-interval"`
+	// ExperimentalWarningApplyDuration is the time duration after which a warning is generated if applying request
+	// takes more time than this value.
+	ExperimentalWarningApplyDuration time.Duration `json:"experimental-warning-apply-duration"`
 
 	// ForceNewCluster starts a new cluster even if previously started; unsafe.
 	ForceNewCluster bool `json:"force-new-cluster"`
@@ -353,10 +373,11 @@ type configYAML struct {
 
 // configJSON has file options that are translated into Config options
 type configJSON struct {
-	LPUrlsJSON string `json:"listen-peer-urls"`
-	LCUrlsJSON string `json:"listen-client-urls"`
-	APUrlsJSON string `json:"initial-advertise-peer-urls"`
-	ACUrlsJSON string `json:"advertise-client-urls"`
+	ListenPeerUrls       string `json:"listen-peer-urls"`
+	ListenClientUrls     string `json:"listen-client-urls"`
+	ListenClientHttpUrls string `json:"listen-client-http-urls"`
+	AdvertisePeerUrls    string `json:"initial-advertise-peer-urls"`
+	AdvertiseClientUrls  string `json:"advertise-client-urls"`
 
 	CORSJSON          string `json:"cors"`
 	HostWhitelistJSON string `json:"host-whitelist"`
@@ -388,8 +409,10 @@ func NewConfig() *Config {
 		SnapshotCount:          etcdserver.DefaultSnapshotCount,
 		SnapshotCatchUpEntries: etcdserver.DefaultSnapshotCatchUpEntries,
 
-		MaxTxnOps:       DefaultMaxTxnOps,
-		MaxRequestBytes: DefaultMaxRequestBytes,
+		MaxTxnOps:                        DefaultMaxTxnOps,
+		MaxRequestBytes:                  DefaultMaxRequestBytes,
+		MaxConcurrentStreams:             DefaultMaxConcurrentStreams,
+		ExperimentalWarningApplyDuration: DefaultWarningApplyDuration,
 
 		GRPCKeepAliveMinTime:  DefaultGRPCKeepAliveMinTime,
 		GRPCKeepAliveInterval: DefaultGRPCKeepAliveInterval,
@@ -399,10 +422,10 @@ func NewConfig() *Config {
 		ElectionMs:                 1000,
 		InitialElectionTickAdvance: true,
 
-		LPUrls: []url.URL{*lpurl},
-		LCUrls: []url.URL{*lcurl},
-		APUrls: []url.URL{*apurl},
-		ACUrls: []url.URL{*acurl},
+		ListenPeerUrls:      []url.URL{*lpurl},
+		ListenClientUrls:    []url.URL{*lcurl},
+		AdvertisePeerUrls:   []url.URL{*apurl},
+		AdvertiseClientUrls: []url.URL{*acurl},
 
 		ClusterState:        ClusterStateFlagNew,
 		InitialClusterToken: "etcd-cluster",
@@ -467,40 +490,49 @@ func (cfg *configYAML) configFromFile(path string) error {
 		return err
 	}
 
-	if cfg.LPUrlsJSON != "" {
-		u, err := types.NewURLs(strings.Split(cfg.LPUrlsJSON, ","))
+	if cfg.configJSON.ListenPeerUrls != "" {
+		u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenPeerUrls, ","))
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unexpected error setting up listen-peer-urls: %v\n", err)
 			os.Exit(1)
 		}
-		cfg.LPUrls = []url.URL(u)
+		cfg.Config.ListenPeerUrls = u
 	}
 
-	if cfg.LCUrlsJSON != "" {
-		u, err := types.NewURLs(strings.Split(cfg.LCUrlsJSON, ","))
+	if cfg.configJSON.ListenClientUrls != "" {
+		u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenClientUrls, ","))
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unexpected error setting up listen-client-urls: %v\n", err)
 			os.Exit(1)
 		}
-		cfg.LCUrls = []url.URL(u)
+		cfg.Config.ListenClientUrls = u
 	}
 
-	if cfg.APUrlsJSON != "" {
-		u, err := types.NewURLs(strings.Split(cfg.APUrlsJSON, ","))
+	if cfg.configJSON.ListenClientHttpUrls != "" {
+		u, err := types.NewURLs(strings.Split(cfg.configJSON.ListenClientHttpUrls, ","))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unexpected error setting up listen-client-http-urls: %v\n", err)
+			os.Exit(1)
+		}
+		cfg.Config.ListenClientHttpUrls = u
+	}
+
+	if cfg.configJSON.AdvertisePeerUrls != "" {
+		u, err := types.NewURLs(strings.Split(cfg.configJSON.AdvertisePeerUrls, ","))
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unexpected error setting up initial-advertise-peer-urls: %v\n", err)
 			os.Exit(1)
 		}
-		cfg.APUrls = []url.URL(u)
+		cfg.Config.AdvertisePeerUrls = u
 	}
 
-	if cfg.ACUrlsJSON != "" {
-		u, err := types.NewURLs(strings.Split(cfg.ACUrlsJSON, ","))
+	if cfg.configJSON.AdvertiseClientUrls != "" {
+		u, err := types.NewURLs(strings.Split(cfg.configJSON.AdvertiseClientUrls, ","))
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unexpected error setting up advertise-peer-urls: %v\n", err)
 			os.Exit(1)
 		}
-		cfg.ACUrls = []url.URL(u)
+		cfg.Config.AdvertiseClientUrls = u
 	}
 
 	if cfg.ListenMetricsUrlsJSON != "" {
@@ -549,39 +581,56 @@ func updateCipherSuites(tls *transport.TLSInfo, ss []string) error {
 		return fmt.Errorf("TLSInfo.CipherSuites is already specified (given %v)", ss)
 	}
 	if len(ss) > 0 {
-		cs := make([]uint16, len(ss))
-		for i, s := range ss {
-			var ok bool
-			cs[i], ok = tlsutil.GetCipherSuite(s)
-			if !ok {
-				return fmt.Errorf("unexpected TLS cipher suite %q", s)
-			}
+		cs, err := tlsutil.GetCipherSuites(ss)
+		if err != nil {
+			return err
 		}
 		tls.CipherSuites = cs
 	}
 	return nil
 }
 
+func updateMinMaxVersions(info *transport.TLSInfo, min, max string) {
+	// Validate() has been called to check the user input, so it should never fail.
+	var err error
+	if info.MinVersion, err = tlsutil.GetTLSVersion(min); err != nil {
+		panic(err)
+	}
+	if info.MaxVersion, err = tlsutil.GetTLSVersion(max); err != nil {
+		panic(err)
+	}
+}
+
 // Validate ensures that '*embed.Config' fields are properly configured.
 func (cfg *Config) Validate() error {
 	if err := cfg.setupLogging(); err != nil {
 		return err
 	}
-	if err := checkBindURLs(cfg.LPUrls); err != nil {
+	if err := checkBindURLs(cfg.ListenPeerUrls); err != nil {
 		return err
 	}
-	if err := checkBindURLs(cfg.LCUrls); err != nil {
+	if err := checkBindURLs(cfg.ListenClientUrls); err != nil {
 		return err
 	}
+	if err := checkBindURLs(cfg.ListenClientHttpUrls); err != nil {
+		return err
+	}
+	if len(cfg.ListenClientHttpUrls) == 0 {
+		if cfg.logger != nil {
+			cfg.logger.Warn("Running http and grpc server on single port. This is not recommended for production.")
+		} else {
+			plog.Warning("Running http and grpc server on single port. This is not recommended for production.")
+		}
+	}
 	if err := checkBindURLs(cfg.ListenMetricsUrls); err != nil {
 		return err
 	}
-	if err := checkHostURLs(cfg.APUrls); err != nil {
-		addrs := cfg.getAPURLs()
+	if err := checkHostURLs(cfg.AdvertisePeerUrls); err != nil {
+		addrs := cfg.getAdvertisePeerUrls()
 		return fmt.Errorf(`--initial-advertise-peer-urls %q must be "host:port" (%v)`, strings.Join(addrs, ","), err)
 	}
-	if err := checkHostURLs(cfg.ACUrls); err != nil {
-		addrs := cfg.getACURLs()
+	if err := checkHostURLs(cfg.AdvertiseClientUrls); err != nil {
+		addrs := cfg.getAdvertiseClientUrls()
 		return fmt.Errorf(`--advertise-client-urls %q must be "host:port" (%v)`, strings.Join(addrs, ","), err)
 	}
 	// Check if conflicting flags are passed.
@@ -614,7 +663,7 @@ func (cfg *Config) Validate() error {
 	}
 
 	// check this last since proxying in etcdmain may make this OK
-	if cfg.LCUrls != nil && cfg.ACUrls == nil {
+	if cfg.ListenClientUrls != nil && cfg.AdvertiseClientUrls == nil {
 		return ErrUnsetAdvertiseClientURLsFlag
 	}
 
@@ -625,6 +674,33 @@ func (cfg *Config) Validate() error {
 		return fmt.Errorf("unknown auto-compaction-mode %q", cfg.AutoCompactionMode)
 	}
 
+	if !cfg.ExperimentalEnableLeaseCheckpointPersist && cfg.ExperimentalEnableLeaseCheckpoint {
+		cfg.logger.Warn("Detected that checkpointing is enabled without persistence. Consider enabling experimental-enable-lease-checkpoint-persist")
+	}
+
+	if cfg.ExperimentalEnableLeaseCheckpointPersist && !cfg.ExperimentalEnableLeaseCheckpoint {
+		return fmt.Errorf("setting experimental-enable-lease-checkpoint-persist requires experimental-enable-lease-checkpoint")
+	}
+
+	minVersion, err := tlsutil.GetTLSVersion(cfg.TlsMinVersion)
+	if err != nil {
+		return err
+	}
+	maxVersion, err := tlsutil.GetTLSVersion(cfg.TlsMaxVersion)
+	if err != nil {
+		return err
+	}
+
+	// maxVersion == 0 means that Go selects the highest available version.
+	if maxVersion != 0 && minVersion > maxVersion {
+		return fmt.Errorf("min version (%s) is greater than max version (%s)", cfg.TlsMinVersion, cfg.TlsMaxVersion)
+	}
+
+	// Check if user attempted to configure ciphers for TLS1.3 only: Go does not support that currently.
+	if minVersion == tls.VersionTLS13 && len(cfg.CipherSuites) > 0 {
+		return fmt.Errorf("cipher suites cannot be configured when only TLS1.3 is enabled")
+	}
+
 	return nil
 }
 
@@ -636,7 +712,7 @@ func (cfg *Config) PeerURLsMapAndToken(which string) (urlsmap types.URLsMap, tok
 		urlsmap = types.URLsMap{}
 		// If using discovery, generate a temporary cluster based on
 		// self's advertised peer URLs
-		urlsmap[cfg.Name] = cfg.APUrls
+		urlsmap[cfg.Name] = cfg.AdvertisePeerUrls
 		token = cfg.Durl
 
 	case cfg.DNSCluster != "":
@@ -692,7 +768,7 @@ func (cfg *Config) GetDNSClusterNames() ([]string, error) {
 
 	// Use both etcd-server-ssl and etcd-server for discovery.
 	// Combine the results if both are available.
-	clusterStrs, cerr = srv.GetCluster("https", "etcd-server-ssl"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.APUrls)
+	clusterStrs, cerr = srv.GetCluster("https", "etcd-server-ssl"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.AdvertisePeerUrls)
 	if cerr != nil {
 		clusterStrs = make([]string, 0)
 	}
@@ -703,13 +779,13 @@ func (cfg *Config) GetDNSClusterNames() ([]string, error) {
 			zap.String("service-name", "etcd-server-ssl"+serviceNameSuffix),
 			zap.String("server-name", cfg.Name),
 			zap.String("discovery-srv", cfg.DNSCluster),
-			zap.Strings("advertise-peer-urls", cfg.getAPURLs()),
+			zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerUrls()),
 			zap.Strings("found-cluster", clusterStrs),
 			zap.Error(cerr),
 		)
 	}
 
-	defaultHTTPClusterStrs, httpCerr := srv.GetCluster("http", "etcd-server"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.APUrls)
+	defaultHTTPClusterStrs, httpCerr := srv.GetCluster("http", "etcd-server"+serviceNameSuffix, cfg.Name, cfg.DNSCluster, cfg.AdvertisePeerUrls)
 	if httpCerr != nil {
 		clusterStrs = append(clusterStrs, defaultHTTPClusterStrs...)
 	}
@@ -720,7 +796,7 @@ func (cfg *Config) GetDNSClusterNames() ([]string, error) {
 			zap.String("service-name", "etcd-server"+serviceNameSuffix),
 			zap.String("server-name", cfg.Name),
 			zap.String("discovery-srv", cfg.DNSCluster),
-			zap.Strings("advertise-peer-urls", cfg.getAPURLs()),
+			zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerUrls()),
 			zap.Strings("found-cluster", clusterStrs),
 			zap.Error(httpCerr),
 		)
@@ -730,15 +806,15 @@ func (cfg *Config) GetDNSClusterNames() ([]string, error) {
 }
 
 func (cfg Config) InitialClusterFromName(name string) (ret string) {
-	if len(cfg.APUrls) == 0 {
+	if len(cfg.AdvertisePeerUrls) == 0 {
 		return ""
 	}
 	n := name
 	if name == "" {
 		n = DefaultName
 	}
-	for i := range cfg.APUrls {
-		ret = ret + "," + n + "=" + cfg.APUrls[i].String()
+	for i := range cfg.AdvertisePeerUrls {
+		ret = ret + "," + n + "=" + cfg.AdvertisePeerUrls[i].String()
 	}
 	return ret[1:]
 }
@@ -747,11 +823,11 @@ func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateF
 func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 
 func (cfg Config) defaultPeerHost() bool {
-	return len(cfg.APUrls) == 1 && cfg.APUrls[0].String() == DefaultInitialAdvertisePeerURLs
+	return len(cfg.AdvertisePeerUrls) == 1 && cfg.AdvertisePeerUrls[0].String() == DefaultInitialAdvertisePeerURLs
 }
 
 func (cfg Config) defaultClientHost() bool {
-	return len(cfg.ACUrls) == 1 && cfg.ACUrls[0].String() == DefaultAdvertiseClientURLs
+	return len(cfg.AdvertiseClientUrls) == 1 && cfg.AdvertiseClientUrls[0].String() == DefaultAdvertiseClientURLs
 }
 
 func (cfg *Config) ClientSelfCert() (err error) {
@@ -766,9 +842,12 @@ func (cfg *Config) ClientSelfCert() (err error) {
 		}
 		return nil
 	}
-	chosts := make([]string, len(cfg.LCUrls))
-	for i, u := range cfg.LCUrls {
-		chosts[i] = u.Host
+	chosts := make([]string, 0, len(cfg.ListenClientUrls)+len(cfg.ListenClientHttpUrls))
+	for _, u := range cfg.ListenClientUrls {
+		chosts = append(chosts, u.Host)
+	}
+	for _, u := range cfg.ListenClientHttpUrls {
+		chosts = append(chosts, u.Host)
 	}
 	cfg.ClientTLSInfo, err = transport.SelfCert(cfg.logger, filepath.Join(cfg.Dir, "fixtures", "client"), chosts)
 	if err != nil {
@@ -789,8 +868,8 @@ func (cfg *Config) PeerSelfCert() (err error) {
 		}
 		return nil
 	}
-	phosts := make([]string, len(cfg.LPUrls))
-	for i, u := range cfg.LPUrls {
+	phosts := make([]string, len(cfg.ListenPeerUrls))
+	for i, u := range cfg.ListenPeerUrls {
 		phosts[i] = u.Host
 	}
 	cfg.PeerTLSInfo, err = transport.SelfCert(cfg.logger, filepath.Join(cfg.Dir, "fixtures", "peer"), phosts)
@@ -818,9 +897,9 @@ func (cfg *Config) UpdateDefaultClusterFromName(defaultInitialCluster string) (s
 	}
 
 	used := false
-	pip, pport := cfg.LPUrls[0].Hostname(), cfg.LPUrls[0].Port()
+	pip, pport := cfg.ListenPeerUrls[0].Hostname(), cfg.ListenPeerUrls[0].Port()
 	if cfg.defaultPeerHost() && pip == "0.0.0.0" {
-		cfg.APUrls[0] = url.URL{Scheme: cfg.APUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, pport)}
+		cfg.AdvertisePeerUrls[0] = url.URL{Scheme: cfg.AdvertisePeerUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, pport)}
 		used = true
 	}
 	// update 'initial-cluster' when only the name is specified (e.g. 'etcd --name=abc')
@@ -828,9 +907,9 @@ func (cfg *Config) UpdateDefaultClusterFromName(defaultInitialCluster string) (s
 		cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
 	}
 
-	cip, cport := cfg.LCUrls[0].Hostname(), cfg.LCUrls[0].Port()
+	cip, cport := cfg.ListenClientUrls[0].Hostname(), cfg.ListenClientUrls[0].Port()
 	if cfg.defaultClientHost() && cip == "0.0.0.0" {
-		cfg.ACUrls[0] = url.URL{Scheme: cfg.ACUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, cport)}
+		cfg.AdvertiseClientUrls[0] = url.URL{Scheme: cfg.AdvertiseClientUrls[0].Scheme, Host: fmt.Sprintf("%s:%s", defaultHostname, cport)}
 		used = true
 	}
 	dhost := defaultHostname
@@ -875,34 +954,42 @@ func checkHostURLs(urls []url.URL) error {
 	return nil
 }
 
-func (cfg *Config) getAPURLs() (ss []string) {
-	ss = make([]string, len(cfg.APUrls))
-	for i := range cfg.APUrls {
-		ss[i] = cfg.APUrls[i].String()
+func (cfg *Config) getAdvertisePeerUrls() (ss []string) {
+	ss = make([]string, len(cfg.AdvertisePeerUrls))
+	for i := range cfg.AdvertisePeerUrls {
+		ss[i] = cfg.AdvertisePeerUrls[i].String()
 	}
 	return ss
 }
 
-func (cfg *Config) getLPURLs() (ss []string) {
-	ss = make([]string, len(cfg.LPUrls))
-	for i := range cfg.LPUrls {
-		ss[i] = cfg.LPUrls[i].String()
+func (cfg *Config) getListenPeerUrls() (ss []string) {
+	ss = make([]string, len(cfg.ListenPeerUrls))
+	for i := range cfg.ListenPeerUrls {
+		ss[i] = cfg.ListenPeerUrls[i].String()
 	}
 	return ss
 }
 
-func (cfg *Config) getACURLs() (ss []string) {
-	ss = make([]string, len(cfg.ACUrls))
-	for i := range cfg.ACUrls {
-		ss[i] = cfg.ACUrls[i].String()
+func (cfg *Config) getAdvertiseClientUrls() (ss []string) {
+	ss = make([]string, len(cfg.AdvertiseClientUrls))
+	for i := range cfg.AdvertiseClientUrls {
+		ss[i] = cfg.AdvertiseClientUrls[i].String()
 	}
 	return ss
 }
 
-func (cfg *Config) getLCURLs() (ss []string) {
-	ss = make([]string, len(cfg.LCUrls))
-	for i := range cfg.LCUrls {
-		ss[i] = cfg.LCUrls[i].String()
+func (cfg *Config) getListenClientUrls() (ss []string) {
+	ss = make([]string, len(cfg.ListenClientUrls))
+	for i := range cfg.ListenClientUrls {
+		ss[i] = cfg.ListenClientUrls[i].String()
+	}
+	return ss
+}
+
+func (cfg *Config) getListenClientHttpUrls() (ss []string) {
+	ss = make([]string, len(cfg.ListenClientHttpUrls))
+	for i := range cfg.ListenClientHttpUrls {
+		ss[i] = cfg.ListenClientHttpUrls[i].String()
 	}
 	return ss
 }

embed/config_logging.go

@@ -196,10 +196,14 @@ func (cfg *Config) setupLogging() error {
 					grpcLogOnce.Do(func() {
 						// debug true, enable info, warning, error
 						// debug false, only discard info
-						var gl grpclog.LoggerV2
-						gl, err = logutil.NewGRPCLoggerV2(copied)
-						if err == nil {
-							grpclog.SetLoggerV2(gl)
+						if cfg.LogLevel == "debug" {
+							var gl grpclog.LoggerV2
+							gl, err = logutil.NewGRPCLoggerV2(copied)
+							if err == nil {
+								grpclog.SetLoggerV2(gl)
+							}
+						} else {
+							grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr))
 						}
 					})
 					return nil
@@ -245,7 +249,11 @@ func (cfg *Config) setupLogging() error {
 					c.loggerWriteSyncer = syncer
 
 					grpcLogOnce.Do(func() {
-						grpclog.SetLoggerV2(logutil.NewGRPCLoggerV2FromZapCore(cr, syncer))
+						if cfg.LogLevel == "debug" {
+							grpclog.SetLoggerV2(logutil.NewGRPCLoggerV2FromZapCore(cr, syncer))
+						} else {
+							grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr))
+						}
 					})
 					return nil
 				}

embed/config_logging_journal_unix.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !windows
 // +build !windows
 
 package embed

embed/config_logging_journal_windows.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build windows
 // +build windows
 
 package embed

embed/config_test.go

@@ -15,6 +15,7 @@
 package embed
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io/ioutil"
 	"net/url"
@@ -22,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"go.etcd.io/etcd/pkg/transport"
 
 	"sigs.k8s.io/yaml"
@@ -75,12 +77,12 @@ func TestConfigFileOtherFields(t *testing.T) {
 func TestUpdateDefaultClusterFromName(t *testing.T) {
 	cfg := NewConfig()
 	defaultInitialCluster := cfg.InitialCluster
-	oldscheme := cfg.APUrls[0].Scheme
-	origpeer := cfg.APUrls[0].String()
-	origadvc := cfg.ACUrls[0].String()
+	oldscheme := cfg.AdvertisePeerUrls[0].Scheme
+	origpeer := cfg.AdvertisePeerUrls[0].String()
+	origadvc := cfg.AdvertiseClientUrls[0].String()
 
 	cfg.Name = "abc"
-	lpport := cfg.LPUrls[0].Port()
+	lpport := cfg.ListenPeerUrls[0].Port()
 
 	// in case of 'etcd --name=abc'
 	exp := fmt.Sprintf("%s=%s://localhost:%s", cfg.Name, oldscheme, lpport)
@@ -89,12 +91,12 @@ func TestUpdateDefaultClusterFromName(t *testing.T) {
 		t.Fatalf("initial-cluster expected %q, got %q", exp, cfg.InitialCluster)
 	}
 	// advertise peer URL should not be affected
-	if origpeer != cfg.APUrls[0].String() {
-		t.Fatalf("advertise peer url expected %q, got %q", origadvc, cfg.APUrls[0].String())
+	if origpeer != cfg.AdvertisePeerUrls[0].String() {
+		t.Fatalf("advertise peer url expected %q, got %q", origadvc, cfg.AdvertisePeerUrls[0].String())
 	}
 	// advertise client URL should not be affected
-	if origadvc != cfg.ACUrls[0].String() {
-		t.Fatalf("advertise client url expected %q, got %q", origadvc, cfg.ACUrls[0].String())
+	if origadvc != cfg.AdvertiseClientUrls[0].String() {
+		t.Fatalf("advertise client url expected %q, got %q", origadvc, cfg.AdvertiseClientUrls[0].String())
 	}
 }
 
@@ -107,17 +109,17 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
 
 	cfg := NewConfig()
 	defaultInitialCluster := cfg.InitialCluster
-	oldscheme := cfg.APUrls[0].Scheme
-	origadvc := cfg.ACUrls[0].String()
+	oldscheme := cfg.AdvertisePeerUrls[0].Scheme
+	origadvc := cfg.AdvertiseClientUrls[0].String()
 
 	cfg.Name = "abc"
-	lpport := cfg.LPUrls[0].Port()
-	cfg.LPUrls[0] = url.URL{Scheme: cfg.LPUrls[0].Scheme, Host: fmt.Sprintf("0.0.0.0:%s", lpport)}
+	lpport := cfg.ListenPeerUrls[0].Port()
+	cfg.ListenPeerUrls[0] = url.URL{Scheme: cfg.ListenPeerUrls[0].Scheme, Host: fmt.Sprintf("0.0.0.0:%s", lpport)}
 	dhost, _ := cfg.UpdateDefaultClusterFromName(defaultInitialCluster)
 	if dhost != defaultHostname {
 		t.Fatalf("expected default host %q, got %q", defaultHostname, dhost)
 	}
-	aphost, apport := cfg.APUrls[0].Hostname(), cfg.APUrls[0].Port()
+	aphost, apport := cfg.AdvertisePeerUrls[0].Hostname(), cfg.AdvertisePeerUrls[0].Port()
 	if apport != lpport {
 		t.Fatalf("advertise peer url got different port %s, expected %s", apport, lpport)
 	}
@@ -130,8 +132,8 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
 	}
 
 	// advertise client URL should not be affected
-	if origadvc != cfg.ACUrls[0].String() {
-		t.Fatalf("advertise-client-url expected %q, got %q", origadvc, cfg.ACUrls[0].String())
+	if origadvc != cfg.AdvertiseClientUrls[0].String() {
+		t.Fatalf("advertise-client-url expected %q, got %q", origadvc, cfg.AdvertiseClientUrls[0].String())
 	}
 }
 
@@ -202,3 +204,80 @@ func TestAutoCompactionModeParse(t *testing.T) {
 		}
 	}
 }
+
+func TestTLSVersionMinMax(t *testing.T) {
+	tests := []struct {
+		name                  string
+		givenTLSMinVersion    string
+		givenTLSMaxVersion    string
+		givenCipherSuites     []string
+		expectError           bool
+		expectedMinTLSVersion uint16
+		expectedMaxTLSVersion uint16
+	}{
+		{
+			name:                  "Minimum TLS version is set",
+			givenTLSMinVersion:    "TLS1.3",
+			expectedMinTLSVersion: tls.VersionTLS13,
+			expectedMaxTLSVersion: 0,
+		},
+		{
+			name:                  "Maximum TLS version is set",
+			givenTLSMaxVersion:    "TLS1.2",
+			expectedMinTLSVersion: 0,
+			expectedMaxTLSVersion: tls.VersionTLS12,
+		},
+		{
+			name:                  "Minimum and Maximum TLS versions are set",
+			givenTLSMinVersion:    "TLS1.3",
+			givenTLSMaxVersion:    "TLS1.3",
+			expectedMinTLSVersion: tls.VersionTLS13,
+			expectedMaxTLSVersion: tls.VersionTLS13,
+		},
+		{
+			name:               "Minimum and Maximum TLS versions are set in reverse order",
+			givenTLSMinVersion: "TLS1.3",
+			givenTLSMaxVersion: "TLS1.2",
+			expectError:        true,
+		},
+		{
+			name:               "Invalid minimum TLS version",
+			givenTLSMinVersion: "invalid version",
+			expectError:        true,
+		},
+		{
+			name:               "Invalid maximum TLS version",
+			givenTLSMaxVersion: "invalid version",
+			expectError:        true,
+		},
+		{
+			name:               "Cipher suites configured for TLS 1.3",
+			givenTLSMinVersion: "TLS1.3",
+			givenCipherSuites:  []string{"TLS_AES_128_GCM_SHA256"},
+			expectError:        true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := NewConfig()
+			cfg.TlsMinVersion = tt.givenTLSMinVersion
+			cfg.TlsMaxVersion = tt.givenTLSMaxVersion
+			cfg.CipherSuites = tt.givenCipherSuites
+
+			err := cfg.Validate()
+			if err != nil {
+				assert.True(t, tt.expectError, "Validate() returned error while expecting success: %v", err)
+				return
+			}
+
+			updateMinMaxVersions(&cfg.PeerTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion)
+			updateMinMaxVersions(&cfg.ClientTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion)
+
+			assert.Equal(t, tt.expectedMinTLSVersion, cfg.PeerTLSInfo.MinVersion)
+			assert.Equal(t, tt.expectedMaxTLSVersion, cfg.PeerTLSInfo.MaxVersion)
+			assert.Equal(t, tt.expectedMinTLSVersion, cfg.ClientTLSInfo.MinVersion)
+			assert.Equal(t, tt.expectedMaxTLSVersion, cfg.ClientTLSInfo.MaxVersion)
+		})
+	}
+}

embed/etcd.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	defaultLog "log"
+	"math"
 	"net"
 	"net/http"
 	"net/url"
@@ -29,6 +30,7 @@ import (
 	"sync"
 	"time"
 
+	"go.etcd.io/etcd/clientv3/credentials"
 	"go.etcd.io/etcd/etcdserver"
 	"go.etcd.io/etcd/etcdserver/api/etcdhttp"
 	"go.etcd.io/etcd/etcdserver/api/rafthttp"
@@ -116,7 +118,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 	if e.cfg.logger != nil {
 		e.cfg.logger.Info(
 			"configuring peer listeners",
-			zap.Strings("listen-peer-urls", e.cfg.getLPURLs()),
+			zap.Strings("listen-peer-urls", e.cfg.getListenPeerUrls()),
 		)
 	}
 	if e.Peers, err = configurePeerListeners(cfg); err != nil {
@@ -126,7 +128,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 	if e.cfg.logger != nil {
 		e.cfg.logger.Info(
 			"configuring client listeners",
-			zap.Strings("listen-client-urls", e.cfg.getLCURLs()),
+			zap.Strings("listen-client-urls", e.cfg.getListenClientUrls()),
 		)
 	}
 	if e.sctxs, err = configureClientListeners(cfg); err != nil {
@@ -163,8 +165,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 
 	srvcfg := etcdserver.ServerConfig{
 		Name:                        cfg.Name,
-		ClientURLs:                  cfg.ACUrls,
-		PeerURLs:                    cfg.APUrls,
+		ClientURLs:                  cfg.AdvertiseClientUrls,
+		PeerURLs:                    cfg.AdvertisePeerUrls,
 		DataDir:                     cfg.Dir,
 		DedicatedWALDir:             cfg.WalDir,
 		SnapshotCount:               cfg.SnapshotCount,
@@ -188,6 +190,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		BackendBatchInterval:        cfg.BackendBatchInterval,
 		MaxTxnOps:                   cfg.MaxTxnOps,
 		MaxRequestBytes:             cfg.MaxRequestBytes,
+		MaxConcurrentStreams:        cfg.MaxConcurrentStreams,
 		StrictReconfigCheck:         cfg.StrictReconfigCheck,
 		ClientCertAuthEnabled:       cfg.ClientTLSInfo.ClientCertAuth,
 		AuthToken:                   cfg.AuthToken,
@@ -207,8 +210,10 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		EnableGRPCGateway:           cfg.EnableGRPCGateway,
 		UnsafeNoFsync:               cfg.UnsafeNoFsync,
 		EnableLeaseCheckpoint:       cfg.ExperimentalEnableLeaseCheckpoint,
+		LeaseCheckpointPersist:      cfg.ExperimentalEnableLeaseCheckpointPersist,
 		CompactionBatchLimit:        cfg.ExperimentalCompactionBatchLimit,
 		WatchProgressNotifyInterval: cfg.ExperimentalWatchProgressNotifyInterval,
+		WarningApplyDuration:        cfg.ExperimentalWarningApplyDuration,
 	}
 	print(e.cfg.logger, *cfg, srvcfg, memberInitialized)
 	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
@@ -244,10 +249,10 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		e.cfg.logger.Info(
 			"now serving peer/client/metrics",
 			zap.String("local-member-id", e.Server.ID().String()),
-			zap.Strings("initial-advertise-peer-urls", e.cfg.getAPURLs()),
-			zap.Strings("listen-peer-urls", e.cfg.getLPURLs()),
-			zap.Strings("advertise-client-urls", e.cfg.getACURLs()),
-			zap.Strings("listen-client-urls", e.cfg.getLCURLs()),
+			zap.Strings("initial-advertise-peer-urls", e.cfg.getAdvertisePeerUrls()),
+			zap.Strings("listen-peer-urls", e.cfg.getListenPeerUrls()),
+			zap.Strings("advertise-client-urls", e.cfg.getAdvertiseClientUrls()),
+			zap.Strings("listen-client-urls", e.cfg.getListenClientUrls()),
 			zap.Strings("listen-metrics-urls", e.cfg.getMetricsURLs()),
 		)
 	}
@@ -319,18 +324,23 @@ func print(lg *zap.Logger, ec Config, sc etcdserver.ServerConfig, memberInitiali
 			zap.String("election-timeout", fmt.Sprintf("%v", time.Duration(sc.ElectionTicks*int(sc.TickMs))*time.Millisecond)),
 			zap.Bool("initial-election-tick-advance", sc.InitialElectionTickAdvance),
 			zap.Uint64("snapshot-count", sc.SnapshotCount),
+			zap.Uint("max-wals", sc.MaxWALFiles),
+			zap.Uint("max-snapshots", sc.MaxSnapFiles),
 			zap.Uint64("snapshot-catchup-entries", sc.SnapshotCatchUpEntries),
-			zap.Strings("initial-advertise-peer-urls", ec.getAPURLs()),
-			zap.Strings("listen-peer-urls", ec.getLPURLs()),
-			zap.Strings("advertise-client-urls", ec.getACURLs()),
-			zap.Strings("listen-client-urls", ec.getLCURLs()),
+			zap.Strings("initial-advertise-peer-urls", ec.getAdvertisePeerUrls()),
+			zap.Strings("listen-peer-urls", ec.getListenPeerUrls()),
+			zap.Strings("advertise-client-urls", ec.getAdvertiseClientUrls()),
+			zap.Strings("listen-client-urls", ec.getListenClientUrls()),
 			zap.Strings("listen-metrics-urls", ec.getMetricsURLs()),
 			zap.Strings("cors", cors),
 			zap.Strings("host-whitelist", hss),
 			zap.String("initial-cluster", sc.InitialPeerURLsMap.String()),
 			zap.String("initial-cluster-state", ec.ClusterState),
 			zap.String("initial-cluster-token", sc.InitialClusterToken),
-			zap.Int64("quota-size-bytes", quota),
+			zap.Int64("quota-backend-bytes", quota),
+			zap.Uint("max-request-bytes", sc.MaxRequestBytes),
+			zap.Uint32("max-concurrent-streams", sc.MaxConcurrentStreams),
+
 			zap.Bool("pre-vote", sc.PreVote),
 			zap.Bool("initial-corrupt-check", sc.InitialCorruptCheck),
 			zap.String("corrupt-check-time-interval", sc.CorruptCheckTime.String()),
@@ -355,8 +365,8 @@ func (e *Etcd) Close() {
 	fields := []zap.Field{
 		zap.String("name", e.cfg.Name),
 		zap.String("data-dir", e.cfg.Dir),
-		zap.Strings("advertise-peer-urls", e.cfg.getAPURLs()),
-		zap.Strings("advertise-client-urls", e.cfg.getACURLs()),
+		zap.Strings("advertise-peer-urls", e.cfg.getAdvertisePeerUrls()),
+		zap.Strings("advertise-client-urls", e.cfg.getAdvertiseClientUrls()),
 	}
 	lg := e.GetLogger()
 	if lg != nil {
@@ -424,7 +434,7 @@ func stopServers(ctx context.Context, ss *servers) {
 	// do not grpc.Server.GracefulStop with TLS enabled etcd server
 	// See https://github.com/grpc/grpc-go/issues/1384#issuecomment-317124531
 	// and https://github.com/etcd-io/etcd/issues/8916
-	if ss.secure {
+	if ss.secure && ss.http != nil {
 		shutdownNow()
 		return
 	}
@@ -463,6 +473,9 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) {
 			plog.Fatalf("could not get certs (%v)", err)
 		}
 	}
+
+	updateMinMaxVersions(&cfg.PeerTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion)
+
 	if !cfg.PeerTLSInfo.Empty() {
 		if cfg.logger != nil {
 			cfg.logger.Info(
@@ -475,7 +488,7 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) {
 		}
 	}
 
-	peers = make([]*peerListener, len(cfg.LPUrls))
+	peers = make([]*peerListener, len(cfg.ListenPeerUrls))
 	defer func() {
 		if err == nil {
 			return
@@ -485,11 +498,11 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) {
 				if cfg.logger != nil {
 					cfg.logger.Warn(
 						"closing peer listener",
-						zap.String("address", cfg.LPUrls[i].String()),
+						zap.String("address", cfg.ListenPeerUrls[i].String()),
 						zap.Error(err),
 					)
 				} else {
-					plog.Info("stopping listening for peers on ", cfg.LPUrls[i].String())
+					plog.Info("stopping listening for peers on ", cfg.ListenPeerUrls[i].String())
 				}
 				ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 				peers[i].close(ctx)
@@ -498,7 +511,7 @@ func configurePeerListeners(cfg *Config) (peers []*peerListener, err error) {
 		}
 	}()
 
-	for i, u := range cfg.LPUrls {
+	for i, u := range cfg.ListenPeerUrls {
 		if u.Scheme == "http" {
 			if !cfg.PeerTLSInfo.Empty() {
 				if cfg.logger != nil {
@@ -600,6 +613,9 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro
 			plog.Fatalf("could not get certs (%v)", err)
 		}
 	}
+
+	updateMinMaxVersions(&cfg.ClientTLSInfo, cfg.TlsMinVersion, cfg.TlsMaxVersion)
+
 	if cfg.EnablePprof {
 		if cfg.logger != nil {
 			cfg.logger.Info("pprof is enabled", zap.String("path", debugutil.HTTPPrefixPProf))
@@ -609,8 +625,7 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro
 	}
 
 	sctxs = make(map[string]*serveCtx)
-	for _, u := range cfg.LCUrls {
-		sctx := newServeCtx(cfg.logger)
+	for _, u := range append(cfg.ListenClientUrls, cfg.ListenClientHttpUrls...) {
 		if u.Scheme == "http" || u.Scheme == "unix" {
 			if !cfg.ClientTLSInfo.Empty() {
 				if cfg.logger != nil {
@@ -630,29 +645,45 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro
 		if (u.Scheme == "https" || u.Scheme == "unixs") && cfg.ClientTLSInfo.Empty() {
 			return nil, fmt.Errorf("TLS key/cert (--cert-file, --key-file) must be provided for client url %s with HTTPS scheme", u.String())
 		}
+	}
 
-		network := "tcp"
-		addr := u.Host
-		if u.Scheme == "unix" || u.Scheme == "unixs" {
-			network = "unix"
-			addr = u.Host + u.Path
+	for _, u := range cfg.ListenClientUrls {
+		addr, secure, network := resolveUrl(u)
+		sctx := sctxs[addr]
+		if sctx == nil {
+			sctx = newServeCtx(cfg.logger)
+			sctxs[addr] = sctx
 		}
+		sctx.secure = sctx.secure || secure
+		sctx.insecure = sctx.insecure || !secure
+		sctx.scheme = u.Scheme
+		sctx.addr = addr
 		sctx.network = network
+	}
+	for _, u := range cfg.ListenClientHttpUrls {
+		addr, secure, network := resolveUrl(u)
 
-		sctx.secure = u.Scheme == "https" || u.Scheme == "unixs"
-		sctx.insecure = !sctx.secure
-		if oldctx := sctxs[addr]; oldctx != nil {
-			oldctx.secure = oldctx.secure || sctx.secure
-			oldctx.insecure = oldctx.insecure || sctx.insecure
-			continue
+		sctx := sctxs[addr]
+		if sctx == nil {
+			sctx = newServeCtx(cfg.logger)
+			sctxs[addr] = sctx
+		} else if !sctx.httpOnly {
+			return nil, fmt.Errorf("cannot bind both --client-listen-urls and --client-listen-http-urls on the same url %s", u.String())
 		}
+		sctx.secure = sctx.secure || secure
+		sctx.insecure = sctx.insecure || !secure
+		sctx.scheme = u.Scheme
+		sctx.addr = addr
+		sctx.network = network
+		sctx.httpOnly = true
+	}
 
-		if sctx.l, err = net.Listen(network, addr); err != nil {
+	for _, sctx := range sctxs {
+		if sctx.l, err = net.Listen(sctx.network, sctx.addr); err != nil {
 			return nil, err
 		}
 		// net.Listener will rewrite ipv4 0.0.0.0 to ipv6 [::], breaking
 		// hosts that disable ipv6. So, use the address given by the user.
-		sctx.addr = addr
 
 		if fdLimit, fderr := runtimeutil.FDLimit(); fderr == nil {
 			if fdLimit <= reservedInternalFDNum {
@@ -669,27 +700,27 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro
 			sctx.l = transport.LimitListener(sctx.l, int(fdLimit-reservedInternalFDNum))
 		}
 
-		if network == "tcp" {
-			if sctx.l, err = transport.NewKeepAliveListener(sctx.l, network, nil); err != nil {
+		if sctx.network == "tcp" {
+			if sctx.l, err = transport.NewKeepAliveListener(sctx.l, sctx.network, nil); err != nil {
 				return nil, err
 			}
 		}
 
-		defer func() {
-			if err == nil {
+		defer func(sctx *serveCtx) {
+			if err == nil || sctx.l == nil {
 				return
 			}
 			sctx.l.Close()
 			if cfg.logger != nil {
 				cfg.logger.Warn(
 					"closing peer listener",
-					zap.String("address", u.Host),
+					zap.String("address", sctx.addr),
 					zap.Error(err),
 				)
 			} else {
-				plog.Info("stopping listening for client requests on ", u.Host)
+				plog.Info("stopping listening for client requests on ", sctx.addr)
 			}
-		}()
+		}(sctx)
 		for k := range cfg.UserHandlers {
 			sctx.userHandlers[k] = cfg.UserHandlers[k]
 		}
@@ -700,11 +731,21 @@ func configureClientListeners(cfg *Config) (sctxs map[string]*serveCtx, err erro
 		if cfg.Debug {
 			sctx.registerTrace()
 		}
-		sctxs[addr] = sctx
 	}
 	return sctxs, nil
 }
 
+func resolveUrl(u url.URL) (addr string, secure bool, network string) {
+	addr = u.Host
+	network = "tcp"
+	if u.Scheme == "unix" || u.Scheme == "unixs" {
+		addr = u.Host + u.Path
+		network = "unix"
+	}
+	secure = u.Scheme == "https" || u.Scheme == "unixs"
+	return addr, secure, network
+}
+
 func (e *Etcd) serveClients() (err error) {
 	if !e.cfg.ClientTLSInfo.Empty() {
 		if e.cfg.logger != nil {
@@ -748,15 +789,69 @@ func (e *Etcd) serveClients() (err error) {
 		}))
 	}
 
+	splitHttp := false
+	for _, sctx := range e.sctxs {
+		if sctx.httpOnly {
+			splitHttp = true
+		}
+	}
+
 	// start client servers in each goroutine
 	for _, sctx := range e.sctxs {
 		go func(s *serveCtx) {
-			e.errHandler(s.serve(e.Server, &e.cfg.ClientTLSInfo, h, e.errHandler, gopts...))
+			e.errHandler(s.serve(e.Server, &e.cfg.ClientTLSInfo, h, e.errHandler, e.grpcGatewayDial(splitHttp), splitHttp, gopts...))
 		}(sctx)
 	}
 	return nil
 }
 
+func (e *Etcd) grpcGatewayDial(splitHttp bool) (grpcDial func(ctx context.Context) (*grpc.ClientConn, error)) {
+	if !e.cfg.EnableGRPCGateway {
+		return nil
+	}
+	sctx := e.pickGrpcGatewayServeContext(splitHttp)
+	addr := sctx.addr
+	if network := sctx.network; network == "unix" {
+		// explicitly define unix network for gRPC socket support
+		addr = fmt.Sprintf("%s://%s", network, addr)
+	}
+
+	opts := []grpc.DialOption{grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(math.MaxInt32))}
+	if sctx.secure {
+		tlscfg, tlsErr := e.cfg.ClientTLSInfo.ServerConfig()
+		if tlsErr != nil {
+			return func(ctx context.Context) (*grpc.ClientConn, error) {
+				return nil, tlsErr
+			}
+		}
+		dtls := tlscfg.Clone()
+		// trust local server
+		dtls.InsecureSkipVerify = true
+		bundle := credentials.NewBundle(credentials.Config{TLSConfig: dtls})
+		opts = append(opts, grpc.WithTransportCredentials(bundle.TransportCredentials()))
+	} else {
+		opts = append(opts, grpc.WithInsecure())
+	}
+
+	return func(ctx context.Context) (*grpc.ClientConn, error) {
+		conn, err := grpc.DialContext(ctx, addr, opts...)
+		if err != nil {
+			sctx.lg.Error("grpc gateway failed to dial", zap.String("addr", addr), zap.Error(err))
+			return nil, err
+		}
+		return conn, err
+	}
+}
+
+func (e *Etcd) pickGrpcGatewayServeContext(splitHttp bool) *serveCtx {
+	for _, sctx := range e.sctxs {
+		if !splitHttp || !sctx.httpOnly {
+			return sctx
+		}
+	}
+	panic("Expect at least one context able to serve grpc")
+}
+
 func (e *Etcd) serveMetrics() (err error) {
 	if e.cfg.Metrics == "extensive" {
 		grpc_prometheus.EnableHandlingTimeHistogram()

embed/serve.go

@@ -23,7 +23,6 @@ import (
 	"net/http"
 	"strings"
 
-	"go.etcd.io/etcd/clientv3/credentials"
 	"go.etcd.io/etcd/etcdserver"
 	"go.etcd.io/etcd/etcdserver/api/v3client"
 	"go.etcd.io/etcd/etcdserver/api/v3election"
@@ -42,17 +41,21 @@ import (
 	"github.com/soheilhy/cmux"
 	"github.com/tmc/grpc-websocket-proxy/wsproxy"
 	"go.uber.org/zap"
+	"golang.org/x/net/http2"
 	"golang.org/x/net/trace"
 	"google.golang.org/grpc"
 )
 
 type serveCtx struct {
-	lg       *zap.Logger
-	l        net.Listener
+	lg *zap.Logger
+	l  net.Listener
+
+	scheme   string
 	addr     string
 	network  string
 	secure   bool
 	insecure bool
+	httpOnly bool
 
 	ctx    context.Context
 	cancel context.CancelFunc
@@ -87,6 +90,8 @@ func (sctx *serveCtx) serve(
 	tlsinfo *transport.TLSInfo,
 	handler http.Handler,
 	errHandler func(error),
+	grpcDialForRestGatewayBackends func(ctx context.Context) (*grpc.ClientConn, error),
+	splitHttp bool,
 	gopts ...grpc.ServerOption) (err error) {
 	logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0)
 	<-s.ReadyNotify()
@@ -96,48 +101,103 @@ func (sctx *serveCtx) serve(
 	}
 
 	m := cmux.New(sctx.l)
+	var server func() error
+	onlyGRPC := splitHttp && !sctx.httpOnly
+	onlyHttp := splitHttp && sctx.httpOnly
+	grpcEnabled := !onlyHttp
+	httpEnabled := !onlyGRPC
+
 	v3c := v3client.New(s)
 	servElection := v3election.NewElectionServer(v3c)
 	servLock := v3lock.NewLockServer(v3c)
 
-	var gs *grpc.Server
-	defer func() {
-		if err != nil && gs != nil {
-			gs.Stop()
+	// Make sure serversC is closed even if we prematurely exit the function.
+	defer close(sctx.serversC)
+	var gwmux *gw.ServeMux
+	if s.Cfg.EnableGRPCGateway {
+		// GRPC gateway connects to grpc server via connection provided by grpc dial.
+		gwmux, err = sctx.registerGateway(grpcDialForRestGatewayBackends)
+		if err != nil {
+			sctx.lg.Error("registerGateway failed", zap.Error(err))
+			return err
 		}
-	}()
+	}
+	var traffic string
+	switch {
+	case onlyGRPC:
+		traffic = "grpc"
+	case onlyHttp:
+		traffic = "http"
+	default:
+		traffic = "grpc+http"
+	}
 
 	if sctx.insecure {
-		gs = v3rpc.Server(s, nil, gopts...)
-		v3electionpb.RegisterElectionServer(gs, servElection)
-		v3lockpb.RegisterLockServer(gs, servLock)
-		if sctx.serviceRegister != nil {
-			sctx.serviceRegister(gs)
-		}
-		grpcl := m.Match(cmux.HTTP2())
-		go func() { errHandler(gs.Serve(grpcl)) }()
-
-		var gwmux *gw.ServeMux
-		if s.Cfg.EnableGRPCGateway {
-			gwmux, err = sctx.registerGateway([]grpc.DialOption{grpc.WithInsecure()})
-			if err != nil {
+		var gs *grpc.Server
+		var srv *http.Server
+		if httpEnabled {
+			httpmux := sctx.createMux(gwmux, handler)
+			srv = &http.Server{
+				Handler:  createAccessController(sctx.lg, s, httpmux),
+				ErrorLog: logger, // do not log user error
+			}
+			if err := configureHttpServer(srv, s.Cfg); err != nil {
+				sctx.lg.Error("Configure http server failed", zap.Error(err))
 				return err
 			}
 		}
+		if grpcEnabled {
+			gs = v3rpc.Server(s, nil, gopts...)
+			v3electionpb.RegisterElectionServer(gs, servElection)
+			v3lockpb.RegisterLockServer(gs, servLock)
+			if sctx.serviceRegister != nil {
+				sctx.serviceRegister(gs)
+			}
+			defer func(gs *grpc.Server) {
+				if err == nil {
+					return
+				}
 
-		httpmux := sctx.createMux(gwmux, handler)
+				if sctx.lg != nil {
+					sctx.lg.Warn("stopping insecure grpc server due to error", zap.Error(err))
+				} else {
+					plog.Warningf("stopping insecure grpc server due to error: %s", err)
+				}
 
-		srvhttp := &http.Server{
-			Handler:  createAccessController(sctx.lg, s, httpmux),
-			ErrorLog: logger, // do not log user error
+				gs.Stop()
+
+				if sctx.lg != nil {
+					sctx.lg.Warn("stopped insecure grpc server due to error", zap.Error(err))
+				} else {
+					plog.Warningf("stopped insecure grpc server due to error: %s", err)
+				}
+			}(gs)
 		}
-		httpl := m.Match(cmux.HTTP1())
-		go func() { errHandler(srvhttp.Serve(httpl)) }()
+		if onlyGRPC {
+			server = func() error {
+				return gs.Serve(sctx.l)
+			}
+		} else {
+			server = m.Serve
 
-		sctx.serversC <- &servers{grpc: gs, http: srvhttp}
+			httpl := m.Match(cmux.HTTP1())
+			go func(srvhttp *http.Server, tlsLis net.Listener) {
+				errHandler(srvhttp.Serve(tlsLis))
+			}(srv, httpl)
+
+			if grpcEnabled {
+				grpcl := m.Match(cmux.HTTP2())
+				go func(gs *grpc.Server, l net.Listener) {
+					errHandler(gs.Serve(l))
+				}(gs, grpcl)
+			}
+		}
+
+		sctx.serversC <- &servers{grpc: gs, http: srv}
 		if sctx.lg != nil {
 			sctx.lg.Info(
 				"serving client traffic insecurely; this is strongly discouraged!",
+				zap.String("traffic", traffic),
 				zap.String("address", sctx.l.Addr().String()),
 			)
 		} else {
@@ -146,50 +206,77 @@ func (sctx *serveCtx) serve(
 	}
 
 	if sctx.secure {
+		var gs *grpc.Server
+		var srv *http.Server
+
 		tlscfg, tlsErr := tlsinfo.ServerConfig()
 		if tlsErr != nil {
 			return tlsErr
 		}
-		gs = v3rpc.Server(s, tlscfg, gopts...)
-		v3electionpb.RegisterElectionServer(gs, servElection)
-		v3lockpb.RegisterLockServer(gs, servLock)
-		if sctx.serviceRegister != nil {
-			sctx.serviceRegister(gs)
-		}
-		handler = grpcHandlerFunc(gs, handler)
 
-		var gwmux *gw.ServeMux
-		if s.Cfg.EnableGRPCGateway {
-			dtls := tlscfg.Clone()
-			// trust local server
-			dtls.InsecureSkipVerify = true
-			bundle := credentials.NewBundle(credentials.Config{TLSConfig: dtls})
-			opts := []grpc.DialOption{grpc.WithTransportCredentials(bundle.TransportCredentials())}
-			gwmux, err = sctx.registerGateway(opts)
-			if err != nil {
+		if grpcEnabled {
+			gs = v3rpc.Server(s, tlscfg, gopts...)
+			v3electionpb.RegisterElectionServer(gs, servElection)
+			v3lockpb.RegisterLockServer(gs, servLock)
+			if sctx.serviceRegister != nil {
+				sctx.serviceRegister(gs)
+			}
+			defer func(gs *grpc.Server) {
+				if err == nil {
+					return
+				}
+
+				if sctx.lg != nil {
+					sctx.lg.Warn("stopping secure grpc server due to error", zap.Error(err))
+				} else {
+					plog.Warningf("stopping secure grpc server due to error: %s", err)
+				}
+
+				gs.Stop()
+
+				if sctx.lg != nil {
+					sctx.lg.Warn("stopped secure grpc server due to error", zap.Error(err))
+				} else {
+					plog.Warningf("stopped secure grpc server due to error: %s", err)
+				}
+			}(gs)
+		}
+		if httpEnabled {
+			if grpcEnabled {
+				handler = grpcHandlerFunc(gs, handler)
+			}
+			httpmux := sctx.createMux(gwmux, handler)
+
+			srv = &http.Server{
+				Handler:   createAccessController(sctx.lg, s, httpmux),
+				TLSConfig: tlscfg,
+				ErrorLog:  logger, // do not log user error
+			}
+			if err := configureHttpServer(srv, s.Cfg); err != nil {
+				sctx.lg.Error("Configure https server failed", zap.Error(err))
 				return err
 			}
 		}
 
-		var tlsl net.Listener
-		tlsl, err = transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo)
-		if err != nil {
-			return err
-		}
-		// TODO: add debug flag; enable logging when debug flag is set
-		httpmux := sctx.createMux(gwmux, handler)
+		if onlyGRPC {
+			server = func() error { return gs.Serve(sctx.l) }
+		} else {
+			server = m.Serve
 
-		srv := &http.Server{
-			Handler:   createAccessController(sctx.lg, s, httpmux),
-			TLSConfig: tlscfg,
-			ErrorLog:  logger, // do not log user error
+			tlsl, err := transport.NewTLSListener(m.Match(cmux.Any()), tlsinfo)
+			if err != nil {
+				return err
+			}
+			go func(srvhttp *http.Server, tlsl net.Listener) {
+				errHandler(srvhttp.Serve(tlsl))
+			}(srv, tlsl)
 		}
-		go func() { errHandler(srv.Serve(tlsl)) }()
 
 		sctx.serversC <- &servers{secure: true, grpc: gs, http: srv}
 		if sctx.lg != nil {
 			sctx.lg.Info(
 				"serving client traffic securely",
+				zap.String("traffic", traffic),
 				zap.String("address", sctx.l.Addr().String()),
 			)
 		} else {
@@ -197,8 +284,16 @@ func (sctx *serveCtx) serve(
 		}
 	}
 
-	close(sctx.serversC)
-	return m.Serve()
+	return server()
+}
+
+func configureHttpServer(srv *http.Server, cfg etcdserver.ServerConfig) error {
+	// todo (ahrtr): should we support configuring other parameters in the future as well?
+	return http2.ConfigureServer(srv, &http2.Server{
+		MaxConcurrentStreams: cfg.MaxConcurrentStreams,
+		// Override to avoid using priority scheduler which is affected by https://github.com/golang/go/issues/58804.
+		NewWriteScheduler: http2.NewRandomWriteScheduler,
+	})
 }
 
 // grpcHandlerFunc returns an http.Handler that delegates to grpcServer on incoming gRPC
@@ -220,16 +315,10 @@ func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Ha
 
 type registerHandlerFunc func(context.Context, *gw.ServeMux, *grpc.ClientConn) error
 
-func (sctx *serveCtx) registerGateway(opts []grpc.DialOption) (*gw.ServeMux, error) {
+func (sctx *serveCtx) registerGateway(dial func(ctx context.Context) (*grpc.ClientConn, error)) (*gw.ServeMux, error) {
 	ctx := sctx.ctx
 
-	addr := sctx.addr
-	if network := sctx.network; network == "unix" {
-		// explicitly define unix network for gRPC socket support
-		addr = fmt.Sprintf("%s://%s", network, addr)
-	}
-
-	conn, err := grpc.DialContext(ctx, addr, opts...)
+	conn, err := dial(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -268,6 +357,18 @@ func (sctx *serveCtx) registerGateway(opts []grpc.DialOption) (*gw.ServeMux, err
 	return gwmux, nil
 }
 
+type wsProxyZapLogger struct {
+	*zap.Logger
+}
+
+func (w wsProxyZapLogger) Warnln(i ...interface{}) {
+	w.Warn(fmt.Sprint(i...))
+}
+
+func (w wsProxyZapLogger) Debugln(i ...interface{}) {
+	w.Debug(fmt.Sprint(i...))
+}
+
 func (sctx *serveCtx) createMux(gwmux *gw.ServeMux, handler http.Handler) *http.ServeMux {
 	httpmux := http.NewServeMux()
 	for path, h := range sctx.userHandlers {
@@ -287,6 +388,7 @@ func (sctx *serveCtx) createMux(gwmux *gw.ServeMux, handler http.Handler) *http.
 					},
 				),
 				wsproxy.WithMaxRespBodyBufferSize(0x7fffffff),
+				wsproxy.WithLogger(wsProxyZapLogger{sctx.lg}),
 			),
 		)
 	}

etcdctl/ctlv2/ctl_cov.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build cov
 // +build cov
 
 package ctlv2

etcdctl/ctlv2/ctl_nocov.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !cov
 // +build !cov
 
 package ctlv2

etcdctl/ctlv3/command/check.go

@@ -311,6 +311,8 @@ func newCheckDatascaleCommand(cmd *cobra.Command, args []string) {
 		ExitWithError(ExitError, errEndpoints)
 	}
 
+	sec := secureCfgFromCmd(cmd)
+
 	ctx, cancel := context.WithCancel(context.Background())
 	resp, err := clients[0].Get(ctx, checkDatascalePrefix, v3.WithPrefix(), v3.WithLimit(1))
 	cancel()
@@ -329,7 +331,7 @@ func newCheckDatascaleCommand(cmd *cobra.Command, args []string) {
 	wg.Add(len(clients))
 
 	// get the process_resident_memory_bytes and process_virtual_memory_bytes before the put operations
-	bytesBefore := endpointMemoryMetrics(eps[0])
+	bytesBefore := endpointMemoryMetrics(eps[0], sec)
 	if bytesBefore == 0 {
 		fmt.Println("FAIL: Could not read process_resident_memory_bytes before the put operations.")
 		os.Exit(ExitError)
@@ -367,7 +369,7 @@ func newCheckDatascaleCommand(cmd *cobra.Command, args []string) {
 	s := <-sc
 
 	// get the process_resident_memory_bytes after the put operations
-	bytesAfter := endpointMemoryMetrics(eps[0])
+	bytesAfter := endpointMemoryMetrics(eps[0], sec)
 	if bytesAfter == 0 {
 		fmt.Println("FAIL: Could not read process_resident_memory_bytes after the put operations.")
 		os.Exit(ExitError)

etcdctl/ctlv3/command/get_command.go

@@ -31,6 +31,7 @@ var (
 	getFromKey     bool
 	getRev         int64
 	getKeysOnly    bool
+	getCountOnly   bool
 	printValueOnly bool
 )
 
@@ -50,6 +51,7 @@ func NewGetCommand() *cobra.Command {
 	cmd.Flags().BoolVar(&getFromKey, "from-key", false, "Get keys that are greater than or equal to the given key using byte compare")
 	cmd.Flags().Int64Var(&getRev, "rev", 0, "Specify the kv revision")
 	cmd.Flags().BoolVar(&getKeysOnly, "keys-only", false, "Get only the keys")
+	cmd.Flags().BoolVar(&getCountOnly, "count-only", false, "Get only the count")
 	cmd.Flags().BoolVar(&printValueOnly, "print-value-only", false, `Only write values when using the "simple" output format`)
 	return cmd
 }
@@ -64,6 +66,12 @@ func getCommandFunc(cmd *cobra.Command, args []string) {
 		ExitWithError(ExitError, err)
 	}
 
+	if getCountOnly {
+		if _, fields := display.(*fieldsPrinter); !fields {
+			ExitWithError(ExitBadArgs, fmt.Errorf("--count-only is only for `--write-out=fields`"))
+		}
+	}
+
 	if printValueOnly {
 		dp, simple := (display).(*simplePrinter)
 		if !simple {
@@ -83,6 +91,10 @@ func getGetOp(args []string) (string, []clientv3.OpOption) {
 		ExitWithError(ExitBadArgs, fmt.Errorf("`--prefix` and `--from-key` cannot be set at the same time, choose one"))
 	}
 
+	if getKeysOnly && getCountOnly {
+		ExitWithError(ExitBadArgs, fmt.Errorf("`--keys-only` and `--count-only` cannot be set at the same time, choose one"))
+	}
+
 	opts := []clientv3.OpOption{}
 	switch getConsistency {
 	case "s":
@@ -159,5 +171,9 @@ func getGetOp(args []string) (string, []clientv3.OpOption) {
 		opts = append(opts, clientv3.WithKeysOnly())
 	}
 
+	if getCountOnly {
+		opts = append(opts, clientv3.WithCountOnly())
+	}
+
 	return key, opts
 }

etcdctl/ctlv3/command/move_leader_command.go

@@ -42,7 +42,8 @@ func transferLeadershipCommandFunc(cmd *cobra.Command, args []string) {
 		ExitWithError(ExitBadArgs, err)
 	}
 
-	c := mustClientFromCmd(cmd)
+	cfg := clientConfigFromCmd(cmd)
+	c := cfg.mustClient()
 	eps := c.Endpoints()
 	c.Close()
 
@@ -52,7 +53,6 @@ func transferLeadershipCommandFunc(cmd *cobra.Command, args []string) {
 	var leaderCli *clientv3.Client
 	var leaderID uint64
 	for _, ep := range eps {
-		cfg := clientConfigFromCmd(cmd)
 		cfg.endpoints = []string{ep}
 		cli := cfg.mustClient()
 		resp, serr := cli.Status(ctx, ep)

etcdctl/ctlv3/command/util.go

@@ -16,6 +16,7 @@ package command
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/hex"
 	"fmt"
 	"io/ioutil"
@@ -90,14 +91,26 @@ func isCommandTimeoutFlagSet(cmd *cobra.Command) bool {
 	return commandTimeoutFlag.Changed
 }
 
-// get the process_resident_memory_bytes from <server:2379>/metrics
-func endpointMemoryMetrics(host string) float64 {
+// get the process_resident_memory_bytes from <server>/metrics
+func endpointMemoryMetrics(host string, scfg *secureCfg) float64 {
 	residentMemoryKey := "process_resident_memory_bytes"
 	var residentMemoryValue string
-	if !strings.HasPrefix(host, `http://`) {
+	if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
 		host = "http://" + host
 	}
 	url := host + "/metrics"
+	if strings.HasPrefix(host, "https://") {
+		// load client certificate
+		cert, err := tls.LoadX509KeyPair(scfg.cert, scfg.key)
+		if err != nil {
+			fmt.Println(fmt.Sprintf("client certificate error: %v", err))
+			return 0.0
+		}
+		http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{
+			Certificates:       []tls.Certificate{cert},
+			InsecureSkipVerify: scfg.insecureSkipVerify,
+		}
+	}
 	resp, err := http.Get(url)
 	if err != nil {
 		fmt.Println(fmt.Sprintf("fetch error: %v", err))

etcdctl/ctlv3/ctl_cov.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build cov
 // +build cov
 
 package ctlv3

etcdctl/ctlv3/ctl_nocov.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !cov
 // +build !cov
 
 package ctlv3

etcdmain/config.go

@@ -29,6 +29,7 @@ import (
 	"go.etcd.io/etcd/embed"
 	"go.etcd.io/etcd/pkg/flags"
 	"go.etcd.io/etcd/pkg/logutil"
+	"go.etcd.io/etcd/pkg/tlsutil"
 	"go.etcd.io/etcd/pkg/types"
 	"go.etcd.io/etcd/version"
 
@@ -140,7 +141,11 @@ func newConfig() *config {
 	)
 	fs.Var(
 		flags.NewUniqueURLsWithExceptions(embed.DefaultListenClientURLs, ""), "listen-client-urls",
-		"List of URLs to listen on for client traffic.",
+		"List of URLs to listen on for client grpc traffic and http as long as --listen-client-http-urls is not specified.",
+	)
+	fs.Var(
+		flags.NewUniqueURLsWithExceptions("", ""), "listen-client-http-urls",
+		"List of URLs to listen on for http only client traffic. Enabling this flag removes http services from --listen-client-urls.",
 	)
 	fs.Var(
 		flags.NewUniqueURLsWithExceptions("", ""),
@@ -163,6 +168,8 @@ func newConfig() *config {
 	fs.DurationVar(&cfg.ec.GRPCKeepAliveInterval, "grpc-keepalive-interval", cfg.ec.GRPCKeepAliveInterval, "Frequency duration of server-to-client ping to check if a connection is alive (0 to disable).")
 	fs.DurationVar(&cfg.ec.GRPCKeepAliveTimeout, "grpc-keepalive-timeout", cfg.ec.GRPCKeepAliveTimeout, "Additional duration of wait before closing a non-responsive connection (0 to disable).")
 
+	fs.Var(flags.NewUint32Value(cfg.ec.MaxConcurrentStreams), "max-concurrent-streams", "Maximum concurrent streams that each client can open at a time.")
+
 	// clustering
 	fs.Var(
 		flags.NewUniqueURLsWithExceptions(embed.DefaultInitialAdvertisePeerURLs, ""),
@@ -182,7 +189,7 @@ func newConfig() *config {
 	fs.StringVar(&cfg.ec.DNSClusterServiceName, "discovery-srv-name", cfg.ec.DNSClusterServiceName, "Service name to query when using DNS discovery.")
 	fs.StringVar(&cfg.ec.InitialCluster, "initial-cluster", cfg.ec.InitialCluster, "Initial cluster configuration for bootstrapping.")
 	fs.StringVar(&cfg.ec.InitialClusterToken, "initial-cluster-token", cfg.ec.InitialClusterToken, "Initial cluster token for the etcd cluster during bootstrap.")
-	fs.Var(cfg.cf.clusterState, "initial-cluster-state", "Initial cluster state ('new' or 'existing').")
+	fs.Var(cfg.cf.clusterState, "initial-cluster-state", "Initial cluster state ('new' when bootstrapping a new cluster or 'existing' when adding new members to an existing cluster). After successful initialization (bootstrapping or adding), flag is ignored on restarts.")
 
 	fs.BoolVar(&cfg.ec.StrictReconfigCheck, "strict-reconfig-check", cfg.ec.StrictReconfigCheck, "Reject reconfiguration requests that would cause quorum loss.")
 	fs.BoolVar(&cfg.ec.EnableV2, "enable-v2", cfg.ec.EnableV2, "Accept etcd V2 client requests.")
@@ -214,6 +221,8 @@ func newConfig() *config {
 	fs.StringVar(&cfg.ec.PeerTLSInfo.AllowedHostname, "peer-cert-allowed-hostname", "", "Allowed TLS hostname for inter peer authentication.")
 	fs.Var(flags.NewStringsValue(""), "cipher-suites", "Comma-separated list of supported TLS cipher suites between client/server and peers (empty will be auto-populated by Go).")
 	fs.BoolVar(&cfg.ec.PeerTLSInfo.SkipClientSANVerify, "experimental-peer-skip-client-san-verification", false, "Skip verification of SAN field in client certificate for peer connections.")
+	fs.StringVar(&cfg.ec.TlsMinVersion, "tls-min-version", string(tlsutil.TLSVersion12), "Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3.")
+	fs.StringVar(&cfg.ec.TlsMaxVersion, "tls-max-version", string(tlsutil.TLSVersionDefault), "Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty defers to Go).")
 
 	fs.Var(
 		flags.NewUniqueURLsWithExceptions("*", "*"),
@@ -255,9 +264,12 @@ func newConfig() *config {
 	fs.DurationVar(&cfg.ec.ExperimentalCorruptCheckTime, "experimental-corrupt-check-time", cfg.ec.ExperimentalCorruptCheckTime, "Duration of time between cluster corruption check passes.")
 	fs.StringVar(&cfg.ec.ExperimentalEnableV2V3, "experimental-enable-v2v3", cfg.ec.ExperimentalEnableV2V3, "v3 prefix for serving emulated v2 state.")
 	fs.StringVar(&cfg.ec.ExperimentalBackendFreelistType, "experimental-backend-bbolt-freelist-type", cfg.ec.ExperimentalBackendFreelistType, "ExperimentalBackendFreelistType specifies the type of freelist that boltdb backend uses(array and map are supported types)")
-	fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpoint, "experimental-enable-lease-checkpoint", false, "Enable to persist lease remaining TTL to prevent indefinite auto-renewal of long lived leases.")
+	fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpoint, "experimental-enable-lease-checkpoint", false, "Enable leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change.")
+	// TODO: delete in v3.7
+	fs.BoolVar(&cfg.ec.ExperimentalEnableLeaseCheckpointPersist, "experimental-enable-lease-checkpoint-persist", false, "Enable persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. Requires experimental-enable-lease-checkpoint to be enabled.")
 	fs.IntVar(&cfg.ec.ExperimentalCompactionBatchLimit, "experimental-compaction-batch-limit", cfg.ec.ExperimentalCompactionBatchLimit, "Sets the maximum revisions deleted in each compaction batch.")
 	fs.DurationVar(&cfg.ec.ExperimentalWatchProgressNotifyInterval, "experimental-watch-progress-notify-interval", cfg.ec.ExperimentalWatchProgressNotifyInterval, "Duration of periodic watch progress notifications.")
+	fs.DurationVar(&cfg.ec.ExperimentalWarningApplyDuration, "experimental-warning-apply-duration", cfg.ec.ExperimentalWarningApplyDuration, "Time duration after which a warning is generated if request takes more time.")
 
 	// unsafe
 	fs.BoolVar(&cfg.ec.UnsafeNoFsync, "unsafe-no-fsync", false, "Disables fsync, unsafe, will cause data loss.")
@@ -324,10 +336,11 @@ func (cfg *config) configFromCmdLine() error {
 		return err
 	}
 
-	cfg.ec.LPUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-peer-urls")
-	cfg.ec.APUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "initial-advertise-peer-urls")
-	cfg.ec.LCUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-urls")
-	cfg.ec.ACUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "advertise-client-urls")
+	cfg.ec.ListenPeerUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-peer-urls")
+	cfg.ec.AdvertisePeerUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "initial-advertise-peer-urls")
+	cfg.ec.ListenClientUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-urls")
+	cfg.ec.ListenClientHttpUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-client-http-urls")
+	cfg.ec.AdvertiseClientUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "advertise-client-urls")
 	cfg.ec.ListenMetricsUrls = flags.UniqueURLsFromFlag(cfg.cf.flagSet, "listen-metrics-urls")
 
 	cfg.ec.CORS = flags.UniqueURLsMapFromFlag(cfg.cf.flagSet, "cors")
@@ -335,6 +348,8 @@ func (cfg *config) configFromCmdLine() error {
 
 	cfg.ec.CipherSuites = flags.StringsFromFlag(cfg.cf.flagSet, "cipher-suites")
 
+	cfg.ec.MaxConcurrentStreams = flags.Uint32FromFlag(cfg.cf.flagSet, "max-concurrent-streams")
+
 	// TODO: remove this in v3.5
 	cfg.ec.DeprecatedLogOutput = flags.UniqueStringsFromFlag(cfg.cf.flagSet, "log-output")
 	cfg.ec.LogOutputs = flags.UniqueStringsFromFlag(cfg.cf.flagSet, "log-outputs")
@@ -346,7 +361,7 @@ func (cfg *config) configFromCmdLine() error {
 	// disable default advertise-client-urls if lcurls is set
 	missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls")
 	if !cfg.mayBeProxy() && missingAC {
-		cfg.ec.ACUrls = nil
+		cfg.ec.AdvertiseClientUrls = nil
 	}
 
 	// disable default initial-cluster if discovery is set

etcdmain/config_test.go

@@ -36,6 +36,7 @@ func TestConfigParsingMemberFlags(t *testing.T) {
 		"-snapshot-count=10",
 		"-listen-peer-urls=http://localhost:8000,https://localhost:8001",
 		"-listen-client-urls=http://localhost:7000,https://localhost:7001",
+		"-listen-client-http-urls=http://localhost:7002,https://localhost:7003",
 		// it should be set if -listen-client-urls is set
 		"-advertise-client-urls=http://localhost:7000,https://localhost:7001",
 	}
@@ -51,14 +52,15 @@ func TestConfigParsingMemberFlags(t *testing.T) {
 
 func TestConfigFileMemberFields(t *testing.T) {
 	yc := struct {
-		Dir           string `json:"data-dir"`
-		MaxSnapFiles  uint   `json:"max-snapshots"`
-		MaxWalFiles   uint   `json:"max-wals"`
-		Name          string `json:"name"`
-		SnapshotCount uint64 `json:"snapshot-count"`
-		LPUrls        string `json:"listen-peer-urls"`
-		LCUrls        string `json:"listen-client-urls"`
-		AcurlsCfgFile string `json:"advertise-client-urls"`
+		Dir                  string `json:"data-dir"`
+		MaxSnapFiles         uint   `json:"max-snapshots"`
+		MaxWalFiles          uint   `json:"max-wals"`
+		Name                 string `json:"name"`
+		SnapshotCount        uint64 `json:"snapshot-count"`
+		ListenPeerUrls       string `json:"listen-peer-urls"`
+		ListenClientUrls     string `json:"listen-client-urls"`
+		ListenClientHttpUrls string `json:"listen-client-http-urls"`
+		AdvertiseClientUrls  string `json:"advertise-client-urls"`
 	}{
 		"testdir",
 		10,
@@ -67,6 +69,7 @@ func TestConfigFileMemberFields(t *testing.T) {
 		10,
 		"http://localhost:8000,https://localhost:8001",
 		"http://localhost:7000,https://localhost:7001",
+		"http://localhost:7002,https://localhost:7003",
 		"http://localhost:7000,https://localhost:7001",
 	}
 
@@ -513,13 +516,14 @@ func mustCreateCfgFile(t *testing.T, b []byte) *os.File {
 
 func validateMemberFlags(t *testing.T, cfg *config) {
 	wcfg := &embed.Config{
-		Dir:           "testdir",
-		LPUrls:        []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}},
-		LCUrls:        []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}},
-		MaxSnapFiles:  10,
-		MaxWalFiles:   10,
-		Name:          "testname",
-		SnapshotCount: 10,
+		Dir:                  "testdir",
+		ListenPeerUrls:       []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}},
+		ListenClientUrls:     []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}},
+		ListenClientHttpUrls: []url.URL{{Scheme: "http", Host: "localhost:7002"}, {Scheme: "https", Host: "localhost:7003"}},
+		MaxSnapFiles:         10,
+		MaxWalFiles:          10,
+		Name:                 "testname",
+		SnapshotCount:        10,
 	}
 
 	if cfg.ec.Dir != wcfg.Dir {
@@ -537,18 +541,21 @@ func validateMemberFlags(t *testing.T, cfg *config) {
 	if cfg.ec.SnapshotCount != wcfg.SnapshotCount {
 		t.Errorf("snapcount = %v, want %v", cfg.ec.SnapshotCount, wcfg.SnapshotCount)
 	}
-	if !reflect.DeepEqual(cfg.ec.LPUrls, wcfg.LPUrls) {
-		t.Errorf("listen-peer-urls = %v, want %v", cfg.ec.LPUrls, wcfg.LPUrls)
+	if !reflect.DeepEqual(cfg.ec.ListenPeerUrls, wcfg.ListenPeerUrls) {
+		t.Errorf("listen-peer-urls = %v, want %v", cfg.ec.ListenPeerUrls, wcfg.ListenPeerUrls)
 	}
-	if !reflect.DeepEqual(cfg.ec.LCUrls, wcfg.LCUrls) {
-		t.Errorf("listen-client-urls = %v, want %v", cfg.ec.LCUrls, wcfg.LCUrls)
+	if !reflect.DeepEqual(cfg.ec.ListenClientUrls, wcfg.ListenClientUrls) {
+		t.Errorf("listen-client-urls = %v, want %v", cfg.ec.ListenClientUrls, wcfg.ListenClientUrls)
+	}
+	if !reflect.DeepEqual(cfg.ec.ListenClientHttpUrls, wcfg.ListenClientHttpUrls) {
+		t.Errorf("listen-client-http-urls = %v, want %v", cfg.ec.ListenClientHttpUrls, wcfg.ListenClientHttpUrls)
 	}
 }
 
 func validateClusteringFlags(t *testing.T, cfg *config) {
 	wcfg := newConfig()
-	wcfg.ec.APUrls = []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}}
-	wcfg.ec.ACUrls = []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}}
+	wcfg.ec.AdvertisePeerUrls = []url.URL{{Scheme: "http", Host: "localhost:8000"}, {Scheme: "https", Host: "localhost:8001"}}
+	wcfg.ec.AdvertiseClientUrls = []url.URL{{Scheme: "http", Host: "localhost:7000"}, {Scheme: "https", Host: "localhost:7001"}}
 	wcfg.ec.ClusterState = embed.ClusterStateFlagExisting
 	wcfg.cf.fallback.Set(fallbackFlagExit)
 	wcfg.ec.InitialCluster = "0=http://localhost:8000"
@@ -566,11 +573,11 @@ func validateClusteringFlags(t *testing.T, cfg *config) {
 	if cfg.ec.InitialClusterToken != wcfg.ec.InitialClusterToken {
 		t.Errorf("initialClusterToken = %v, want %v", cfg.ec.InitialClusterToken, wcfg.ec.InitialClusterToken)
 	}
-	if !reflect.DeepEqual(cfg.ec.APUrls, wcfg.ec.APUrls) {
-		t.Errorf("initial-advertise-peer-urls = %v, want %v", cfg.ec.APUrls, wcfg.ec.APUrls)
+	if !reflect.DeepEqual(cfg.ec.AdvertisePeerUrls, wcfg.ec.AdvertisePeerUrls) {
+		t.Errorf("initial-advertise-peer-urls = %v, want %v", cfg.ec.AdvertisePeerUrls, wcfg.ec.AdvertisePeerUrls)
 	}
-	if !reflect.DeepEqual(cfg.ec.ACUrls, wcfg.ec.ACUrls) {
-		t.Errorf("advertise-client-urls = %v, want %v", cfg.ec.ACUrls, wcfg.ec.ACUrls)
+	if !reflect.DeepEqual(cfg.ec.AdvertiseClientUrls, wcfg.ec.AdvertiseClientUrls) {
+		t.Errorf("advertise-client-urls = %v, want %v", cfg.ec.AdvertiseClientUrls, wcfg.ec.AdvertiseClientUrls)
 	}
 }
 

etcdmain/etcd.go

@@ -251,7 +251,7 @@ func startEtcdOrProxyV2() {
 					plog.Infof("forgot to set --initial-cluster flag?")
 				}
 			}
-			if types.URLs(cfg.ec.APUrls).String() == embed.DefaultInitialAdvertisePeerURLs {
+			if types.URLs(cfg.ec.AdvertisePeerUrls).String() == embed.DefaultInitialAdvertisePeerURLs {
 				if lg != nil {
 					lg.Warn("forgot to set --initial-advertise-peer-urls?")
 				} else {
@@ -507,11 +507,11 @@ func startProxy(cfg *config) error {
 
 	// setup self signed certs when serving https
 	cHosts, cTLS := []string{}, false
-	for _, u := range cfg.ec.LCUrls {
+	for _, u := range cfg.ec.ListenClientUrls {
 		cHosts = append(cHosts, u.Host)
 		cTLS = cTLS || u.Scheme == "https"
 	}
-	for _, u := range cfg.ec.ACUrls {
+	for _, u := range cfg.ec.AdvertiseClientUrls {
 		cHosts = append(cHosts, u.Host)
 		cTLS = cTLS || u.Scheme == "https"
 	}
@@ -528,7 +528,7 @@ func startProxy(cfg *config) error {
 	}
 
 	// Start a proxy server goroutine for each listen address
-	for _, u := range cfg.ec.LCUrls {
+	for _, u := range cfg.ec.ListenClientUrls {
 		l, err := transport.NewListener(u.Host, u.Scheme, &listenerTLS)
 		if err != nil {
 			return err

etcdmain/grpc_proxy.go

@@ -38,6 +38,7 @@ import (
 	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
 	"go.etcd.io/etcd/pkg/debugutil"
 	"go.etcd.io/etcd/pkg/logutil"
+	"go.etcd.io/etcd/pkg/tlsutil"
 	"go.etcd.io/etcd/pkg/transport"
 	"go.etcd.io/etcd/proxy/grpcproxy"
 
@@ -45,6 +46,7 @@ import (
 	"github.com/soheilhy/cmux"
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
+	"golang.org/x/net/http2"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
 )
@@ -69,11 +71,12 @@ var (
 
 	// tls for clients connecting to proxy
 
-	grpcProxyListenCA      string
-	grpcProxyListenCert    string
-	grpcProxyListenKey     string
-	grpcProxyListenAutoTLS bool
-	grpcProxyListenCRL     string
+	grpcProxyListenCA           string
+	grpcProxyListenCert         string
+	grpcProxyListenKey          string
+	grpcProxyListenCipherSuites []string
+	grpcProxyListenAutoTLS      bool
+	grpcProxyListenCRL          string
 
 	grpcProxyAdvertiseClientURL string
 	grpcProxyResolverPrefix     string
@@ -86,6 +89,8 @@ var (
 	grpcProxyEnableOrdering bool
 
 	grpcProxyDebug bool
+
+	maxConcurrentStreams uint32
 )
 
 const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024
@@ -137,6 +142,7 @@ func newGRPCProxyStartCommand() *cobra.Command {
 	cmd.Flags().StringVar(&grpcProxyListenCert, "cert-file", "", "identify secure connections to the proxy using this TLS certificate file")
 	cmd.Flags().StringVar(&grpcProxyListenKey, "key-file", "", "identify secure connections to the proxy using this TLS key file")
 	cmd.Flags().StringVar(&grpcProxyListenCA, "trusted-ca-file", "", "verify certificates of TLS-enabled secure proxy using this CA bundle")
+	cmd.Flags().StringSliceVar(&grpcProxyListenCipherSuites, "listen-cipher-suites", grpcProxyListenCipherSuites, "Comma-separated list of supported TLS cipher suites between client/proxy (empty will be auto-populated by Go).")
 	cmd.Flags().BoolVar(&grpcProxyListenAutoTLS, "auto-tls", false, "proxy TLS using generated certificates")
 	cmd.Flags().StringVar(&grpcProxyListenCRL, "client-crl-file", "", "proxy client certificate revocation list file.")
 
@@ -146,6 +152,8 @@ func newGRPCProxyStartCommand() *cobra.Command {
 
 	cmd.Flags().BoolVar(&grpcProxyDebug, "debug", false, "Enable debug-level logging for grpc-proxy.")
 
+	cmd.Flags().Uint32Var(&maxConcurrentStreams, "max-concurrent-streams", math.MaxUint32, "Maximum concurrent streams that each client can open at a time.")
+
 	return &cmd
 }
 
@@ -171,20 +179,27 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
 	}
 	grpclog.SetLoggerV2(gl)
 
-	tlsinfo := newTLS(grpcProxyListenCA, grpcProxyListenCert, grpcProxyListenKey)
-	if tlsinfo == nil && grpcProxyListenAutoTLS {
+	tlsInfo := newTLS(grpcProxyListenCA, grpcProxyListenCert, grpcProxyListenKey)
+	if len(grpcProxyListenCipherSuites) > 0 {
+		cs, err := tlsutil.GetCipherSuites(grpcProxyListenCipherSuites)
+		if err != nil {
+			log.Fatal(err)
+		}
+		tlsInfo.CipherSuites = cs
+	}
+	if tlsInfo == nil && grpcProxyListenAutoTLS {
 		host := []string{"https://" + grpcProxyListenAddr}
 		dir := filepath.Join(grpcProxyDataDir, "fixtures", "proxy")
 		autoTLS, err := transport.SelfCert(lg, dir, host)
 		if err != nil {
 			log.Fatal(err)
 		}
-		tlsinfo = &autoTLS
+		tlsInfo = &autoTLS
 	}
-	if tlsinfo != nil {
-		lg.Info("gRPC proxy server TLS", zap.String("tls-info", fmt.Sprintf("%+v", tlsinfo)))
+	if tlsInfo != nil {
+		lg.Info("gRPC proxy server TLS", zap.String("tls-info", fmt.Sprintf("%+v", tlsInfo)))
 	}
-	m := mustListenCMux(lg, tlsinfo)
+	m := mustListenCMux(lg, tlsInfo)
 	grpcl := m.Match(cmux.HTTP2())
 	defer func() {
 		grpcl.Close()
@@ -194,13 +209,20 @@ func startGRPCProxy(cmd *cobra.Command, args []string) {
 	client := mustNewClient(lg)
 	httpClient := mustNewHTTPClient(lg)
 
-	srvhttp, httpl := mustHTTPListener(lg, m, tlsinfo, client)
+	srvhttp, httpl := mustHTTPListener(lg, m, tlsInfo, client)
+
+	if err := http2.ConfigureServer(srvhttp, &http2.Server{
+		MaxConcurrentStreams: maxConcurrentStreams,
+	}); err != nil {
+		lg.Fatal("Failed to configure the http server", zap.Error(err))
+	}
+
 	errc := make(chan error)
 	go func() { errc <- newGRPCProxyServer(lg, client).Serve(grpcl) }()
 	go func() { errc <- srvhttp.Serve(httpl) }()
 	go func() { errc <- m.Serve() }()
 	if len(grpcProxyMetricsListenAddr) > 0 {
-		mhttpl := mustMetricsListener(lg, tlsinfo)
+		mhttpl := mustMetricsListener(lg, tlsInfo)
 		go func() {
 			mux := http.NewServeMux()
 			grpcproxy.HandleMetrics(mux, httpClient, client.Endpoints())
@@ -326,7 +348,7 @@ func mustListenCMux(lg *zap.Logger, tlsinfo *transport.TLSInfo) cmux.CMux {
 
 func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server {
 	if grpcProxyEnableOrdering {
-		vf := ordering.NewOrderViolationSwitchEndpointClosure(*client)
+		vf := ordering.NewOrderViolationSwitchEndpointClosure(client)
 		client.KV = ordering.NewKV(client.KV, vf)
 		lg.Info("waiting for linearized read from cluster to recover ordering")
 		for {
@@ -350,12 +372,12 @@ func newGRPCProxyServer(lg *zap.Logger, client *clientv3.Client) *grpc.Server {
 	}
 
 	kvp, _ := grpcproxy.NewKvProxy(client)
-	watchp, _ := grpcproxy.NewWatchProxy(client)
+	watchp, _ := grpcproxy.NewWatchProxy(client.Ctx(), client)
 	if grpcProxyResolverPrefix != "" {
 		grpcproxy.Register(client, grpcProxyResolverPrefix, grpcProxyAdvertiseClientURL, grpcProxyResolverTTL)
 	}
 	clusterp, _ := grpcproxy.NewClusterProxy(client, grpcProxyAdvertiseClientURL, grpcProxyResolverPrefix)
-	leasep, _ := grpcproxy.NewLeaseProxy(client)
+	leasep, _ := grpcproxy.NewLeaseProxy(client.Ctx(), client)
 	mainp := grpcproxy.NewMaintenanceProxy(client)
 	authp := grpcproxy.NewAuthProxy(client)
 	electionp := grpcproxy.NewElectionProxy(client)

etcdmain/help.go

@@ -62,7 +62,9 @@ Member:
   --listen-peer-urls 'http://localhost:2380'
     List of URLs to listen on for peer traffic.
   --listen-client-urls 'http://localhost:2379'
-    List of URLs to listen on for client traffic.
+    List of URLs to listen on for client grpc traffic and http as long as --listen-client-http-urls is not specified.
+  --listen-client-http-urls ''
+    List of URLs to listen on for http only client traffic. Enabling this flag removes http services from --listen-client-urls.
   --max-snapshots '` + strconv.Itoa(embed.DefaultMaxSnapshots) + `'
     Maximum number of snapshot files to retain (0 is unlimited).
   --max-wals '` + strconv.Itoa(embed.DefaultMaxWALs) + `'
@@ -77,6 +79,8 @@ Member:
     Maximum number of operations permitted in a transaction.
   --max-request-bytes '1572864'
     Maximum client request size in bytes the server will accept.
+  --max-concurrent-streams 'math.MaxUint32'
+    Maximum concurrent streams that each client can open at a time.
   --grpc-keepalive-min-time '5s'
     Minimum duration interval that a client should wait before pinging server.
   --grpc-keepalive-interval '2h'
@@ -90,7 +94,8 @@ Clustering:
   --initial-cluster 'default=http://localhost:2380'
     Initial cluster configuration for bootstrapping.
   --initial-cluster-state 'new'
-    Initial cluster state ('new' or 'existing').
+    Initial cluster state ('new' when bootstrapping a new cluster or 'existing' when adding new members to an existing cluster).
+    After successful initialization (bootstrapping or adding), flag is ignored on restarts.
   --initial-cluster-token 'etcd-cluster'
     Initial cluster token for the etcd cluster during bootstrap.
     Specifying this can protect you from unintended cross-cluster interaction when running multiple clusters.
@@ -156,6 +161,10 @@ Security:
     Comma-separated whitelist of origins for CORS, or cross-origin resource sharing, (empty or * means allow all).
   --host-whitelist '*'
     Acceptable hostnames from HTTP client requests, if server is not secure (empty or * means allow all).
+  --tls-min-version 'TLS1.2'
+    Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3.
+  --tls-max-version ''
+    Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty will be auto-populated by Go).
 
 Auth:
   --auth-token 'simple'
@@ -212,6 +221,8 @@ Experimental feature:
     Skip verification of SAN field in client certificate for peer connections.
   --experimental-watch-progress-notify-interval '10m'
     Duration of periodical watch progress notification.
+  --experimental-warning-apply-duration '100ms'
+    Warning is generated if requests take more than this duration.
 
 Unsafe feature:
   --force-new-cluster 'false'

etcdserver/api/etcdhttp/metrics.go

@@ -36,7 +36,7 @@ const (
 // HandleMetricsHealth registers metrics and health handlers.
 func HandleMetricsHealth(mux *http.ServeMux, srv etcdserver.ServerV2) {
 	mux.Handle(PathMetrics, promhttp.Handler())
-	mux.Handle(PathHealth, NewHealthHandler(func() Health { return checkHealth(srv) }))
+	mux.Handle(PathHealth, NewHealthHandler(func(excludedAlarms AlarmSet) Health { return checkHealth(srv, excludedAlarms) }))
 }
 
 // HandlePrometheus registers prometheus handler on '/metrics'.
@@ -45,7 +45,7 @@ func HandlePrometheus(mux *http.ServeMux) {
 }
 
 // NewHealthHandler handles '/health' requests.
-func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
+func NewHealthHandler(hfunc func(excludedAlarms AlarmSet) Health) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodGet {
 			w.Header().Set("Allow", http.MethodGet)
@@ -53,7 +53,8 @@ func NewHealthHandler(hfunc func() Health) http.HandlerFunc {
 			plog.Warningf("/health error (status code %d)", http.StatusMethodNotAllowed)
 			return
 		}
-		h := hfunc()
+		excludedAlarms := getExcludedAlarms(r)
+		h := hfunc(excludedAlarms)
 		d, _ := json.Marshal(h)
 		if h.Health != "true" {
 			http.Error(w, string(d), http.StatusServiceUnavailable)
@@ -90,16 +91,38 @@ type Health struct {
 	Health string `json:"health"`
 }
 
+type AlarmSet map[string]struct{}
+
+func getExcludedAlarms(r *http.Request) (alarms AlarmSet) {
+	alarms = make(map[string]struct{}, 2)
+	alms, found := r.URL.Query()["exclude"]
+	if found {
+		for _, alm := range alms {
+			if len(alms) == 0 {
+				continue
+			}
+			alarms[alm] = struct{}{}
+		}
+	}
+	return alarms
+}
+
 // TODO: server NOSPACE, etcdserver.ErrNoLeader in health API
 
-func checkHealth(srv etcdserver.ServerV2) Health {
+func checkHealth(srv etcdserver.ServerV2, excludedAlarms AlarmSet) Health {
 	h := Health{Health: "true"}
 
 	as := srv.Alarms()
 	if len(as) > 0 {
-		h.Health = "false"
 		for _, v := range as {
-			plog.Warningf("/health error due to an alarm %s", v.String())
+			alarmName := v.Alarm.String()
+			if _, found := excludedAlarms[alarmName]; found {
+				plog.Debugf("/health excluded alarm %s", v.String())
+				continue
+			}
+			h.Health = "false"
+			plog.Warningf("/health error due to %s", v.String())
+			return h
 		}
 	}
 
@@ -122,7 +145,7 @@ func checkHealth(srv etcdserver.ServerV2) Health {
 
 	if h.Health == "true" {
 		healthSuccess.Inc()
-		plog.Infof("/health OK (status code %d)", http.StatusOK)
+		plog.Debugf("/health OK (status code %d)", http.StatusOK)
 	} else {
 		healthFailed.Inc()
 	}

etcdserver/api/etcdhttp/metrics_test.go

@@ -0,0 +1,157 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcdhttp
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"go.etcd.io/etcd/etcdserver"
+	stats "go.etcd.io/etcd/etcdserver/api/v2stats"
+	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
+	"go.etcd.io/etcd/pkg/testutil"
+	"go.etcd.io/etcd/pkg/types"
+	"go.etcd.io/etcd/raft"
+)
+
+type fakeStats struct{}
+
+func (s *fakeStats) SelfStats() []byte   { return nil }
+func (s *fakeStats) LeaderStats() []byte { return nil }
+func (s *fakeStats) StoreStats() []byte  { return nil }
+
+type fakeServerV2 struct {
+	fakeServer
+	stats.Stats
+	health string
+}
+
+func (s *fakeServerV2) Leader() types.ID {
+	if s.health == "true" {
+		return 1
+	}
+	return types.ID(raft.None)
+}
+func (s *fakeServerV2) Do(ctx context.Context, r pb.Request) (etcdserver.Response, error) {
+	if s.health == "true" {
+		return etcdserver.Response{}, nil
+	}
+	return etcdserver.Response{}, fmt.Errorf("fail health check")
+}
+func (s *fakeServerV2) ClientCertAuthEnabled() bool { return false }
+
+func TestHealthHandler(t *testing.T) {
+	// define the input and expected output
+	// input: alarms, and healthCheckURL
+	tests := []struct {
+		alarms         []*pb.AlarmMember
+		healthCheckURL string
+		statusCode     int
+		health         string
+	}{
+		{
+			[]*pb.AlarmMember{},
+			"/health",
+			http.StatusOK,
+			"true",
+		},
+		{
+			[]*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}},
+			"/health",
+			http.StatusServiceUnavailable,
+			"false",
+		},
+		{
+			[]*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}},
+			"/health?exclude=NOSPACE",
+			http.StatusOK,
+			"true",
+		},
+		{
+			[]*pb.AlarmMember{},
+			"/health?exclude=NOSPACE",
+			http.StatusOK,
+			"true",
+		},
+		{
+			[]*pb.AlarmMember{{MemberID: uint64(1), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(2), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(3), Alarm: pb.AlarmType_NOSPACE}},
+			"/health?exclude=NOSPACE",
+			http.StatusOK,
+			"true",
+		},
+		{
+			[]*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}},
+			"/health?exclude=NOSPACE",
+			http.StatusServiceUnavailable,
+			"false",
+		},
+		{
+			[]*pb.AlarmMember{{MemberID: uint64(0), Alarm: pb.AlarmType_NOSPACE}, {MemberID: uint64(1), Alarm: pb.AlarmType_CORRUPT}},
+			"/health?exclude=NOSPACE&exclude=CORRUPT",
+			http.StatusOK,
+			"true",
+		},
+	}
+
+	for i, tt := range tests {
+		func() {
+			mux := http.NewServeMux()
+			HandleMetricsHealth(mux, &fakeServerV2{
+				fakeServer: fakeServer{alarms: tt.alarms},
+				Stats:      &fakeStats{},
+				health:     tt.health,
+			})
+			ts := httptest.NewServer(mux)
+			defer ts.Close()
+
+			res, err := ts.Client().Do(&http.Request{Method: http.MethodGet, URL: testutil.MustNewURL(t, ts.URL+tt.healthCheckURL)})
+			if err != nil {
+				t.Errorf("fail serve http request %s %v in test case #%d", tt.healthCheckURL, err, i+1)
+			}
+			if res == nil {
+				t.Errorf("got nil http response with http request %s in test case #%d", tt.healthCheckURL, i+1)
+				return
+			}
+			if res.StatusCode != tt.statusCode {
+				t.Errorf("want statusCode %d but got %d in test case #%d", tt.statusCode, res.StatusCode, i+1)
+			}
+			health, err := parseHealthOutput(res.Body)
+			if err != nil {
+				t.Errorf("fail parse health check output %v", err)
+			}
+			if health.Health != tt.health {
+				t.Errorf("want health %s but got %s", tt.health, health.Health)
+			}
+		}()
+	}
+}
+
+func parseHealthOutput(body io.Reader) (Health, error) {
+	obj := Health{}
+	d, derr := ioutil.ReadAll(body)
+	if derr != nil {
+		return obj, derr
+	}
+	if err := json.Unmarshal(d, &obj); err != nil {
+		return obj, err
+	}
+	return obj, nil
+}

etcdserver/api/etcdhttp/peer_test.go

@@ -58,6 +58,7 @@ func (c *fakeCluster) Version() *semver.Version              { return nil }
 
 type fakeServer struct {
 	cluster api.Cluster
+	alarms  []*pb.AlarmMember
 }
 
 func (s *fakeServer) AddMember(ctx context.Context, memb membership.Member) ([]*membership.Member, error) {
@@ -74,7 +75,7 @@ func (s *fakeServer) PromoteMember(ctx context.Context, id uint64) ([]*membershi
 }
 func (s *fakeServer) ClusterVersion() *semver.Version { return nil }
 func (s *fakeServer) Cluster() api.Cluster            { return s.cluster }
-func (s *fakeServer) Alarms() []*pb.AlarmMember       { return nil }
+func (s *fakeServer) Alarms() []*pb.AlarmMember       { return s.alarms }
 
 var fakeRaftHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("test data"))

etcdserver/api/membership/cluster.go

@@ -763,16 +763,21 @@ func ValidateClusterAndAssignIDs(lg *zap.Logger, local *RaftCluster, existing *R
 	if len(ems) != len(lms) {
 		return fmt.Errorf("member count is unequal")
 	}
-	sort.Sort(MembersByPeerURLs(ems))
-	sort.Sort(MembersByPeerURLs(lms))
 
 	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
 	defer cancel()
 	for i := range ems {
-		if ok, err := netutil.URLStringsEqual(ctx, lg, ems[i].PeerURLs, lms[i].PeerURLs); !ok {
-			return fmt.Errorf("unmatched member while checking PeerURLs (%v)", err)
+		var err error
+		ok := false
+		for j := range lms {
+			if ok, err = netutil.URLStringsEqual(ctx, lg, ems[i].PeerURLs, lms[j].PeerURLs); ok {
+				lms[j].ID = ems[i].ID
+				break
+			}
+		}
+		if !ok {
+			return fmt.Errorf("PeerURLs: no match found for existing member (%v, %v), last resolver error (%v)", ems[i].ID, ems[i].PeerURLs, err)
 		}
-		lms[i].ID = ems[i].ID
 	}
 	local.members = make(map[types.ID]*Member)
 	for _, m := range lms {

etcdserver/api/snap/snappb/snap.pb.go

@@ -2,13 +2,15 @@
 // source: snap.proto
 
 /*
-	Package snappb is a generated protocol buffer package.
+Package snappb is a generated protocol buffer package.
 
-	It is generated from these files:
-		snap.proto
+It is generated from these files:
 
-	It has these top-level messages:
-		Snapshot
+	snap.proto
+
+It has these top-level messages:
+
+	Snapshot
 */
 package snappb
 

etcdserver/api/v2store/node_extern_test.go

@@ -104,5 +104,5 @@ func TestNodeExternClone(t *testing.T) {
 func sameSlice(a, b []*NodeExtern) bool {
 	ah := (*reflect.SliceHeader)(unsafe.Pointer(&a))
 	bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	return *ah == *bh
+	return ah.Data == bh.Data && ah.Len == bh.Len && ah.Cap == bh.Cap
 }

etcdserver/api/v2store/stats_test.go

@@ -98,7 +98,7 @@ func TestStoreStatsDeleteFail(t *testing.T) {
 	testutil.AssertEqual(t, uint64(1), s.Stats.DeleteFail, "")
 }
 
-//Ensure that the number of expirations is recorded in the stats.
+// Ensure that the number of expirations is recorded in the stats.
 func TestStoreStatsExpireCount(t *testing.T) {
 	s := newStore()
 	fc := newFakeClock()

etcdserver/api/v2store/store_test.go

@@ -844,7 +844,7 @@ func TestStoreWatchSlowConsumer(t *testing.T) {
 	s.Watch("/foo", true, true, 0) // stream must be true
 	// Fill watch channel with 100 events
 	for i := 1; i <= 100; i++ {
-		s.Set("/foo", false, string(i), v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // ok
+		s.Set("/foo", false, string(rune(i)), v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // ok
 	}
 	// testutil.AssertEqual(t, s.WatcherHub.count, int64(1))
 	s.Set("/foo", false, "101", v2store.TTLOptionSet{ExpireTime: v2store.Permanent}) // ok

etcdserver/api/v2store/store_v2_test.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !v2v3
 // +build !v2v3
 
 package v2store_test

etcdserver/api/v2store/store_v2v3_test.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build v2v3
 // +build v2v3
 
 package v2store_test

etcdserver/api/v3client/doc.go

@@ -41,5 +41,4 @@
 //	if err != nil {
 //		// handle error!
 //	}
-//
 package v3client

etcdserver/api/v3election/v3electionpb/v3election.pb.go

@@ -2,21 +2,23 @@
 // source: v3election.proto
 
 /*
-	Package v3electionpb is a generated protocol buffer package.
+Package v3electionpb is a generated protocol buffer package.
 
-	It is generated from these files:
-		v3election.proto
+It is generated from these files:
 
-	It has these top-level messages:
-		CampaignRequest
-		CampaignResponse
-		LeaderKey
-		LeaderRequest
-		LeaderResponse
-		ResignRequest
-		ResignResponse
-		ProclaimRequest
-		ProclaimResponse
+	v3election.proto
+
+It has these top-level messages:
+
+	CampaignRequest
+	CampaignResponse
+	LeaderKey
+	LeaderRequest
+	LeaderResponse
+	ResignRequest
+	ResignResponse
+	ProclaimRequest
+	ProclaimResponse
 */
 package v3electionpb
 

etcdserver/api/v3lock/v3lockpb/v3lock.pb.go

@@ -2,16 +2,18 @@
 // source: v3lock.proto
 
 /*
-	Package v3lockpb is a generated protocol buffer package.
+Package v3lockpb is a generated protocol buffer package.
 
-	It is generated from these files:
-		v3lock.proto
+It is generated from these files:
 
-	It has these top-level messages:
-		LockRequest
-		LockResponse
-		UnlockRequest
-		UnlockResponse
+	v3lock.proto
+
+It has these top-level messages:
+
+	LockRequest
+	LockResponse
+	UnlockRequest
+	UnlockResponse
 */
 package v3lockpb
 

etcdserver/api/v3rpc/grpc.go

@@ -31,7 +31,6 @@ import (
 
 const (
 	grpcOverheadBytes = 512 * 1024
-	maxStreams        = math.MaxUint32
 	maxSendBytes      = math.MaxInt32
 )
 
@@ -53,7 +52,7 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, gopts ...grpc.ServerOptio
 	)))
 	opts = append(opts, grpc.MaxRecvMsgSize(int(s.Cfg.MaxRequestBytes+grpcOverheadBytes)))
 	opts = append(opts, grpc.MaxSendMsgSize(maxSendBytes))
-	opts = append(opts, grpc.MaxConcurrentStreams(maxStreams))
+	opts = append(opts, grpc.MaxConcurrentStreams(s.Cfg.MaxConcurrentStreams))
 	grpcServer := grpc.NewServer(append(opts, gopts...)...)
 
 	pb.RegisterKVServer(grpcServer, NewQuotaKVServer(s))

etcdserver/api/v3rpc/interceptor.go

@@ -217,8 +217,8 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor
 					return rpctypes.ErrGRPCNoLeader
 				}
 
-				cctx, cancel := context.WithCancel(ss.Context())
-				ss = serverStreamWithCtx{ctx: cctx, cancel: &cancel, ServerStream: ss}
+				ctx := newCancellableContext(ss.Context())
+				ss = serverStreamWithCtx{ctx: ctx, ServerStream: ss}
 
 				smap.mu.Lock()
 				smap.streams[ss] = struct{}{}
@@ -228,7 +228,8 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor
 					smap.mu.Lock()
 					delete(smap.streams, ss)
 					smap.mu.Unlock()
-					cancel()
+					// TODO: investigate whether the reason for cancellation here is useful to know
+					ctx.Cancel(nil)
 				}()
 			}
 		}
@@ -237,10 +238,52 @@ func newStreamInterceptor(s *etcdserver.EtcdServer) grpc.StreamServerInterceptor
 	}
 }
 
+// cancellableContext wraps a context with new cancellable context that allows a
+// specific cancellation error to be preserved and later retrieved using the
+// Context.Err() function. This is so downstream context users can disambiguate
+// the reason for the cancellation which could be from the client (for example)
+// or from this interceptor code.
+type cancellableContext struct {
+	context.Context
+
+	lock         sync.RWMutex
+	cancel       context.CancelFunc
+	cancelReason error
+}
+
+func newCancellableContext(parent context.Context) *cancellableContext {
+	ctx, cancel := context.WithCancel(parent)
+	return &cancellableContext{
+		Context: ctx,
+		cancel:  cancel,
+	}
+}
+
+// Cancel stores the cancellation reason and then delegates to context.WithCancel
+// against the parent context.
+func (c *cancellableContext) Cancel(reason error) {
+	c.lock.Lock()
+	c.cancelReason = reason
+	c.lock.Unlock()
+	c.cancel()
+}
+
+// Err will return the preserved cancel reason error if present, and will
+// otherwise return the underlying error from the parent context.
+func (c *cancellableContext) Err() error {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	if c.cancelReason != nil {
+		return c.cancelReason
+	}
+	return c.Context.Err()
+}
+
 type serverStreamWithCtx struct {
 	grpc.ServerStream
-	ctx    context.Context
-	cancel *context.CancelFunc
+
+	// ctx is used so that we can preserve a reason for cancellation.
+	ctx *cancellableContext
 }
 
 func (ssc serverStreamWithCtx) Context() context.Context { return ssc.ctx }
@@ -272,7 +315,7 @@ func monitorLeader(s *etcdserver.EtcdServer) *streamsMap {
 					smap.mu.Lock()
 					for ss := range smap.streams {
 						if ssWithCtx, ok := ss.(serverStreamWithCtx); ok {
-							(*ssWithCtx.cancel)()
+							ssWithCtx.ctx.Cancel(rpctypes.ErrGRPCNoLeader)
 							<-ss.Context().Done()
 						}
 					}

etcdserver/api/v3rpc/rpctypes/error.go

@@ -35,6 +35,8 @@ var (
 	ErrGRPCLeaseExist       = status.New(codes.FailedPrecondition, "etcdserver: lease already exists").Err()
 	ErrGRPCLeaseTTLTooLarge = status.New(codes.OutOfRange, "etcdserver: too large lease TTL").Err()
 
+	ErrGRPCWatchCanceled = status.New(codes.Canceled, "etcdserver: watch canceled").Err()
+
 	ErrGRPCMemberExist            = status.New(codes.FailedPrecondition, "etcdserver: member ID already exist").Err()
 	ErrGRPCPeerURLExist           = status.New(codes.FailedPrecondition, "etcdserver: Peer URLs already exists").Err()
 	ErrGRPCMemberNotEnoughStarted = status.New(codes.FailedPrecondition, "etcdserver: re-configuration failed due to not enough started members").Err()
@@ -56,12 +58,14 @@ var (
 	ErrGRPCRoleNotFound         = status.New(codes.FailedPrecondition, "etcdserver: role name not found").Err()
 	ErrGRPCRoleEmpty            = status.New(codes.InvalidArgument, "etcdserver: role name is empty").Err()
 	ErrGRPCAuthFailed           = status.New(codes.InvalidArgument, "etcdserver: authentication failed, invalid user ID or password").Err()
+	ErrGRPCPermissionNotGiven   = status.New(codes.InvalidArgument, "etcdserver: permission not given").Err()
 	ErrGRPCPermissionDenied     = status.New(codes.PermissionDenied, "etcdserver: permission denied").Err()
 	ErrGRPCRoleNotGranted       = status.New(codes.FailedPrecondition, "etcdserver: role is not granted to the user").Err()
 	ErrGRPCPermissionNotGranted = status.New(codes.FailedPrecondition, "etcdserver: permission is not granted to the role").Err()
 	ErrGRPCAuthNotEnabled       = status.New(codes.FailedPrecondition, "etcdserver: authentication is not enabled").Err()
 	ErrGRPCInvalidAuthToken     = status.New(codes.Unauthenticated, "etcdserver: invalid auth token").Err()
 	ErrGRPCInvalidAuthMgmt      = status.New(codes.InvalidArgument, "etcdserver: invalid auth management").Err()
+	ErrGRPCAuthOldRevision      = status.New(codes.InvalidArgument, "etcdserver: revision of auth store is old").Err()
 
 	ErrGRPCNoLeader                   = status.New(codes.Unavailable, "etcdserver: no leader").Err()
 	ErrGRPCNotLeader                  = status.New(codes.FailedPrecondition, "etcdserver: not leader").Err()
@@ -71,6 +75,7 @@ var (
 	ErrGRPCTimeout                    = status.New(codes.Unavailable, "etcdserver: request timed out").Err()
 	ErrGRPCTimeoutDueToLeaderFail     = status.New(codes.Unavailable, "etcdserver: request timed out, possibly due to previous leader failure").Err()
 	ErrGRPCTimeoutDueToConnectionLost = status.New(codes.Unavailable, "etcdserver: request timed out, possibly due to connection lost").Err()
+	ErrGRPCTimeoutWaitAppliedIndex    = status.New(codes.Unavailable, "etcdserver: request timed out, waiting for the applied index took too long").Err()
 	ErrGRPCUnhealthy                  = status.New(codes.Unavailable, "etcdserver: unhealthy cluster").Err()
 	ErrGRPCCorrupt                    = status.New(codes.DataLoss, "etcdserver: corrupt cluster").Err()
 	ErrGPRCNotSupportedForLearner     = status.New(codes.Unavailable, "etcdserver: rpc not supported for learner").Err()
@@ -119,6 +124,7 @@ var (
 		ErrorDesc(ErrGRPCAuthNotEnabled):       ErrGRPCAuthNotEnabled,
 		ErrorDesc(ErrGRPCInvalidAuthToken):     ErrGRPCInvalidAuthToken,
 		ErrorDesc(ErrGRPCInvalidAuthMgmt):      ErrGRPCInvalidAuthMgmt,
+		ErrorDesc(ErrGRPCAuthOldRevision):      ErrGRPCAuthOldRevision,
 
 		ErrorDesc(ErrGRPCNoLeader):                   ErrGRPCNoLeader,
 		ErrorDesc(ErrGRPCNotLeader):                  ErrGRPCNotLeader,
@@ -128,6 +134,7 @@ var (
 		ErrorDesc(ErrGRPCTimeout):                    ErrGRPCTimeout,
 		ErrorDesc(ErrGRPCTimeoutDueToLeaderFail):     ErrGRPCTimeoutDueToLeaderFail,
 		ErrorDesc(ErrGRPCTimeoutDueToConnectionLost): ErrGRPCTimeoutDueToConnectionLost,
+		ErrorDesc(ErrGRPCTimeoutWaitAppliedIndex):    ErrGRPCTimeoutWaitAppliedIndex,
 		ErrorDesc(ErrGRPCUnhealthy):                  ErrGRPCUnhealthy,
 		ErrorDesc(ErrGRPCCorrupt):                    ErrGRPCCorrupt,
 		ErrorDesc(ErrGPRCNotSupportedForLearner):     ErrGPRCNotSupportedForLearner,
@@ -177,6 +184,7 @@ var (
 	ErrPermissionNotGranted = Error(ErrGRPCPermissionNotGranted)
 	ErrAuthNotEnabled       = Error(ErrGRPCAuthNotEnabled)
 	ErrInvalidAuthToken     = Error(ErrGRPCInvalidAuthToken)
+	ErrAuthOldRevision      = Error(ErrGRPCAuthOldRevision)
 	ErrInvalidAuthMgmt      = Error(ErrGRPCInvalidAuthMgmt)
 
 	ErrNoLeader                   = Error(ErrGRPCNoLeader)
@@ -187,6 +195,7 @@ var (
 	ErrTimeout                    = Error(ErrGRPCTimeout)
 	ErrTimeoutDueToLeaderFail     = Error(ErrGRPCTimeoutDueToLeaderFail)
 	ErrTimeoutDueToConnectionLost = Error(ErrGRPCTimeoutDueToConnectionLost)
+	ErrTimeoutWaitAppliedIndex    = Error(ErrGRPCTimeoutWaitAppliedIndex)
 	ErrUnhealthy                  = Error(ErrGRPCUnhealthy)
 	ErrCorrupt                    = Error(ErrGRPCCorrupt)
 	ErrBadLeaderTransferee        = Error(ErrGRPCBadLeaderTransferee)

etcdserver/api/v3rpc/util.go

@@ -53,6 +53,7 @@ var toGRPCErrorMap = map[error]error{
 	etcdserver.ErrTimeout:                    rpctypes.ErrGRPCTimeout,
 	etcdserver.ErrTimeoutDueToLeaderFail:     rpctypes.ErrGRPCTimeoutDueToLeaderFail,
 	etcdserver.ErrTimeoutDueToConnectionLost: rpctypes.ErrGRPCTimeoutDueToConnectionLost,
+	etcdserver.ErrTimeoutWaitAppliedIndex:    rpctypes.ErrGRPCTimeoutWaitAppliedIndex,
 	etcdserver.ErrUnhealthy:                  rpctypes.ErrGRPCUnhealthy,
 	etcdserver.ErrKeyNotFound:                rpctypes.ErrGRPCKeyNotFound,
 	etcdserver.ErrCorrupt:                    rpctypes.ErrGRPCCorrupt,
@@ -71,12 +72,14 @@ var toGRPCErrorMap = map[error]error{
 	auth.ErrRoleNotFound:         rpctypes.ErrGRPCRoleNotFound,
 	auth.ErrRoleEmpty:            rpctypes.ErrGRPCRoleEmpty,
 	auth.ErrAuthFailed:           rpctypes.ErrGRPCAuthFailed,
+	auth.ErrPermissionNotGiven:   rpctypes.ErrGRPCPermissionNotGiven,
 	auth.ErrPermissionDenied:     rpctypes.ErrGRPCPermissionDenied,
 	auth.ErrRoleNotGranted:       rpctypes.ErrGRPCRoleNotGranted,
 	auth.ErrPermissionNotGranted: rpctypes.ErrGRPCPermissionNotGranted,
 	auth.ErrAuthNotEnabled:       rpctypes.ErrGRPCAuthNotEnabled,
 	auth.ErrInvalidAuthToken:     rpctypes.ErrGRPCInvalidAuthToken,
 	auth.ErrInvalidAuthMgmt:      rpctypes.ErrGRPCInvalidAuthMgmt,
+	auth.ErrAuthOldRevision:      rpctypes.ErrGRPCAuthOldRevision,
 }
 
 func togRPCError(err error) error {

etcdserver/api/v3rpc/watch.go

@@ -16,12 +16,14 @@ package v3rpc
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"math/rand"
 	"sync"
 	"time"
 
 	"go.etcd.io/etcd/auth"
+	"go.etcd.io/etcd/clientv3"
 	"go.etcd.io/etcd/etcdserver"
 	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
 	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
@@ -143,6 +145,10 @@ type serverWatchStream struct {
 	// records fragmented watch IDs
 	fragment map[mvcc.WatchID]bool
 
+	// indicates whether we have an outstanding global progress
+	// notification to send
+	deferredProgress bool
+
 	// closec indicates the stream is closed.
 	closec chan struct{}
 
@@ -172,6 +178,8 @@ func (ws *watchServer) Watch(stream pb.Watch_WatchServer) (err error) {
 		prevKV:   make(map[mvcc.WatchID]bool),
 		fragment: make(map[mvcc.WatchID]bool),
 
+		deferredProgress: false,
+
 		closec: make(chan struct{}),
 	}
 
@@ -206,15 +214,25 @@ func (ws *watchServer) Watch(stream pb.Watch_WatchServer) (err error) {
 		}
 	}()
 
+	// TODO: There's a race here. When a stream  is closed (e.g. due to a cancellation),
+	// the underlying error (e.g. a gRPC stream error) may be returned and handled
+	// through errc if the recv goroutine finishes before the send goroutine.
+	// When the recv goroutine wins, the stream error is retained. When recv loses
+	// the race, the underlying error is lost (unless the root error is propagated
+	// through Context.Err() which is not always the case (as callers have to decide
+	// to implement a custom context to do so). The stdlib context package builtins
+	// may be insufficient to carry semantically useful errors around and should be
+	// revisited.
 	select {
 	case err = <-errc:
+		if err == context.Canceled {
+			err = rpctypes.ErrGRPCWatchCanceled
+		}
 		close(sws.ctrlStream)
-
 	case <-stream.Context().Done():
 		err = stream.Context().Err()
-		// the only server-side cancellation is noleader for now.
 		if err == context.Canceled {
-			err = rpctypes.ErrGRPCNoLeader
+			err = rpctypes.ErrGRPCWatchCanceled
 		}
 	}
 
@@ -222,16 +240,16 @@ func (ws *watchServer) Watch(stream pb.Watch_WatchServer) (err error) {
 	return err
 }
 
-func (sws *serverWatchStream) isWatchPermitted(wcr *pb.WatchCreateRequest) bool {
+func (sws *serverWatchStream) isWatchPermitted(wcr *pb.WatchCreateRequest) error {
 	authInfo, err := sws.ag.AuthInfoFromCtx(sws.gRPCStream.Context())
 	if err != nil {
-		return false
+		return err
 	}
 	if authInfo == nil {
 		// if auth is enabled, IsRangePermitted() can cause an error
 		authInfo = &auth.AuthInfo{}
 	}
-	return sws.ag.AuthStore().IsRangePermitted(authInfo, wcr.Key, wcr.RangeEnd) == nil
+	return sws.ag.AuthStore().IsRangePermitted(authInfo, wcr.Key, wcr.RangeEnd)
 }
 
 func (sws *serverWatchStream) recvLoop() error {
@@ -265,13 +283,29 @@ func (sws *serverWatchStream) recvLoop() error {
 				creq.RangeEnd = []byte{}
 			}
 
-			if !sws.isWatchPermitted(creq) {
+			err := sws.isWatchPermitted(creq)
+			if err != nil {
+				var cancelReason string
+				switch err {
+				case auth.ErrInvalidAuthToken:
+					cancelReason = rpctypes.ErrGRPCInvalidAuthToken.Error()
+				case auth.ErrAuthOldRevision:
+					cancelReason = rpctypes.ErrGRPCAuthOldRevision.Error()
+				case auth.ErrUserEmpty:
+					cancelReason = rpctypes.ErrGRPCUserEmpty.Error()
+				default:
+					if err != auth.ErrPermissionDenied {
+						sws.lg.Error("unexpected error code", zap.Error(err))
+					}
+					cancelReason = rpctypes.ErrGRPCPermissionDenied.Error()
+				}
+
 				wr := &pb.WatchResponse{
 					Header:       sws.newResponseHeader(sws.watchStream.Rev()),
-					WatchId:      creq.WatchId,
+					WatchId:      clientv3.InvalidWatchID,
 					Canceled:     true,
 					Created:      true,
-					CancelReason: rpctypes.ErrGRPCPermissionDenied.Error(),
+					CancelReason: cancelReason,
 				}
 
 				select {
@@ -302,7 +336,10 @@ func (sws *serverWatchStream) recvLoop() error {
 					sws.fragment[id] = true
 				}
 				sws.mu.Unlock()
+			} else {
+				id = clientv3.InvalidWatchID
 			}
+
 			wr := &pb.WatchResponse{
 				Header:   sws.newResponseHeader(wsrev),
 				WatchId:  int64(id),
@@ -337,10 +374,16 @@ func (sws *serverWatchStream) recvLoop() error {
 			}
 		case *pb.WatchRequest_ProgressRequest:
 			if uv.ProgressRequest != nil {
-				sws.ctrlStream <- &pb.WatchResponse{
-					Header:  sws.newResponseHeader(sws.watchStream.Rev()),
-					WatchId: -1, // response is not associated with any WatchId and will be broadcast to all watch channels
+				sws.mu.Lock()
+				// Ignore if deferred progress notification is already in progress
+				if !sws.deferredProgress {
+					// Request progress for all watchers,
+					// force generation of a response
+					if !sws.watchStream.RequestProgressAll() {
+						sws.deferredProgress = true
+					}
 				}
+				sws.mu.Unlock()
 			}
 		default:
 			// we probably should not shutdown the entire stream when
@@ -390,7 +433,7 @@ func (sws *serverWatchStream) sendLoop() {
 			sws.mu.RUnlock()
 			for i := range evs {
 				events[i] = &evs[i]
-				if needPrevKV {
+				if needPrevKV && !isCreateEvent(evs[i]) {
 					opt := mvcc.RangeOptions{Rev: evs[i].Kv.ModRevision - 1}
 					r, err := sws.watchable.Range(evs[i].Kv.Key, nil, opt)
 					if err == nil && len(r.KVs) != 0 {
@@ -408,11 +451,15 @@ func (sws *serverWatchStream) sendLoop() {
 				Canceled:        canceled,
 			}
 
-			if _, okID := ids[wresp.WatchID]; !okID {
-				// buffer if id not yet announced
-				wrs := append(pending[wresp.WatchID], wr)
-				pending[wresp.WatchID] = wrs
-				continue
+			// Progress notifications can have WatchID -1
+			// if they announce on behalf of multiple watchers
+			if wresp.WatchID != clientv3.InvalidWatchID {
+				if _, okID := ids[wresp.WatchID]; !okID {
+					// buffer if id not yet announced
+					wrs := append(pending[wresp.WatchID], wr)
+					pending[wresp.WatchID] = wrs
+					continue
+				}
 			}
 
 			mvcc.ReportEventReceived(len(evs))
@@ -451,6 +498,11 @@ func (sws *serverWatchStream) sendLoop() {
 				// elide next progress update if sent a key update
 				sws.progress[wresp.WatchID] = false
 			}
+			if sws.deferredProgress {
+				if sws.watchStream.RequestProgressAll() {
+					sws.deferredProgress = false
+				}
+			}
 			sws.mu.Unlock()
 
 		case c, ok := <-sws.ctrlStream:
@@ -478,7 +530,12 @@ func (sws *serverWatchStream) sendLoop() {
 
 			// track id creation
 			wid := mvcc.WatchID(c.WatchId)
-			if c.Canceled {
+
+			if !(!(c.Canceled && c.Created) || wid == clientv3.InvalidWatchID) {
+				panic(fmt.Sprintf("unexpected watchId: %d, wanted: %d, since both 'Canceled' and 'Created' are true", wid, clientv3.InvalidWatchID))
+			}
+
+			if c.Canceled && wid != clientv3.InvalidWatchID {
 				delete(ids, wid)
 				continue
 			}
@@ -524,6 +581,10 @@ func (sws *serverWatchStream) sendLoop() {
 	}
 }
 
+func isCreateEvent(e mvccpb.Event) bool {
+	return e.Type == mvccpb.PUT && e.Kv.CreateRevision == e.Kv.ModRevision
+}
+
 func sendFragments(
 	wr *pb.WatchResponse,
 	maxRequestBytes int,

etcdserver/apply.go

@@ -33,10 +33,6 @@ import (
 	"go.uber.org/zap"
 )
 
-const (
-	warnApplyDuration = 100 * time.Millisecond
-)
-
 type applyResult struct {
 	resp proto.Message
 	err  error
@@ -115,7 +111,7 @@ func (s *EtcdServer) newApplierV3() applierV3 {
 func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
 	ar := &applyResult{}
 	defer func(start time.Time) {
-		warnOfExpensiveRequest(a.s.getLogger(), start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
+		warnOfExpensiveRequest(a.s.getLogger(), a.s.Cfg.WarningApplyDuration, start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
 		if ar.err != nil {
 			warnOfFailedRequest(a.s.getLogger(), start, &pb.InternalRaftStringer{Request: r}, ar.resp, ar.err)
 		}
@@ -185,7 +181,7 @@ func (a *applierV3backend) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (resp *pb.Pu
 	trace = traceutil.New("put",
 		a.s.getLogger(),
 		traceutil.Field{Key: "key", Value: string(p.Key)},
-		traceutil.Field{Key: "req_size", Value: proto.Size(p)},
+		traceutil.Field{Key: "req_size", Value: p.Size()},
 	)
 	val, leaseID := p.Value, lease.LeaseID(p.Lease)
 	if txn == nil {

etcdserver/apply_auth.go

@@ -176,18 +176,29 @@ func (aa *authApplierV3) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevoke
 }
 
 func (aa *authApplierV3) checkLeasePuts(leaseID lease.LeaseID) error {
-	lease := aa.lessor.Lookup(leaseID)
-	if lease != nil {
-		for _, key := range lease.Keys() {
-			if err := aa.as.IsPutPermitted(&aa.authInfo, []byte(key)); err != nil {
-				return err
-			}
-		}
+	l := aa.lessor.Lookup(leaseID)
+	if l != nil {
+		return aa.checkLeasePutsKeys(l)
 	}
 
 	return nil
 }
 
+func (aa *authApplierV3) checkLeasePutsKeys(l *lease.Lease) error {
+	// early return for most-common scenario of either disabled auth or admin user.
+	// IsAdminPermitted also checks whether auth is enabled
+	if err := aa.as.IsAdminPermitted(&aa.authInfo); err == nil {
+		return nil
+	}
+
+	for _, key := range l.Keys() {
+		if err := aa.as.IsPutPermitted(&aa.authInfo, []byte(key)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (aa *authApplierV3) UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse, error) {
 	err := aa.as.IsAdminPermitted(&aa.authInfo)
 	if err != nil && r.Name != aa.authInfo.Username {

etcdserver/apply_auth_test.go

@@ -0,0 +1,115 @@
+// Copyright 2023 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcdserver
+
+import (
+	"testing"
+	"time"
+
+	"go.etcd.io/etcd/auth"
+	"go.etcd.io/etcd/auth/authpb"
+	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
+	"go.etcd.io/etcd/lease"
+	"golang.org/x/crypto/bcrypt"
+
+	betesting "go.etcd.io/etcd/mvcc/backend"
+
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestCheckLeasePutsKeys(t *testing.T) {
+	lg := zaptest.NewLogger(t)
+
+	b, _ := betesting.NewDefaultTmpBackend()
+	defer b.Close()
+
+	simpleTokenTTLDefault := 300 * time.Second
+	tokenTypeSimple := "simple"
+	dummyIndexWaiter := func(index uint64) <-chan struct{} {
+		ch := make(chan struct{}, 1)
+		go func() {
+			ch <- struct{}{}
+		}()
+		return ch
+	}
+
+	tp, _ := auth.NewTokenProvider(zaptest.NewLogger(t), tokenTypeSimple, dummyIndexWaiter, simpleTokenTTLDefault)
+	as := auth.NewAuthStore(lg, b, tp, bcrypt.MinCost)
+
+	aa := authApplierV3{as: as}
+	assert.NoError(t, aa.checkLeasePutsKeys(lease.NewLease(lease.LeaseID(1), 3600)), "auth is disabled, should allow puts")
+	assert.NoError(t, enableAuthAndCreateRoot(aa.as), "error while enabling auth")
+	aa.authInfo = auth.AuthInfo{Username: "root"}
+	assert.NoError(t, aa.checkLeasePutsKeys(lease.NewLease(lease.LeaseID(1), 3600)), "auth is enabled, should allow puts for root")
+
+	l := lease.NewLease(lease.LeaseID(1), 3600)
+	l.SetLeaseItem(lease.LeaseItem{Key: "a"})
+	aa.authInfo = auth.AuthInfo{Username: "bob", Revision: 0}
+	assert.ErrorIs(t, aa.checkLeasePutsKeys(l), auth.ErrUserEmpty, "auth is enabled, should not allow bob, non existing at rev 0")
+	aa.authInfo = auth.AuthInfo{Username: "bob", Revision: 1}
+	assert.ErrorIs(t, aa.checkLeasePutsKeys(l), auth.ErrAuthOldRevision, "auth is enabled, old revision")
+
+	aa.authInfo = auth.AuthInfo{Username: "bob", Revision: aa.as.Revision()}
+	assert.ErrorIs(t, aa.checkLeasePutsKeys(l), auth.ErrPermissionDenied, "auth is enabled, bob does not have permissions, bob does not exist")
+	_, err := aa.as.UserAdd(&pb.AuthUserAddRequest{Name: "bob", Options: &authpb.UserAddOptions{NoPassword: true}})
+	assert.NoError(t, err, "bob should be added without error")
+	aa.authInfo = auth.AuthInfo{Username: "bob", Revision: aa.as.Revision()}
+	assert.ErrorIs(t, aa.checkLeasePutsKeys(l), auth.ErrPermissionDenied, "auth is enabled, bob exists yet does not have permissions")
+
+	// allow bob to access "a"
+	_, err = aa.as.RoleAdd(&pb.AuthRoleAddRequest{Name: "bobsrole"})
+	assert.NoError(t, err, "bobsrole should be added without error")
+	_, err = aa.as.RoleGrantPermission(&pb.AuthRoleGrantPermissionRequest{
+		Name: "bobsrole",
+		Perm: &authpb.Permission{
+			PermType: authpb.READWRITE,
+			Key:      []byte("a"),
+			RangeEnd: nil,
+		},
+	})
+	assert.NoError(t, err, "bobsrole should be granted permissions without error")
+	_, err = aa.as.UserGrantRole(&pb.AuthUserGrantRoleRequest{
+		User: "bob",
+		Role: "bobsrole",
+	})
+	assert.NoError(t, err, "bob should be granted bobsrole without error")
+
+	aa.authInfo = auth.AuthInfo{Username: "bob", Revision: aa.as.Revision()}
+	assert.NoError(t, aa.checkLeasePutsKeys(l), "bob should be able to access key 'a'")
+
+}
+
+func enableAuthAndCreateRoot(as auth.AuthStore) error {
+	_, err := as.UserAdd(&pb.AuthUserAddRequest{
+		Name:     "root",
+		Password: "root",
+		Options:  &authpb.UserAddOptions{NoPassword: false}})
+	if err != nil {
+		return err
+	}
+
+	_, err = as.RoleAdd(&pb.AuthRoleAddRequest{Name: "root"})
+	if err != nil {
+		return err
+	}
+
+	_, err = as.UserGrantRole(&pb.AuthUserGrantRoleRequest{User: "root", Role: "root"})
+	if err != nil {
+		return err
+	}
+
+	return as.AuthEnable()
+}

etcdserver/apply_v2.go

@@ -119,7 +119,7 @@ func (s *EtcdServer) applyV2Request(r *RequestV2) Response {
 		stringer:    r,
 		alternative: func() string { return fmt.Sprintf("id:%d,method:%s,path:%s", r.ID, r.Method, r.Path) },
 	}
-	defer warnOfExpensiveRequest(s.getLogger(), time.Now(), stringer, nil, nil)
+	defer warnOfExpensiveRequest(s.getLogger(), s.Cfg.WarningApplyDuration, time.Now(), stringer, nil, nil)
 
 	switch r.Method {
 	case "POST":

etcdserver/config.go

@@ -119,6 +119,12 @@ type ServerConfig struct {
 	// MaxRequestBytes is the maximum request size to send over raft.
 	MaxRequestBytes uint
 
+	// MaxConcurrentStreams specifies the maximum number of concurrent
+	// streams that each client can open at a time.
+	MaxConcurrentStreams uint32
+
+	WarningApplyDuration time.Duration
+
 	StrictReconfigCheck bool
 
 	// ClientCertAuthEnabled is true when cert has been signed by the client CA.
@@ -152,10 +158,12 @@ type ServerConfig struct {
 
 	ForceNewCluster bool
 
-	// EnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases.
+	// EnableLeaseCheckpoint enables leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change.
 	EnableLeaseCheckpoint bool
 	// LeaseCheckpointInterval time.Duration is the wait duration between lease checkpoints.
 	LeaseCheckpointInterval time.Duration
+	// LeaseCheckpointPersist enables persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled.
+	LeaseCheckpointPersist bool
 
 	EnableGRPCGateway bool
 

etcdserver/corrupt.go

@@ -202,13 +202,17 @@ func (s *EtcdServer) checkHashKV() error {
 	}
 
 	alarmed := false
-	mismatch := func(id uint64) {
+	mismatch := func(id types.ID) {
 		if alarmed {
 			return
 		}
 		alarmed = true
+		// It isn't clear which member's data is corrupted, so we
+		// intentionally set the memberID as 0. We will identify
+		// the corrupted members using quorum in 3.6. Please see
+		// discussion in https://github.com/etcd-io/etcd/pull/14828.
 		a := &pb.AlarmRequest{
-			MemberID: id,
+			MemberID: 0,
 			Action:   pb.AlarmRequest_ACTIVATE,
 			Alarm:    pb.AlarmType_CORRUPT,
 		}
@@ -231,7 +235,7 @@ func (s *EtcdServer) checkHashKV() error {
 		} else {
 			plog.Warningf("mismatched hashes %d and %d for revision %d", h, h2, rev)
 		}
-		mismatch(uint64(s.ID()))
+		mismatch(s.ID())
 	}
 
 	checkedCount := 0
@@ -240,7 +244,6 @@ func (s *EtcdServer) checkHashKV() error {
 			continue
 		}
 		checkedCount++
-		id := p.resp.Header.MemberId
 
 		// leader expects follower's latest revision less than or equal to leader's
 		if p.resp.Header.Revision > rev2 {
@@ -249,16 +252,16 @@ func (s *EtcdServer) checkHashKV() error {
 					"revision from follower must be less than or equal to leader's",
 					zap.Int64("leader-revision", rev2),
 					zap.Int64("follower-revision", p.resp.Header.Revision),
-					zap.String("follower-peer-id", types.ID(id).String()),
+					zap.String("follower-peer-id", p.id.String()),
 				)
 			} else {
 				plog.Warningf(
 					"revision %d from member %v, expected at most %d",
 					p.resp.Header.Revision,
-					types.ID(id),
+					p.id,
 					rev2)
 			}
-			mismatch(id)
+			mismatch(p.id)
 		}
 
 		// leader expects follower's latest compact revision less than or equal to leader's
@@ -268,17 +271,17 @@ func (s *EtcdServer) checkHashKV() error {
 					"compact revision from follower must be less than or equal to leader's",
 					zap.Int64("leader-compact-revision", crev2),
 					zap.Int64("follower-compact-revision", p.resp.CompactRevision),
-					zap.String("follower-peer-id", types.ID(id).String()),
+					zap.String("follower-peer-id", p.id.String()),
 				)
 			} else {
 				plog.Warningf(
 					"compact revision %d from member %v, expected at most %d",
 					p.resp.CompactRevision,
-					types.ID(id),
+					p.id,
 					crev2,
 				)
 			}
-			mismatch(id)
+			mismatch(p.id)
 		}
 
 		// follower's compact revision is leader's old one, then hashes must match
@@ -290,18 +293,18 @@ func (s *EtcdServer) checkHashKV() error {
 					zap.Uint32("leader-hash", h),
 					zap.Int64("follower-compact-revision", p.resp.CompactRevision),
 					zap.Uint32("follower-hash", p.resp.Hash),
-					zap.String("follower-peer-id", types.ID(id).String()),
+					zap.String("follower-peer-id", p.id.String()),
 				)
 			} else {
 				plog.Warningf(
 					"hash %d at revision %d from member %v, expected hash %d",
 					p.resp.Hash,
 					rev,
-					types.ID(id),
+					p.id,
 					h,
 				)
 			}
-			mismatch(id)
+			mismatch(p.id)
 		}
 	}
 	if lg != nil {

etcdserver/errors.go

@@ -26,6 +26,7 @@ var (
 	ErrTimeout                    = errors.New("etcdserver: request timed out")
 	ErrTimeoutDueToLeaderFail     = errors.New("etcdserver: request timed out, possibly due to previous leader failure")
 	ErrTimeoutDueToConnectionLost = errors.New("etcdserver: request timed out, possibly due to connection lost")
+	ErrTimeoutWaitAppliedIndex    = errors.New("etcdserver: request timed out, waiting for the applied index took too long")
 	ErrTimeoutLeaderTransfer      = errors.New("etcdserver: request timed out, leader transfer took too long")
 	ErrLeaderChanged              = errors.New("etcdserver: leader changed")
 	ErrNotEnoughStartedMembers    = errors.New("etcdserver: re-configuration failed due to not enough started members")

etcdserver/etcdserverpb/etcdserver.pb.go

@@ -2,111 +2,113 @@
 // source: etcdserver.proto
 
 /*
-	Package etcdserverpb is a generated protocol buffer package.
+Package etcdserverpb is a generated protocol buffer package.
 
-	It is generated from these files:
-		etcdserver.proto
-		raft_internal.proto
-		rpc.proto
+It is generated from these files:
 
-	It has these top-level messages:
-		Request
-		Metadata
-		RequestHeader
-		InternalRaftRequest
-		EmptyResponse
-		InternalAuthenticateRequest
-		ResponseHeader
-		RangeRequest
-		RangeResponse
-		PutRequest
-		PutResponse
-		DeleteRangeRequest
-		DeleteRangeResponse
-		RequestOp
-		ResponseOp
-		Compare
-		TxnRequest
-		TxnResponse
-		CompactionRequest
-		CompactionResponse
-		HashRequest
-		HashKVRequest
-		HashKVResponse
-		HashResponse
-		SnapshotRequest
-		SnapshotResponse
-		WatchRequest
-		WatchCreateRequest
-		WatchCancelRequest
-		WatchProgressRequest
-		WatchResponse
-		LeaseGrantRequest
-		LeaseGrantResponse
-		LeaseRevokeRequest
-		LeaseRevokeResponse
-		LeaseCheckpoint
-		LeaseCheckpointRequest
-		LeaseCheckpointResponse
-		LeaseKeepAliveRequest
-		LeaseKeepAliveResponse
-		LeaseTimeToLiveRequest
-		LeaseTimeToLiveResponse
-		LeaseLeasesRequest
-		LeaseStatus
-		LeaseLeasesResponse
-		Member
-		MemberAddRequest
-		MemberAddResponse
-		MemberRemoveRequest
-		MemberRemoveResponse
-		MemberUpdateRequest
-		MemberUpdateResponse
-		MemberListRequest
-		MemberListResponse
-		MemberPromoteRequest
-		MemberPromoteResponse
-		DefragmentRequest
-		DefragmentResponse
-		MoveLeaderRequest
-		MoveLeaderResponse
-		AlarmRequest
-		AlarmMember
-		AlarmResponse
-		StatusRequest
-		StatusResponse
-		AuthEnableRequest
-		AuthDisableRequest
-		AuthenticateRequest
-		AuthUserAddRequest
-		AuthUserGetRequest
-		AuthUserDeleteRequest
-		AuthUserChangePasswordRequest
-		AuthUserGrantRoleRequest
-		AuthUserRevokeRoleRequest
-		AuthRoleAddRequest
-		AuthRoleGetRequest
-		AuthUserListRequest
-		AuthRoleListRequest
-		AuthRoleDeleteRequest
-		AuthRoleGrantPermissionRequest
-		AuthRoleRevokePermissionRequest
-		AuthEnableResponse
-		AuthDisableResponse
-		AuthenticateResponse
-		AuthUserAddResponse
-		AuthUserGetResponse
-		AuthUserDeleteResponse
-		AuthUserChangePasswordResponse
-		AuthUserGrantRoleResponse
-		AuthUserRevokeRoleResponse
-		AuthRoleAddResponse
-		AuthRoleGetResponse
-		AuthRoleListResponse
-		AuthUserListResponse
-		AuthRoleDeleteResponse
-		AuthRoleGrantPermissionResponse
-		AuthRoleRevokePermissionResponse
+	etcdserver.proto
+	raft_internal.proto
+	rpc.proto
+
+It has these top-level messages:
+
+	Request
+	Metadata
+	RequestHeader
+	InternalRaftRequest
+	EmptyResponse
+	InternalAuthenticateRequest
+	ResponseHeader
+	RangeRequest
+	RangeResponse
+	PutRequest
+	PutResponse
+	DeleteRangeRequest
+	DeleteRangeResponse
+	RequestOp
+	ResponseOp
+	Compare
+	TxnRequest
+	TxnResponse
+	CompactionRequest
+	CompactionResponse
+	HashRequest
+	HashKVRequest
+	HashKVResponse
+	HashResponse
+	SnapshotRequest
+	SnapshotResponse
+	WatchRequest
+	WatchCreateRequest
+	WatchCancelRequest
+	WatchProgressRequest
+	WatchResponse
+	LeaseGrantRequest
+	LeaseGrantResponse
+	LeaseRevokeRequest
+	LeaseRevokeResponse
+	LeaseCheckpoint
+	LeaseCheckpointRequest
+	LeaseCheckpointResponse
+	LeaseKeepAliveRequest
+	LeaseKeepAliveResponse
+	LeaseTimeToLiveRequest
+	LeaseTimeToLiveResponse
+	LeaseLeasesRequest
+	LeaseStatus
+	LeaseLeasesResponse
+	Member
+	MemberAddRequest
+	MemberAddResponse
+	MemberRemoveRequest
+	MemberRemoveResponse
+	MemberUpdateRequest
+	MemberUpdateResponse
+	MemberListRequest
+	MemberListResponse
+	MemberPromoteRequest
+	MemberPromoteResponse
+	DefragmentRequest
+	DefragmentResponse
+	MoveLeaderRequest
+	MoveLeaderResponse
+	AlarmRequest
+	AlarmMember
+	AlarmResponse
+	StatusRequest
+	StatusResponse
+	AuthEnableRequest
+	AuthDisableRequest
+	AuthenticateRequest
+	AuthUserAddRequest
+	AuthUserGetRequest
+	AuthUserDeleteRequest
+	AuthUserChangePasswordRequest
+	AuthUserGrantRoleRequest
+	AuthUserRevokeRoleRequest
+	AuthRoleAddRequest
+	AuthRoleGetRequest
+	AuthUserListRequest
+	AuthRoleListRequest
+	AuthRoleDeleteRequest
+	AuthRoleGrantPermissionRequest
+	AuthRoleRevokePermissionRequest
+	AuthEnableResponse
+	AuthDisableResponse
+	AuthenticateResponse
+	AuthUserAddResponse
+	AuthUserGetResponse
+	AuthUserDeleteResponse
+	AuthUserChangePasswordResponse
+	AuthUserGrantRoleResponse
+	AuthUserRevokeRoleResponse
+	AuthRoleAddResponse
+	AuthRoleGetResponse
+	AuthRoleListResponse
+	AuthUserListResponse
+	AuthRoleDeleteResponse
+	AuthRoleGrantPermissionResponse
+	AuthRoleRevokePermissionResponse
 */
 package etcdserverpb
 

etcdserver/quota.go

@@ -135,8 +135,14 @@ func NewBackendQuota(s *EtcdServer, name string) Quota {
 }
 
 func (b *backendQuota) Available(v interface{}) bool {
+	cost := b.Cost(v)
+	// if there are no mutating requests, it's safe to pass through
+	if cost == 0 {
+		return true
+	}
+
 	// TODO: maybe optimize backend.Size()
-	return b.s.Backend().Size()+int64(b.Cost(v)) < b.maxBackendBytes
+	return b.s.Backend().Size()+int64(cost) < b.maxBackendBytes
 }
 
 func (b *backendQuota) Cost(v interface{}) int {

etcdserver/raft.go

@@ -215,6 +215,18 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					notifyc:  notifyc,
 				}
 
+				waitWALSync := shouldWaitWALSync(rd)
+				if waitWALSync {
+					// gofail: var raftBeforeSaveWaitWalSync struct{}
+					if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
+						if r.lg != nil {
+							r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err))
+						} else {
+							plog.Fatalf("failed to save state and entries error: %v", err)
+						}
+					}
+				}
+
 				updateCommittedIndex(&ap, rh)
 
 				select {
@@ -245,12 +257,14 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					// gofail: var raftAfterSaveSnap struct{}
 				}
 
-				// gofail: var raftBeforeSave struct{}
-				if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
-					if r.lg != nil {
-						r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err))
-					} else {
-						plog.Fatalf("failed to save state and entries error: %v", err)
+				if !waitWALSync {
+					// gofail: var raftBeforeSave struct{}
+					if err := r.storage.Save(rd.HardState, rd.Entries); err != nil {
+						if r.lg != nil {
+							r.lg.Fatal("failed to save Raft hard state and entries", zap.Error(err))
+						} else {
+							plog.Fatalf("failed to save state and entries error: %v", err)
+						}
 					}
 				}
 				if !raft.IsEmptyHardState(rd.HardState) {
@@ -342,6 +356,43 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 	}()
 }
 
+// For a cluster with only one member, the raft may send both the
+// unstable entries and committed entries to etcdserver, and there
+// may have overlapped log entries between them.
+//
+// etcd responds to the client once it finishes (actually partially)
+// the applying workflow. But when the client receives the response,
+// it doesn't mean etcd has already successfully saved the data,
+// including BoltDB and WAL, because:
+//  1. etcd commits the boltDB transaction periodically instead of on each request;
+//  2. etcd saves WAL entries in parallel with applying the committed entries.
+//
+// Accordingly, it might run into a situation of data loss when the etcd crashes
+// immediately after responding to the client and before the boltDB and WAL
+// successfully save the data to disk.
+// Note that this issue can only happen for clusters with only one member.
+//
+// For clusters with multiple members, it isn't an issue, because etcd will
+// not commit & apply the data before it being replicated to majority members.
+// When the client receives the response, it means the data must have been applied.
+// It further means the data must have been committed.
+// Note: for clusters with multiple members, the raft will never send identical
+// unstable entries and committed entries to etcdserver.
+//
+// Refer to https://github.com/etcd-io/etcd/issues/14370.
+func shouldWaitWALSync(rd raft.Ready) bool {
+	if len(rd.CommittedEntries) == 0 || len(rd.Entries) == 0 {
+		return false
+	}
+
+	// Check if there is overlap between unstable and committed entries
+	// assuming that their index and term are only incrementing.
+	lastCommittedEntry := rd.CommittedEntries[len(rd.CommittedEntries)-1]
+	firstUnstableEntry := rd.Entries[0]
+	return lastCommittedEntry.Term > firstUnstableEntry.Term ||
+		(lastCommittedEntry.Term == firstUnstableEntry.Term && lastCommittedEntry.Index >= firstUnstableEntry.Index)
+}
+
 func updateCommittedIndex(ap *apply, rh *raftReadyHandler) {
 	var ci uint64
 	if len(ap.entries) != 0 {

etcdserver/raft_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"go.etcd.io/etcd/etcdserver/api/membership"
 	"go.etcd.io/etcd/pkg/mock/mockstorage"
 	"go.etcd.io/etcd/pkg/pbutil"
@@ -267,3 +268,79 @@ func TestProcessDuplicatedAppRespMessage(t *testing.T) {
 		t.Errorf("count = %d, want %d", got, want)
 	}
 }
+
+func TestShouldWaitWALSync(t *testing.T) {
+	testcases := []struct {
+		name            string
+		unstableEntries []raftpb.Entry
+		commitedEntries []raftpb.Entry
+		expectedResult  bool
+	}{
+		{
+			name:            "both entries are nil",
+			unstableEntries: nil,
+			commitedEntries: nil,
+			expectedResult:  false,
+		},
+		{
+			name:            "both entries are empty slices",
+			unstableEntries: []raftpb.Entry{},
+			commitedEntries: []raftpb.Entry{},
+			expectedResult:  false,
+		},
+		{
+			name:            "one nil and the other empty",
+			unstableEntries: nil,
+			commitedEntries: []raftpb.Entry{},
+			expectedResult:  false,
+		},
+		{
+			name:            "one nil and the other has data",
+			unstableEntries: nil,
+			commitedEntries: []raftpb.Entry{{Term: 4, Index: 10, Type: raftpb.EntryNormal, Data: []byte{0x11, 0x22, 0x33}}},
+			expectedResult:  false,
+		},
+		{
+			name:            "one empty and the other has data",
+			unstableEntries: []raftpb.Entry{},
+			commitedEntries: []raftpb.Entry{{Term: 4, Index: 10, Type: raftpb.EntryNormal, Data: []byte{0x11, 0x22, 0x33}}},
+			expectedResult:  false,
+		},
+		{
+			name:            "has different term and index",
+			unstableEntries: []raftpb.Entry{{Term: 5, Index: 11, Type: raftpb.EntryNormal, Data: []byte{0x11, 0x22, 0x33}}},
+			commitedEntries: []raftpb.Entry{{Term: 4, Index: 10, Type: raftpb.EntryNormal, Data: []byte{0x11, 0x22, 0x33}}},
+			expectedResult:  false,
+		},
+		{
+			name:            "has identical data",
+			unstableEntries: []raftpb.Entry{{Term: 4, Index: 10, Type: raftpb.EntryNormal, Data: []byte{0x11, 0x22, 0x33}}},
+			commitedEntries: []raftpb.Entry{{Term: 4, Index: 10, Type: raftpb.EntryNormal, Data: []byte{0x11, 0x22, 0x33}}},
+			expectedResult:  true,
+		},
+		{
+			name: "has overlapped entry",
+			unstableEntries: []raftpb.Entry{
+				{Term: 4, Index: 10, Type: raftpb.EntryNormal, Data: []byte{0x11, 0x22, 0x33}},
+				{Term: 4, Index: 11, Type: raftpb.EntryNormal, Data: []byte{0x44, 0x55, 0x66}},
+				{Term: 4, Index: 12, Type: raftpb.EntryNormal, Data: []byte{0x77, 0x88, 0x99}},
+			},
+			commitedEntries: []raftpb.Entry{
+				{Term: 4, Index: 8, Type: raftpb.EntryNormal, Data: []byte{0x07, 0x08, 0x09}},
+				{Term: 4, Index: 9, Type: raftpb.EntryNormal, Data: []byte{0x10, 0x11, 0x12}},
+				{Term: 4, Index: 10, Type: raftpb.EntryNormal, Data: []byte{0x11, 0x22, 0x33}},
+			},
+			expectedResult: true,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			shouldWALSync := shouldWaitWALSync(raft.Ready{
+				Entries:          tc.unstableEntries,
+				CommittedEntries: tc.commitedEntries,
+			})
+			assert.Equal(t, tc.expectedResult, shouldWALSync)
+		})
+	}
+}