Merge branch 'main' into remove_e2e_calc

2023-03-30 16:46:31 +13:00 · 2023-03-30 16:46:31 +13:00 · 5faad23812
parent 0bd94bbae5 c41e47fd5f
commit 5faad23812
53 changed files with 673 additions and 331 deletions
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@ -19,13 +19,15 @@ jobs:
          - linux-ppc64le
          - linux-s390x
    steps:
-      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
      - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
        with:
          go-version: "1.19.7"
      - env:
          TARGET: ${{ matrix.target }}
        run: |
+          set -euo pipefail
+
          echo "${TARGET}"
          case "${TARGET}" in
            linux-amd64)
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@ -41,11 +41,11 @@ jobs:

    steps:
    - name: Checkout repository
-      uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
-      uses: github/codeql-action/init@168b99b3c22180941ae7dbdd5f5c9678ede476ba # v2.2.7
+      uses: github/codeql-action/init@04df1262e6247151b5ac09cd2c303ac36ad3f62b # v2.2.9
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
@ -56,7 +56,7 @@ jobs:
    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
-      uses: github/codeql-action/autobuild@168b99b3c22180941ae7dbdd5f5c9678ede476ba # v2.2.7
+      uses: github/codeql-action/autobuild@04df1262e6247151b5ac09cd2c303ac36ad3f62b # v2.2.9

    # ℹ️ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl
@ -70,4 +70,4 @@ jobs:
    #   make release

    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@168b99b3c22180941ae7dbdd5f5c9678ede476ba # v2.2.7
+      uses: github/codeql-action/analyze@04df1262e6247151b5ac09cd2c303ac36ad3f62b # v2.2.9
--- a/.github/workflows/contrib.yaml
+++ b/.github/workflows/contrib.yaml
@ -5,8 +5,11 @@ jobs:
  test:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        go-version: "1.19.7"
-    - run: make -C contrib/mixin tools test
+    - run: |
+        set -euo pipefail
+
+        make -C contrib/mixin tools test
--- a/.github/workflows/coverage.yaml
+++ b/.github/workflows/coverage.yaml
@ -10,7 +10,7 @@ jobs:
        target:
        - linux-amd64-coverage
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        go-version: "1.19.7"
--- a/.github/workflows/e2e-arm64.yaml
+++ b/.github/workflows/e2e-arm64.yaml
@ -14,7 +14,7 @@ jobs:
        target:
        - linux-arm64-e2e
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        ref: main
@ -23,11 +23,12 @@ jobs:
    - env:
        TARGET: ${{ matrix.target }}
      run: |
+        set -euo pipefail
+
        echo "${TARGET}"
        case "${TARGET}" in
          linux-arm64-e2e)
-            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
            ;;
          *)
            echo "Failed to find target"
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@ -11,7 +11,7 @@ jobs:
        - linux-amd64-e2e
        - linux-386-e2e
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        go-version: "1.19.7"
@ -19,15 +19,15 @@ jobs:
    - env:
        TARGET: ${{ matrix.target }}
      run: |
+        set -euo pipefail
+
        echo "${TARGET}"
        case "${TARGET}" in
          linux-amd64-e2e)
-            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build release e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
            ;;
          linux-386-e2e)
-            GOARCH=386 PASSES='build e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            GOARCH=386 PASSES='build e2e' CPU='4' EXPECT_DEBUG='true' COVER='false' RACE='true' ./scripts/test.sh
            ;;
          *)
            echo "Failed to find target"
--- a/.github/workflows/fuzzing.yaml
+++ b/.github/workflows/fuzzing.yaml
@ -9,11 +9,14 @@ jobs:
    env:
      TARGET_PATH: ./server/etcdserver/api/v3rpc
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        go-version: "1.19.7"
-    - run: GOARCH=amd64 CPU=4 make fuzz
+    - run: |
+        set -euo pipefail
+
+        GOARCH=amd64 CPU=4 make fuzz
    - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
      if: failure()
      with:
--- a/.github/workflows/govuln.yaml
+++ b/.github/workflows/govuln.yaml
@ -5,9 +5,12 @@ jobs:
  test:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
      - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
        with:
          go-version: "1.19.7"
      - run: date
-      - run: go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./...
+      - run: |
+          set -euo pipefail
+
+          go install golang.org/x/vuln/cmd/govulncheck@latest && govulncheck ./...
--- a/.github/workflows/grpcproxy.yaml
+++ b/.github/workflows/grpcproxy.yaml
@ -11,7 +11,7 @@ jobs:
        - linux-amd64-grpcproxy-integration
        - linux-amd64-grpcproxy-e2e
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        go-version: "1.19.7"
@ -19,15 +19,15 @@ jobs:
    - env:
        TARGET: ${{ matrix.target }}
      run: |
+        set -euo pipefail
+
        echo "${TARGET}"
        case "${TARGET}" in
          linux-amd64-grpcproxy-integration)
-            PASSES='build grpcproxy_integration'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build grpcproxy_integration'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh
            ;;
          linux-amd64-grpcproxy-e2e)
-            PASSES='build grpcproxy_e2e'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh 2>&1 | tee test.log
-            ! grep -E "(--- FAIL:|FAIL:|DATA RACE|panic: test timed out|appears to have leaked)" -B50 -A10 test.log
+            PASSES='build grpcproxy_e2e'  CPU='4' COVER='false' RACE='true' ./scripts/test.sh
            ;;
          *)
            echo "Failed to find target"
--- a/.github/workflows/measure-test-flakiness.yaml
+++ b/.github/workflows/measure-test-flakiness.yaml
@ -2,7 +2,7 @@ name: Measure Test Flakiness

 on:
  schedule:
-    - cron: "0 0 * * 0"
+    - cron: "0 0 * * 0" # run every Sunday at midnight

 permissions: read-all

@ -11,7 +11,12 @@ jobs:
    name: Measure Test Flakiness
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
-    - run: "./scripts/measure-test-flakiness.sh"
-      env:
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
+    - env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        set -euo pipefail
+
+        ./scripts/measure-test-flakiness.sh
+        make bin/etcd-test-analyzer
+        bin/etcd-test-analyzer run -token $GITHUB_TOKEN -max-age=168h -workflow Tests -branch main
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@ -5,11 +5,14 @@ jobs:
  main:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        go-version: "1.19.7"
-    - run: |
+    - name: release
+      run: |
+        set -euo pipefail
+
        git config --global user.email "github-action@etcd.io"
        git config --global user.name "Github Action"
        gpg --batch --gen-key <<EOF
@ -23,3 +26,6 @@ jobs:
        Expire-Date: 0
        EOF
        DRY_RUN=true ./scripts/release.sh --no-upload --no-docker-push --in-place 3.6.99
+    - name: test-image
+      run: |
+        VERSION=3.6.99 ./scripts/test_images.sh
--- a/.github/workflows/robustness-template.yaml
+++ b/.github/workflows/robustness-template.yaml
@ -21,7 +21,7 @@ jobs:
    timeout-minutes: 210
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        go-version: '1.19.7'
@ -29,22 +29,27 @@ jobs:
      env:
        GITHUB_REF: ${{ inputs.ref }}
      run: |
+        set -euo pipefail
+
        case "${GITHUB_REF}" in
          release-3.5)
-            make build-failpoints-release-3.5
-            ./bin/etcd --version
+            make /tmp/etcd-release-3.5-failpoints/bin/etcd
+            cp /tmp/etcd-release-3.5-failpoints/bin/etcd bin/etcd
            ;;
          release-3.4)
-            make build-failpoints-release-3.4
-            ./bin/etcd --version
+            make /tmp/etcd-release-3.4-failpoints/bin/etcd
+            cp /tmp/etcd-release-3.4-failpoints/bin/etcd bin/etcd
            ;;
          *)
            make gofail-enable
            make build
            ;;
          esac
+        ./bin/etcd --version
    - name: test-robustness
      run: |
+        set -euo pipefail
+
        # Use --failfast to avoid overriding report generated by failed test
        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count ${{ inputs.count }} --timeout ${{ inputs.testTimeout }} --failfast --run TestRobustness' RESULTS_DIR=/tmp/results make test-robustness
    - uses: actions/upload-artifact@v2
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@ -22,7 +22,7 @@ jobs:

    steps:
      - name: "Checkout code"
-        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # tag=v3.0.0
+        uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # tag=v3.0.0
        with:
          persist-credentials: false

@ -49,6 +49,6 @@ jobs:
      
      # Upload the results to GitHub's code scanning dashboard.
      - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@168b99b3c22180941ae7dbdd5f5c9678ede476ba # tag=v1.0.26
+        uses: github/codeql-action/upload-sarif@04df1262e6247151b5ac09cd2c303ac36ad3f62b # tag=v1.0.26
        with:
          sarif_file: results.sarif
--- a/.github/workflows/static-analysis.yaml
+++ b/.github/workflows/static-analysis.yaml
@ -5,7 +5,7 @@ jobs:
  run:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
      - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
        with:
          go-version: "1.19.7"
@ -18,5 +18,11 @@ jobs:
        with:
          version: '3.14.0'
          repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: make verify
-      - run: make fix
+      - run: |
+          set -euo pipefail
+
+          make verify
+      - run: |
+          set -euo pipefail
+
+          make fix
--- a/.github/workflows/tests-arm64.yaml
+++ b/.github/workflows/tests-arm64.yaml
@ -16,7 +16,7 @@ jobs:
        - linux-arm64-integration-4-cpu
        - linux-arm64-unit-4-cpu-race
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        ref: main
@ -25,6 +25,8 @@ jobs:
    - env:
        TARGET: ${{ matrix.target }}
      run: |
+        set -euo pipefail
+
        mkdir "${TARGET}"
        export JUNIT_REPORT_DIR=$(realpath ${TARGET})
        case "${TARGET}" in
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@ -14,7 +14,7 @@ jobs:
        - linux-amd64-unit-4-cpu-race
        - linux-386-unit-1-cpu
    steps:
-    - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
    - uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
      with:
        go-version: "1.19.7"
@ -22,6 +22,8 @@ jobs:
    - env:
        TARGET: ${{ matrix.target }}
      run: |
+        set -euo pipefail
+
        mkdir "${TARGET}"
        export JUNIT_REPORT_DIR=$(realpath ${TARGET})
        case "${TARGET}" in
--- a/CHANGELOG/CHANGELOG-3.4.md
+++ b/CHANGELOG/CHANGELOG-3.4.md
@ -6,10 +6,17 @@ Previous change logs can be found at [CHANGELOG-3.3](https://github.com/etcd-io/

 ## v3.4.25 (TBD)

+### etcd server
+- Fix [server/embed: fix data race when starting both secure & insecure gRPC servers on the same address](https://github.com/etcd-io/etcd/pull/15518)
+- Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15486) to enable support for TLS 1.3.
+
 ### Go
 - Require [Go 1.19+](https://github.com/etcd-io/etcd/pull/15333).
 - Compile with [Go 1.19+](https://go.dev/doc/devel/release#go1.19)

+### Package `clientv3`
+- Reverted the fix to [auth invalid token and old revision errors in watch](https://github.com/etcd-io/etcd/pull/15542).
+
 <hr>

 ## v3.4.24 (2023-02-16)
--- a/CHANGELOG/CHANGELOG-3.5.md
+++ b/CHANGELOG/CHANGELOG-3.5.md
@ -6,6 +6,10 @@ Previous change logs can be found at [CHANGELOG-3.4](https://github.com/etcd-io/

 ## v3.5.8 (TBD)

+### etcd server
+- Fix [server/embed: fix data race when starting both secure & insecure gRPC servers on the same address](https://github.com/etcd-io/etcd/pull/15517)
+- Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15483) to enable support for TLS 1.3.
+
 ### Package `client/pkg/v3`
 - Fix [aligning zap log timestamp resolution to microseconds](https://github.com/etcd-io/etcd/pull/15240). Etcd now uses zap timestamp format: `2006-01-02T15:04:05.999999Z0700` (microsecond instead of milliseconds precision).

--- a/CHANGELOG/CHANGELOG-3.6.md
+++ b/CHANGELOG/CHANGELOG-3.6.md
@ -74,6 +74,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0).
 - Add [field `hash_revision` into `HashKVResponse`](https://github.com/etcd-io/etcd/pull/14537).
 - Add [`etcd --experimental-snapshot-catch-up-entries`](https://github.com/etcd-io/etcd/pull/15033) flag to configure number of entries for a slow follower to catch up after compacting the the raft storage entries and defaults to 5k. 
 - Decreased [`--snapshot-count` default value from 100,000 to 10,000](https://github.com/etcd-io/etcd/pull/15408)
+- Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15156) to enable support for TLS 1.3.

 ### etcd grpc-proxy

--- a/Documentation/postmortems/v3.5-data-inconsistency.md
+++ b/Documentation/postmortems/v3.5-data-inconsistency.md
@ -115,18 +115,18 @@ To reflect this action items should have assigned priority:
 * P1 - Important for long term success of the project. Blocks v3.6 release.
 * P2 - Stretch  goals that would be nice to have for v3.6, however should not be blocking.

-| Action Item                                                                         | Type     | Priority | Bug                                          |
-|-------------------------------------------------------------------------------------|----------|----------|----------------------------------------------|
-| etcd testing can reproduce historical data inconsistency issues                     | Prevent  | P0       | https://github.com/etcd-io/etcd/issues/14045 |
-| etcd detects data corruption by default                                             | Detect   | P0       | https://github.com/etcd-io/etcd/issues/14039 |
-| etcd testing is high quality, easy to maintain and expand                           | Prevent  | P1       | https://github.com/etcd-io/etcd/issues/13637 |
-| etcd apply code should be easy to understand and validate correctness               | Prevent  | P1       |                                              |
-| Critical etcd features are not abandoned when contributors move on                  | Prevent  | P1       | https://github.com/etcd-io/etcd/issues/13775 |
-| etcd is continuously qualified with failure injection                               | Prevent  | P1       |                                              |
-| etcd can reliably detect data corruption (hash is linearizable)                     | Detect   | P1       |                                              |
-| etcd checks consistency of snapshots sent between leader and followers              | Detect   | P1       | https://github.com/etcd-io/etcd/issues/13973 |
-| etcd recovery from data inconsistency procedures are documented and tested          | Mitigate | P1       |                                              |
-| etcd can imminently detect and recover from data corruption (implement Merkle root) | Mitigate | P2       | https://github.com/etcd-io/etcd/issues/13839 |
+| Action Item                                                                         | Type     | Priority | Bug                                          | Status |
+|-------------------------------------------------------------------------------------|----------|----------|----------------------------------------------|--------|
+| etcd testing can reproduce historical data inconsistency issues                     | Prevent  | P0       | https://github.com/etcd-io/etcd/issues/14045 | DONE   |
+| etcd detects data corruption by default                                             | Detect   | P0       | https://github.com/etcd-io/etcd/issues/14039 | DONE   |
+| etcd testing is high quality, easy to maintain and expand                           | Prevent  | P1       | https://github.com/etcd-io/etcd/issues/13637 |        |
+| etcd apply code should be easy to understand and validate correctness               | Prevent  | P1       |                                              |        |
+| Critical etcd features are not abandoned when contributors move on                  | Prevent  | P1       | https://github.com/etcd-io/etcd/issues/13775 | DONE   |
+| etcd is continuously qualified with failure injection                               | Prevent  | P1       | https://github.com/etcd-io/etcd/pull/14911   | DONE   |
+| etcd can reliably detect data corruption (hash is linearizable)                     | Detect   | P1       |                                              |        |
+| etcd checks consistency of snapshots sent between leader and followers              | Detect   | P1       | https://github.com/etcd-io/etcd/issues/13973 | DONE   |
+| etcd recovery from data inconsistency procedures are documented and tested          | Mitigate | P1       |                                              |        |
+| etcd can imminently detect and recover from data corruption (implement Merkle root) | Mitigate | P2       | https://github.com/etcd-io/etcd/issues/13839 |        |

 ## Timeline

--- a/61
+++ b/61
@ -1,3 +1,6 @@
+all: build
+include tests/robustness/makefile.mk
+
 .PHONY: build
 build:
 	GO_BUILD_FLAGS="${GO_BUILD_FLAGS} -v" ./scripts/build.sh
@ -9,6 +12,17 @@ build:
 tools:
 	GO_BUILD_FLAGS="${GO_BUILD_FLAGS} -v" ./scripts/build_tools.sh

+TEMP_TEST_ANALYZER_DIR=/tmp/etcd-test-analyzer
+TEST_ANALYZER_BIN=${PWD}/bin
+bin/etcd-test-analyzer: $(TEMP_TEST_ANALYZER_DIR)/*
+	make -C ${TEMP_TEST_ANALYZER_DIR} build
+	mkdir -p ${TEST_ANALYZER_BIN}
+	install ${TEMP_TEST_ANALYZER_DIR}/bin/etcd-test-analyzer ${TEST_ANALYZER_BIN}
+	${TEST_ANALYZER_BIN}/etcd-test-analyzer -h
+
+$(TEMP_TEST_ANALYZER_DIR)/*:
+	git clone "https://github.com/endocrimes/etcd-test-analyzer.git" ${TEMP_TEST_ANALYZER_DIR}
+
 # Tests

 GO_TEST_FLAGS?=
@ -113,53 +127,6 @@ verify-genproto:
 verify-goimport:
 	PASSES="goimport" ./scripts/test.sh

-# Failpoints
-
-GOFAIL_VERSION = $(shell cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail)
-
-.PHONY: gofail-enable
-gofail-enable: install-gofail
-	gofail enable server/etcdserver/ server/storage/backend/ server/storage/mvcc/ server/storage/wal/
-	cd ./server && go get go.etcd.io/gofail@${GOFAIL_VERSION}
-	cd ./etcdutl && go get go.etcd.io/gofail@${GOFAIL_VERSION}
-	cd ./etcdctl && go get go.etcd.io/gofail@${GOFAIL_VERSION}
-	cd ./tests && go get go.etcd.io/gofail@${GOFAIL_VERSION}
-
-.PHONY: gofail-disable
-gofail-disable: install-gofail
-	gofail disable server/etcdserver/ server/storage/backend/ server/storage/mvcc/ server/storage/wal/
-	cd ./server && go mod tidy
-	cd ./etcdutl && go mod tidy
-	cd ./etcdctl && go mod tidy
-	cd ./tests && go mod tidy
-
-.PHONY: install-gofail
-install-gofail:
-	cd tools/mod; go install go.etcd.io/gofail@${GOFAIL_VERSION}
-
-build-failpoints-release-3.5:
-	rm -rf /tmp/etcd-release-3.5/
-	mkdir -p /tmp/etcd-release-3.5/
-	cd /tmp/etcd-release-3.5/; \
-	  git clone --depth 1 --branch release-3.5 https://github.com/etcd-io/etcd.git .; \
-	  go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
-	  (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
-	  (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
-	  (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
-	  FAILPOINTS=true ./build;
-	mkdir -p ./bin
-	cp /tmp/etcd-release-3.5/bin/etcd ./bin/etcd
-
-build-failpoints-release-3.4:
-	rm -rf /tmp/etcd-release-3.4/
-	mkdir -p /tmp/etcd-release-3.4/
-	cd /tmp/etcd-release-3.4/; \
-	  git clone --depth 1 --branch release-3.4 https://github.com/etcd-io/etcd.git .; \
-	  go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
-	  FAILPOINTS=true ./build;
-	mkdir -p ./bin
-	cp /tmp/etcd-release-3.4/bin/etcd ./bin/etcd
-
 # Cleanup

 clean:
--- a/bill-of-materials.json
+++ b/bill-of-materials.json
@ -396,7 +396,7 @@
 		]
 	},
 	{
-		"project": "github.com/stretchr/testify/assert",
+		"project": "github.com/stretchr/testify",
 		"licenses": [
 			{
 				"type": "MIT License",
--- a/contrib/mixin/mixin.libsonnet
+++ b/contrib/mixin/mixin.libsonnet
@ -241,7 +241,7 @@
          {
            alert: 'etcdDatabaseHighFragmentationRatio',
            expr: |||
-              (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5
+              (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes > 104857600
            ||| % $._config,
            'for': '10m',
            labels: {
--- a/contrib/mixin/test.yaml
+++ b/contrib/mixin/test.yaml
@ -143,13 +143,13 @@ tests:
  - interval: 1m
    input_series:
      - series: 'etcd_mvcc_db_total_size_in_use_in_bytes{job="etcd",instance="10.10.10.0"}'
-        values: '30000+0x10'
+        values: '300000000+0x10'
      - series: 'etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.0"}'
-        values: '100000+0x10'
+        values: '1000000000+0x10'
      - series: 'etcd_mvcc_db_total_size_in_use_in_bytes{job="etcd",instance="10.10.10.1"}'
-        values: '70000+0x10'
+        values: '700000000+0x10'
      - series: 'etcd_mvcc_db_total_size_in_bytes{job="etcd",instance="10.10.10.1"}'
-        values: '100000+0x10'
+        values: '1000000000+0x10'
    alert_rule_test:
      - eval_time: 11m
        alertname: etcdDatabaseHighFragmentationRatio
--- a/etcd.conf.yml.sample
+++ b/etcd.conf.yml.sample
@ -138,3 +138,9 @@ force-new-cluster: false

 auto-compaction-mode: periodic
 auto-compaction-retention: "1"
+
+# Limit etcd to a specific set of tls cipher suites
+cipher-suites: [
+  TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+  TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+]
--- a/scripts/build-binary.sh
+++ b/scripts/build-binary.sh
@ -1,19 +1,17 @@
 #!/usr/bin/env bash

-set -e
+set -euo pipefail

 source ./scripts/test_lib.sh

-VER=$1
+VER=${1:-}
 REPOSITORY="${REPOSITORY:-git@github.com:etcd-io/etcd.git}"

-if [ -z "$1" ]; then
+if [ -z "$VER" ]; then
  echo "Usage: ${0} VERSION" >> /dev/stderr
  exit 255
 fi

-set -u
-
 function setup_env {
  local ver=${1}
  local proj=${2}
@ -60,7 +58,7 @@ function main {
  cd release
  setup_env "${VER}" "${proj}"

-  tarcmd=tar
+  local tarcmd=tar
  if [[ $(go env GOOS) == "darwin" ]]; then
      echo "Please use linux machine for release builds."
    exit 1
--- a/scripts/build-docker.sh
+++ b/scripts/build-docker.sh
@ -1,17 +1,23 @@
 #!/usr/bin/env bash

-set -e
+set -euo pipefail

 if [ "$#" -ne 1 ]; then
  echo "Usage: $0 VERSION" >&2
  exit 1
 fi

+VERSION=${1}
+if [ -z "$VERSION" ]; then
+  echo "Usage: ${0} VERSION" >&2
+  exit 1
+fi
+
 ARCH=$(go env GOARCH)
-VERSION="${1}-${ARCH}"
+VERSION="${VERSION}-${ARCH}"
 DOCKERFILE="Dockerfile-release.${ARCH}"

-if [ -z "${BINARYDIR}" ]; then
+if [ -z "${BINARYDIR:-}" ]; then
  RELEASE="etcd-${1}"-$(go env GOOS)-$(go env GOARCH)
  BINARYDIR="${RELEASE}"
  TARFILE="${RELEASE}.tar.gz"
@ -34,7 +40,7 @@ cp "${BINARYDIR}"/etcd "${BINARYDIR}"/etcdctl "${BINARYDIR}"/etcdutl "${IMAGEDIR

 cat ./"${DOCKERFILE}" > "${IMAGEDIR}"/Dockerfile

-if [ -z "$TAG" ]; then
+if [ -z "${TAG:-}" ]; then
    docker build -t "gcr.io/etcd-development/etcd:${VERSION}" "${IMAGEDIR}"
    docker build -t "quay.io/coreos/etcd:${VERSION}" "${IMAGEDIR}"
 else
--- a/scripts/build-release.sh
+++ b/scripts/build-release.sh
@ -3,11 +3,11 @@
 # Build all release binaries and images to directory ./release.
 # Run from repository root.
 #
-set -e
+set -euo pipefail

 source ./scripts/test_lib.sh

-VERSION=$1
+VERSION=${1:-}
 if [ -z "${VERSION}" ]; then
  echo "Usage: ${0} VERSION" >> /dev/stderr
  exit 255
--- a/scripts/build.sh
+++ b/scripts/build.sh
@ -3,6 +3,8 @@
 # This scripts build the etcd binaries
 # To build the tools, run `build_tools.sh`

+set -euo pipefail
+
 source ./scripts/test_lib.sh
 source ./scripts/build_lib.sh

--- a/scripts/build_lib.sh
+++ b/scripts/build_lib.sh
@ -1,18 +1,26 @@
 #!/usr/bin/env bash

+set -euo pipefail
+
 source ./scripts/test_lib.sh

 GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound")
 VERSION_SYMBOL="${ROOT_MODULE}/api/v3/version.GitSHA"

+# use go env if noset
+GOOS=${GOOS:-$(go env GOOS)}
+GOARCH=${GOARCH:-$(go env GOARCH)}
+
+GO_BUILD_FLAGS=${GO_BUILD_FLAGS:-}
+
 # Set GO_LDFLAGS="-s" for building without symbols for debugging.
 # shellcheck disable=SC2206
-GO_LDFLAGS=(${GO_LDFLAGS} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
+GO_LDFLAGS=(${GO_LDFLAGS:-} "-X=${VERSION_SYMBOL}=${GIT_SHA}")
 GO_BUILD_ENV=("CGO_ENABLED=0" "GO_BUILD_FLAGS=${GO_BUILD_FLAGS}" "GOOS=${GOOS}" "GOARCH=${GOARCH}")

 etcd_build() {
  out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi

  run rm -f "${out}/etcd"
  (
@ -62,7 +70,7 @@ etcd_build() {

 tools_build() {
  out="bin"
-  if [[ -n "${BINDIR}" ]]; then out="${BINDIR}"; fi
+  if [[ -n "${BINDIR:-}" ]]; then out="${BINDIR}"; fi
  tools_path="tools/benchmark
    tools/etcd-dump-db
    tools/etcd-dump-logs
--- a/scripts/build_tools.sh
+++ b/scripts/build_tools.sh
@ -1,5 +1,7 @@
 #!/usr/bin/env bash

+set -euo pipefail
+
 source ./scripts/test_lib.sh
 source ./scripts/build_lib.sh

--- a/scripts/fix.sh
+++ b/scripts/fix.sh
@ -1,6 +1,6 @@
 #!/usr/bin/env bash

-set -e
+set -euo pipefail

 # Top level problems with modules can lead to test_lib being not functional
 go mod tidy
--- a/scripts/fuzzing.sh
+++ b/scripts/fuzzing.sh
@ -1,5 +1,7 @@
 #!/usr/bin/env bash
-set -e
+
+set -euo pipefail
+
 source ./scripts/test_lib.sh

 GO_CMD="go"
--- a/scripts/genproto.sh
+++ b/scripts/genproto.sh
@ -3,7 +3,8 @@
 # Generate all etcd protobuf bindings.
 # Run from repository root directory named etcd.
 #
-set -e
+set -euo pipefail
+
 shopt -s globstar

 if ! [[ "$0" =~ scripts/genproto.sh ]]; then
@ -98,7 +99,7 @@ log_callout -e "\\nRunning swagger ..."
 run_go_tool github.com/hexfusion/schwag -input=Documentation/dev-guide/apispec/swagger/rpc.swagger.json


-if [ "$1" != "--skip-protodoc" ]; then
+if [ "${1:-}" != "--skip-protodoc" ]; then
  log_callout "protodoc is auto-generating grpc API reference documentation..."

  # API reference
--- a/scripts/install-marker.sh
+++ b/scripts/install-marker.sh
@ -1,10 +1,10 @@
 #!/usr/bin/env bash

-set -e
+set -euo pipefail

-ARCH=$1
+ARCH=${1:-}

-if [ -z "$1" ]; then
+if [ -z "$ARCH" ]; then
    echo "Usage: ${0} [amd64 or darwin], defaulting to 'amd64'" >> /dev/stderr
    ARCH=amd64
 fi
--- a/scripts/measure-test-flakiness.sh
+++ b/scripts/measure-test-flakiness.sh
@ -1,9 +1,8 @@
-#!/bin/bash
+#!/usr/bin/env bash

-set -e
-set -o pipefail
+set -euo pipefail

-if [[ -z ${GITHUB_TOKEN} ]]
+if [[ -z ${GITHUB_TOKEN:-} ]]
 then
    echo "Please set the \$GITHUB_TOKEN environment variable for the script to work"
    exit 1
--- a/scripts/release_mod.sh
+++ b/scripts/release_mod.sh
@ -10,7 +10,7 @@
 #
 # % DRY_RUN=false REMOTE_REPO="origin" ./scripts/release_mod.sh push_mod_tags

-set -e
+set -euo pipefail

 source ./scripts/test_lib.sh

@ -55,7 +55,7 @@ function mod_tidy_fix {
 function update_versions_cmd() {
  assert_no_git_modifications || return 2

-  if [ -z "${TARGET_VERSION}" ]; then
+  if [ -z "${TARGET_VERSION:-}" ]; then
    log_error "TARGET_VERSION environment variable not set. Set it to e.g. v3.5.10-alpha.0"
    return 2
  fi
@ -89,7 +89,7 @@ function get_gpg_key {
 function push_mod_tags_cmd {
  assert_no_git_modifications || return 2

-  if [ -z "${REMOTE_REPO}" ]; then
+  if [ -z "${REMOTE_REPO:-}" ]; then
    log_error "REMOTE_REPO environment variable not set"
    return 2
  fi
--- a/scripts/test.sh
+++ b/scripts/test.sh
@ -36,6 +36,7 @@ set -e
 # Consider command as failed when any component of the pipe fails:
 # https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash
 set -o pipefail
+set -o nounset

 # The test script is not supposed to make any changes to the files
 # e.g. add/update missing dependencies. Such divergences should be 
@ -46,6 +47,8 @@ export ETCD_VERIFY=all
 source ./scripts/test_lib.sh
 source ./scripts/build_lib.sh

+OUTPUT_FILE=${OUTPUT_FILE:-""}
+
 if [ -n "${OUTPUT_FILE}" ]; then
  log_callout "Dumping output to: ${OUTPUT_FILE}"
  exec > >(tee -a "${OUTPUT_FILE}") 2>&1
@ -55,12 +58,12 @@ PASSES=${PASSES:-"gofmt bom dep build unit"}
 PKG=${PKG:-}
 SHELLCHECK_VERSION=${SHELLCHECK_VERSION:-"v0.8.0"}

-if [ -z "$GOARCH" ]; then
+if [ -z "${GOARCH:-}" ]; then
  GOARCH=$(go env GOARCH);
 fi

 # determine whether target supports race detection
-if [ -z "${RACE}" ] ; then
+if [ -z "${RACE:-}" ] ; then
  if [ "$GOARCH" == "amd64" ]; then
    RACE="--race"
  else
@ -72,14 +75,14 @@ fi

 # This options make sense for cases where SUT (System Under Test) is compiled by test.
 COMMON_TEST_FLAGS=("${RACE}")
-if [[ -n "${CPU}" ]]; then
+if [[ -n "${CPU:-}" ]]; then
  COMMON_TEST_FLAGS+=("--cpu=${CPU}")
 fi 

 log_callout "Running with ${COMMON_TEST_FLAGS[*]}"

 RUN_ARG=()
-if [ -n "${TESTCASE}" ]; then
+if [ -n "${TESTCASE:-}" ]; then
  RUN_ARG=("-run=${TESTCASE}")
 fi

@ -120,13 +123,13 @@ function integration_pass {

 function e2e_pass {
  # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
-  run_for_module "tests" go_test "./e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
+  run_for_module "tests" go_test "./e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@" || return $?
  run_for_module "tests" go_test "./common/..." "keep_going" : --tags=e2e -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
 }

 function robustness_pass {
  # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
-  run_for_module "tests" go_test "./robustness/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
+  run_for_module "tests" go_test "./robustness" "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
 }

 function integration_e2e_pass {
@ -249,7 +252,7 @@ function merge_cov {

 function cov_pass {
  # shellcheck disable=SC2153
-  if [ -z "$COVERDIR" ]; then
+  if [ -z "${COVERDIR:-}" ]; then
    log_error "COVERDIR undeclared"
    return 255
  fi
@ -352,6 +355,7 @@ function markdown_you_find_eschew_you {
 }

 function markdown_you_pass {
+  # TODO: ./CONTRIBUTING.md:## Get your pull request reviewed
  generic_checker markdown_you_find_eschew_you
 }

@ -367,31 +371,42 @@ function govet_pass {
 }

 function govet_shadow_pass {
+  # TODO: we should ignore the generated packages?
+  #
+  # stderr: etcdserverpb/gw/rpc.pb.gw.go:2100:3: declaration of "ctx" shadows declaration at line 2005
  local shadow
  shadow=$(tool_get_bin "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow")
  run_for_modules generic_checker run go vet -all -vettool="${shadow}"
 }

 function unparam_pass {
+  # TODO: transport/listener.go:129:60: newListenConfig - result 1 (error) is always nil
  run_for_modules generic_checker run_go_tool "mvdan.cc/unparam"
 }

 function staticcheck_pass {
+  # TODO: we should upgrade pb or ignore the pb package
+  #
+  # versionpb/version.pb.go:69:15: proto.RegisterFile is deprecated: Use protoregistry.GlobalFiles.RegisterFile instead.  (SA1019)
  run_for_modules generic_checker run_go_tool "honnef.co/go/tools/cmd/staticcheck"
 }

 function revive_pass {
+  # TODO: etcdserverpb/raft_internal_stringer.go:15:1: should have a package comment
  run_for_modules generic_checker run_go_tool "github.com/mgechev/revive" -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..." -exclude "out/..."
 }

 function unconvert_pass {
+  # TODO: pb package should be filtered out.
  run_for_modules generic_checker run_go_tool "github.com/mdempsky/unconvert" unconvert -v
 }

 function ineffassign_per_package {
-  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
+  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module)
  local gofiles=()
-  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
+  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module)
+
+  # TODO: ineffassign should work with package instead of files
  run_go_tool github.com/gordonklaus/ineffassign "${gofiles[@]}"
 }

@ -400,13 +415,14 @@ function ineffassign_pass {
 }

 function nakedret_pass {
+  # TODO: nakedret should work with -set_exit_status
  run_for_modules generic_checker run_go_tool "github.com/alexkohler/nakedret"
 }

 function license_header_per_module {
-  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
+  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module)
  local gofiles=()
-  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
+  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module)
  run_go_tool "github.com/google/addlicense" --check "${gofiles[@]}"
 }

@ -415,9 +431,9 @@ function license_header_pass {
 }

 function receiver_name_for_package {
-  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
+  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module)
  local gofiles=()
-  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
+  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module)

  recvs=$(grep 'func ([^*]' "${gofiles[@]}"  | tr  ':' ' ' |  \
    awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\\.]*go//g" |  sort  | uniq  | \
@ -441,9 +457,9 @@ function receiver_name_pass {
 # checks spelling and comments in the 'package' in the current module
 #
 function goword_for_package {
-  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
+  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module)
  local gofiles=()
-  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
+  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module)
  
  local gowordRes

@ -542,7 +558,7 @@ function release_pass {
  rm -f ./bin/etcd-last-release
  # to grab latest patch release; bump this up for every minor release
  UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.5.*" | head -1 | cut -d- -f1)
-  if [ -n "$MANUAL_VER" ]; then
+  if [ -n "${MANUAL_VER:-}" ]; then
    # in case, we need to test against different version
    UPGRADE_VER=$MANUAL_VER
  fi
@ -594,6 +610,7 @@ function mod_tidy_for_module {
    log_error "${PWD}/go.mod is not in sync with 'go mod tidy'"
    return 255
  fi
+  set -e
 }

 function mod_tidy_pass {
@ -609,8 +626,8 @@ function genproto_pass {
 }

 function goimport_for_module {
-  GOFILES=$(run go list  --f "{{with \$d:=.}}{{range .GoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...)
-  TESTGOFILES=$(run go list  --f "{{with \$d:=.}}{{range .TestGoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...)
+  GOFILES=$(run go list  --f "{{with \$d:=.}}{{range .GoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) || return 2
+  TESTGOFILES=$(run go list  --f "{{with \$d:=.}}{{range .TestGoFiles}}{{\$d.Dir}}/{{.}}{{\"\n\"}}{{end}}{{end}}" ./...) || return 2
  cd "${ETCD_ROOT_DIR}/tools/mod"
  FILESNEEDSFIX=$(echo "${GOFILES}" "${TESTGOFILES}" | grep -v '.gw.go' | grep -v '.pb.go' | xargs -n 100 go run golang.org/x/tools/cmd/goimports -l -local go.etcd.io)
  if [ -n "$FILESNEEDSFIX" ]; then
--- a/scripts/test_images.sh
+++ b/scripts/test_images.sh
@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
+IFS=$'\n\t'
+
+source ./scripts/test_lib.sh
+source ./scripts/build_lib.sh
+
+function startContainer {
+    # run docker in the background
+    docker run -d --rm --name "${RUN_NAME}" "${IMAGE}"
+
+    # wait for etcd daemon to bootstrap
+    sleep 5
+}
+
+function runVersionCheck {
+    Out=$(docker run --rm "${IMAGE}" "${@}")
+    foundVersion=$(echo "$Out" | head -1 | rev  | cut -d" "  -f 1 | rev )
+    if [[ "${foundVersion}" != "${VERSION}" ]]; then
+        echo "error: Invalid Version. Got $foundVersion, expected $VERSION. Error: $Out"
+        exit 1
+    fi
+}
+
+# Can't proceed without docker
+if ! command -v docker >/dev/null; then
+    log_error "cannot find docker"
+    exit 1
+fi
+
+# You can't run darwin binaries in linux containers
+if [[ $(go env GOOS) == "darwin" ]]; then
+    echo "Please use linux machine for release builds."
+    exit 1
+fi
+
+# Pick defaults based on release workflow
+ARCH=$(go env GOARCH)
+REPOSITARY=${REPOSITARY:-"gcr.io/etcd-development/etcd"}
+if [ -n "$VERSION" ]; then
+    # Expected Format: v3.6.99-amd64
+    TAG=v"${VERSION}"-"${ARCH}"
+else
+    echo "Terminating test, VERSION not supplied"
+    exit 1
+fi
+IMAGE=${IMAGE:-"${REPOSITARY}:${TAG}"}
+
+# ETCD related values
+RUN_NAME="test_etcd"
+KEY="foo"
+VALUE="bar"
+
+if [[ "$(docker images -q "${IMAGE}" 2> /dev/null)" == "" ]]; then
+    echo "${IMAGE} not present locally"
+    exit 1
+fi
+
+# Version check
+runVersionCheck "/usr/local/bin/etcd" "--version"
+runVersionCheck "/usr/local/bin/etcdctl" "version"
+runVersionCheck "/usr/local/bin/etcdutl" "version"
+
+startContainer
+# stop container
+trap 'docker stop "${RUN_NAME}"' EXIT
+
+
+# Put/Get check
+PUT=$(docker exec "${RUN_NAME}" /usr/local/bin/etcdctl put "${KEY}" "${VALUE}")
+if [ "${PUT}" != "OK" ]; then
+    echo "Problem with Putting in etcd"
+    exit 1
+fi
+
+GET=$(docker exec "${RUN_NAME}" /usr/local/bin/etcdctl get "$KEY" --print-value-only)
+if [ "${GET}" != "${VALUE}" ]; then
+    echo "Problem with getting foo bar in etcd. Got ${GET}"
+    exit 1
+fi
+
+echo "Succesfully tested etcd local image ${TAG}"
+
--- a/scripts/test_lib.sh
+++ b/scripts/test_lib.sh
@ -105,7 +105,7 @@ function relativePath {

 ####   Discovery of files/packages within a go module #####

-# go_srcs_in_module [package]
+# go_srcs_in_module
 # returns list of all not-generated go sources in the current (dir) module.
 function go_srcs_in_module {
  go list -f "{{with \$c:=.}}{{range \$f:=\$c.GoFiles  }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{range \$f:=\$c.TestGoFiles  }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{range \$f:=\$c.XTestGoFiles  }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{end}}" ./... | grep -vE "(\\.pb\\.go|\\.pb\\.gw.go)"
@ -212,7 +212,7 @@ function run_for_modules {
 }

 junitFilenamePrefix() {
-  if [[ -z "${JUNIT_REPORT_DIR}" ]]; then
+  if [[ -z "${JUNIT_REPORT_DIR:-}" ]]; then
    echo ""
    return
  fi
@ -222,7 +222,7 @@ junitFilenamePrefix() {
 }

 function produce_junit_xmlreport {
-  local -r junit_filename_prefix=$1
+  local -r junit_filename_prefix=${1:-}
  if [[ -z "${junit_filename_prefix}" ]]; then
    return
  fi
@ -232,7 +232,7 @@ function produce_junit_xmlreport {

  # Ensure that gotestsum is run without cross-compiling
  run_go_tool gotest.tools/gotestsum --junitfile "${junit_xml_filename}" --raw-command cat "${junit_filename_prefix}"*.stdout || exit 1
-  if [ "${VERBOSE}" != "1" ]; then
+  if [ "${VERBOSE:-}" != "1" ]; then
    rm "${junit_filename_prefix}"*.stdout
  fi

@ -290,7 +290,7 @@ function go_test {

  junit_filename_prefix=$(junitFilenamePrefix)

-  if [ "${VERBOSE}" == "1" ]; then
+  if [ "${VERBOSE:-}" == "1" ]; then
    goTestFlags="-v"
  fi

@ -315,7 +315,7 @@ function go_test {
    additional_flags=$(${flags_for_package_func} ${pkg})

    # shellcheck disable=SC2206
-    local cmd=( go test ${goTestFlags} ${additional_flags} "$@" ${pkg} )
+    local cmd=( go test ${goTestFlags} ${additional_flags} ${pkg} "$@" )

    # shellcheck disable=SC2086
    if ! run env ${goTestEnv} ETCD_VERIFY="${ETCD_VERIFY}" "${cmd[@]}" | tee ${junit_filename_prefix:+"${junit_filename_prefix}.stdout"} | grep --binary-files=text "${go_test_grep_pattern}" ; then
--- a/scripts/update_dep.sh
+++ b/scripts/update_dep.sh
@ -9,6 +9,8 @@
 #
 # Updates version of given dependency in all the modules that depend on the mod.

+set -euo pipefail
+
 source ./scripts/test_lib.sh

 mod="$1"
--- a/scripts/updatebom.sh
+++ b/scripts/updatebom.sh
@ -1,6 +1,7 @@
 #!/usr/bin/env bash

-set -e
+set -euo pipefail
+
 source ./scripts/test_lib.sh

 function bom_fixlet {
--- a/server/verify/verify.go
+++ b/server/verify/verify.go
@ -19,13 +19,15 @@ import (

 	"go.uber.org/zap"

+	"go.etcd.io/raft/v3/raftpb"
+
+	"go.etcd.io/etcd/client/pkg/v3/fileutil"
 	"go.etcd.io/etcd/client/pkg/v3/verify"
 	"go.etcd.io/etcd/server/v3/storage/backend"
 	"go.etcd.io/etcd/server/v3/storage/datadir"
 	"go.etcd.io/etcd/server/v3/storage/schema"
 	wal2 "go.etcd.io/etcd/server/v3/storage/wal"
 	"go.etcd.io/etcd/server/v3/storage/wal/walpb"
-	"go.etcd.io/raft/v3/raftpb"
 )

 const ENV_VERIFY_VALUE_STORAGE_WAL verify.VerificationType = "storage_wal"
@ -53,6 +55,11 @@ func Verify(cfg Config) error {
 		lg = zap.NewNop()
 	}

+	if !fileutil.Exist(datadir.ToBackendFileName(cfg.DataDir)) {
+		lg.Info("verification skipped due to non exist db file")
+		return nil
+	}
+
 	var err error
 	lg.Info("verification of persisted state", zap.String("data-dir", cfg.DataDir))
 	defer func() {
--- a/tests/common/auth_test.go
+++ b/tests/common/auth_test.go
@ -629,6 +629,99 @@ func TestAuthTestInvalidMgmt(t *testing.T) {
 	})
 }

+func TestAuthLeaseRevoke(t *testing.T) {
+	testRunner.BeforeTest(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1}))
+	defer clus.Close()
+	cc := testutils.MustClient(clus.Client())
+	testutils.ExecuteUntil(ctx, t, func() {
+		require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth")
+		rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword)))
+
+		lresp, err := rootAuthClient.Grant(ctx, 10)
+		require.NoError(t, err)
+		err = rootAuthClient.Put(ctx, "key", "value", config.PutOptions{LeaseID: lresp.ID})
+		require.NoError(t, err)
+
+		_, err = rootAuthClient.Revoke(ctx, lresp.ID)
+		require.NoError(t, err)
+
+		_, err = rootAuthClient.Get(ctx, "key", config.GetOptions{})
+		require.NoError(t, err)
+	})
+}
+
+func TestAuthRoleGet(t *testing.T) {
+	testRunner.BeforeTest(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1}))
+	defer clus.Close()
+	cc := testutils.MustClient(clus.Client())
+	testutils.ExecuteUntil(ctx, t, func() {
+		require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth")
+		rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword)))
+		testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword)))
+
+		resp, err := rootAuthClient.RoleGet(ctx, testRoleName)
+		require.NoError(t, err)
+		requireRolePermissionEqual(t, testRole, resp.Perm)
+
+		// test-user can get the information of test-role because it belongs to the role
+		resp, err = testUserAuthClient.RoleGet(ctx, testRoleName)
+		require.NoError(t, err)
+		requireRolePermissionEqual(t, testRole, resp.Perm)
+		// test-user cannot get the information of root because it doesn't belong to the role
+		_, err = testUserAuthClient.RoleGet(ctx, rootRoleName)
+		require.ErrorContains(t, err, PermissionDenied)
+	})
+}
+
+func TestAuthUserGet(t *testing.T) {
+	testRunner.BeforeTest(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1}))
+	defer clus.Close()
+	cc := testutils.MustClient(clus.Client())
+	testutils.ExecuteUntil(ctx, t, func() {
+		require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth")
+		rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword)))
+		testUserAuthClient := testutils.MustClient(clus.Client(WithAuth(testUserName, testPassword)))
+
+		resp, err := rootAuthClient.UserGet(ctx, testUserName)
+		require.NoError(t, err)
+		requireUserRolesEqual(t, testUser, resp.Roles)
+
+		// test-user can get the information of test-user itself
+		resp, err = testUserAuthClient.UserGet(ctx, testUserName)
+		require.NoError(t, err)
+		requireUserRolesEqual(t, testUser, resp.Roles)
+		// test-user cannot get the information of root
+		_, err = testUserAuthClient.UserGet(ctx, rootUserName)
+		require.ErrorContains(t, err, PermissionDenied)
+	})
+}
+
+func TestAuthRoleList(t *testing.T) {
+	testRunner.BeforeTest(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	clus := testRunner.NewCluster(ctx, t, config.WithClusterConfig(config.ClusterConfig{ClusterSize: 1}))
+	defer clus.Close()
+	cc := testutils.MustClient(clus.Client())
+	testutils.ExecuteUntil(ctx, t, func() {
+		require.NoErrorf(t, setupAuth(cc, []authRole{testRole}, []authUser{rootUser, testUser}), "failed to enable auth")
+		rootAuthClient := testutils.MustClient(clus.Client(WithAuth(rootUserName, rootPassword)))
+
+		resp, err := rootAuthClient.RoleList(ctx)
+		require.NoError(t, err)
+		requireUserRolesEqual(t, testUser, resp.Roles)
+	})
+}
+
 func mustAbsPath(path string) string {
 	abs, err := filepath.Abs(path)
 	if err != nil {
--- a/tests/common/auth_util.go
+++ b/tests/common/auth_util.go
@ -17,7 +17,11 @@ package common
 import (
 	"context"
 	"fmt"
+	"testing"

+	"github.com/stretchr/testify/require"
+
+	"go.etcd.io/etcd/api/v3/authpb"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/tests/v3/framework/config"
 	"go.etcd.io/etcd/tests/v3/framework/interfaces"
@ -106,3 +110,15 @@ func setupAuth(c interfaces.Client, roles []authRole, users []authUser) error {

 	return nil
 }
+
+func requireRolePermissionEqual(t *testing.T, expectRole authRole, actual []*authpb.Permission) {
+	require.Equal(t, 1, len(actual))
+	require.Equal(t, expectRole.permission, clientv3.PermissionType(actual[0].PermType))
+	require.Equal(t, expectRole.key, string(actual[0].Key))
+	require.Equal(t, expectRole.keyEnd, string(actual[0].RangeEnd))
+}
+
+func requireUserRolesEqual(t *testing.T, expectUser authUser, actual []string) {
+	require.Equal(t, 1, len(actual))
+	require.Equal(t, expectUser.role, actual[0])
+}
--- a/tests/e2e/ctl_v3_auth_test.go
+++ b/tests/e2e/ctl_v3_auth_test.go
@ -29,14 +29,10 @@ import (

 func TestCtlV3AuthMemberUpdate(t *testing.T) { testCtl(t, authTestMemberUpdate) }
 func TestCtlV3AuthFromKeyPerm(t *testing.T)  { testCtl(t, authTestFromKeyPerm) }
-func TestCtlV3AuthAndWatch(t *testing.T)     { testCtl(t, authTestWatch) }
-func TestCtlV3AuthAndWatchJWT(t *testing.T)  { testCtl(t, authTestWatch, withCfg(*e2e.NewConfigJWT())) }

-func TestCtlV3AuthLeaseRevoke(t *testing.T) { testCtl(t, authLeaseTestLeaseRevoke) }
-
-func TestCtlV3AuthRoleGet(t *testing.T)  { testCtl(t, authTestRoleGet) }
-func TestCtlV3AuthUserGet(t *testing.T)  { testCtl(t, authTestUserGet) }
-func TestCtlV3AuthRoleList(t *testing.T) { testCtl(t, authTestRoleList) }
+// TestCtlV3AuthAndWatch TODO https://github.com/etcd-io/etcd/issues/7988 is the blocker of migration to common/auth_test.go
+func TestCtlV3AuthAndWatch(t *testing.T)    { testCtl(t, authTestWatch) }
+func TestCtlV3AuthAndWatchJWT(t *testing.T) { testCtl(t, authTestWatch, withCfg(*e2e.NewConfigJWT())) }

 func TestCtlV3AuthDefrag(t *testing.T) { testCtl(t, authTestDefrag) }
 func TestCtlV3AuthEndpointHealth(t *testing.T) {
@ -230,44 +226,6 @@ func authTestFromKeyPerm(cx ctlCtx) {
 	}
 }

-func leaseTestGrantLeasesList(cx ctlCtx) error {
-	id, err := ctlV3LeaseGrant(cx, 10)
-	if err != nil {
-		return fmt.Errorf("ctlV3LeaseGrant error (%v)", err)
-	}
-
-	cmdArgs := append(cx.PrefixArgs(), "lease", "list")
-	proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap)
-	if err != nil {
-		return fmt.Errorf("lease list failed (%v)", err)
-	}
-	_, err = proc.Expect(id)
-	if err != nil {
-		return fmt.Errorf("lease id not in returned list (%v)", err)
-	}
-	return proc.Close()
-}
-
-func authLeaseTestLeaseRevoke(cx ctlCtx) {
-	cx.user, cx.pass = "root", "root"
-	authSetupTestUser(cx)
-
-	// put with TTL 10 seconds and revoke
-	leaseID, err := ctlV3LeaseGrant(cx, 10)
-	if err != nil {
-		cx.t.Fatalf("ctlV3LeaseGrant error (%v)", err)
-	}
-	if err := ctlV3Put(cx, "key", "val", leaseID); err != nil {
-		cx.t.Fatalf("ctlV3Put error (%v)", err)
-	}
-	if err := ctlV3LeaseRevoke(cx, leaseID); err != nil {
-		cx.t.Fatalf("ctlV3LeaseRevoke error (%v)", err)
-	}
-	if err := ctlV3GetWithErr(cx, []string{"key"}, []string{"retrying of unary invoker failed"}); err != nil { // expect errors
-		cx.t.Fatalf("ctlV3GetWithErr error (%v)", err)
-	}
-}
-
 func authTestWatch(cx ctlCtx) {
 	if err := authEnable(cx); err != nil {
 		cx.t.Fatal(err)
@ -344,77 +302,6 @@ func authTestWatch(cx ctlCtx) {

 }

-func authTestRoleGet(cx ctlCtx) {
-	if err := authEnable(cx); err != nil {
-		cx.t.Fatal(err)
-	}
-	cx.user, cx.pass = "root", "root"
-	authSetupTestUser(cx)
-
-	expected := []string{
-		"Role test-role",
-		"KV Read:", "foo",
-		"KV Write:", "foo",
-	}
-	if err := e2e.SpawnWithExpects(append(cx.PrefixArgs(), "role", "get", "test-role"), cx.envMap, expected...); err != nil {
-		cx.t.Fatal(err)
-	}
-
-	// test-user can get the information of test-role because it belongs to the role
-	cx.user, cx.pass = "test-user", "pass"
-	if err := e2e.SpawnWithExpects(append(cx.PrefixArgs(), "role", "get", "test-role"), cx.envMap, expected...); err != nil {
-		cx.t.Fatal(err)
-	}
-
-	// test-user cannot get the information of root because it doesn't belong to the role
-	expected = []string{
-		"Error: etcdserver: permission denied",
-	}
-	err := e2e.SpawnWithExpects(append(cx.PrefixArgs(), "role", "get", "root"), cx.envMap, expected...)
-	require.ErrorContains(cx.t, err, "permission denied")
-}
-
-func authTestUserGet(cx ctlCtx) {
-	if err := authEnable(cx); err != nil {
-		cx.t.Fatal(err)
-	}
-	cx.user, cx.pass = "root", "root"
-	authSetupTestUser(cx)
-
-	expected := []string{
-		"User: test-user",
-		"Roles: test-role",
-	}
-
-	if err := e2e.SpawnWithExpects(append(cx.PrefixArgs(), "user", "get", "test-user"), cx.envMap, expected...); err != nil {
-		cx.t.Fatal(err)
-	}
-
-	// test-user can get the information of test-user itself
-	cx.user, cx.pass = "test-user", "pass"
-	if err := e2e.SpawnWithExpects(append(cx.PrefixArgs(), "user", "get", "test-user"), cx.envMap, expected...); err != nil {
-		cx.t.Fatal(err)
-	}
-
-	// test-user cannot get the information of root
-	expected = []string{
-		"Error: etcdserver: permission denied",
-	}
-	err := e2e.SpawnWithExpects(append(cx.PrefixArgs(), "user", "get", "root"), cx.envMap, expected...)
-	require.ErrorContains(cx.t, err, "permission denied")
-}
-
-func authTestRoleList(cx ctlCtx) {
-	if err := authEnable(cx); err != nil {
-		cx.t.Fatal(err)
-	}
-	cx.user, cx.pass = "root", "root"
-	authSetupTestUser(cx)
-	if err := e2e.SpawnWithExpectWithEnv(append(cx.PrefixArgs(), "role", "list"), cx.envMap, "test-role"); err != nil {
-		cx.t.Fatal(err)
-	}
-}
-
 func authTestDefrag(cx ctlCtx) {
 	maintenanceInitKeys(cx)

--- a/tests/robustness/README.md
+++ b/tests/robustness/README.md
@ -0,0 +1,71 @@
+# etcd Robustness Testing
+
+Purpose of etcd robustness tests is to validate that etcd upholds
+[API guarantees] and [watch guarantees] under any condition or failure.
+
+Robustness tests achieve that comparing etcd cluster behavior against a simplified model.
+Multiple test encompass different etcd cluster setups, client traffic types and failures experienced by cluster.
+During a single test we create a cluster and inject failures while sending and recording client traffic.
+Correctness is validated by running collected history of client operations against the etcd model and a set of validators.
+Upon failure tests generate a report that can be used to attribute whether failure was caused by bug in etcd or test framework. 
+
+[API guarantees]: https://etcd.io/docs/latest/learning/api_guarantees/
+[watch guarantees]: https://etcd.io/docs/latest/learning/api/#watch-streams
+
+## Running locally 
+
+1. Build etcd with failpoints
+    ```bash
+    make gofail-enable
+    make build
+    make gofail-disable
+    ```
+2. Run the tests
+
+    ```bash
+    make test-robustness
+    ```
+   
+    Optionally you can pass environment variables:
+    * `GO_TEST_FLAGS` - to pass additional arguments to `go test`. 
+      It is recommended to run tests multiple times with failfast enabled. this can be done by setting `GO_TEST_FLAGS='--count=100 --failfast'`.
+    * `EXPECT_DEBUG=true` - to get logs from the cluster.
+    * `RESULTS_DIR` - to change location where results report will be saved.
+
+## Analysing failure
+
+If robustness tests fails we want to analyse the report to confirm if the issue is on etcd side. Location of this report
+is included in test logs. One of log lines should look like:
+```
+    history.go:34: Model is not linearizable
+    logger.go:130: 2023-03-18T12:18:03.244+0100 INFO    Saving member data dir  {"member": "TestRobustnessIssue14370-test-0", "path": "/tmp/TestRobustness_Issue14370/TestRobustnessIssue14370-test-0"}
+    logger.go:130: 2023-03-18T12:18:03.244+0100 INFO    Saving watch responses  {"path": "/tmp/TestRobustness_Issue14370/TestRobustnessIssue14370-test-0/responses.json"}
+    logger.go:130: 2023-03-18T12:18:03.247+0100 INFO    Saving watch events     {"path": "/tmp/TestRobustness_Issue14370/TestRobustnessIssue14370-test-0/events.json"}
+    logger.go:130: 2023-03-18T12:18:03.248+0100 INFO    Saving operation history        {"path": "/tmp/TestRobustness_Issue14370/full-history.json"}
+    logger.go:130: 2023-03-18T12:18:03.252+0100 INFO    Saving operation history        {"path": "/tmp/TestRobustness_Issue14370/patched-history.json"}
+    logger.go:130: 2023-03-18T12:18:03.256+0100 INFO    Saving visualization    {"path": "/tmp/TestRobustness_Issue14370/history.html"}
+```
+
+Report includes multiple types of files:
+* Member db files, can be used to verify disk/memory corruption.
+* Watch responses saved as json, can be used to validate [watch guarantees].
+* Operation history saved as both html visualization and a json, can be used to validate [API guarantees].
+
+### Example analysis of linearization issue
+
+Let's analyse issue [#14370].
+To reproduce the issue by yourself run `make test-robustness-issue14370`.
+After a couple of tries robustness tests should report `Model is not linearizable` and save report locally.
+Lineralization issues are easiest to analyse via history visualization. 
+Open `/tmp/TestRobustness_Issue14370/history.html` file in your browser.
+Jump to the error in linearization by clicking `[ jump to first error ]` on the top of the page.
+
+You should see a graph similar to the one on the image below.
+![issue14370](./issue14370.png)
+
+Last correct request (connected with grey line) is a `Put` request that succeeded and got revision `168`.
+All following requests are invalid (connected with red line) as they have revision `167`. 
+Etcd guarantee that revision is non-decreasing, so this shows a bug in etcd as there is no way revision should decrease.
+This is consistent with the root cause of [#14370] as it was issue with process crash causing last write to be lost.
+
+[#14370]: https://github.com/etcd-io/etcd/issues/14370
--- a/tests/robustness/issue14370.png
+++ b/tests/robustness/issue14370.png
--- a/tests/robustness/makefile.mk
+++ b/tests/robustness/makefile.mk
@ -0,0 +1,92 @@
+# Reproduce historical issues
+
+.PHONY: test-robustness-issue14370
+test-robustness-issue14370: /tmp/etcd-v3.5.4-failpoints/bin
+	GO_TEST_FLAGS='-v --run=TestRobustness/Issue14370 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.4-failpoints/bin' make test-robustness && \
+	 echo "Failed to reproduce" || echo "Successful reproduction"
+
+.PHONY: test-robustness-issue13766
+test-robustness-issue13766: /tmp/etcd-v3.5.2-failpoints/bin
+	GO_TEST_FLAGS='-v --run=TestRobustness/Issue13766 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.2-failpoints/bin' make test-robustness && \
+	 echo "Failed to reproduce" || echo "Successful reproduction"
+
+.PHONY: test-robustness-issue14685
+test-robustness-issue14685: /tmp/etcd-v3.5.5-failpoints/bin
+	GO_TEST_FLAGS='-v --run=TestRobustness/Issue14685 --count 100 --failfast --bin-dir=/tmp/etcd-v3.5.5-failpoints/bin' make test-robustness && \
+	 echo "Failed to reproduce" || echo "Successful reproduction"
+
+# Failpoints
+
+GOFAIL_VERSION = $(shell cd tools/mod && go list -m -f {{.Version}} go.etcd.io/gofail)
+
+.PHONY: gofail-enable
+gofail-enable: install-gofail
+	gofail enable server/etcdserver/ server/storage/backend/ server/storage/mvcc/ server/storage/wal/
+	cd ./server && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+	cd ./etcdutl && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+	cd ./etcdctl && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+	cd ./tests && go get go.etcd.io/gofail@${GOFAIL_VERSION}
+
+.PHONY: gofail-disable
+gofail-disable: install-gofail
+	gofail disable server/etcdserver/ server/storage/backend/ server/storage/mvcc/ server/storage/wal/
+	cd ./server && go mod tidy
+	cd ./etcdutl && go mod tidy
+	cd ./etcdctl && go mod tidy
+	cd ./tests && go mod tidy
+
+.PHONY: install-gofail
+install-gofail:
+	cd tools/mod; go install go.etcd.io/gofail@${GOFAIL_VERSION}
+
+# Build previous releases for robustness tests
+
+/tmp/etcd-v3.6.0-failpoints/bin:
+	rm -rf /tmp/etcd-v3.6.0-failpoints/
+	mkdir -p /tmp/etcd-v3.6.0-failpoints/
+	cd /tmp/etcd-v3.6.0-failpoints/; \
+	  git clone --depth 1 --branch main https://github.com/etcd-io/etcd.git .; \
+	  make gofail-enable; \
+	  make build;
+
+/tmp/etcd-v3.5.2-failpoints/bin:
+/tmp/etcd-v3.5.4-failpoints/bin:
+/tmp/etcd-v3.5.5-failpoints/bin:
+/tmp/etcd-v3.5.%-failpoints/bin:
+	rm -rf /tmp/etcd-v3.5.$*-failpoints/
+	mkdir -p /tmp/etcd-v3.5.$*-failpoints/
+	cd /tmp/etcd-v3.5.$*-failpoints/; \
+	  git clone --depth 1 --branch v3.5.$* https://github.com/etcd-io/etcd.git .; \
+	  go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
+	  (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  FAILPOINTS=true ./build;
+
+/tmp/etcd-release-3.5-failpoints/bin/etcd:
+	rm -rf /tmp/etcd-release-3.5-failpoints/
+	mkdir -p /tmp/etcd-release-3.5-failpoints/
+	cd /tmp/etcd-release-3.5-failpoints/; \
+	  git clone --depth 1 --branch release-3.5 https://github.com/etcd-io/etcd.git .; \
+	  go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
+	  (cd server; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  (cd etcdctl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  (cd etcdutl; go get go.etcd.io/gofail@${GOFAIL_VERSION}); \
+	  FAILPOINTS=true ./build;
+
+/tmp/etcd-v3.4.23-failpoints/bin:
+/tmp/etcd-v3.4.%-failpoints/bin:
+	rm -rf /tmp/etcd-v3.4.$*-failpoints/
+	mkdir -p /tmp/etcd-v3.4.$*-failpoints/
+	cd /tmp/etcd-v3.4.$*-failpoints/; \
+	  git clone --depth 1 --branch v3.4.$* https://github.com/etcd-io/etcd.git .; \
+	  go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
+	  FAILPOINTS=true ./build;
+
+/tmp/etcd-release-3.4-failpoints/bin/etcd:
+	rm -rf /tmp/etcd-release-3.4-failpoints/
+	mkdir -p /tmp/etcd-release-3.4-failpoints/
+	cd /tmp/etcd-release-3.4-failpoints/; \
+	  git clone --depth 1 --branch release-3.4 https://github.com/etcd-io/etcd.git .; \
+	  go get go.etcd.io/gofail@${GOFAIL_VERSION}; \
+	  FAILPOINTS=true ./build;
--- a/tests/robustness/watch.go
+++ b/tests/robustness/watch.go
@ -85,61 +85,84 @@ func validateWatchResponses(t *testing.T, responses [][]watchResponse, expectPro
 }

 func validateMemberWatchResponses(t *testing.T, responses []watchResponse, expectProgressNotify bool) {
-	var (
-		gotProgressNotify                = false
-		lastEventModRevision       int64 = 1 // The event.Kv.ModRevision in the latest event.
-		lastHeadRevision           int64 = 1 // The resp.Header.Revision in last watch response.
-		lastProgressNotifyRevision int64 = 0 // The resp.Header.Revision in the last progress notify watch response.
-	)
+	// Validate watch is correctly configured to ensure proper testing
+	validateGotAtLeastOneProgressNotify(t, responses, expectProgressNotify)
+
+	// Validate etcd watch properties defined in https://etcd.io/docs/v3.6/learning/api/#watch-streams
+	validateOrderedAndReliable(t, responses)
+	validateUnique(t, responses)
+	validateAtomic(t, responses)
+	// Validate kubernetes usage of watch
+	validateRenewable(t, responses)
+}
+
+func validateGotAtLeastOneProgressNotify(t *testing.T, responses []watchResponse, expectProgressNotify bool) {
+	var gotProgressNotify = false
+	var lastHeadRevision int64 = 1
+	for _, resp := range responses {
+		if resp.IsProgressNotify() && resp.Header.Revision == lastHeadRevision {
+			gotProgressNotify = true
+			break
+		}
+		lastHeadRevision = resp.Header.Revision
+	}
+	if gotProgressNotify != expectProgressNotify {
+		t.Errorf("Expected at least one progress notify: %v, got: %v", expectProgressNotify, gotProgressNotify)
+	}
+}
+
+func validateRenewable(t *testing.T, responses []watchResponse) {
+	var lastProgressNotifyRevision int64 = 0
+	for _, resp := range responses {
+		for _, event := range resp.Events {
+			if event.Kv.ModRevision <= lastProgressNotifyRevision {
+				t.Errorf("BROKE: Renewable - watch can renewed using revision in last progress notification; Progress notification guarantees that previous events have been already delivered, eventRevision: %d, progressNotifyRevision: %d", event.Kv.ModRevision, lastProgressNotifyRevision)
+			}
+		}
+		if resp.IsProgressNotify() {
+			lastProgressNotifyRevision = resp.Header.Revision
+		}
+	}
+}
+
+func validateOrderedAndReliable(t *testing.T, responses []watchResponse) {
+	var lastEventRevision int64 = 1
+	for _, resp := range responses {
+		for _, event := range resp.Events {
+			if event.Kv.ModRevision != lastEventRevision && event.Kv.ModRevision != lastEventRevision+1 {
+				t.Errorf("BROKE: Reliable - a sequence of events will never drop any subsequence of events; if there are events ordered in time as a < b < c, then if the watch receives events a and c, it is guaranteed to receive b, lastRevision: %d, currentRevision: %d", lastEventRevision, event.Kv.ModRevision)
+			}
+			lastEventRevision = event.Kv.ModRevision
+		}
+	}
+}
+
+func validateUnique(t *testing.T, responses []watchResponse) {
 	type revisionKey struct {
 		revision int64
 		key      string
 	}
 	uniqueOperations := map[revisionKey]struct{}{}
-
 	for _, resp := range responses {
-		if resp.Header.Revision < lastHeadRevision {
-			t.Errorf("Server revision should never decrease, lastHeadRevision: %d, resp.Header.Revision: %d",
-				lastHeadRevision, resp.Header.Revision)
-		}
-
-		if resp.IsProgressNotify() && resp.Header.Revision == lastHeadRevision {
-			gotProgressNotify = true
-		}
-		if resp.Header.Revision == lastHeadRevision && len(resp.Events) != 0 {
-			t.Errorf("Got two non-empty responses about same revision")
-		}
-		if len(resp.Events) > 0 && resp.Events[0].Kv.ModRevision <= lastEventModRevision {
-			t.Errorf("Events with same revision should not be split between multiple responses, lastResponseModRevision: %d, firstEventModRevision: %d", lastEventModRevision, resp.Events[0].Kv.ModRevision)
-		}
-
 		for _, event := range resp.Events {
 			rk := revisionKey{key: string(event.Kv.Key), revision: event.Kv.ModRevision}
 			if _, found := uniqueOperations[rk]; found {
-				t.Errorf("BROKE: Within the:wq same revision key can only be modified once. Suspecting duplicate watch event. key: %q, revision: %d", rk.key, rk.revision)
+				t.Errorf("BROKE: Unique - an event will never appear on a watch twice, key: %q, revision: %d", rk.key, rk.revision)
 			}
 			uniqueOperations[rk] = struct{}{}
-			if event.Kv.ModRevision <= lastProgressNotifyRevision {
-				t.Errorf("BROKE: etcd will not send progress notification to a watcher until it has synced all events. So a watcher will never receive an event with a lower `ModRevision` than the last progressNotification's revision, eventRevision: %d, progressNotifyRevision: %d", event.Kv.ModRevision, lastProgressNotifyRevision)
-			}
-			if event.Kv.ModRevision != lastEventModRevision && event.Kv.ModRevision != lastEventModRevision+1 {
-				t.Errorf("Event mod revision should stay the same or increment by one, last: %d, current: %d", lastEventModRevision, event.Kv.ModRevision)
-			}
-			lastEventModRevision = event.Kv.ModRevision
 		}
-
-		if resp.Header.Revision < lastEventModRevision {
-			t.Errorf("Event revision should never exceed the server's revision, lastEventRevision: %d, resp.Header.Revision: %d",
-				lastEventModRevision, resp.Header.Revision)
-		}
-
-		if resp.IsProgressNotify() {
-			lastProgressNotifyRevision = resp.Header.Revision
-		}
-		lastHeadRevision = resp.Header.Revision
 	}
-	if gotProgressNotify != expectProgressNotify {
-		t.Errorf("Expected progress notify: %v, got: %v", expectProgressNotify, gotProgressNotify)
+}
+
+func validateAtomic(t *testing.T, responses []watchResponse) {
+	var lastEventRevision int64 = 1
+	for _, resp := range responses {
+		if len(resp.Events) > 0 {
+			if resp.Events[0].Kv.ModRevision == lastEventRevision {
+				t.Errorf("BROKE: Atomic - a list of events is guaranteed to encompass complete revisions; updates in the same revision over multiple keys will not be split over several lists of events, previousListEventRevision: %d, currentListEventRevision: %d", lastEventRevision, resp.Events[0].Kv.ModRevision)
+			}
+			lastEventRevision = resp.Events[len(resp.Events)-1].Kv.ModRevision
+		}
 	}
 }

--- a/tools/mod/go.mod
+++ b/tools/mod/go.mod
@ -14,7 +14,7 @@ require (
 	github.com/hexfusion/schwag v0.0.0-20211117114134-3ceb0191ccbf
 	github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f
 	github.com/mgechev/revive v1.3.1
-	github.com/mikefarah/yq/v4 v4.31.2
+	github.com/mikefarah/yq/v4 v4.33.1
 	go.etcd.io/gofail v0.1.0
 	go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116
 	go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a
@ -50,8 +50,8 @@ require (
 	github.com/go-openapi/strfmt v0.21.0 // indirect
 	github.com/go-openapi/swag v0.19.15 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/goccy/go-json v0.10.0 // indirect
-	github.com/goccy/go-yaml v1.9.8 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/goccy/go-yaml v1.10.0 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
@ -68,6 +68,7 @@ require (
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/spf13/cobra v1.6.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
--- a/tools/mod/go.sum
+++ b/tools/mod/go.sum
@ -110,10 +110,10 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
-github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
-github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.9.8 h1:5gMyLUeU1/6zl+WFfR1hN7D2kf+1/eRGa7DFtToiBvQ=
-github.com/goccy/go-yaml v1.9.8/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-yaml v1.10.0 h1:rBi+5HGuznOxx0JZ+60LDY85gc0dyIJCIMvsMJTKSKQ=
+github.com/goccy/go-yaml v1.10.0/go.mod h1:h/18Lr6oSQ3mvmqFoWmQ47KChOgpfHpTyIHl3yVmpiY=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
@ -132,8 +132,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -195,8 +196,8 @@ github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 h1:zpIH83+oKzcpryru8c
 github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg=
 github.com/mgechev/revive v1.3.1 h1:OlQkcH40IB2cGuprTPcjB0iIUddgVZgGmDX3IAMR8D4=
 github.com/mgechev/revive v1.3.1/go.mod h1:YlD6TTWl2B8A103R9KWJSPVI9DrEf+oqr15q21Ld+5I=
-github.com/mikefarah/yq/v4 v4.31.2 h1:uOAj+ymBvvta+nOI56Z+v4Z3AUI4a6LXUf+8klVkLV4=
-github.com/mikefarah/yq/v4 v4.31.2/go.mod h1:0grgy4sm131n6gj9LSY2Qo3vHdmOlBeEJXiy3wwdcIs=
+github.com/mikefarah/yq/v4 v4.33.1 h1:lqdWrUrNRpcsZkhBefh8fwjWG81i56pKvl8SRVE65JI=
+github.com/mikefarah/yq/v4 v4.33.1/go.mod h1:7jHWghYUFVJAtUzz/E/gPNMarI+LPsqC6pJfeouINLk=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@ -210,6 +211,8 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
+github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
+github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -244,6 +247,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
--- a/tools/mod/install_all.sh
+++ b/tools/mod/install_all.sh
@ -1,4 +1,6 @@
 #!/usr/bin/env bash

+set -euo pipefail
+
 cd ./tools/mod || exit 2
 go list --tags tools -f '{{ join .Imports "\n" }}' | xargs go install