etcd/test

#!/usr/bin/env bash
#
# Run all etcd tests
# ./test
# ./test -v
#
#
# Run specified test pass
#
# $ PASSES=unit ./test
# $ PASSES=integration ./test
#
#
# Run tests for one package
# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT
# flag for different expectation
#
# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
# $ PASSES=integration PKG=client/integration TIMEOUT=1m ./test
#
#
# Run specified unit tests in one package
# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
# to run only "TestNew", set "TESTCASE="\bTestNew\b""
#
# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
# $ PASSES=integration PKG=client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
#
#
# Run code coverage
# COVERDIR must either be a absolute path or a relative path to the etcd root
# $ COVERDIR=coverage PASSES="build_cov cov" ./test
set -e

source ./build

# build before setting up test GOPATH
if [[ "${PASSES}" == *"functional"* ]]; then
	./functional/build
fi

if [ -z "$PASSES" ]; then
	PASSES="fmt bom dep build unit"
fi

USERPKG=${PKG:-}

# Invoke ./tests/cover.test.bash for HTML output
COVER=${COVER:-"-cover"}

# Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
IGNORE_PKGS="(vendor/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)"
INTEGRATION_PKGS="(integration|tests/e2e|contrib|functional)"

# all github.com/etcd-io/etcd/whatever pkgs that are not auto-generated / tools
# shellcheck disable=SC1117
PKGS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | grep -vE "(tools/|contrib/|tests/e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo)
# pkg1,pkg2,pkg3
PKGS_COMMA=${PKGS// /,}

# shellcheck disable=SC1117
TEST_PKGS=$(find . -name \*_test.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")

# shellcheck disable=SC1117
FORMATTABLE=$(find . -name \*.go | while read -r a; do echo "$(dirname "$a")/*.go"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")

TESTABLE_AND_FORMATTABLE=$(echo "$TEST_PKGS" | grep -vE "$INTEGRATION_PKGS")

# check if user provided PKG override
if [ -z "${USERPKG}" ]; then
	TEST=$TESTABLE_AND_FORMATTABLE
	FMT=$FORMATTABLE
else
	# strip out leading dotslashes and trailing slashes from PKG=./foo/
	TEST=${USERPKG/#./}
	TEST=${TEST/#\//}
	TEST=${TEST/%\//}
	# only run gofmt on packages provided by user
	FMT="$TEST"
fi

# shellcheck disable=SC2206
FMT=($FMT)
if [ "${VERBOSE}" == "1" ]; then
	# shellcheck disable=SC2128
	echo "Running with FMT:" "${FMT[@]}"
fi

# prepend REPO_PATH to each local package
split=$TEST
TEST=""
for a in $split; do TEST="$TEST ${REPO_PATH}/${a}"; done

# shellcheck disable=SC2206
TEST=($TEST)
if [ "${VERBOSE}" == "1" ]; then
	# shellcheck disable=SC2128
	echo "Running with TEST:" "${TEST[@]}"
fi

# TODO: 'rafttest' is failing with unused
STATIC_ANALYSIS_PATHS=$(find . -name \*.go ! -path './vendor/*' ! -path './gopath.proto/*' ! -path '*pb/*' | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS")
# shellcheck disable=SC2206
STATIC_ANALYSIS_PATHS=($STATIC_ANALYSIS_PATHS)
if [ "${VERBOSE}" == "1" ]; then
	# shellcheck disable=SC2128
	echo "Running with STATIC_ANALYSIS_PATHS:" "${STATIC_ANALYSIS_PATHS[@]}"
fi

if [ -z "$GOARCH" ]; then
	GOARCH=$(go env GOARCH);
fi

# determine the number of CPUs to use for Go tests
TEST_CPUS="1,2,4"
if [ -n "${CPU}" ]; then
	TEST_CPUS="${CPU}"
fi
echo "Running with TEST_CPUS:" "${TEST_CPUS}"

# determine whether target supports race detection
if [ "$GOARCH" == "amd64" ]; then
	RACE="--race"
fi

RUN_ARG=""
if [ -n "${TESTCASE}" ]; then
	RUN_ARG="-run=${TESTCASE}"
fi

function unit_pass {
	echo "Running unit tests..."
	GO_TEST_FLAG=""
	if [ "${VERBOSE}" == "1" ]; then
		GO_TEST_FLAG="-v"
	fi
	if [ "${VERBOSE}" == "2" ]; then
		GO_TEST_FLAG="-v"
		export CLIENT_DEBUG=1
	fi

	if [ "${RUN_ARG}" == "" ]; then
	    RUN_ARG="-run=Test"
	fi

	# check if user provided time out, especially useful when just run one test case
	# expectation could be different
	USERTIMEOUT=""
	if [ -z "${TIMEOUT}" ]; then
		USERTIMEOUT="3m"
	else
		USERTIMEOUT="${TIMEOUT}"
	fi
	go test -mod=mod ${GO_TEST_FLAG} -timeout "${USERTIMEOUT}"  "${COVER}" ${RACE} -cpu "${TEST_CPUS}" ${RUN_ARG} "$@" "${TEST[@]}"
}

function integration_pass {
	echo "Running integration tests..."

	# check if user provided time out, especially useful when just run one test case
	# expectation could be different
	USERTIMEOUT=""
	if [ -z "${TIMEOUT}" ]; then
		USERTIMEOUT="30m"
	else
		USERTIMEOUT="${TIMEOUT}"
	fi

	# if TESTCASE and PKG set, run specified test case in specified PKG
	# if TESTCASE set, PKG not set, run specified test case in all integration and integration_extra packages
	# if TESTCASE not set, PKG set, run all test cases in specified package
	# if TESTCASE not set, PKG not set, run all tests in all integration and integration_extra packages
	if [ -z "${TESTCASE}" ] && [ -z "${USERPKG}" ]; then
		go test -mod=mod -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration"
		integration_extra "$@"
	else
		if [ -z "${USERPKG}" ]; then
			INTEGTESTPKG=("${REPO_PATH}/integration"
						  "${REPO_PATH}/client/integration"
						  "${REPO_PATH}/clientv3/integration"
						  "${REPO_PATH}/contrib/raftexample")
		else
			INTEGTESTPKG=("${TEST[@]}")
		fi
		go test -mod=mod -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}"  "$@" "${INTEGTESTPKG[@]}"
	fi
}

function integration_extra {
	go test -mod=mod -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/client/integration"
	go test -mod=mod -timeout 25m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration"
	go test -mod=mod -timeout 1m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/contrib/raftexample"
	go test -mod=mod -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/etcdserver/api/v2store"
	go test -mod=mod -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" -run=Example "$@" "${TEST[@]}"
}

function functional_pass {
  	# Clean up any data and logs from previous runs
  	rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup

	for a in 1 2 3; do
		./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 &
		pid="$!"
		agent_pids="${agent_pids} $pid"
	done

	for a in 1 2 3; do
		echo "Waiting for 'etcd-agent' on ${a}9027..."
		while ! nc -z localhost ${a}9027; do
			sleep 1
		done
	done

	echo "functional test START!"
	./bin/etcd-tester --config ./functional.yaml && echo "'etcd-tester' succeeded"
	ETCD_TESTER_EXIT_CODE=$?
	echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}

	# shellcheck disable=SC2206
	agent_pids=($agent_pids)
	kill -s TERM "${agent_pids[@]}" || true

	if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
		printf "\n"
		echo "FAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
		tail -1000 /tmp/etcd-functional-1/etcd.log

		printf "\n"
		echo "FAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
		tail -1000 /tmp/etcd-functional-2/etcd.log

		printf "\n"
		echo "FAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
		tail -1000 /tmp/etcd-functional-3/etcd.log

		echo "--- FAIL: exit code" ${ETCD_TESTER_EXIT_CODE}
		exit ${ETCD_TESTER_EXIT_CODE}
	fi
	echo "functional test PASS!"
}

function cov_pass {
	echo "Running code coverage..."
	# install gocovmerge before running code coverage from github.com/wadey/gocovmerge
	# gocovmerge merges coverage files
	if ! command -v gocovmerge >/dev/null; then
		echo "gocovmerge not installed"
		exit 255
	fi

	if [ -z "$COVERDIR" ]; then
		echo "COVERDIR undeclared"
		exit 255
	fi

	if [ ! -f "bin/etcd_test" ]; then
		echo "etcd_test binary not found"
		exit 255
	fi

	mkdir -p "$COVERDIR"

	# run code coverage for unit and integration tests
	GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 30m"
	# shellcheck disable=SC2206
	GOCOVFLAGS=($GOCOVFLAGS)
	failed=""
	for t in $(echo "${TEST_PKGS}" | grep -vE "(tests/e2e|functional)"); do
		tf=$(echo "$t" | tr / _)
		# cache package compilation data for faster repeated builds
		go test -mod=mod "${GOCOVFLAGS[@]}" -i "${REPO_PATH}/$t" || true
		# uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
		go test -mod=mod "${GOCOVFLAGS[@]}" -run=Test -coverprofile "$COVERDIR/${tf}.coverprofile"  "${REPO_PATH}/$t" || failed="$failed $t"
	done

	# v2v3 tests
	go test -mod=mod -tags v2v3 "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/store-v2v3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed store-v2v3"

	# proxy tests
	go test -mod=mod -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_integration.coverprofile" "${REPO_PATH}/integration" || failed="$failed proxy-integration"
	go test -mod=mod -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_clientv3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed proxy-clientv3/integration"

	# run code coverage for e2e tests
	# use 30m timeout because e2e coverage takes longer
	# due to many tests cause etcd process to wait
	# on leadership transfer timeout during gracefully shutdown
	echo Testing tests/e2e without proxy...
	go test -mod=mod -tags cov -timeout 30m -v "${REPO_PATH}/tests/e2e" || failed="$failed tests/e2e"
	echo Testing tests/e2e with proxy...
	go test -mod=mod -tags "cov cluster_proxy" -timeout 30m -v "${REPO_PATH}/tests/e2e" || failed="$failed tests/e2e-proxy"

	# incrementally merge to get coverage data even if some coverage files are corrupted
	# optimistically assume etcdserver package's coverage file is OK since gocovmerge
	# expects to start with a non-empty file
	cp "$COVERDIR"/etcdserver.coverprofile "$COVERDIR"/cover.out
	for f in "$COVERDIR"/*.coverprofile; do
		echo "merging test coverage file ${f}"
		gocovmerge "$f" "$COVERDIR"/cover.out  >"$COVERDIR"/cover.tmp || failed="$failed $f"
		if [ -s "$COVERDIR"/cover.tmp ]; then
			mv "$COVERDIR"/cover.tmp "$COVERDIR"/cover.out
		fi
	done
	# strip out generated files (using GNU-style sed)
	sed --in-place '/generated.go/d' "$COVERDIR"/cover.out || true

	echo -e "\nTo generate coverage report use:"
	echo -e "  go tool cover --html=$COVERDIR/cover.out\n"

	# held failures to generate the full coverage file, now fail
	if [ -n "$failed" ]; then
		for f in $failed; do
			echo "--- FAIL:" "$f"
		done
		exit 255
	fi
}

function e2e_pass {
	echo "Running e2e tests..."

	# check if user provided time out, especially useful when just run one test case
	# expectation could be different
	USERTIMEOUT=""
	if [ -z "${TIMEOUT}" ]; then
		USERTIMEOUT="30m"
	else
		USERTIMEOUT="${TIMEOUT}"
	fi

	go test -mod=mod -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}"  "$@" "${REPO_PATH}/tests/e2e"
}

function integration_e2e_pass {
	echo "Running integration and e2e tests..."

	go test -mod=mod -timeout 30m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/tests/e2e" &
	e2epid="$!"
	go test -mod=mod -timeout 30m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration" &
	intpid="$!"
	wait $e2epid
	wait $intpid
	integration_extra "$@"
}

function grpcproxy_pass {
	go test -mod=mod -timeout 30m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration"
	go test -mod=mod -timeout 30m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration"
	go test -mod=mod -timeout 30m -v -tags cluster_proxy "$@" "${REPO_PATH}/tests/e2e"
}

function release_pass {
	rm -f ./bin/etcd-last-release
	# to grab latest patch release; bump this up for every minor release
	UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
	if [ -n "$MANUAL_VER" ]; then
		# in case, we need to test against different version
		UPGRADE_VER=$MANUAL_VER
	fi
	if [[ -z ${UPGRADE_VER} ]]; then
		UPGRADE_VER="v3.3.0"
		echo "fallback to" ${UPGRADE_VER}
	fi

	local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
	echo "Downloading $file"

	set +e
	curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
	local result=$?
	set -e
	case $result in
		0)	;;
		*)	echo "--- FAIL:" ${result}
			exit $result
			;;
	esac

	tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
	mkdir -p ./bin
	mv /tmp/etcd ./bin/etcd-last-release
}

function shellcheck_pass {
	if command -v shellcheck >/dev/null; then
		shellcheckResult=$(shellcheck -fgcc build test scripts/*.sh 2>&1 || true)
		if [ -n "${shellcheckResult}" ]; then
			echo -e "shellcheck checking failed:\\n${shellcheckResult}"
			exit 255
		fi
	fi
}

function markdown_you_pass {
	# eschew you
	yous=$(find . -name \*.md ! -path './vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + || true)
	if [ -n "$yous" ]; then
		echo -e "found 'you' in documentation:\\n${yous}"
		exit 255
	fi
}

function markdown_marker_pass {
	# TODO: check other markdown files when marker handles headers with '[]'
	if command -v marker >/dev/null; then
		markerResult=$(marker --skip-http --root ./Documentation 2>&1 || true)
		if [ -n "${markerResult}" ]; then
			echo -e "marker checking failed:\\n${markerResult}"
			exit 255
		fi
	else
		echo "Skipping marker..."
	fi
}

function goword_pass {
	if command -v goword >/dev/null; then
		# get all go files to process
		gofiles=$(find "${FMT[@]}" -iname '*.go' 2>/dev/null)
		# shellcheck disable=SC2206
		gofiles_all=($gofiles)
		# ignore tests and protobuf files
		# shellcheck disable=SC1117
		gofiles=$(echo "${gofiles_all[@]}" | sort | uniq | sed "s/ /\n/g" | grep -vE "(\\_test.go|\\.pb\\.go)")
		# shellcheck disable=SC2206
		gofiles=($gofiles)
		# only check for broken exported godocs
		gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort)
		if [ -n "$gowordRes" ]; then
			echo -e "goword checking failed:\\n${gowordRes}"
			exit 255
		fi
		# check some spelling
		gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort)
		if [ -n "$gowordRes" ]; then
			echo -e "goword checking failed:\\n${gowordRes}"
			exit 255
		fi
	else
		echo "Skipping goword..."
	fi
}

function gofmt_pass {
	fmtRes=$(gofmt -l -s -d "${FMT[@]}")
	if [ -n "${fmtRes}" ]; then
		echo -e "gofmt checking failed:\\n${fmtRes}"
		exit 255
	fi
}

function govet_pass {
	vetRes=$(go vet -mod=mod "${TEST[@]}")
	if [ -n "${vetRes}" ]; then
		echo -e "govet checking failed:\\n${vetRes}"
		exit 255
	fi
}

function govet_shadow_pass {
	fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.")
	# shellcheck disable=SC2206
	fmtpkgs=($fmtpkgs)
	# Golang 1.12 onwards the experimental -shadow option is no longer available with go vet
	./scripts/install_tool.sh golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
	export PATH=${GOPATH}/bin:${PATH}
	shadow_tool=$(which shadow)
	vetRes=$(go vet -all -vettool="${shadow_tool}" "${TEST[@]}")
	if [ -n "${vetRes}" ]; then
		echo -e "govet -shadow checking failed:\\n${vetRes}"
		exit 255
	fi
}

function unparam_pass {
	if command -v unparam >/dev/null; then
		unparamResult=$(unparam "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
		if [ -n "${unparamResult}" ]; then
			echo -e "unparam checking failed:\\n${unparamResult}"
			exit 255
		fi
	else
		echo "Skipping unparam..."
	fi
}

function staticcheck_pass {
	if command -v staticcheck >/dev/null; then
		staticcheckResult=$(staticcheck "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
		if [ -n "${staticcheckResult}" ]; then
			# TODO: resolve these after go1.8 migration
			# See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
			STATIC_CHECK_MASK="S(A|T)(1002|1005|1006|1008|1012|1019|1032|2002|4003|4006)"
			if echo "${staticcheckResult}" | grep -vE "$STATIC_CHECK_MASK"; then
				echo -e "staticcheck checking failed:\\n${staticcheckResult}"
				exit 255
			else
				suppressed=$(echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c)
				echo -e "staticcheck suppressed warnings:\\n${suppressed}"
			fi
		fi
	else
		echo "Skipping staticcheck..."
	fi
}

function revive_pass {
	if command -v revive >/dev/null; then
		reviveResult=$(revive -config ./tests/revive.toml -exclude "vendor/..." ./... 2>&1 || true)
		if [ -n "${reviveResult}" ]; then
			echo -e "revive checking failed:\\n${reviveResult}"
			exit 255
		fi
	else
		echo "Skipping revive..."
	fi
}

function unconvert_pass {
	if command -v unconvert >/dev/null; then
		unconvertResult=$(unconvert -v "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
		if [ -n "${unconvertResult}" ]; then
			echo -e "unconvert checking failed:\\n${unconvertResult}"
			exit 255
		fi
	else
		echo "Skipping unconvert..."
	fi
}

function ineffassign_pass {
	if command -v ineffassign >/dev/null; then
		ineffassignResult=$(ineffassign "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
		if [ -n "${ineffassignResult}" ]; then
			echo -e "ineffassign checking failed:\\n${ineffassignResult}"
			exit 255
		fi
	else
		echo "Skipping ineffassign..."
	fi
}

function nakedret_pass {
	if command -v nakedret >/dev/null; then
		nakedretResult=$(nakedret "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
		if [ -n "${nakedretResult}" ]; then
			echo -e "nakedret checking failed:\\n${nakedretResult}"
			exit 255
		fi
	else
		echo "Skipping nakedret..."
	fi
}

function license_header_pass {
	licRes=""
	files=$(find . -type f -iname '*.go' ! -path './vendor/*' ! -path './gopath.proto/*')
	for file in $files; do
		if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
			licRes="${licRes}"$(echo -e "  ${file}")
		fi
	done
	if [ -n "${licRes}" ]; then
		echo -e "license header checking failed:\\n${licRes}"
		exit 255
	fi
}

function receiver_name_pass {
	# shellcheck disable=SC1117
	recvs=$(grep 'func ([^*]' {*,*/*,*/*/*}.go  | grep -Ev "(generated|pb/)" | tr  ':' ' ' |  \
		awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" |  sort  | uniq  | \
		grep -Ev  "(Descriptor|Proto|_)"  | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
	if [ -n "${recvs}" ]; then
		# shellcheck disable=SC2206
		recvs=($recvs)
		for recv in "${recvs[@]}"; do
			echo "Mismatched receiver for $recv..."
			grep "$recv" "${FMT[@]}" | grep 'func ('
		done
		exit 255
	fi
}

function commit_title_pass {
	git log --oneline "$(git merge-base HEAD master)"...HEAD | while read -r l; do
		commitMsg=$(echo "$l" | cut -f2- -d' ')
		if [[ "$commitMsg" == Merge* ]]; then
			# ignore "Merge pull" commits
			continue
		fi
		if [[ "$commitMsg" == Revert* ]]; then
			# ignore revert commits
			continue
		fi

		pkgPrefix=$(echo "$commitMsg" | cut -f1 -d':')
		spaceCommas=$(echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0)
		commaSpaces=$(echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0)
		if [[ $(echo "$commitMsg" | grep -c ":..*") == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
			echo "$l"...
			echo "Expected commit title format '<package>{\", \"<package>}: <description>'"
			echo "Got: $l"
			exit 255
		fi
	done
}

# tools gosimple,unused,staticheck,unconvert,ineffasign,nakedret
# are not module-aware. See https://github.com/golang/go/issues/24661
# The module-aware versions need to be used when they become available
function fmt_pass {
	toggle_failpoints disable

    # TODO: add "unparam"
	for p in shellcheck \
			markdown_you \
			goword \
			gofmt \
			govet \
			revive \
			license_header \
			receiver_name \
			commit_title \
			; do
		echo "'$p' started at $(date)"
		"${p}"_pass "$@"
		echo "'$p' completed at $(date)"
	done
}

function bom_pass {
	if ! command -v license-bill-of-materials >/dev/null; then
		return
	fi
	if [ "${GO111MODULE}" == "on" ]; then
		# license-bill-off-materials calls "go list std cmd" which cannot handle modules
		# Please see https://github.com/golang/go/issues/26924
		echo "Skipping license-bill-of-materials with go modules..."
		return
	fi
	echo "Checking bill of materials..."
	# https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
	GOFLAGS="-mod=mod" license-bill-of-materials \
		--override-file bill-of-materials.override.json \
		go.etcd.io/etcd/v3 go.etcd.io/etcd/v3/etcdctl >bom-now.json || true
	if ! diff bill-of-materials.json bom-now.json; then
		echo "vendored licenses do not match given bill of materials"
		exit 255
	fi
	rm bom-now.json
}

function dep_pass {
	echo "Checking package dependencies..."
	# don't pull in etcdserver package
	pushd clientv3 >/dev/null
	badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)"
	deps=$(go list -f '{{ .Deps }}'  | sed 's/ /\n/g' | grep -E "${badpkg}" || echo "")
	popd >/dev/null
	if [ -n "$deps" ]; then
		echo -e "clientv3 has masked dependencies:\\n${deps}"
		exit 255
	fi
}

function build_cov_pass {
	out="bin"
	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
	go test -mod=mod -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcd_test"

	go test -mod=mod -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcdctl_test" "${REPO_PATH}/etcdctl"
}

# fail fast on static tests
function build_pass {
	echo "Checking build..."
	GO_BUILD_FLAGS="-v -mod=mod" etcd_build
	GO_BUILD_FLAGS="-v -mod=mod" tools_build
}

for pass in $PASSES; do
	echo "Starting '$pass' pass at $(date)"
	"${pass}"_pass "$@"
	echo "Finished '$pass' pass at $(date)"
done

echo "Success"