./test: Refactoring of test script for modularization
This refactoring offers following benefits: - Unified way how go test commands are being called (in terms of flags intepretation) - Uses standard go mechanisms (like go lists) to find files/packages that are subject for test. The mechanism are module aware. - Added instruction how to install tools needed for the tests/checkers. - Added colors to the output to make it easier to spot any failure. Confirmed to work using: - COVERDIR="./coverage" CPU="4" RACE=false COVER=false PASSES="build build_cov cov" ./test - CPU="4" RACE=false COVER=false PASSES="e2e functional integration" ./test - COVERDIR="./coverage" COVER="false" CPU="4" RACE="false" PASSES="fmt build unit build_cov integration e2e integration_e2e grpcproxy cov" ./test - PASSES=unit PKG=./wal TIMEOUT=1m ./test - PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test - PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test - PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test - PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test - COVERDIR=coverage PASSES="build_cov cov" ./testrelease-3.5
Piotr Tabor
parent
31426b0041
commit
f1d4593241
|
|
@ -17,5 +17,6 @@ hack/tls-setup/certs
|
|||
/contrib/raftexample/raftexample
|
||||
/contrib/raftexample/raftexample-*
|
||||
/vendor
|
||||
/tests/e2e/default.proxy
|
||||
|
||||
*.bak
|
||||
|
|
|
|||
20
.travis.yml
20
.travis.yml
|
|
@ -78,19 +78,22 @@ script:
|
|||
/bin/bash -c "GOARCH=amd64 PASSES='fmt bom dep' ./test"
|
||||
;;
|
||||
linux-amd64-integration-1-cpu)
|
||||
# TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
|
||||
docker run --rm \
|
||||
--volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
|
||||
/bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' ./test"
|
||||
/bin/bash -c "GOARCH=amd64 CPU=1 PASSES='integration' RACE='false' ./test"
|
||||
;;
|
||||
linux-amd64-integration-2-cpu)
|
||||
# TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
|
||||
docker run --rm \
|
||||
--volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
|
||||
/bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' ./test"
|
||||
/bin/bash -c "GOARCH=amd64 CPU=2 PASSES='integration' RACE='false' ./test"
|
||||
;;
|
||||
linux-amd64-integration-4-cpu)
|
||||
# TODO: Reenable 'race' when https://github.com/etcd-io/etcd/issues/12336 fixed.
|
||||
docker run --rm \
|
||||
--volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
|
||||
/bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' ./test"
|
||||
/bin/bash -c "GOARCH=amd64 CPU=4 PASSES='integration' RACE='false' ./test"
|
||||
;;
|
||||
linux-amd64-functional)
|
||||
docker run --rm \
|
||||
|
|
@ -100,7 +103,7 @@ script:
|
|||
linux-amd64-unit)
|
||||
docker run --rm \
|
||||
--volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
|
||||
/bin/bash -c "GOARCH=amd64 PASSES='unit' ./test"
|
||||
/bin/bash -c "GOARCH=amd64 PASSES='unit' ./test -p=2"
|
||||
;;
|
||||
all-build)
|
||||
docker run --rm \
|
||||
|
|
@ -115,17 +118,18 @@ script:
|
|||
&& GO_BUILD_FLAGS='-v -mod=readonly' GOARCH=s390x ./build"
|
||||
;;
|
||||
linux-amd64-grpcproxy)
|
||||
sudo HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy'" make docker-test
|
||||
# TODO: Reenable race when https://github.com/etcd-io/etcd/issues/12336 fixed.
|
||||
sudo HOST_TMP_DIR=/tmp TEST_OPTS="PASSES='build grpcproxy' VERBOSE='1' CPU='4' COVER='false' RACE='false'" make docker-test
|
||||
;;
|
||||
linux-amd64-coverage)
|
||||
sudo HOST_TMP_DIR=/tmp make docker-test-coverage
|
||||
sudo HOST_TMP_DIR=/tmp TEST_OPTS="VERBOSE='1'" make docker-test-coverage
|
||||
;;
|
||||
linux-amd64-fmt-unit-go-tip)
|
||||
GOARCH=amd64 PASSES='fmt unit' ./test
|
||||
GOARCH=amd64 PASSES='fmt unit' ./test -p=2
|
||||
;;
|
||||
linux-386-unit)
|
||||
docker run --rm \
|
||||
--volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
|
||||
/bin/bash -c "GOARCH=386 PASSES='unit' ./test"
|
||||
/bin/bash -c "GOARCH=386 PASSES='unit' ./test -p=2"
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
|
|
@ -33,8 +33,6 @@ import (
|
|||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// TODO(ptabor): This is integration test. Skip it in --short and move to integration tests directory.
|
||||
|
||||
// TestSnapshotV3RestoreSingle tests single node cluster restoring
|
||||
// from a snapshot file.
|
||||
func TestSnapshotV3RestoreSingle(t *testing.T) {
|
||||
|
|
@ -194,6 +192,8 @@ type kv struct {
|
|||
|
||||
// creates a snapshot file and returns the file path.
|
||||
func createSnapshotFile(t *testing.T, kvs []kv) string {
|
||||
testutil.SkipTestIfShortMode(t,
|
||||
"Snapshot creation tests are depending on embedded etcServer so are integration-level tests.")
|
||||
clusterN := 1
|
||||
urls := newEmbedURLs(clusterN * 2)
|
||||
cURLs, pURLs := urls[:clusterN], urls[clusterN:]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
// Copyright 2016 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build cluster_proxy
|
||||
|
||||
// The purpose of this (empty) package is too keep following test working:
|
||||
// # go test -tags=cluster_proxy ./integration/embed
|
||||
package embed_test
|
||||
|
|
@ -0,0 +1,194 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
ETCD_ROOT_DIR="$(pwd)"
|
||||
|
||||
#### Convenient IO methods #####
|
||||
|
||||
COLOR_RED='\033[0;31m'
|
||||
COLOR_ORANGE='\033[0;33m'
|
||||
COLOR_GREEN='\033[0;32m'
|
||||
COLOR_LIGHTCYAN='\033[0;36m'
|
||||
|
||||
COLOR_NONE='\033[0m' # No Color
|
||||
|
||||
function log_error {
|
||||
>&2 echo -n -e "${COLOR_RED}"
|
||||
>&2 echo "$@"
|
||||
>&2 echo -n -e "${COLOR_NONE}"
|
||||
}
|
||||
|
||||
function log_warning {
|
||||
>&2 echo -n -e "${COLOR_ORANGE}"
|
||||
>&2 echo "$@"
|
||||
>&2 echo -n -e "${COLOR_NONE}"
|
||||
}
|
||||
|
||||
function log_callout {
|
||||
>&2 echo -n -e "${COLOR_LIGHTCYAN}"
|
||||
>&2 echo "$@"
|
||||
>&2 echo -n -e "${COLOR_NONE}"
|
||||
}
|
||||
|
||||
function log_success {
|
||||
>&2 echo -n -e "${COLOR_GREEN}"
|
||||
>&2 echo "$@"
|
||||
>&2 echo -n -e "${COLOR_NONE}"
|
||||
}
|
||||
|
||||
|
||||
#### Discovery of files/packages within a go module #####
|
||||
|
||||
# go_srcs_in_module [package]
|
||||
# returns list of all not-generated go sources in the current (dir) module.
|
||||
function go_srcs_in_module {
|
||||
go fmt -n "$1" | grep -Eo "([^ ]*)$" | grep -vE "(\\_test.go|\\.pb\\.go|\\.pb\\.gw.go)"
|
||||
}
|
||||
|
||||
# pkgs_in_module [optional:package_pattern]
|
||||
# returns list of all packages in the current (dir) module.
|
||||
# if the package_pattern is given, its being resolved.
|
||||
function pkgs_in_module {
|
||||
go list -mod=mod "${1:-./...}";
|
||||
}
|
||||
|
||||
function filter_out_integration_style_tests {
|
||||
grep -Ev '/(tests/e2e|integration|functional)(/|$)'
|
||||
}
|
||||
|
||||
#### Running actions against multiple modules ####
|
||||
|
||||
# run [command...] - runs given command, printing it first and
|
||||
# again if it failed (in RED). Use to wrap important test commands
|
||||
# that user might want to re-execute to shorten the feedback loop when fixing
|
||||
# the test.
|
||||
function run {
|
||||
local rpath
|
||||
rpath=$(realpath "--relative-to=${ETCD_ROOT_DIR}" "${PWD}")
|
||||
local repro="$*"
|
||||
if [ "${rpath}" != "." ]; then
|
||||
repro="(cd ${rpath} && ${repro})"
|
||||
fi
|
||||
|
||||
log_callout "% ${repro}"
|
||||
"${@}"
|
||||
local error_code=$?
|
||||
if [ ${error_code} != 0 ]; then
|
||||
log_error -e "FAIL: (code:${error_code}):\n % ${repro}"
|
||||
return ${error_code}
|
||||
fi
|
||||
}
|
||||
|
||||
# run_for_module [module] [cmd]
|
||||
# executes given command in the given module for given pkgs.
|
||||
# module_name - "." (in future: tests, client, server)
|
||||
# cmd - cmd to be executed - that takes package as last argument
|
||||
function run_for_module {
|
||||
local module=${1:-"."}
|
||||
shift 1
|
||||
(
|
||||
cd "${module}" && "$@"
|
||||
)
|
||||
}
|
||||
|
||||
# run_for_modules [cmd]
|
||||
# run given command across all modules and packages
|
||||
# (unless the set is limited using ${PKG} or / ${USERMOD})
|
||||
function run_for_modules {
|
||||
local pkg="${PKG:-./...}"
|
||||
if [ -z "${USERMOD}" ]; then
|
||||
run_for_module "." "$@" "${pkg}" || return "$?"
|
||||
else
|
||||
run_for_module "${USERMOD}" "$@" "${pkg}" || return "$?"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
#### Running go test ########
|
||||
|
||||
# go_test [packages] [mode] [flags_for_package_func] [$@]
|
||||
# [mode] supports 3 states:
|
||||
# - "parallel": fastest as concurrently processes multiple packages, but silent
|
||||
# till the last package. See: https://github.com/golang/go/issues/2731
|
||||
# - "keep_going" : executes tests package by package, but postpones reporting error to the last
|
||||
# - "fail_fast" : executes tests packages 1 by 1, exits on the first failure.
|
||||
#
|
||||
# [flags_for_package_func] is a name of function that takes list of packages as parameter
|
||||
# and computes additional flags to the go_test commands.
|
||||
# Use 'true' or ':' if you dont need additional arguments.
|
||||
#
|
||||
# depends on the VERBOSE top-level variable.
|
||||
#
|
||||
# Example:
|
||||
# go_test "./..." "keep_going" ":" --short
|
||||
#
|
||||
# The function returns != 0 code in case of test failure.
|
||||
function go_test {
|
||||
local packages="${1}"
|
||||
local mode="${2}"
|
||||
local flags_for_package_func="${3}"
|
||||
|
||||
shift 3
|
||||
|
||||
local goTestFlags=""
|
||||
local goTestEnv=""
|
||||
if [ "${VERBOSE}" == "1" ]; then
|
||||
goTestFlags="-v"
|
||||
fi
|
||||
if [ "${VERBOSE}" == "2" ]; then
|
||||
goTestFlags="-v"
|
||||
goTestEnv="CLIENT_DEBUG=1"
|
||||
fi
|
||||
|
||||
# Expanding patterns (like ./...) into list of packages
|
||||
|
||||
local unpacked_packages=("${packages}")
|
||||
if [ "${mode}" != "parallel" ]; then
|
||||
# shellcheck disable=SC2207
|
||||
# shellcheck disable=SC2086
|
||||
if ! unpacked_packages=($(go list ${packages})); then
|
||||
log_error "Cannot resolve packages: ${packages}"
|
||||
return 255
|
||||
fi
|
||||
fi
|
||||
|
||||
local failures=""
|
||||
|
||||
# execution of tests against packages:
|
||||
for pkg in "${unpacked_packages[@]}"; do
|
||||
local additional_flags
|
||||
# shellcheck disable=SC2086
|
||||
additional_flags=$(${flags_for_package_func} ${pkg})
|
||||
|
||||
# shellcheck disable=SC2206
|
||||
local cmd=( go test ${goTestFlags} ${additional_flags} "$@" ${pkg} )
|
||||
|
||||
if ! run env ${goTestEnv} "${cmd[@]}" ; then
|
||||
if [ "${mode}" != "keep_going" ]; then
|
||||
return 2
|
||||
else
|
||||
failures=("${failures[@]}" "${pkg}")
|
||||
fi
|
||||
fi
|
||||
echo
|
||||
done
|
||||
|
||||
if [ -n "${failures[*]}" ] ; then
|
||||
log_error -e "ERROR: Tests for following packages failed:\n ${failures[*]}"
|
||||
return 2
|
||||
fi
|
||||
}
|
||||
|
||||
#### Other ####
|
||||
|
||||
# tool_exists [tool] [instruction]
|
||||
# Checks whether given [tool] is installed. In case of failure,
|
||||
# prints a warning with installation [instruction] and returns !=0 code.
|
||||
function tool_exists {
|
||||
local tool="${1}"
|
||||
local instruction="${2}"
|
||||
if ! command -v "${tool}" >/dev/null; then
|
||||
log_warning "Tool: '${tool}' not found on PATH. ${instruction}"
|
||||
return 255
|
||||
fi
|
||||
}
|
||||
|
||||
775
test
775
test
|
|
@ -16,8 +16,7 @@
|
|||
# flag for different expectation
|
||||
#
|
||||
# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
|
||||
# $ PASSES=integration PKG=client/integration TIMEOUT=1m ./test
|
||||
#
|
||||
# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test
|
||||
#
|
||||
# Run specified unit tests in one package
|
||||
# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
|
||||
|
|
@ -25,7 +24,7 @@
|
|||
#
|
||||
# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
|
||||
# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
|
||||
# $ PASSES=integration PKG=client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
|
||||
# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
|
||||
#
|
||||
#
|
||||
# Run code coverage
|
||||
|
|
@ -39,168 +38,109 @@ set -e
|
|||
export GOFLAGS=-mod=readonly
|
||||
|
||||
source ./build
|
||||
source ./scripts/test_lib.sh
|
||||
|
||||
# build before setting up test GOPATH
|
||||
if [[ "${PASSES}" == *"functional"* ]]; then
|
||||
./functional/build
|
||||
fi
|
||||
|
||||
if [ -z "$PASSES" ]; then
|
||||
PASSES="fmt bom dep build unit"
|
||||
fi
|
||||
|
||||
USERPKG=${PKG:-}
|
||||
PASSES=${PASSES:-"fmt bom dep build unit"}
|
||||
PKG=${PKG:-}
|
||||
|
||||
# Invoke ./tests/cover.test.bash for HTML output
|
||||
COVER=${COVER:-"-cover"}
|
||||
|
||||
# Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
|
||||
IGNORE_PKGS="(vendor/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)"
|
||||
INTEGRATION_PKGS="(integration|tests/e2e|contrib|functional)"
|
||||
|
||||
# all github.com/etcd-io/etcd/whatever pkgs that are not auto-generated / tools
|
||||
# shellcheck disable=SC1117
|
||||
PKGS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | grep -vE "(tools/|contrib/|tests/e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo)
|
||||
# pkg1,pkg2,pkg3
|
||||
PKGS_COMMA=${PKGS// /,}
|
||||
|
||||
# shellcheck disable=SC1117
|
||||
TEST_PKGS=$(find . -name \*_test.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")
|
||||
|
||||
# shellcheck disable=SC1117
|
||||
FORMATTABLE=$(find . -name \*.go | while read -r a; do echo "$(dirname "$a")/*.go"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")
|
||||
|
||||
TESTABLE_AND_FORMATTABLE=$(echo "$TEST_PKGS" | grep -vE "$INTEGRATION_PKGS")
|
||||
|
||||
# check if user provided PKG override
|
||||
if [ -z "${USERPKG}" ]; then
|
||||
TEST=$TESTABLE_AND_FORMATTABLE
|
||||
FMT=$FORMATTABLE
|
||||
else
|
||||
# strip out leading dotslashes and trailing slashes from PKG=./foo/
|
||||
TEST=${USERPKG/#./}
|
||||
TEST=${TEST/#\//}
|
||||
TEST=${TEST/%\//}
|
||||
# only run gofmt on packages provided by user
|
||||
FMT="$TEST"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2206
|
||||
FMT=($FMT)
|
||||
if [ "${VERBOSE}" == "1" ]; then
|
||||
# shellcheck disable=SC2128
|
||||
echo "Running with FMT:" "${FMT[@]}"
|
||||
fi
|
||||
|
||||
# prepend REPO_PATH to each local package
|
||||
split=$TEST
|
||||
TEST=""
|
||||
for a in $split; do TEST="$TEST ${REPO_PATH}/${a}"; done
|
||||
|
||||
# shellcheck disable=SC2206
|
||||
TEST=($TEST)
|
||||
if [ "${VERBOSE}" == "1" ]; then
|
||||
# shellcheck disable=SC2128
|
||||
echo "Running with TEST:" "${TEST[@]}"
|
||||
fi
|
||||
|
||||
# TODO: 'rafttest' is failing with unused
|
||||
STATIC_ANALYSIS_PATHS=$(find . -name \*.go ! -path './vendor/*' ! -path './gopath.proto/*' ! -path '*pb/*' | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS")
|
||||
# shellcheck disable=SC2206
|
||||
STATIC_ANALYSIS_PATHS=($STATIC_ANALYSIS_PATHS)
|
||||
if [ "${VERBOSE}" == "1" ]; then
|
||||
# shellcheck disable=SC2128
|
||||
echo "Running with STATIC_ANALYSIS_PATHS:" "${STATIC_ANALYSIS_PATHS[@]}"
|
||||
fi
|
||||
COVER="--cover=${COVER:-true}"
|
||||
|
||||
if [ -z "$GOARCH" ]; then
|
||||
GOARCH=$(go env GOARCH);
|
||||
fi
|
||||
|
||||
# determine the number of CPUs to use for Go tests
|
||||
TEST_CPUS="1,2,4"
|
||||
if [ -n "${CPU}" ]; then
|
||||
TEST_CPUS="${CPU}"
|
||||
fi
|
||||
echo "Running with TEST_CPUS:" "${TEST_CPUS}"
|
||||
CPU=${CPU:-"1,2,4"}
|
||||
echo "Running with CPU:" "${CPU}"
|
||||
|
||||
# determine whether target supports race detection
|
||||
if [ -z "${RACE}" ] ; then
|
||||
if [ "$GOARCH" == "amd64" ]; then
|
||||
RACE="--race"
|
||||
fi
|
||||
|
||||
RUN_ARG=""
|
||||
if [ -n "${TESTCASE}" ]; then
|
||||
RUN_ARG="-run=${TESTCASE}"
|
||||
fi
|
||||
|
||||
function unit_pass {
|
||||
echo "Running unit tests..."
|
||||
GO_TEST_FLAG=""
|
||||
if [ "${VERBOSE}" == "1" ]; then
|
||||
GO_TEST_FLAG="-v"
|
||||
fi
|
||||
if [ "${VERBOSE}" == "2" ]; then
|
||||
GO_TEST_FLAG="-v"
|
||||
export CLIENT_DEBUG=1
|
||||
fi
|
||||
|
||||
if [ "${RUN_ARG}" == "" ]; then
|
||||
RUN_ARG="-run=Test"
|
||||
fi
|
||||
|
||||
# check if user provided time out, especially useful when just run one test case
|
||||
# expectation could be different
|
||||
USERTIMEOUT=""
|
||||
if [ -z "${TIMEOUT}" ]; then
|
||||
USERTIMEOUT="3m"
|
||||
else
|
||||
USERTIMEOUT="${TIMEOUT}"
|
||||
RACE="--race=false"
|
||||
fi
|
||||
go test ${GO_TEST_FLAG} -timeout "${USERTIMEOUT}" "${COVER}" ${RACE} -cpu "${TEST_CPUS}" ${RUN_ARG} "$@" "${TEST[@]}"
|
||||
else
|
||||
RACE="--race=${RACE:-true}"
|
||||
fi
|
||||
|
||||
# This options make sense for cases where SUT (System Under Test) is compiled by test.
|
||||
COMMON_TEST_FLAGS=("-cpu=${CPU}" "${RACE}" "${COVER}")
|
||||
|
||||
RUN_ARG=()
|
||||
if [ -n "${TESTCASE}" ]; then
|
||||
RUN_ARG=("-run=${TESTCASE}")
|
||||
fi
|
||||
|
||||
function build_pass {
|
||||
log_callout "Building etcd"
|
||||
GO_BUILD_FLAGS="-v" etcd_build "${@}"
|
||||
GO_BUILD_FLAGS="-v" tools_build "${@}"
|
||||
}
|
||||
|
||||
function integration_pass {
|
||||
echo "Running integration tests..."
|
||||
################# REGULAR TESTS ################################################
|
||||
|
||||
# check if user provided time out, especially useful when just run one test case
|
||||
# expectation could be different
|
||||
USERTIMEOUT=""
|
||||
if [ -z "${TIMEOUT}" ]; then
|
||||
USERTIMEOUT="30m"
|
||||
else
|
||||
USERTIMEOUT="${TIMEOUT}"
|
||||
fi
|
||||
# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs]
|
||||
function run_unit_tests {
|
||||
local pkgs="${1:-./...}"
|
||||
shift 1
|
||||
# shellcheck disable=SC2086
|
||||
go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
|
||||
}
|
||||
|
||||
# if TESTCASE and PKG set, run specified test case in specified PKG
|
||||
# if TESTCASE set, PKG not set, run specified test case in all integration and integration_extra packages
|
||||
# if TESTCASE not set, PKG set, run all test cases in specified package
|
||||
# if TESTCASE not set, PKG not set, run all tests in all integration and integration_extra packages
|
||||
if [ -z "${TESTCASE}" ] && [ -z "${USERPKG}" ]; then
|
||||
go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration"
|
||||
integration_extra "$@"
|
||||
else
|
||||
if [ -z "${USERPKG}" ]; then
|
||||
INTEGTESTPKG=("${REPO_PATH}/integration"
|
||||
"${REPO_PATH}/client/integration"
|
||||
"${REPO_PATH}/clientv3/integration"
|
||||
"${REPO_PATH}/contrib/raftexample")
|
||||
else
|
||||
INTEGTESTPKG=("${TEST[@]}")
|
||||
fi
|
||||
go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}" "$@" "${INTEGTESTPKG[@]}"
|
||||
fi
|
||||
function unit_pass {
|
||||
run_for_modules run_unit_tests "$@"
|
||||
}
|
||||
|
||||
function integration_extra {
|
||||
go test -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/client/integration"
|
||||
go test -timeout 25m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration"
|
||||
go test -timeout 1m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/contrib/raftexample"
|
||||
go test -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/etcdserver/api/v2store"
|
||||
go test -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" -run=Example "$@" "${TEST[@]}"
|
||||
if [ -z "${PKG}" ] ; then
|
||||
if [[ -z "${RUN_ARG[*]}" ]]; then
|
||||
go_test "./client/... ./clientv3/..." "parallel" : -timeout="${TIMEOUT:-5m}" "${COMMON_TEST_FLAGS[@]}" --run=Example "$@" || return $?
|
||||
fi
|
||||
|
||||
go_test "./etcdserver/api/v2store/..." "parallel" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
|
||||
else
|
||||
log_warning "integration_extra ignored when PKG is specified"
|
||||
fi
|
||||
}
|
||||
|
||||
function integration_pass {
|
||||
local pkgs=${USERPKG:-"./integration/... ./clientv3/snapshot/... ./client/integration/... ./clientv3/integration/... ./contrib/raftexample"}
|
||||
go_test "${pkgs}" "keep_going" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
|
||||
integration_extra "$@"
|
||||
}
|
||||
|
||||
function e2e_pass {
|
||||
# e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
|
||||
run_for_module "." go_test "./tests/e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
|
||||
}
|
||||
|
||||
function integration_e2e_pass {
|
||||
run_pass "integration" "${@}"
|
||||
run_pass "e2e" "${@}"
|
||||
}
|
||||
|
||||
# generic_checker [cmd...]
|
||||
# executes given command in the current module, and clearly fails if it
|
||||
# failed or returned output.
|
||||
function generic_checker {
|
||||
local cmd=("$@")
|
||||
if ! output=$("${cmd[@]}"); then
|
||||
echo "${output}"
|
||||
log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)"
|
||||
return 255
|
||||
fi
|
||||
if [ -n "${output}" ]; then
|
||||
echo "${output}"
|
||||
log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)"
|
||||
return 255
|
||||
fi
|
||||
}
|
||||
|
||||
function functional_pass {
|
||||
./functional/build
|
||||
|
||||
# Clean up any data and logs from previous runs
|
||||
rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup
|
||||
|
||||
|
|
@ -218,7 +158,7 @@ function functional_pass {
|
|||
done
|
||||
|
||||
echo "functional test START!"
|
||||
./bin/etcd-tester --config ./functional.yaml && echo "'etcd-tester' succeeded"
|
||||
run ./bin/etcd-tester --config ./functional.yaml && log_success "'etcd-tester' succeeded"
|
||||
ETCD_TESTER_EXIT_CODE=$?
|
||||
echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
|
||||
|
||||
|
|
@ -227,368 +167,257 @@ function functional_pass {
|
|||
kill -s TERM "${agent_pids[@]}" || true
|
||||
|
||||
if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
|
||||
printf "\n"
|
||||
echo "FAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
|
||||
log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
|
||||
tail -1000 /tmp/etcd-functional-1/etcd.log
|
||||
|
||||
printf "\n"
|
||||
echo "FAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
|
||||
log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
|
||||
tail -1000 /tmp/etcd-functional-2/etcd.log
|
||||
|
||||
printf "\n"
|
||||
echo "FAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
|
||||
log_error -e "\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
|
||||
tail -1000 /tmp/etcd-functional-3/etcd.log
|
||||
|
||||
echo "--- FAIL: exit code" ${ETCD_TESTER_EXIT_CODE}
|
||||
exit ${ETCD_TESTER_EXIT_CODE}
|
||||
log_error "--- FAIL: exit code" ${ETCD_TESTER_EXIT_CODE}
|
||||
return ${ETCD_TESTER_EXIT_CODE}
|
||||
fi
|
||||
echo "functional test PASS!"
|
||||
log_success "functional test PASS!"
|
||||
}
|
||||
|
||||
function grpcproxy_pass {
|
||||
go_test "./integration ./clientv3/integration ./tests/e2e" "fail_fast" : \
|
||||
-timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@"
|
||||
}
|
||||
|
||||
################# COVERAGE #####################################################
|
||||
|
||||
# Builds artifacts used by tests/e2e in coverage mode.
|
||||
function build_cov_pass {
|
||||
local out="${BINDIR:-./bin}"
|
||||
run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcd_test"
|
||||
run go test -tags cov -c -covermode=set -coverpkg="./..." -o "${out}/etcdctl_test" "./etcdctl"
|
||||
}
|
||||
|
||||
# pkg_to_coverflag [prefix] [pkgs]
|
||||
# produces name of .coverprofile file to be used for tests of this package
|
||||
function pkg_to_coverprofileflag {
|
||||
local prefix="${1}"
|
||||
local pkgs="${2}"
|
||||
local pkgs_normalized
|
||||
pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
|
||||
echo -n "-coverprofile=${coverdir}/${prefix}_${pkgs_normalized}.coverprofile"
|
||||
}
|
||||
|
||||
function cov_pass {
|
||||
echo "Running code coverage..."
|
||||
# install gocovmerge before running code coverage from github.com/wadey/gocovmerge
|
||||
# gocovmerge merges coverage files
|
||||
if ! command -v gocovmerge >/dev/null; then
|
||||
echo "gocovmerge not installed"
|
||||
exit 255
|
||||
log_error "gocovmerge not installed (need: ./scripts/install_tool.sh github.com/gyuho/gocovmerge)"
|
||||
return 255
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2153
|
||||
if [ -z "$COVERDIR" ]; then
|
||||
echo "COVERDIR undeclared"
|
||||
exit 255
|
||||
log_error "COVERDIR undeclared"
|
||||
return 255
|
||||
fi
|
||||
|
||||
if [ ! -f "bin/etcd_test" ]; then
|
||||
echo "etcd_test binary not found"
|
||||
exit 255
|
||||
log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test"
|
||||
return 255
|
||||
fi
|
||||
|
||||
mkdir -p "$COVERDIR"
|
||||
local coverdir
|
||||
coverdir=$(readlink -f "${COVERDIR}")
|
||||
mkdir -p "${coverdir}"
|
||||
rm -f "${coverdir}/*.coverprofile" "${coverdir}/cover.*"
|
||||
|
||||
# run code coverage for unit and integration tests
|
||||
GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 30m"
|
||||
# shellcheck disable=SC2206
|
||||
GOCOVFLAGS=($GOCOVFLAGS)
|
||||
failed=""
|
||||
for t in $(echo "${TEST_PKGS}" | grep -vE "(tests/e2e|functional)"); do
|
||||
tf=$(echo "$t" | tr / _)
|
||||
# cache package compilation data for faster repeated builds
|
||||
go test "${GOCOVFLAGS[@]}" -i "${REPO_PATH}/$t" || true
|
||||
# uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
|
||||
go test "${GOCOVFLAGS[@]}" -run=Test -coverprofile "$COVERDIR/${tf}.coverprofile" "${REPO_PATH}/$t" || failed="$failed $t"
|
||||
done
|
||||
local covpkgs
|
||||
covpkgs=$(pkgs_in_module "./..." | filter_out_integration_style_tests)
|
||||
local coverpkg_comma
|
||||
coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',')
|
||||
local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma")
|
||||
|
||||
# v2v3 tests
|
||||
go test -tags v2v3 "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/store-v2v3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed store-v2v3"
|
||||
local failed=""
|
||||
|
||||
# proxy tests
|
||||
go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_integration.coverprofile" "${REPO_PATH}/integration" || failed="$failed proxy-integration"
|
||||
go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_clientv3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed proxy-clientv3/integration"
|
||||
log_callout "Collecting coverage from unit tests ..."
|
||||
go_test "./..." "keep_going" "pkg_to_coverprofileflag unit" -short -timeout=30m \
|
||||
"${gocov_build_flags[@]}" "$@" || failed="$failed unit"
|
||||
|
||||
# run code coverage for e2e tests
|
||||
# use 30m timeout because e2e coverage takes longer
|
||||
# due to many tests cause etcd process to wait
|
||||
# on leadership transfer timeout during gracefully shutdown
|
||||
echo Testing tests/e2e without proxy...
|
||||
go test -tags cov -timeout 30m -v "${REPO_PATH}/tests/e2e" || failed="$failed tests/e2e"
|
||||
echo Testing tests/e2e with proxy...
|
||||
go test -tags "cov cluster_proxy" -timeout 30m -v "${REPO_PATH}/tests/e2e" || failed="$failed tests/e2e-proxy"
|
||||
log_callout "Collecting coverage from integration tests ..."
|
||||
go_test "./integration ./clientv3/integration ./client/integration" "keep_going" "pkg_to_coverprofileflag integration" \
|
||||
-timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration"
|
||||
# integration-extra
|
||||
go_test "./client/... ./clientv3/..." "keep_going" "pkg_to_coverprofileflag integration_example" \
|
||||
-timeout=5m --run=Example "${gocov_build_flags[@]}" "$@" || failed="$failed integration-examples"
|
||||
# integration-store-v2
|
||||
go_test "./etcdserver/api/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \
|
||||
-tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed etcdserver_api_v2store"
|
||||
# integration_cluster_proxy
|
||||
go_test "./clientv3/... ./client/integration/... ./clientv3/snapshot/... ./integration/..." "keep_going" "pkg_to_coverprofileflag integration_cluster_proxy" \
|
||||
-tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy"
|
||||
|
||||
log_callout "Collecting coverage from e2e tests ..."
|
||||
# We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here,
|
||||
# as the coverage is colleced from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
|
||||
go_test "./tests/e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"
|
||||
|
||||
log_callout "Collecting coverage from e2e tests with proxy ..."
|
||||
go_test "./tests/e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"
|
||||
|
||||
log_callout "Merging coverage results ..."
|
||||
local cover_out_file="${coverdir}/cover.out"
|
||||
# gocovmerge requires not-empty test to start with:
|
||||
echo "mode: set" > "${cover_out_file}"
|
||||
|
||||
# incrementally merge to get coverage data even if some coverage files are corrupted
|
||||
# optimistically assume etcdserver package's coverage file is OK since gocovmerge
|
||||
# expects to start with a non-empty file
|
||||
cp "$COVERDIR"/etcdserver.coverprofile "$COVERDIR"/cover.out
|
||||
for f in "$COVERDIR"/*.coverprofile; do
|
||||
for f in "${coverdir}"/*.coverprofile; do
|
||||
echo "merging test coverage file ${f}"
|
||||
gocovmerge "$f" "$COVERDIR"/cover.out >"$COVERDIR"/cover.tmp || failed="$failed $f"
|
||||
if [ -s "$COVERDIR"/cover.tmp ]; then
|
||||
mv "$COVERDIR"/cover.tmp "$COVERDIR"/cover.out
|
||||
gocovmerge "${f}" "${cover_out_file}" > "${coverdir}/cover.tmp" || failed="$failed gocovmerge:$f"
|
||||
if [ -s "${coverdir}"/cover.tmp ]; then
|
||||
mv "${coverdir}/cover.tmp" "${cover_out_file}"
|
||||
fi
|
||||
done
|
||||
# strip out generated files (using GNU-style sed)
|
||||
sed --in-place '/generated.go/d' "$COVERDIR"/cover.out || true
|
||||
|
||||
echo -e "\nTo generate coverage report use:"
|
||||
echo -e " go tool cover --html=$COVERDIR/cover.out\n"
|
||||
sed --in-place '/generated.go/d' "${cover_out_file}" || true
|
||||
|
||||
# held failures to generate the full coverage file, now fail
|
||||
if [ -n "$failed" ]; then
|
||||
for f in $failed; do
|
||||
echo "--- FAIL:" "$f"
|
||||
log_error "--- FAIL:" "$f"
|
||||
done
|
||||
exit 255
|
||||
fi
|
||||
}
|
||||
|
||||
function e2e_pass {
|
||||
echo "Running e2e tests..."
|
||||
|
||||
# check if user provided time out, especially useful when just run one test case
|
||||
# expectation could be different
|
||||
USERTIMEOUT=""
|
||||
if [ -z "${TIMEOUT}" ]; then
|
||||
USERTIMEOUT="30m"
|
||||
else
|
||||
USERTIMEOUT="${TIMEOUT}"
|
||||
log_warning "Despite failures, you can see partial report:"
|
||||
log_warning " go tool cover -html ${cover_out_file}"
|
||||
return 255
|
||||
fi
|
||||
|
||||
go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}" "$@" "${REPO_PATH}/tests/e2e"
|
||||
log_success "done :) [see report: go tool cover -html ${cover_out_file}]"
|
||||
}
|
||||
|
||||
function integration_e2e_pass {
|
||||
echo "Running integration and e2e tests..."
|
||||
######### Code formatting checkers #############################################
|
||||
|
||||
go test -timeout 30m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/tests/e2e" &
|
||||
e2epid="$!"
|
||||
go test -timeout 30m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration" &
|
||||
intpid="$!"
|
||||
wait $e2epid
|
||||
wait $intpid
|
||||
integration_extra "$@"
|
||||
}
|
||||
function fmt_pass {
|
||||
toggle_failpoints disable
|
||||
|
||||
function grpcproxy_pass {
|
||||
go test -timeout 30m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration"
|
||||
go test -timeout 30m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration"
|
||||
go test -timeout 30m -v -tags cluster_proxy "$@" "${REPO_PATH}/tests/e2e"
|
||||
}
|
||||
|
||||
function release_pass {
|
||||
rm -f ./bin/etcd-last-release
|
||||
# to grab latest patch release; bump this up for every minor release
|
||||
UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
|
||||
if [ -n "$MANUAL_VER" ]; then
|
||||
# in case, we need to test against different version
|
||||
UPGRADE_VER=$MANUAL_VER
|
||||
fi
|
||||
if [[ -z ${UPGRADE_VER} ]]; then
|
||||
UPGRADE_VER="v3.3.0"
|
||||
echo "fallback to" ${UPGRADE_VER}
|
||||
fi
|
||||
|
||||
local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
|
||||
echo "Downloading $file"
|
||||
|
||||
set +e
|
||||
curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
|
||||
local result=$?
|
||||
set -e
|
||||
case $result in
|
||||
0) ;;
|
||||
*) echo "--- FAIL:" ${result}
|
||||
exit $result
|
||||
;;
|
||||
esac
|
||||
|
||||
tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
|
||||
mkdir -p ./bin
|
||||
mv /tmp/etcd ./bin/etcd-last-release
|
||||
# TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret"
|
||||
# after resolving ore-existing errors.
|
||||
for p in shellcheck \
|
||||
markdown_you \
|
||||
goword \
|
||||
gofmt \
|
||||
govet \
|
||||
revive \
|
||||
license_header \
|
||||
receiver_name \
|
||||
commit_title \
|
||||
mod_tidy \
|
||||
; do
|
||||
run_pass "${p}" "${@}"
|
||||
done
|
||||
}
|
||||
|
||||
function shellcheck_pass {
|
||||
if command -v shellcheck >/dev/null; then
|
||||
shellcheckResult=$(shellcheck -fgcc build test scripts/*.sh 2>&1 || true)
|
||||
if [ -n "${shellcheckResult}" ]; then
|
||||
echo -e "shellcheck checking failed:\\n${shellcheckResult}"
|
||||
exit 255
|
||||
fi
|
||||
else
|
||||
echo "Shellcheck not found !!! Check: https://github.com/koalaman/shellcheck#installing"
|
||||
if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then
|
||||
generic_checker run shellcheck -fgcc build test scripts/*.sh
|
||||
fi
|
||||
}
|
||||
|
||||
function markdown_you_find_eschew_you {
|
||||
find . -name \*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + || true
|
||||
}
|
||||
|
||||
function markdown_you_pass {
|
||||
# eschew you
|
||||
yous=$(find . -name \*.md ! -path './vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + || true)
|
||||
if [ -n "$yous" ]; then
|
||||
echo -e "found 'you' in documentation:\\n${yous}"
|
||||
exit 255
|
||||
fi
|
||||
generic_checker markdown_you_find_eschew_you
|
||||
}
|
||||
|
||||
function markdown_marker_pass {
|
||||
# TODO: check other markdown files when marker handles headers with '[]'
|
||||
if command -v marker >/dev/null; then
|
||||
markerResult=$(marker --skip-http --root ./Documentation 2>&1 || true)
|
||||
if [ -n "${markerResult}" ]; then
|
||||
echo -e "marker checking failed:\\n${markerResult}"
|
||||
exit 255
|
||||
fi
|
||||
else
|
||||
echo "Skipping marker..."
|
||||
fi
|
||||
}
|
||||
|
||||
function goword_pass {
|
||||
if command -v goword >/dev/null; then
|
||||
# get all go files to process
|
||||
gofiles=$(find "${FMT[@]}" -iname '*.go' 2>/dev/null)
|
||||
# shellcheck disable=SC2206
|
||||
gofiles_all=($gofiles)
|
||||
# ignore tests and protobuf files
|
||||
# shellcheck disable=SC1117
|
||||
gofiles=$(echo "${gofiles_all[@]}" | sort | uniq | sed "s/ /\n/g" | grep -vE "(\\_test.go|\\.pb\\.go)")
|
||||
# shellcheck disable=SC2206
|
||||
gofiles=($gofiles)
|
||||
# only check for broken exported godocs
|
||||
gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort)
|
||||
if [ -n "$gowordRes" ]; then
|
||||
echo -e "goword checking failed:\\n${gowordRes}"
|
||||
exit 255
|
||||
fi
|
||||
# check some spelling
|
||||
gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort)
|
||||
if [ -n "$gowordRes" ]; then
|
||||
echo -e "goword checking failed:\\n${gowordRes}"
|
||||
exit 255
|
||||
fi
|
||||
else
|
||||
echo "Skipping goword..."
|
||||
fi
|
||||
}
|
||||
|
||||
function gofmt_pass {
|
||||
fmtRes=$(gofmt -l -s -d "${FMT[@]}")
|
||||
if [ -n "${fmtRes}" ]; then
|
||||
echo -e "gofmt checking failed:\\n${fmtRes}"
|
||||
exit 255
|
||||
if tool_exists "marker" "https://crates.io/crates/marker"; then
|
||||
generic_checker run marker --skip-http --root ./Documentation 2>&1
|
||||
fi
|
||||
}
|
||||
|
||||
function govet_pass {
|
||||
vetRes=$(go vet "${TEST[@]}")
|
||||
if [ -n "${vetRes}" ]; then
|
||||
echo -e "govet checking failed:\\n${vetRes}"
|
||||
exit 255
|
||||
fi
|
||||
run_for_modules generic_checker run go vet
|
||||
}
|
||||
|
||||
function govet_shadow_pass {
|
||||
fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.")
|
||||
# shellcheck disable=SC2206
|
||||
fmtpkgs=($fmtpkgs)
|
||||
# Golang 1.12 onwards the experimental -shadow option is no longer available with go vet
|
||||
./scripts/install_tool.sh golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
|
||||
export PATH=${GOPATH}/bin:${PATH}
|
||||
shadow_tool=$(which shadow)
|
||||
vetRes=$(go vet -all -vettool="${shadow_tool}" "${TEST[@]}")
|
||||
if [ -n "${vetRes}" ]; then
|
||||
echo -e "govet -shadow checking failed:\\n${vetRes}"
|
||||
exit 255
|
||||
if tool_exists "shadow" "./scripts/install_tool.sh golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow"; then
|
||||
run_for_modules generic_checker run go vet -all -vettool="$(command -v shadow)"
|
||||
fi
|
||||
}
|
||||
|
||||
function unparam_pass {
|
||||
if command -v unparam >/dev/null; then
|
||||
unparamResult=$(unparam "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
|
||||
if [ -n "${unparamResult}" ]; then
|
||||
echo -e "unparam checking failed:\\n${unparamResult}"
|
||||
exit 255
|
||||
fi
|
||||
else
|
||||
echo "Skipping unparam..."
|
||||
if tool_exists "unparam" "./scripts/install_tool.sh mvdan.cc/unparam"; then
|
||||
run_for_modules generic_checker run unparam
|
||||
fi
|
||||
}
|
||||
|
||||
function staticcheck_pass {
|
||||
if command -v staticcheck >/dev/null; then
|
||||
staticcheckResult=$(staticcheck "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
|
||||
if [ -n "${staticcheckResult}" ]; then
|
||||
# TODO: resolve these after go1.8 migration
|
||||
# See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
|
||||
STATIC_CHECK_MASK="S(A|T)(1002|1005|1006|1008|1012|1019|1032|2002|4003|4006)"
|
||||
if echo "${staticcheckResult}" | grep -vE "$STATIC_CHECK_MASK"; then
|
||||
echo -e "staticcheck checking failed:\\n${staticcheckResult}"
|
||||
exit 255
|
||||
else
|
||||
suppressed=$(echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c)
|
||||
echo -e "staticcheck suppressed warnings:\\n${suppressed}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "Skipping staticcheck..."
|
||||
if tool_exists "staticcheck" "./scripts/install_tool.sh honnef.co/go/tools/cmd/staticcheck"; then
|
||||
run_for_modules generic_checker run staticcheck
|
||||
fi
|
||||
}
|
||||
|
||||
function revive_pass {
|
||||
if command -v revive >/dev/null; then
|
||||
reviveResult=$(revive -config ./tests/revive.toml -exclude "vendor/..." ./... 2>&1 || true)
|
||||
if [ -n "${reviveResult}" ]; then
|
||||
echo -e "revive checking failed:\\n${reviveResult}"
|
||||
exit 255
|
||||
fi
|
||||
else
|
||||
echo "Skipping revive..."
|
||||
if tool_exists "revive" "./scripts/install_tool.sh github.com/mgechev/revive"; then
|
||||
run_for_modules generic_checker run revive -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..."
|
||||
fi
|
||||
}
|
||||
|
||||
function unconvert_pass {
|
||||
if command -v unconvert >/dev/null; then
|
||||
unconvertResult=$(unconvert -v "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
|
||||
if [ -n "${unconvertResult}" ]; then
|
||||
echo -e "unconvert checking failed:\\n${unconvertResult}"
|
||||
exit 255
|
||||
fi
|
||||
else
|
||||
echo "Skipping unconvert..."
|
||||
if tool_exists "unconvert" "./scripts/install_tool.sh github.com/mdempsky/unconvert"; then
|
||||
run_for_modules generic_checker run unconvert -v
|
||||
fi
|
||||
}
|
||||
|
||||
function ineffassign_per_package {
|
||||
mapfile -t gofiles < <(go_srcs_in_module "$1")
|
||||
run ineffassign "${gofiles[@]}"
|
||||
}
|
||||
|
||||
function ineffassign_pass {
|
||||
if command -v ineffassign >/dev/null; then
|
||||
ineffassignResult=$(ineffassign "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
|
||||
if [ -n "${ineffassignResult}" ]; then
|
||||
echo -e "ineffassign checking failed:\\n${ineffassignResult}"
|
||||
exit 255
|
||||
fi
|
||||
else
|
||||
echo "Skipping ineffassign..."
|
||||
if tool_exists "ineffassign" "./scripts/install_tool.sh github.com/gordonklaus/ineffassign"; then
|
||||
run_for_modules generic_checker ineffassign_per_package
|
||||
fi
|
||||
}
|
||||
|
||||
function nakedret_pass {
|
||||
if command -v nakedret >/dev/null; then
|
||||
nakedretResult=$(nakedret "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
|
||||
if [ -n "${nakedretResult}" ]; then
|
||||
echo -e "nakedret checking failed:\\n${nakedretResult}"
|
||||
exit 255
|
||||
fi
|
||||
else
|
||||
echo "Skipping nakedret..."
|
||||
if tool_exists "nakedret" "./scripts/install_tool.sh github.com/alexkohler/nakedret"; then
|
||||
run_for_modules generic_checker run nakedret
|
||||
fi
|
||||
}
|
||||
|
||||
function license_header_pass {
|
||||
licRes=""
|
||||
files=$(find . -type f -iname '*.go' ! -path './vendor/*' ! -path './gopath.proto/*')
|
||||
for file in $files; do
|
||||
mapfile -t gofiles < <(go_srcs_in_module "$1")
|
||||
for file in "${gofiles[@]}"; do
|
||||
if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
|
||||
licRes="${licRes}"$(echo -e " ${file}")
|
||||
fi
|
||||
done
|
||||
if [ -n "${licRes}" ]; then
|
||||
echo -e "license header checking failed:\\n${licRes}"
|
||||
exit 255
|
||||
log_error -e "license header checking failed:\\n${licRes}"
|
||||
return 255
|
||||
fi
|
||||
}
|
||||
|
||||
function receiver_name_pass {
|
||||
# shellcheck disable=SC1117
|
||||
recvs=$(grep 'func ([^*]' {*,*/*,*/*/*}.go | grep -Ev "(generated|pb/)" | tr ':' ' ' | \
|
||||
function receiver_name_for_package {
|
||||
mapfile -t gofiles < <(go_srcs_in_module "$1")
|
||||
recvs=$(grep 'func ([^*]' "${gofiles[@]}" | tr ':' ' ' | \
|
||||
awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" | sort | uniq | \
|
||||
grep -Ev "(Descriptor|Proto|_)" | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
|
||||
if [ -n "${recvs}" ]; then
|
||||
# shellcheck disable=SC2206
|
||||
recvs=($recvs)
|
||||
for recv in "${recvs[@]}"; do
|
||||
echo "Mismatched receiver for $recv..."
|
||||
grep "$recv" "${FMT[@]}" | grep 'func ('
|
||||
log_error "Mismatched receiver for $recv..."
|
||||
grep "$recv" "${gofiles[@]}" | grep 'func ('
|
||||
done
|
||||
exit 255
|
||||
return 255
|
||||
fi
|
||||
}
|
||||
|
||||
function receiver_name_pass {
|
||||
run_for_modules receiver_name_for_package
|
||||
}
|
||||
|
||||
function commit_title_pass {
|
||||
git log --oneline "$(git merge-base HEAD master)"...HEAD | while read -r l; do
|
||||
commitMsg=$(echo "$l" | cut -f2- -d' ')
|
||||
|
|
@ -605,86 +434,111 @@ function commit_title_pass {
|
|||
spaceCommas=$(echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0)
|
||||
commaSpaces=$(echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0)
|
||||
if [[ $(echo "$commitMsg" | grep -c ":..*") == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
|
||||
echo "$l"...
|
||||
echo "Expected commit title format '<package>{\", \"<package>}: <description>'"
|
||||
echo "Got: $l"
|
||||
exit 255
|
||||
log_error "$l"...
|
||||
log_error "Expected commit title format '<package>{\", \"<package>}: <description>'"
|
||||
log_error "Got: $l"
|
||||
return 255
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# tools gosimple,unused,staticheck,unconvert,ineffasign,nakedret
|
||||
# are not module-aware. See https://github.com/golang/go/issues/24661
|
||||
# The module-aware versions need to be used when they become available
|
||||
function fmt_pass {
|
||||
toggle_failpoints disable
|
||||
# goword_for_package package
|
||||
# checks spelling and comments in the 'package' in the current module
|
||||
function goword_for_package {
|
||||
mapfile -t gofiles < <(go_srcs_in_module "$1")
|
||||
# only check for broke exported godocs
|
||||
gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort)
|
||||
if [ -n "$gowordRes" ]; then
|
||||
log_error -e "goword checking failed:\\n${gowordRes}"
|
||||
return 255
|
||||
fi
|
||||
# check some spelling
|
||||
gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort)
|
||||
if [ -n "$gowordRes" ]; then
|
||||
log_error -e "goword checking failed:\\n${gowordRes}"
|
||||
return 255
|
||||
fi
|
||||
}
|
||||
|
||||
# TODO: add "unparam"
|
||||
for p in shellcheck \
|
||||
markdown_you \
|
||||
goword \
|
||||
gofmt \
|
||||
govet \
|
||||
revive \
|
||||
license_header \
|
||||
receiver_name \
|
||||
commit_title \
|
||||
mod_tidy \
|
||||
; do
|
||||
echo "'$p' started at $(date)"
|
||||
"${p}"_pass "$@"
|
||||
echo "'$p' completed at $(date)"
|
||||
done
|
||||
function goword_pass {
|
||||
if tool_exists "goword" "./scripts_install_tool.sh github.com/chzchzchz/goword"; then
|
||||
run_for_modules goword_for_package
|
||||
fi
|
||||
}
|
||||
|
||||
function go_fmt_for_package {
|
||||
# We utilize 'go fmt' to find all files suitable for formatting,
|
||||
# but reuse full power gofmt to perform just RO check.
|
||||
go fmt -n "$1" | sed 's| -w | -d |g' | sh
|
||||
}
|
||||
|
||||
function gofmt_pass {
|
||||
run_for_modules generic_checker go_fmt_for_package
|
||||
}
|
||||
|
||||
function bom_pass {
|
||||
if ! command -v license-bill-of-materials >/dev/null; then
|
||||
log_warning "./license-bill-of-materials not FOUND"
|
||||
log_warning "USE: ./scripts/install_tool.sh github.com/coreos/license-bill-of-materials"
|
||||
return
|
||||
fi
|
||||
if [ "${GO111MODULE}" == "on" ]; then
|
||||
# license-bill-off-materials calls "go list std cmd" which cannot handle modules
|
||||
# Please see https://github.com/golang/go/issues/26924
|
||||
echo "Skipping license-bill-of-materials with go modules..."
|
||||
return
|
||||
fi
|
||||
echo "Checking bill of materials..."
|
||||
log_callout "Checking bill of materials..."
|
||||
# https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
|
||||
license-bill-of-materials \
|
||||
run license-bill-of-materials \
|
||||
--override-file bill-of-materials.override.json \
|
||||
go.etcd.io/etcd/v3 go.etcd.io/etcd/v3/etcdctl >bom-now.json || true
|
||||
if ! diff bill-of-materials.json bom-now.json; then
|
||||
echo "vendored licenses do not match given bill of materials"
|
||||
exit 255
|
||||
log_error "modularized licenses do not match given bill of materials"
|
||||
return 255
|
||||
fi
|
||||
rm bom-now.json
|
||||
}
|
||||
|
||||
######## VARIOUS CHECKERS ######################################################
|
||||
|
||||
function dep_pass {
|
||||
echo "Checking package dependencies..."
|
||||
log_callout "Checking package dependencies..."
|
||||
# don't pull in etcdserver package
|
||||
pushd clientv3 >/dev/null
|
||||
badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)"
|
||||
deps=$(go list -f '{{ .Deps }}' | sed 's/ /\n/g' | grep -E "${badpkg}" || echo "")
|
||||
popd >/dev/null
|
||||
if [ -n "$deps" ]; then
|
||||
echo -e "clientv3 has masked dependencies:\\n${deps}"
|
||||
exit 255
|
||||
log_error -e "clientv3 has masked dependencies:\\n${deps}"
|
||||
return 255
|
||||
fi
|
||||
}
|
||||
|
||||
function build_cov_pass {
|
||||
out="bin"
|
||||
if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
|
||||
go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcd_test"
|
||||
function release_pass {
|
||||
rm -f ./bin/etcd-last-release
|
||||
# to grab latest patch release; bump this up for every minor release
|
||||
UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
|
||||
if [ -n "$MANUAL_VER" ]; then
|
||||
# in case, we need to test against different version
|
||||
UPGRADE_VER=$MANUAL_VER
|
||||
fi
|
||||
if [[ -z ${UPGRADE_VER} ]]; then
|
||||
UPGRADE_VER="v3.3.0"
|
||||
log_warning "fallback to" ${UPGRADE_VER}
|
||||
fi
|
||||
|
||||
go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcdctl_test" "${REPO_PATH}/etcdctl"
|
||||
}
|
||||
local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
|
||||
log_callout "Downloading $file"
|
||||
|
||||
# fail fast on static tests
|
||||
function build_pass {
|
||||
echo "Checking build..."
|
||||
GO_BUILD_FLAGS="-v" etcd_build
|
||||
GO_BUILD_FLAGS="-v" tools_build
|
||||
set +e
|
||||
curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
|
||||
local result=$?
|
||||
set -e
|
||||
case $result in
|
||||
0) ;;
|
||||
*) log_error "--- FAIL:" ${result}
|
||||
return $result
|
||||
;;
|
||||
esac
|
||||
|
||||
tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
|
||||
mkdir -p ./bin
|
||||
mv /tmp/etcd ./bin/etcd-last-release
|
||||
}
|
||||
|
||||
function mod_tidy_pass {
|
||||
|
|
@ -710,20 +564,31 @@ function mod_tidy_pass {
|
|||
mv "${tmpModDir}/go.sum" "./go.sum"
|
||||
|
||||
if [ "${tmpFileGoModInSync}" -ne 0 ]; then
|
||||
echo "./go.mod is not in sync with 'go mod tidy'"
|
||||
exit 255
|
||||
log_error "./go.mod is not in sync with 'go mod tidy'"
|
||||
return 255
|
||||
fi
|
||||
if [ "${tmpFileGoSumInSync}" -ne 0 ]; then
|
||||
echo "./go.sum is not in sync with 'go mod tidy'"
|
||||
log_error "./go.sum is not in sync with 'rm go.sum; go mod tidy'"
|
||||
return 255
|
||||
fi
|
||||
}
|
||||
|
||||
########### MAIN ###############################################################
|
||||
|
||||
function run_pass {
|
||||
local pass="${1}"
|
||||
shift 1
|
||||
log_callout -e "\n'${pass}' started at $(date)"
|
||||
if "${pass}_pass" "$@" ; then
|
||||
log_success "'${pass}' completed at $(date)"
|
||||
else
|
||||
log_error "FAIL: '${pass}' failed at $(date)"
|
||||
exit 255
|
||||
fi
|
||||
}
|
||||
|
||||
for pass in $PASSES; do
|
||||
echo "Starting '$pass' pass at $(date)"
|
||||
"${pass}"_pass "$@"
|
||||
echo "Finished '$pass' pass at $(date)"
|
||||
run_pass "${pass}" "${@}"
|
||||
done
|
||||
|
||||
echo "Success"
|
||||
|
||||
log_success "SUCCESS"
|
||||
|
|
|
|||
Loading…
Reference in New Issue