2020-09-20 13:25:49 +03:00
#!/usr/bin/env bash
2020-11-27 13:11:09 +03:00
ROOT_MODULE = "go.etcd.io/etcd"
2020-11-04 16:59:00 +03:00
2020-11-27 13:11:09 +03:00
if [ [ " $( go list) " != " ${ ROOT_MODULE } /v3 " ] ] ; then
echo " must be run from ' ${ ROOT_MODULE } /v3' module directory "
2020-10-26 12:42:25 +03:00
exit 255
2020-10-13 20:30:22 +03:00
fi
2021-01-21 09:35:26 +03:00
function set_root_dir {
ETCD_ROOT_DIR = $( go list -f '{{.Dir}}' " ${ ROOT_MODULE } /v3 " )
}
set_root_dir
2020-09-20 13:25:49 +03:00
#### Convenient IO methods #####
COLOR_RED = '\033[0;31m'
COLOR_ORANGE = '\033[0;33m'
COLOR_GREEN = '\033[0;32m'
COLOR_LIGHTCYAN = '\033[0;36m'
2020-10-13 20:30:22 +03:00
COLOR_BLUE = '\033[0;94m'
COLOR_MAGENTA = '\033[95m'
COLOR_BOLD = '\033[1m'
2020-09-20 13:25:49 +03:00
COLOR_NONE = '\033[0m' # No Color
2021-01-21 09:35:26 +03:00
2020-09-20 13:25:49 +03:00
function log_error {
2020-10-13 20:30:22 +03:00
>& 2 echo -n -e " ${ COLOR_BOLD } ${ COLOR_RED } "
2020-09-20 13:25:49 +03:00
>& 2 echo " $@ "
>& 2 echo -n -e " ${ COLOR_NONE } "
}
function log_warning {
>& 2 echo -n -e " ${ COLOR_ORANGE } "
>& 2 echo " $@ "
>& 2 echo -n -e " ${ COLOR_NONE } "
}
function log_callout {
>& 2 echo -n -e " ${ COLOR_LIGHTCYAN } "
>& 2 echo " $@ "
>& 2 echo -n -e " ${ COLOR_NONE } "
}
2020-10-13 20:30:22 +03:00
function log_cmd {
>& 2 echo -n -e " ${ COLOR_BLUE } "
>& 2 echo " $@ "
>& 2 echo -n -e " ${ COLOR_NONE } "
}
2020-09-20 13:25:49 +03:00
function log_success {
>& 2 echo -n -e " ${ COLOR_GREEN } "
>& 2 echo " $@ "
>& 2 echo -n -e " ${ COLOR_NONE } "
}
2020-11-04 16:59:00 +03:00
function log_info {
>& 2 echo -n -e " ${ COLOR_NONE } "
>& 2 echo " $@ "
>& 2 echo -n -e " ${ COLOR_NONE } "
}
2020-10-26 18:43:15 +03:00
# From http://stackoverflow.com/a/12498485
function relativePath {
# both $1 and $2 are absolute paths beginning with /
# returns relative path to $2 from $1
local source = $1
local target = $2
local commonPart = $source
local result = ""
2022-01-26 14:11:10 +03:00
while [ [ " ${ target # " $commonPart " } " = = " ${ target } " ] ] ; do
2020-10-26 18:43:15 +03:00
# no match, means that candidate common part is not correct
# go up one level (reduce common part)
commonPart = " $( dirname " $commonPart " ) "
# and record that we went back, with correct / handling
if [ [ -z $result ] ] ; then
result = ".."
else
result = " ../ $result "
fi
done
if [ [ $commonPart = = "/" ] ] ; then
# special case for root (no common path)
result = " $result / "
fi
# since we now have identified the common part,
# compute the non-common part
2022-01-26 14:11:10 +03:00
local forwardPart = " ${ target # " $commonPart " } "
2020-10-26 18:43:15 +03:00
# and now stick all parts together
if [ [ -n $result ] ] && [ [ -n $forwardPart ] ] ; then
result = " $result $forwardPart "
elif [ [ -n $forwardPart ] ] ; then
# extra slash removal
result = " ${ forwardPart : 1 } "
fi
echo " $result "
}
2020-09-20 13:25:49 +03:00
#### Discovery of files/packages within a go module #####
# go_srcs_in_module [package]
# returns list of all not-generated go sources in the current (dir) module.
function go_srcs_in_module {
2022-11-23 21:17:11 +03:00
go list -f "{{with \$c:=.}}{{range \$f:=\$c.GoFiles }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{range \$f:=\$c.TestGoFiles }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{range \$f:=\$c.XTestGoFiles }}{{\$c.Dir}}/{{\$f}}{{\"\n\"}}{{end}}{{end}}" ./... | grep -vE "(\\.pb\\.go|\\.pb\\.gw.go)"
2020-09-20 13:25:49 +03:00
}
# pkgs_in_module [optional:package_pattern]
# returns list of all packages in the current (dir) module.
# if the package_pattern is given, its being resolved.
function pkgs_in_module {
go list -mod= mod " ${ 1 :- ./... } " ;
}
2020-11-04 16:59:00 +03:00
# Prints subdirectory (from the repo root) for the current module.
function module_subdir {
relativePath " ${ ETCD_ROOT_DIR } " " ${ PWD } "
}
2020-09-20 13:25:49 +03:00
#### Running actions against multiple modules ####
# run [command...] - runs given command, printing it first and
# again if it failed (in RED). Use to wrap important test commands
# that user might want to re-execute to shorten the feedback loop when fixing
# the test.
function run {
local rpath
2020-10-10 03:32:15 +03:00
local command
2020-11-04 16:59:00 +03:00
rpath = $( module_subdir)
2020-10-10 03:32:15 +03:00
# Quoting all components as the commands are fully copy-parsable:
command = ( " ${ @ } " )
command = ( " ${ command [@]@Q } " )
2020-11-04 16:59:00 +03:00
if [ [ " ${ rpath } " != "." && " ${ rpath } " != "" ] ] ; then
2020-10-10 03:32:15 +03:00
repro = " (cd ${ rpath } && ${ command [*] } ) "
else
repro = " ${ command [*] } "
2020-09-20 13:25:49 +03:00
fi
2020-10-13 20:30:22 +03:00
log_cmd " % ${ repro } "
2021-01-19 22:00:40 +03:00
" ${ @ } " 2> >( while read -r line; do echo -e " ${ COLOR_NONE } stderr: ${ COLOR_MAGENTA } ${ line } ${ COLOR_NONE } " >& 2; done )
2020-09-20 13:25:49 +03:00
local error_code = $?
2020-10-01 15:34:58 +03:00
if [ ${ error_code } -ne 0 ] ; then
2021-01-30 01:31:44 +03:00
log_error -e " FAIL: (code: ${ error_code } ):\\n % ${ repro } "
2020-09-20 13:25:49 +03:00
return ${ error_code }
fi
}
# run_for_module [module] [cmd]
# executes given command in the given module for given pkgs.
# module_name - "." (in future: tests, client, server)
# cmd - cmd to be executed - that takes package as last argument
function run_for_module {
local module = ${ 1 :- "." }
shift 1
(
2020-10-13 20:30:22 +03:00
cd " ${ ETCD_ROOT_DIR } / ${ module } " && " $@ "
2020-09-20 13:25:49 +03:00
)
}
2021-01-19 17:28:21 +03:00
function module_dirs( ) {
2022-12-02 08:56:01 +03:00
echo "api pkg client/pkg client/v2 client/v3 server etcdutl etcdctl tests ."
2021-01-19 17:28:21 +03:00
}
2021-01-21 09:35:26 +03:00
# maybe_run [cmd...] runs given command depending on the DRY_RUN flag.
function maybe_run( ) {
if ${ DRY_RUN } ; then
2021-01-30 01:31:44 +03:00
log_warning -e " # DRY_RUN:\\n % ${ * } "
2021-01-21 09:35:26 +03:00
else
run " ${ @ } "
fi
}
2020-10-07 16:29:50 +03:00
function modules( ) {
2020-10-21 00:07:21 +03:00
modules = (
2020-11-27 13:11:09 +03:00
" ${ ROOT_MODULE } /api/v3 "
" ${ ROOT_MODULE } /pkg/v3 "
2021-04-05 23:31:07 +03:00
" ${ ROOT_MODULE } /client/pkg/v3 "
2020-11-27 13:11:09 +03:00
" ${ ROOT_MODULE } /client/v2 "
" ${ ROOT_MODULE } /client/v3 "
" ${ ROOT_MODULE } /server/v3 "
2021-05-14 14:50:31 +03:00
" ${ ROOT_MODULE } /etcdutl/v3 "
2020-11-27 13:11:09 +03:00
" ${ ROOT_MODULE } /etcdctl/v3 "
" ${ ROOT_MODULE } /tests/v3 "
" ${ ROOT_MODULE } /v3 " )
2020-10-21 00:07:21 +03:00
echo " ${ modules [@] } "
2020-10-07 16:29:50 +03:00
}
function modules_exp( ) {
for m in $( modules) ; do
echo -n " ${ m } /... "
done
}
2020-09-20 13:25:49 +03:00
# run_for_modules [cmd]
# run given command across all modules and packages
# (unless the set is limited using ${PKG} or / ${USERMOD})
function run_for_modules {
local pkg = " ${ PKG :- ./... } "
2021-02-24 19:56:45 +03:00
if [ -z " ${ USERMOD :- } " ] ; then
2021-01-19 17:28:21 +03:00
for m in $( module_dirs) ; do
run_for_module " ${ m } " " $@ " " ${ pkg } " || return " $? "
done
2020-09-20 13:25:49 +03:00
else
run_for_module " ${ USERMOD } " " $@ " " ${ pkg } " || return " $? "
fi
}
2021-06-15 22:18:48 +03:00
junitFilenamePrefix( ) {
if [ [ -z " ${ JUNIT_REPORT_DIR } " ] ] ; then
echo ""
return
fi
mkdir -p " ${ JUNIT_REPORT_DIR } "
DATE = $( date +%s | base64 | head -c 15 )
echo " ${ JUNIT_REPORT_DIR } /junit_ $DATE "
}
function produce_junit_xmlreport {
local -r junit_filename_prefix = $1
if [ [ -z " ${ junit_filename_prefix } " ] ] ; then
return
fi
local junit_xml_filename
junit_xml_filename = " ${ junit_filename_prefix } .xml "
2021-06-28 17:22:59 +03:00
# Ensure that gotestsum is run without cross-compiling
run_go_tool gotest.tools/gotestsum --junitfile " ${ junit_xml_filename } " --raw-command cat " ${ junit_filename_prefix } " *.stdout || exit 1
2021-06-15 22:18:48 +03:00
if [ " ${ VERBOSE } " != "1" ] ; then
rm " ${ junit_filename_prefix } " *.stdout
fi
log_callout " Saved JUnit XML test report to ${ junit_xml_filename } "
}
2020-09-20 13:25:49 +03:00
#### Running go test ########
# go_test [packages] [mode] [flags_for_package_func] [$@]
# [mode] supports 3 states:
# - "parallel": fastest as concurrently processes multiple packages, but silent
# till the last package. See: https://github.com/golang/go/issues/2731
# - "keep_going" : executes tests package by package, but postpones reporting error to the last
# - "fail_fast" : executes tests packages 1 by 1, exits on the first failure.
#
# [flags_for_package_func] is a name of function that takes list of packages as parameter
# and computes additional flags to the go_test commands.
# Use 'true' or ':' if you dont need additional arguments.
#
# depends on the VERBOSE top-level variable.
#
# Example:
# go_test "./..." "keep_going" ":" --short
#
# The function returns != 0 code in case of test failure.
function go_test {
local packages = " ${ 1 } "
local mode = " ${ 2 } "
local flags_for_package_func = " ${ 3 } "
2021-06-15 22:18:48 +03:00
local junit_filename_prefix
2020-09-20 13:25:49 +03:00
shift 3
local goTestFlags = ""
local goTestEnv = ""
2021-06-15 22:18:48 +03:00
##### Create a junit-style XML test report in this directory if set. #####
JUNIT_REPORT_DIR = ${ JUNIT_REPORT_DIR :- }
# If JUNIT_REPORT_DIR is unset, and ARTIFACTS is set, then have them match.
if [ [ -z " ${ JUNIT_REPORT_DIR :- } " && -n " ${ ARTIFACTS :- } " ] ] ; then
export JUNIT_REPORT_DIR = " ${ ARTIFACTS } "
fi
# Used to filter verbose test output.
go_test_grep_pattern = ".*"
if [ [ -n " ${ JUNIT_REPORT_DIR } " ] ] ; then
goTestFlags += "-v "
goTestFlags += "-json "
# Show only summary lines by matching lines like "status package/test"
go_test_grep_pattern = "^[^[:space:]]\+[[:space:]]\+[^[:space:]]\+/[^[[:space:]]\+"
fi
junit_filename_prefix = $( junitFilenamePrefix)
2020-09-20 13:25:49 +03:00
if [ " ${ VERBOSE } " = = "1" ] ; then
goTestFlags = "-v"
fi
# Expanding patterns (like ./...) into list of packages
local unpacked_packages = ( " ${ packages } " )
if [ " ${ mode } " != "parallel" ] ; then
# shellcheck disable=SC2207
# shellcheck disable=SC2086
if ! unpacked_packages = ( $( go list ${ packages } ) ) ; then
log_error " Cannot resolve packages: ${ packages } "
return 255
fi
fi
local failures = ""
# execution of tests against packages:
for pkg in " ${ unpacked_packages [@] } " ; do
local additional_flags
# shellcheck disable=SC2086
additional_flags = $( ${ flags_for_package_func } ${ pkg } )
# shellcheck disable=SC2206
local cmd = ( go test ${ goTestFlags } ${ additional_flags } " $@ " ${ pkg } )
2021-03-15 23:23:38 +03:00
# shellcheck disable=SC2086
2022-04-18 00:18:25 +03:00
if ! run env ${ goTestEnv } ETCD_VERIFY = " ${ ETCD_VERIFY } " " ${ cmd [@] } " | tee ${ junit_filename_prefix : + " ${ junit_filename_prefix } .stdout " } | grep --binary-files= text " ${ go_test_grep_pattern } " ; then
2020-09-20 13:25:49 +03:00
if [ " ${ mode } " != "keep_going" ] ; then
2021-06-15 22:18:48 +03:00
produce_junit_xmlreport " ${ junit_filename_prefix } "
2020-09-20 13:25:49 +03:00
return 2
else
failures = ( " ${ failures [@] } " " ${ pkg } " )
fi
fi
2021-06-15 22:18:48 +03:00
produce_junit_xmlreport " ${ junit_filename_prefix } "
2020-09-20 13:25:49 +03:00
done
if [ -n " ${ failures [*] } " ] ; then
2021-01-30 01:31:44 +03:00
log_error -e " ERROR: Tests for following packages failed:\\n ${ failures [*] } "
2020-09-20 13:25:49 +03:00
return 2
fi
}
#### Other ####
# tool_exists [tool] [instruction]
# Checks whether given [tool] is installed. In case of failure,
# prints a warning with installation [instruction] and returns !=0 code.
2020-10-08 20:48:35 +03:00
#
# WARNING: This depend on "any" version of the 'binary' that might be tricky
# from hermetic build perspective. For go binaries prefer 'tool_go_run'
2020-09-20 13:25:49 +03:00
function tool_exists {
local tool = " ${ 1 } "
local instruction = " ${ 2 } "
if ! command -v " ${ tool } " >/dev/null; then
log_warning " Tool: ' ${ tool } ' not found on PATH. ${ instruction } "
return 255
fi
}
2020-10-08 20:48:35 +03:00
# tool_get_bin [tool] - returns absolute path to a tool binary (or returns error)
function tool_get_bin {
local tool = " $1 "
2022-02-06 09:23:28 +03:00
local pkg_part = " $1 "
2020-10-08 20:48:35 +03:00
if [ [ " $tool " = = *"@" * ] ] ; then
2022-02-06 09:23:28 +03:00
pkg_part = $( echo " ${ tool } " | cut -d'@' -f1)
2020-10-13 20:30:22 +03:00
# shellcheck disable=SC2086
2022-02-06 09:23:28 +03:00
run go install ${ GOBINARGS :- } " ${ tool } " || return 2
2020-10-08 20:48:35 +03:00
else
2020-10-13 20:30:22 +03:00
# shellcheck disable=SC2086
2022-02-06 09:23:28 +03:00
run_for_module ./tools/mod run go install ${ GOBINARGS :- } " ${ tool } " || return 2
2020-10-08 20:48:35 +03:00
fi
2022-02-06 09:23:28 +03:00
# remove the version suffix, such as removing "/v3" from "go.etcd.io/etcd/v3".
local cmd_base_name
cmd_base_name = $( basename " ${ pkg_part } " )
if [ [ ${ cmd_base_name } = ~ ^v[ 0-9] *$ ] ] ; then
pkg_part = $( dirname " ${ pkg_part } " )
fi
2022-04-18 00:18:25 +03:00
run_for_module ./tools/mod go list -f '{{.Target}}' " ${ pkg_part } "
2020-10-08 20:48:35 +03:00
}
# tool_pkg_dir [pkg] - returns absolute path to a directory that stores given pkg.
# The pkg versions must be defined in ./tools/mod directory.
function tool_pkg_dir {
run_for_module ./tools/mod run go list -f '{{.Dir}}' " ${ 1 } "
}
# tool_get_bin [tool]
function run_go_tool {
local cmdbin
2021-06-28 17:22:59 +03:00
if ! cmdbin = $( GOARCH = "" tool_get_bin " ${ 1 } " ) ; then
2022-04-18 00:18:25 +03:00
log_warning " Failed to install tool ' ${ 1 } ' "
2020-10-08 20:48:35 +03:00
return 2
fi
shift 1
2021-06-28 17:22:59 +03:00
GOARCH = "" run " ${ cmdbin } " " $@ " || return 2
2020-10-08 20:48:35 +03:00
}
2020-11-04 16:59:00 +03:00
# assert_no_git_modifications fails if there are any uncommited changes.
function assert_no_git_modifications {
log_callout "Making sure everything is committed."
if ! git diff --cached --exit-code; then
log_error "Found staged by uncommited changes. Do commit/stash your changes first."
return 2
fi
if ! git diff --exit-code; then
log_error "Found unstaged and uncommited changes. Do commit/stash your changes first."
return 2
fi
}
2020-11-27 13:11:09 +03:00
# makes sure that the current branch is in sync with the origin branch:
# - no uncommitted nor unstaged changes
# - no differencing commits in relation to the origin/$branch
function git_assert_branch_in_sync {
local branch
2021-02-24 19:56:45 +03:00
# TODO: When git 2.22 popular, change to:
# branch=$(git branch --show-current)
2022-11-26 14:17:52 +03:00
branch = $( run git rev-parse --abbrev-ref HEAD)
log_callout " Verify the current branch ' ${ branch } ' is clean "
2020-11-27 13:11:09 +03:00
if [ [ $( run git status --porcelain --untracked-files= no) ] ] ; then
log_error " The workspace in ' $( pwd ) ' for branch: ${ branch } has uncommitted changes "
2021-01-21 09:35:26 +03:00
log_error " Consider cleaning up / renaming this directory or (cd $( pwd ) && git reset --hard) "
2020-11-27 13:11:09 +03:00
return 2
fi
2022-11-26 14:17:52 +03:00
log_callout " Verify the current branch ' ${ branch } ' is in sync with the 'origin/ ${ branch } ' "
2021-01-21 09:35:26 +03:00
if [ -n " ${ branch } " ] ; then
ref_local = $( run git rev-parse " ${ branch } " )
ref_origin = $( run git rev-parse " origin/ ${ branch } " )
if [ " x ${ ref_local } " != " x ${ ref_origin } " ] ; then
log_error " In workspace ' $( pwd ) ' the branch: ${ branch } diverges from the origin. "
log_error " Consider cleaning up / renaming this directory or (cd $( pwd ) && git reset --hard origin/ ${ branch } ) "
return 2
fi
else
log_warning "Cannot verify consistency with the origin, as git is on detached branch."
2020-11-27 13:11:09 +03:00
fi
}