From dcc2b9cfdad583538ebd2bfe16db7833c6b453b7 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 22 Mar 2018 02:05:48 -0700 Subject: [PATCH 1/6] hack/scripts/Makefile: fix docker commands Signed-off-by: Gyuho Lee --- hack/scripts-dev/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/scripts-dev/Makefile b/hack/scripts-dev/Makefile index 1fb652262..0e1207051 100644 --- a/hack/scripts-dev/Makefile +++ b/hack/scripts-dev/Makefile @@ -36,8 +36,8 @@ restart-docker: service docker restart delete-docker-images: - docker rm --force $(docker ps -a -q) || true - docker rmi --force $(docker images -q) || true + docker rm --force `docker ps -a -q` || true + docker rmi --force `docker images -q` || true From a2a22a650605b8eda2e0b8b92c2e95756b682877 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 22 Mar 2018 02:36:46 -0700 Subject: [PATCH 2/6] Dockerfile-test: use "fedora" as base image Signed-off-by: Gyuho Lee --- Dockerfile-functional-tester | 33 ++++++++------------------ Dockerfile-test | 45 +++++++++++++----------------------- 2 files changed, 26 insertions(+), 52 deletions(-) diff --git a/Dockerfile-functional-tester b/Dockerfile-functional-tester index 5c11d07cf..34a1f8941 100644 --- a/Dockerfile-functional-tester +++ b/Dockerfile-functional-tester @@ -1,27 +1,14 @@ -FROM ubuntu:17.10 +FROM fedora:28 -RUN rm /bin/sh && ln -s /bin/bash /bin/sh -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - -RUN apt-get -y update \ - && apt-get -y install \ - build-essential \ - gcc \ - apt-utils \ - pkg-config \ - software-properties-common \ - apt-transport-https \ - libssl-dev \ - sudo \ - bash \ - curl \ - wget \ - tar \ - git \ - && apt-get -y update \ - && apt-get -y upgrade \ - && apt-get -y autoremove \ - && apt-get -y autoclean +RUN dnf check-update || true \ + && dnf install --assumeyes \ + git curl wget mercurial meld gcc gcc-c++ which \ + gcc automake autoconf dh-autoreconf libtool libtool-ltdl \ + tar unzip gzip \ + && dnf check-update || true \ + && dnf upgrade --assumeyes || true \ + && dnf autoremove --assumeyes || true \ + && dnf clean all || true ENV GOROOT /usr/local/go ENV GOPATH /go diff --git a/Dockerfile-test b/Dockerfile-test index fd7a61470..70ee40d40 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,33 +1,16 @@ -FROM ubuntu:17.10 +FROM fedora:28 -RUN rm /bin/sh && ln -s /bin/bash /bin/sh -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - -RUN apt-get -y update \ - && apt-get -y install \ - build-essential \ - gcc \ - apt-utils \ - pkg-config \ - software-properties-common \ - apt-transport-https \ - libssl-dev \ - sudo \ - bash \ - curl \ - wget \ - tar \ - git \ - netcat \ - libaspell-dev \ - libhunspell-dev \ - hunspell-en-us \ - aspell-en \ - shellcheck \ - && apt-get -y update \ - && apt-get -y upgrade \ - && apt-get -y autoremove \ - && apt-get -y autoclean +RUN dnf check-update || true \ + && dnf install --assumeyes \ + git curl wget mercurial meld gcc gcc-c++ which \ + gcc automake autoconf dh-autoreconf libtool libtool-ltdl \ + tar unzip gzip \ + aspell-devel aspell-en hunspell hunspell-devel hunspell-en hunspell-en-US ShellCheck || true \ + && dnf check-update || true \ + && dnf upgrade --assumeyes || true \ + && dnf autoremove --assumeyes || true \ + && dnf clean all || true \ + && dnf reinstall which || true ENV GOROOT /usr/local/go ENV GOPATH /go @@ -44,6 +27,10 @@ WORKDIR ${GOPATH}/src/github.com/coreos/etcd ADD ./scripts/install-marker.sh /tmp/install-marker.sh +# manually link "goword" dependency +# ldconfig -v | grep hunspell +RUN ln -s /lib64/libhunspell-1.6.so /lib64/libhunspell.so + RUN go get -v -u -tags spell github.com/chzchzchz/goword \ && go get -v -u github.com/coreos/license-bill-of-materials \ && go get -v -u honnef.co/go/tools/cmd/gosimple \ From 9c995fab9b3b7cfff0907500c7b6400a0dfbf672 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 22 Mar 2018 10:39:23 -0700 Subject: [PATCH 3/6] test: fix shellcheck Signed-off-by: Gyuho Lee --- test | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/test b/test index 48cf905ab..6549d6b97 100755 --- a/test +++ b/test @@ -54,12 +54,17 @@ IGNORE_PKGS="(vendor/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)" INTEGRATION_PKGS="(integration|e2e|contrib|functional-tester)" # all github.com/coreos/etcd/whatever pkgs that are not auto-generated / tools +# shellcheck disable=SC1117 PKGS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | grep -vE "(tools/|contrib/|e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo) # pkg1,pkg2,pkg3 PKGS_COMMA=${PKGS// /,} +# shellcheck disable=SC1117 TEST_PKGS=$(find . -name \*_test.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g") + +# shellcheck disable=SC1117 FORMATTABLE=$(find . -name \*.go | while read -r a; do echo "$(dirname "$a")/*.go"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g") + TESTABLE_AND_FORMATTABLE=$(echo "$TEST_PKGS" | grep -vE "$INTEGRATION_PKGS") # check if user provided PKG override @@ -74,17 +79,22 @@ else # only run gofmt on packages provided by user FMT="$TEST" fi + +# shellcheck disable=SC2206 FMT=($FMT) # prepend REPO_PATH to each local package split=$TEST TEST="" for a in $split; do TEST="$TEST ${REPO_PATH}/${a}"; done + +# shellcheck disable=SC2206 TEST=($TEST) # TODO: 'client' pkg fails with gosimple from generated files # TODO: 'rafttest' is failing with unused STATIC_ANALYSIS_PATHS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | grep -v 'client') +# shellcheck disable=SC2206 STATIC_ANALYSIS_PATHS=($STATIC_ANALYSIS_PATHS) if [ -z "$GOARCH" ]; then @@ -203,6 +213,7 @@ function functional_pass { echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE} echo "Waiting for processes to exit" + # shellcheck disable=SC2206 agent_pids=($agent_pids) kill -s TERM "${agent_pids[@]}" for a in "${agent_pids[@]}"; do wait "$a" || true; done @@ -236,6 +247,7 @@ function cov_pass { # run code coverage for unit and integration tests GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 15m" + # shellcheck disable=SC2206 GOCOVFLAGS=($GOCOVFLAGS) failed="" for t in $(echo "${TEST_PKGS}" | grep -vE "(e2e|functional-tester)"); do @@ -354,7 +366,7 @@ function shellcheck_pass { if which shellcheck >/dev/null; then shellcheckResult=$(shellcheck -fgcc build test scripts/* 2>&1 || true) if [ -n "${shellcheckResult}" ]; then - echo -e "shellcheck checking failed:\n${shellcheckResult}" + echo -e "shellcheck checking failed:\\n${shellcheckResult}" exit 255 fi fi @@ -364,7 +376,7 @@ function markdown_you_pass { # eschew you yous=$(find . -name \*.md -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + | grep -v /v2/ || true) if [ ! -z "$yous" ]; then - echo -e "found 'you' in documentation:\n${yous}" + echo -e "found 'you' in documentation:\\n${yous}" exit 255 fi } @@ -374,7 +386,7 @@ function markdown_marker_pass { if which marker >/dev/null; then markerResult=$(marker --skip-http --root ./Documentation 2>&1 || true) if [ -n "${markerResult}" ]; then - echo -e "marker checking failed:\n${markerResult}" + echo -e "marker checking failed:\\n${markerResult}" exit 255 fi else @@ -386,20 +398,23 @@ function goword_pass { if which goword >/dev/null; then # get all go files to process gofiles=$(find "${FMT[@]}" -iname '*.go' 2>/dev/null) + # shellcheck disable=SC2206 gofiles_all=($gofiles) # ignore tests and protobuf files + # shellcheck disable=SC1117 gofiles=$(echo "${gofiles_all[@]}" | sort | uniq | sed "s/ /\n/g" | grep -vE "(\\_test.go|\\.pb\\.go)") + # shellcheck disable=SC2206 gofiles=($gofiles) # only check for broken exported godocs gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort) if [ ! -z "$gowordRes" ]; then - echo -e "goword checking failed:\n${gowordRes}" + echo -e "goword checking failed:\\n${gowordRes}" exit 255 fi # check some spelling gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort) if [ ! -z "$gowordRes" ]; then - echo -e "goword checking failed:\n${gowordRes}" + echo -e "goword checking failed:\\n${gowordRes}" exit 255 fi else @@ -410,7 +425,7 @@ function goword_pass { function gofmt_pass { fmtRes=$(gofmt -l -s -d "${FMT[@]}") if [ -n "${fmtRes}" ]; then - echo -e "gofmt checking failed:\n${fmtRes}" + echo -e "gofmt checking failed:\\n${fmtRes}" exit 255 fi } @@ -418,17 +433,18 @@ function gofmt_pass { function govet_pass { vetRes=$(go vet "${TEST[@]}") if [ -n "${vetRes}" ]; then - echo -e "govet checking failed:\n${vetRes}" + echo -e "govet checking failed:\\n${vetRes}" exit 255 fi } function govet_shadow_pass { fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.") + # shellcheck disable=SC2206 fmtpkgs=($fmtpkgs) vetRes=$(go tool vet -all -shadow "${fmtpkgs[@]}" 2>&1 | grep -v '/gw/' || true) if [ -n "${vetRes}" ]; then - echo -e "govet -all -shadow checking failed:\n${vetRes}" + echo -e "govet -all -shadow checking failed:\\n${vetRes}" exit 255 fi } @@ -437,7 +453,7 @@ function gosimple_pass { if which gosimple >/dev/null; then gosimpleResult=$(gosimple "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${gosimpleResult}" ]; then - echo -e "gosimple checking failed:\n${gosimpleResult}" + echo -e "gosimple checking failed:\\n${gosimpleResult}" exit 255 fi else @@ -449,7 +465,7 @@ function unused_pass { if which unused >/dev/null; then unusedResult=$(unused "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${unusedResult}" ]; then - echo -e "unused checking failed:\n${unusedResult}" + echo -e "unused checking failed:\\n${unusedResult}" exit 255 fi else @@ -465,11 +481,11 @@ function staticcheck_pass { # See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck STATIC_CHECK_MASK="SA(1012|1019|2002)" if echo "${staticcheckResult}" | grep -vE "$STATIC_CHECK_MASK"; then - echo -e "staticcheck checking failed:\n${staticcheckResult}" + echo -e "staticcheck checking failed:\\n${staticcheckResult}" exit 255 else suppressed=$(echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c) - echo -e "staticcheck suppressed warnings:\n${suppressed}" + echo -e "staticcheck suppressed warnings:\\n${suppressed}" fi fi else @@ -481,7 +497,7 @@ function ineffassign_pass { if which ineffassign >/dev/null; then ineffassignResult=$(ineffassign "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${ineffassignResult}" ]; then - echo -e "ineffassign checking failed:\n${ineffassignResult}" + echo -e "ineffassign checking failed:\\n${ineffassignResult}" exit 255 fi else @@ -493,7 +509,7 @@ function nakedret_pass { if which nakedret >/dev/null; then nakedretResult=$(nakedret "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${nakedretResult}" ]; then - echo -e "nakedret checking failed:\n${nakedretResult}" + echo -e "nakedret checking failed:\\n${nakedretResult}" exit 255 fi else @@ -510,16 +526,18 @@ function license_header_pass { fi done if [ -n "${licRes}" ]; then - echo -e "license header checking failed:\n${licRes}" + echo -e "license header checking failed:\\n${licRes}" exit 255 fi } function receiver_name_pass { + # shellcheck disable=SC1117 recvs=$(grep 'func ([^*]' {*,*/*,*/*/*}.go | grep -Ev "(generated|pb/)" | tr ':' ' ' | \ awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" | sort | uniq | \ grep -Ev "(Descriptor|Proto|_)" | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ') if [ -n "${recvs}" ]; then + # shellcheck disable=SC2206 recvs=($recvs) for recv in "${recvs[@]}"; do echo "Mismatched receiver for $recv..." @@ -601,7 +619,7 @@ function dep_pass { deps=$(go list -f '{{ .Deps }}' | sed 's/ /\n/g' | grep -E "${badpkg}" || echo "") popd >/dev/null if [ ! -z "$deps" ]; then - echo -e "clientv3 has masked dependencies:\n${deps}" + echo -e "clientv3 has masked dependencies:\\n${deps}" exit 255 fi } From 906a40bcedbfdddb6c301a1766e51048029b76ec Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 22 Mar 2018 10:39:31 -0700 Subject: [PATCH 4/6] scripts/genproto: fix shellcheck Signed-off-by: Gyuho Lee --- scripts/genproto.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/genproto.sh b/scripts/genproto.sh index 650cbd806..a7437b8a2 100755 --- a/scripts/genproto.sh +++ b/scripts/genproto.sh @@ -59,11 +59,17 @@ popd for dir in ${DIRS}; do pushd "${dir}" protoc --gofast_out=plugins=grpc,import_prefix=github.com/coreos/:. -I=".:${GOGOPROTO_PATH}:${COREOS_ROOT}:${GRPC_GATEWAY_ROOT}/third_party/googleapis" ./*.proto - sed -i.bak -E "s/github\.com\/coreos\/(gogoproto|github\.com|golang\.org|google\.golang\.org)/\1/g" ./*.pb.go + # shellcheck disable=SC1117 + sed -i.bak -E 's/github\.com\/coreos\/(gogoproto|github\.com|golang\.org|google\.golang\.org)/\1/g' ./*.pb.go + # shellcheck disable=SC1117 sed -i.bak -E 's/github\.com\/coreos\/(errors|fmt|io)/\1/g' ./*.pb.go + # shellcheck disable=SC1117 sed -i.bak -E 's/import _ \"gogoproto\"//g' ./*.pb.go + # shellcheck disable=SC1117 sed -i.bak -E 's/import fmt \"fmt\"//g' ./*.pb.go + # shellcheck disable=SC1117 sed -i.bak -E 's/import _ \"github\.com\/coreos\/google\/api\"//g' ./*.pb.go + # shellcheck disable=SC1117 sed -i.bak -E 's/import _ \"google\.golang\.org\/genproto\/googleapis\/api\/annotations\"//g' ./*.pb.go rm -f ./*.bak goimports -w ./*.pb.go @@ -86,12 +92,14 @@ for pb in etcdserverpb/rpc api/v3lock/v3lockpb/v3lock api/v3election/v3electionp pkg=$(basename "${pkgpath}") gwfile="${protobase}.pb.gw.go" sed -i.bak -E "s/package $pkg/package gw/g" ${gwfile} + # shellcheck disable=SC1117 sed -i.bak -E "s/protoReq /&$pkg\./g" ${gwfile} sed -i.bak -E "s/, client /, client $pkg./g" ${gwfile} sed -i.bak -E "s/Client /, client $pkg./g" ${gwfile} sed -i.bak -E "s/[^(]*Client, runtime/${pkg}.&/" ${gwfile} sed -i.bak -E "s/New[A-Za-z]*Client/${pkg}.&/" ${gwfile} # darwin doesn't like newlines in sed... + # shellcheck disable=SC1117 sed -i.bak -E "s|import \(|& \"github.com/coreos/etcd/${pkgpath}\"|" ${gwfile} mkdir -p "${pkgpath}"/gw/ go fmt ${gwfile} From f6c9f767e595989bfcd5b940e2d8c69559f36b6b Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 22 Mar 2018 10:48:17 -0700 Subject: [PATCH 5/6] Dockerfile-test: install "netcat" Signed-off-by: Gyuho Lee --- Dockerfile-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-test b/Dockerfile-test index 70ee40d40..44f768da3 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -5,7 +5,7 @@ RUN dnf check-update || true \ git curl wget mercurial meld gcc gcc-c++ which \ gcc automake autoconf dh-autoreconf libtool libtool-ltdl \ tar unzip gzip \ - aspell-devel aspell-en hunspell hunspell-devel hunspell-en hunspell-en-US ShellCheck || true \ + aspell-devel aspell-en hunspell hunspell-devel hunspell-en hunspell-en-US ShellCheck nc || true \ && dnf check-update || true \ && dnf upgrade --assumeyes || true \ && dnf autoremove --assumeyes || true \ From be50343652dee915c996c18d63daaddc232475b1 Mon Sep 17 00:00:00 2001 From: Gyuho Lee Date: Thu, 22 Mar 2018 11:01:26 -0700 Subject: [PATCH 6/6] words: whitelist more words Signed-off-by: Gyuho Lee --- .words | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.words b/.words index 58135f9ff..7a9a2e32f 100644 --- a/.words +++ b/.words @@ -19,6 +19,7 @@ cancelation cluster_proxy defragment defragmenting +deleter errClientDisconnected etcd gRPC @@ -31,6 +32,8 @@ keepalive keepalives keyspace linearization +liveness +linearized localhost mutex prefetching @@ -43,6 +46,18 @@ serializable statusError teardown too_many_pings +transactional uncontended unprefixed unlisting +nondeterministically +atomics +transferee +Balancer +lexicographically +lexically +accessors +unbuffered +nils +reconnection +mutators