etcdctl: use health endpoint to greatly simplify health checking
parent
0a2d2b8b9d
commit
8d410bdfcb
|
@ -2,12 +2,10 @@ package command
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"sort"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
|
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
|
||||||
|
@ -42,124 +40,60 @@ func handleClusterHealth(c *cli.Context) {
|
||||||
handleError(ExitServerError, err)
|
handleError(ExitServerError, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: update members when forever is set.
|
hc := http.Client{
|
||||||
|
Transport: tr,
|
||||||
|
}
|
||||||
|
|
||||||
mi := mustNewMembersAPI(c)
|
mi := mustNewMembersAPI(c)
|
||||||
ms, err := mi.List(context.TODO())
|
ms, err := mi.List(context.TODO())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println("cluster may be unhealthy: failed to list members")
|
fmt.Println("cluster may be unhealthy: failed to list members")
|
||||||
handleError(ExitServerError, err)
|
handleError(ExitServerError, err)
|
||||||
}
|
}
|
||||||
cl := make([]string, 0)
|
|
||||||
for _, m := range ms {
|
|
||||||
cl = append(cl, m.ClientURLs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
// check the /health endpoint of all members first
|
health := false
|
||||||
|
for _, m := range ms {
|
||||||
|
checked := false
|
||||||
|
for _, url := range m.ClientURLs {
|
||||||
|
resp, err := hc.Get(url + "/health")
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to check the health of member %s on %s: %v\n", m.ID, url, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
ep, rs0, err := getLeaderStatus(tr, cl)
|
result := struct{ Health string }{}
|
||||||
if err != nil {
|
d := json.NewDecoder(resp.Body)
|
||||||
fmt.Println("cluster may be unhealthy: failed to connect", cl)
|
err = d.Decode(&result)
|
||||||
if forever {
|
resp.Body.Close()
|
||||||
time.Sleep(10 * time.Second)
|
if err != nil {
|
||||||
continue
|
fmt.Printf("failed to check the health of member %s on %s: %v\n", m.ID, url, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
checked = true
|
||||||
|
if result.Health == "true" {
|
||||||
|
checked = true
|
||||||
|
fmt.Printf("member %s is healthy: got healthy result from %s\n", m.ID, url)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("member %s is unhealthy: got unhealthy result from %s\n", m.ID, url)
|
||||||
|
}
|
||||||
|
break
|
||||||
}
|
}
|
||||||
os.Exit(1)
|
if !checked {
|
||||||
}
|
fmt.Printf("member %s is unreachable: %v are all unreachable\n", m.ID, m.ClientURLs)
|
||||||
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
|
|
||||||
// are all the members makeing progress?
|
|
||||||
_, rs1, err := getLeaderStatus(tr, []string{ep})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("cluster is unhealthy")
|
|
||||||
if forever {
|
|
||||||
time.Sleep(10 * time.Second)
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
|
if health {
|
||||||
if rs1.Commit > rs0.Commit {
|
fmt.Println("cluster is healthy")
|
||||||
fmt.Printf("cluster is healthy: raft is making progress [commit index: %v->%v]\n", rs0.Commit, rs1.Commit)
|
|
||||||
} else {
|
} else {
|
||||||
fmt.Printf("cluster is unhealthy: raft is not making progress [commit index: %v]\n", rs0.Commit)
|
fmt.Println("cluster is unhealthy")
|
||||||
}
|
|
||||||
fmt.Printf("leader is %v\n", rs0.Lead)
|
|
||||||
|
|
||||||
var prints []string
|
|
||||||
|
|
||||||
for id, pr0 := range rs0.Progress {
|
|
||||||
pr1, ok := rs1.Progress[id]
|
|
||||||
if !ok {
|
|
||||||
// TODO: forever should handle configuration change.
|
|
||||||
fmt.Println("Cluster configuration changed during health checking. Please retry.")
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
if pr1.Match <= pr0.Match {
|
|
||||||
prints = append(prints, fmt.Sprintf("member %s is unhealthy: raft is not making progress [match: %v->%v]\n", id, pr0.Match, pr1.Match))
|
|
||||||
} else {
|
|
||||||
prints = append(prints, fmt.Sprintf("member %s is healthy: raft is making progress [match: %v->%v]\n", id, pr0.Match, pr1.Match))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Strings(prints)
|
|
||||||
for _, p := range prints {
|
|
||||||
fmt.Print(p)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !forever {
|
if !forever {
|
||||||
return
|
break
|
||||||
}
|
}
|
||||||
|
fmt.Printf("\nnext check after 10 second...\n\n")
|
||||||
time.Sleep(10 * time.Second)
|
time.Sleep(10 * time.Second)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type raftStatus struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Term uint64 `json:"term"`
|
|
||||||
Vote string `json:"vote"`
|
|
||||||
Commit uint64 `json:"commit"`
|
|
||||||
Lead string `json:"lead"`
|
|
||||||
RaftState string `json:"raftState"`
|
|
||||||
Progress map[string]struct {
|
|
||||||
Match uint64 `json:"match"`
|
|
||||||
Next uint64 `json:"next"`
|
|
||||||
State string `json:"state"`
|
|
||||||
} `json:"progress"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type vars struct {
|
|
||||||
RaftStatus raftStatus `json:"raft.status"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func getLeaderStatus(tr *http.Transport, endpoints []string) (string, raftStatus, error) {
|
|
||||||
// TODO: use new etcd client
|
|
||||||
httpclient := http.Client{
|
|
||||||
Transport: tr,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, ep := range endpoints {
|
|
||||||
resp, err := httpclient.Get(ep + "/debug/vars")
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
vs := &vars{}
|
|
||||||
d := json.NewDecoder(resp.Body)
|
|
||||||
err = d.Decode(vs)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if vs.RaftStatus.Lead != vs.RaftStatus.ID {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return ep, vs.RaftStatus, nil
|
|
||||||
}
|
|
||||||
return "", raftStatus{}, errors.New("no leader")
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in New Issue