2014-09-21 22:26:59 +04:00
|
|
|
package discovery
|
|
|
|
|
|
|
|
import (
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
2014-10-08 00:58:58 +04:00
|
|
|
"log"
|
2014-10-04 16:20:45 +04:00
|
|
|
"net/http"
|
|
|
|
"net/url"
|
2014-09-21 22:26:59 +04:00
|
|
|
"path"
|
|
|
|
"sort"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
2014-10-04 16:20:45 +04:00
|
|
|
"time"
|
2014-09-21 22:26:59 +04:00
|
|
|
|
|
|
|
"github.com/coreos/etcd/client"
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
2014-10-08 00:58:58 +04:00
|
|
|
ErrInvalidURL = errors.New("discovery: invalid URL")
|
|
|
|
ErrBadSizeKey = errors.New("discovery: size key is bad")
|
|
|
|
ErrSizeNotFound = errors.New("discovery: size key not found")
|
|
|
|
ErrTokenNotFound = errors.New("discovery: token not found")
|
|
|
|
ErrDuplicateID = errors.New("discovery: found duplicate id")
|
|
|
|
ErrFullCluster = errors.New("discovery: cluster is full")
|
|
|
|
ErrTooManyRetries = errors.New("discovery: too many retries")
|
2014-10-08 05:20:45 +04:00
|
|
|
)
|
2014-10-08 00:58:58 +04:00
|
|
|
|
2014-10-08 05:20:45 +04:00
|
|
|
const (
|
2014-10-08 00:58:58 +04:00
|
|
|
// Number of retries discovery will attempt before giving up and erroring out.
|
|
|
|
nRetries = uint(3)
|
2014-09-21 22:26:59 +04:00
|
|
|
)
|
|
|
|
|
2014-10-04 16:20:45 +04:00
|
|
|
type Discoverer interface {
|
|
|
|
Discover() (string, error)
|
|
|
|
}
|
|
|
|
|
2014-09-21 22:26:59 +04:00
|
|
|
type discovery struct {
|
|
|
|
cluster string
|
2014-10-08 15:58:53 +04:00
|
|
|
id uint64
|
2014-10-04 16:20:45 +04:00
|
|
|
config string
|
2014-09-21 22:26:59 +04:00
|
|
|
c client.Client
|
2014-10-08 00:58:58 +04:00
|
|
|
retries uint
|
|
|
|
url *url.URL
|
2014-10-09 03:41:59 +04:00
|
|
|
|
|
|
|
// Injectable for testing. nil means Seconds.
|
|
|
|
timeoutTimescale time.Duration
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-10-08 15:58:53 +04:00
|
|
|
func New(durl string, id uint64, config string) (Discoverer, error) {
|
2014-10-04 16:20:45 +04:00
|
|
|
u, err := url.Parse(durl)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
token := u.Path
|
|
|
|
u.Path = ""
|
2014-10-08 00:58:58 +04:00
|
|
|
c, err := client.NewHTTPClient(&http.Transport{}, u.String(), time.Second*5)
|
2014-10-04 16:20:45 +04:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
// discovery service redirects /[key] to /v2/keys/[key]
|
|
|
|
// set the prefix of client to "" to handle this
|
2014-10-08 00:58:58 +04:00
|
|
|
c.SetPrefix("")
|
2014-10-04 16:20:45 +04:00
|
|
|
return &discovery{
|
2014-10-09 03:41:59 +04:00
|
|
|
cluster: token,
|
|
|
|
id: id,
|
|
|
|
config: config,
|
|
|
|
c: c,
|
|
|
|
url: u,
|
|
|
|
timeoutTimescale: time.Second,
|
2014-10-04 16:20:45 +04:00
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *discovery) Discover() (string, error) {
|
2014-09-22 09:34:39 +04:00
|
|
|
// fast path: if the cluster is full, returns the error
|
|
|
|
// do not need to register itself to the cluster in this
|
|
|
|
// case.
|
|
|
|
if _, _, err := d.checkCluster(); err != nil {
|
2014-10-04 16:20:45 +04:00
|
|
|
return "", err
|
2014-09-22 09:34:39 +04:00
|
|
|
}
|
|
|
|
|
2014-09-21 22:26:59 +04:00
|
|
|
if err := d.createSelf(); err != nil {
|
2014-10-08 05:20:45 +04:00
|
|
|
// Fails, even on a timeout, if createSelf times out.
|
|
|
|
// TODO(barakmich): Retrying the same node might want to succeed here
|
|
|
|
// (ie, createSelf should be idempotent for discovery).
|
2014-10-04 16:20:45 +04:00
|
|
|
return "", err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
nodes, size, err := d.checkCluster()
|
|
|
|
if err != nil {
|
2014-10-04 16:20:45 +04:00
|
|
|
return "", err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
all, err := d.waitNodes(nodes, size)
|
|
|
|
if err != nil {
|
2014-10-04 16:20:45 +04:00
|
|
|
return "", err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-10-04 16:20:45 +04:00
|
|
|
return nodesToCluster(all), nil
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func (d *discovery) createSelf() error {
|
2014-10-04 16:20:45 +04:00
|
|
|
resp, err := d.c.Create(d.selfKey(), d.config, -1)
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// ensure self appears on the server we connected to
|
2014-09-22 22:13:49 +04:00
|
|
|
w := d.c.Watch(d.selfKey(), resp.Node.CreatedIndex)
|
2014-09-23 02:38:19 +04:00
|
|
|
_, err = w.Next()
|
|
|
|
return err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
func (d *discovery) checkCluster() (client.Nodes, int, error) {
|
2014-09-23 01:25:35 +04:00
|
|
|
configKey := path.Join("/", d.cluster, "_config")
|
2014-09-23 00:54:21 +04:00
|
|
|
// find cluster size
|
|
|
|
resp, err := d.c.Get(path.Join(configKey, "size"))
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
2014-09-23 00:54:21 +04:00
|
|
|
if err == client.ErrKeyNoExist {
|
|
|
|
return nil, 0, ErrSizeNotFound
|
|
|
|
}
|
2014-10-08 00:58:58 +04:00
|
|
|
if err == client.ErrTimeout {
|
|
|
|
return d.checkClusterRetry()
|
|
|
|
}
|
2014-09-21 22:26:59 +04:00
|
|
|
return nil, 0, err
|
|
|
|
}
|
2014-09-23 00:54:21 +04:00
|
|
|
size, err := strconv.Atoi(resp.Node.Value)
|
|
|
|
if err != nil {
|
|
|
|
return nil, 0, ErrBadSizeKey
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
2014-09-23 00:54:21 +04:00
|
|
|
|
|
|
|
resp, err = d.c.Get(d.cluster)
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
2014-10-08 00:58:58 +04:00
|
|
|
if err == client.ErrTimeout {
|
|
|
|
return d.checkClusterRetry()
|
|
|
|
}
|
2014-09-23 00:54:21 +04:00
|
|
|
return nil, 0, err
|
|
|
|
}
|
|
|
|
nodes := make(client.Nodes, 0)
|
|
|
|
// append non-config keys to nodes
|
|
|
|
for _, n := range resp.Node.Nodes {
|
2014-10-04 16:20:45 +04:00
|
|
|
if !strings.Contains(n.Key, configKey) {
|
2014-09-23 00:54:21 +04:00
|
|
|
nodes = append(nodes, n)
|
|
|
|
}
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-09-23 02:38:19 +04:00
|
|
|
snodes := sortableNodes{nodes}
|
2014-09-23 00:54:21 +04:00
|
|
|
sort.Sort(snodes)
|
2014-09-21 22:26:59 +04:00
|
|
|
|
|
|
|
// find self position
|
|
|
|
for i := range nodes {
|
2014-10-04 16:20:45 +04:00
|
|
|
if strings.Contains(nodes[i].Key, d.selfKey()) {
|
2014-09-21 22:26:59 +04:00
|
|
|
break
|
|
|
|
}
|
|
|
|
if i >= size-1 {
|
|
|
|
return nil, size, ErrFullCluster
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nodes, size, nil
|
|
|
|
}
|
|
|
|
|
2014-10-08 00:58:58 +04:00
|
|
|
func (d *discovery) logAndBackoffForRetry(step string) {
|
|
|
|
d.retries++
|
2014-10-09 03:41:59 +04:00
|
|
|
retryTime := d.timeoutTimescale * (0x1 << d.retries)
|
2014-10-08 00:58:58 +04:00
|
|
|
log.Println("discovery: during", step, "connection to", d.url, "timed out, retrying in", retryTime)
|
|
|
|
time.Sleep(retryTime)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (d *discovery) checkClusterRetry() (client.Nodes, int, error) {
|
|
|
|
if d.retries < nRetries {
|
|
|
|
d.logAndBackoffForRetry("cluster status check")
|
|
|
|
return d.checkCluster()
|
|
|
|
}
|
|
|
|
return nil, 0, ErrTooManyRetries
|
|
|
|
}
|
|
|
|
|
2014-10-08 05:20:45 +04:00
|
|
|
func (d *discovery) waitNodesRetry() (client.Nodes, error) {
|
|
|
|
if d.retries < nRetries {
|
|
|
|
d.logAndBackoffForRetry("waiting for other nodes")
|
|
|
|
nodes, n, err := d.checkCluster()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return d.waitNodes(nodes, n)
|
|
|
|
}
|
|
|
|
return nil, ErrTooManyRetries
|
|
|
|
}
|
|
|
|
|
2014-09-21 22:26:59 +04:00
|
|
|
func (d *discovery) waitNodes(nodes client.Nodes, size int) (client.Nodes, error) {
|
|
|
|
if len(nodes) > size {
|
|
|
|
nodes = nodes[:size]
|
|
|
|
}
|
2014-10-04 16:20:45 +04:00
|
|
|
w := d.c.RecursiveWatch(d.cluster, nodes[len(nodes)-1].ModifiedIndex+1)
|
2014-09-21 22:26:59 +04:00
|
|
|
all := make(client.Nodes, len(nodes))
|
|
|
|
copy(all, nodes)
|
|
|
|
// wait for others
|
|
|
|
for len(all) < size {
|
|
|
|
resp, err := w.Next()
|
|
|
|
if err != nil {
|
2014-10-08 05:20:45 +04:00
|
|
|
if err == client.ErrTimeout {
|
|
|
|
return d.waitNodesRetry()
|
|
|
|
}
|
2014-09-21 22:26:59 +04:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
all = append(all, resp.Node)
|
|
|
|
}
|
|
|
|
return all, nil
|
|
|
|
}
|
|
|
|
|
2014-09-22 22:13:49 +04:00
|
|
|
func (d *discovery) selfKey() string {
|
|
|
|
return path.Join("/", d.cluster, fmt.Sprintf("%d", d.id))
|
|
|
|
}
|
|
|
|
|
2014-10-04 16:20:45 +04:00
|
|
|
func nodesToCluster(ns client.Nodes) string {
|
2014-09-21 22:26:59 +04:00
|
|
|
s := make([]string, len(ns))
|
|
|
|
for i, n := range ns {
|
|
|
|
s[i] = n.Value
|
|
|
|
}
|
2014-10-04 16:20:45 +04:00
|
|
|
return strings.Join(s, ",")
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-09-23 02:38:19 +04:00
|
|
|
type sortableNodes struct{ client.Nodes }
|
2014-09-21 22:26:59 +04:00
|
|
|
|
2014-09-23 02:38:19 +04:00
|
|
|
func (ns sortableNodes) Len() int { return len(ns.Nodes) }
|
|
|
|
func (ns sortableNodes) Less(i, j int) bool {
|
2014-09-21 22:26:59 +04:00
|
|
|
return ns.Nodes[i].CreatedIndex < ns.Nodes[j].CreatedIndex
|
|
|
|
}
|
2014-09-23 02:38:19 +04:00
|
|
|
func (ns sortableNodes) Swap(i, j int) { ns.Nodes[i], ns.Nodes[j] = ns.Nodes[j], ns.Nodes[i] }
|