2016-05-13 06:51:48 +03:00
|
|
|
// Copyright 2015 The etcd Authors
|
2015-01-25 06:19:16 +03:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2014-10-18 02:41:22 +04:00
|
|
|
|
2015-11-18 07:54:10 +03:00
|
|
|
// Package discovery provides an implementation of the cluster discovery that
|
|
|
|
// is used by etcd.
|
2014-09-21 22:26:59 +04:00
|
|
|
package discovery
|
|
|
|
|
|
|
|
import (
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
2014-12-11 07:02:45 +03:00
|
|
|
"math"
|
2015-06-20 02:50:44 +03:00
|
|
|
"net"
|
2014-10-04 16:20:45 +04:00
|
|
|
"net/http"
|
|
|
|
"net/url"
|
2014-09-21 22:26:59 +04:00
|
|
|
"path"
|
|
|
|
"sort"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
2014-10-04 16:20:45 +04:00
|
|
|
"time"
|
2014-09-21 22:26:59 +04:00
|
|
|
|
2014-10-17 07:23:07 +04:00
|
|
|
"github.com/coreos/etcd/client"
|
2014-10-31 03:31:14 +03:00
|
|
|
"github.com/coreos/etcd/pkg/types"
|
2016-03-23 03:10:28 +03:00
|
|
|
"github.com/coreos/pkg/capnslog"
|
|
|
|
"github.com/jonboulle/clockwork"
|
|
|
|
"golang.org/x/net/context"
|
2014-09-21 22:26:59 +04:00
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
2015-06-11 19:48:10 +03:00
|
|
|
plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "discovery")
|
|
|
|
|
2015-07-15 02:46:49 +03:00
|
|
|
ErrInvalidURL = errors.New("discovery: invalid URL")
|
|
|
|
ErrBadSizeKey = errors.New("discovery: size key is bad")
|
|
|
|
ErrSizeNotFound = errors.New("discovery: size key not found")
|
|
|
|
ErrTokenNotFound = errors.New("discovery: token not found")
|
|
|
|
ErrDuplicateID = errors.New("discovery: found duplicate id")
|
|
|
|
ErrDuplicateName = errors.New("discovery: found duplicate name")
|
|
|
|
ErrFullCluster = errors.New("discovery: cluster is full")
|
|
|
|
ErrTooManyRetries = errors.New("discovery: too many retries")
|
|
|
|
ErrBadDiscoveryEndpoint = errors.New("discovery: bad discovery endpoint")
|
2014-10-08 05:20:45 +04:00
|
|
|
)
|
2014-10-08 00:58:58 +04:00
|
|
|
|
2014-12-11 07:02:45 +03:00
|
|
|
var (
|
2014-10-08 00:58:58 +04:00
|
|
|
// Number of retries discovery will attempt before giving up and erroring out.
|
2014-12-11 07:02:45 +03:00
|
|
|
nRetries = uint(math.MaxUint32)
|
2014-09-21 22:26:59 +04:00
|
|
|
)
|
|
|
|
|
2014-11-06 09:45:01 +03:00
|
|
|
// JoinCluster will connect to the discovery service at the given url, and
|
|
|
|
// register the server represented by the given id and config to the cluster
|
2014-11-07 03:35:24 +03:00
|
|
|
func JoinCluster(durl, dproxyurl string, id types.ID, config string) (string, error) {
|
|
|
|
d, err := newDiscovery(durl, dproxyurl, id)
|
2014-11-06 09:45:01 +03:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return d.joinCluster(config)
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetCluster will connect to the discovery service at the given url and
|
|
|
|
// retrieve a string describing the cluster
|
2014-11-07 03:35:24 +03:00
|
|
|
func GetCluster(durl, dproxyurl string) (string, error) {
|
|
|
|
d, err := newDiscovery(durl, dproxyurl, 0)
|
2014-11-06 09:45:01 +03:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return d.getCluster()
|
2014-10-04 16:20:45 +04:00
|
|
|
}
|
|
|
|
|
2014-09-21 22:26:59 +04:00
|
|
|
type discovery struct {
|
|
|
|
cluster string
|
2014-10-31 03:31:14 +03:00
|
|
|
id types.ID
|
2014-10-24 04:11:04 +04:00
|
|
|
c client.KeysAPI
|
2014-10-08 00:58:58 +04:00
|
|
|
retries uint
|
|
|
|
url *url.URL
|
2014-10-09 03:41:59 +04:00
|
|
|
|
2014-10-15 22:53:05 +04:00
|
|
|
clock clockwork.Clock
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-11-07 03:35:24 +03:00
|
|
|
// newProxyFunc builds a proxy function from the given string, which should
|
|
|
|
// represent a URL that can be used as a proxy. It performs basic
|
|
|
|
// sanitization of the URL and returns any error encountered.
|
|
|
|
func newProxyFunc(proxy string) (func(*http.Request) (*url.URL, error), error) {
|
2014-10-13 22:24:23 +04:00
|
|
|
if proxy == "" {
|
|
|
|
return nil, nil
|
|
|
|
}
|
|
|
|
// Do a small amount of URL sanitization to help the user
|
|
|
|
// Derived from net/http.ProxyFromEnvironment
|
|
|
|
proxyURL, err := url.Parse(proxy)
|
|
|
|
if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
|
|
|
|
// proxy was bogus. Try prepending "http://" to it and
|
|
|
|
// see if that parses correctly. If not, we ignore the
|
|
|
|
// error and complain about the original one
|
|
|
|
var err2 error
|
|
|
|
proxyURL, err2 = url.Parse("http://" + proxy)
|
|
|
|
if err2 == nil {
|
|
|
|
err = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
|
|
|
|
}
|
|
|
|
|
2015-06-11 19:48:10 +03:00
|
|
|
plog.Infof("using proxy %q", proxyURL.String())
|
2014-10-13 22:24:23 +04:00
|
|
|
return http.ProxyURL(proxyURL), nil
|
|
|
|
}
|
|
|
|
|
2014-11-07 03:35:24 +03:00
|
|
|
func newDiscovery(durl, dproxyurl string, id types.ID) (*discovery, error) {
|
2014-10-04 16:20:45 +04:00
|
|
|
u, err := url.Parse(durl)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
token := u.Path
|
|
|
|
u.Path = ""
|
2014-11-07 03:35:24 +03:00
|
|
|
pf, err := newProxyFunc(dproxyurl)
|
2014-10-13 22:24:23 +04:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-01-27 04:26:15 +03:00
|
|
|
cfg := client.Config{
|
2015-06-20 02:50:44 +03:00
|
|
|
Transport: &http.Transport{
|
|
|
|
Proxy: pf,
|
|
|
|
Dial: (&net.Dialer{
|
|
|
|
Timeout: 30 * time.Second,
|
|
|
|
KeepAlive: 30 * time.Second,
|
|
|
|
}).Dial,
|
2015-07-15 02:19:11 +03:00
|
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
|
|
// TODO: add ResponseHeaderTimeout back when watch on discovery service writes header early
|
2015-06-20 02:50:44 +03:00
|
|
|
},
|
2015-01-27 00:49:50 +03:00
|
|
|
Endpoints: []string{u.String()},
|
|
|
|
}
|
|
|
|
c, err := client.New(cfg)
|
2014-10-04 16:20:45 +04:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-01-28 00:31:57 +03:00
|
|
|
dc := client.NewKeysAPIWithPrefix(c, "")
|
2014-10-04 16:20:45 +04:00
|
|
|
return &discovery{
|
2014-10-15 22:53:05 +04:00
|
|
|
cluster: token,
|
2014-10-31 20:59:39 +03:00
|
|
|
c: dc,
|
2014-11-06 09:45:01 +03:00
|
|
|
id: id,
|
2014-10-15 22:53:05 +04:00
|
|
|
url: u,
|
|
|
|
clock: clockwork.NewRealClock(),
|
2014-10-04 16:20:45 +04:00
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2014-11-06 09:45:01 +03:00
|
|
|
func (d *discovery) joinCluster(config string) (string, error) {
|
|
|
|
// fast path: if the cluster is full, return the error
|
|
|
|
// do not need to register to the cluster in this case.
|
2014-12-11 07:02:45 +03:00
|
|
|
if _, _, _, err := d.checkCluster(); err != nil {
|
2014-10-04 16:20:45 +04:00
|
|
|
return "", err
|
2014-09-22 09:34:39 +04:00
|
|
|
}
|
|
|
|
|
2014-11-06 09:45:01 +03:00
|
|
|
if err := d.createSelf(config); err != nil {
|
2014-10-08 05:20:45 +04:00
|
|
|
// Fails, even on a timeout, if createSelf times out.
|
|
|
|
// TODO(barakmich): Retrying the same node might want to succeed here
|
|
|
|
// (ie, createSelf should be idempotent for discovery).
|
2014-10-04 16:20:45 +04:00
|
|
|
return "", err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-12-11 07:02:45 +03:00
|
|
|
nodes, size, index, err := d.checkCluster()
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
2014-10-04 16:20:45 +04:00
|
|
|
return "", err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-12-11 07:02:45 +03:00
|
|
|
all, err := d.waitNodes(nodes, size, index)
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
2014-10-04 16:20:45 +04:00
|
|
|
return "", err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2015-07-03 04:16:49 +03:00
|
|
|
return nodesToCluster(all, size)
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-11-06 09:45:01 +03:00
|
|
|
func (d *discovery) getCluster() (string, error) {
|
2014-12-11 07:02:45 +03:00
|
|
|
nodes, size, index, err := d.checkCluster()
|
2014-11-05 00:09:24 +03:00
|
|
|
if err != nil {
|
|
|
|
if err == ErrFullCluster {
|
2015-07-03 04:16:49 +03:00
|
|
|
return nodesToCluster(nodes, size)
|
2014-11-05 00:09:24 +03:00
|
|
|
}
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
2014-12-11 07:02:45 +03:00
|
|
|
all, err := d.waitNodes(nodes, size, index)
|
2014-11-05 00:09:24 +03:00
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
2015-07-03 04:16:49 +03:00
|
|
|
return nodesToCluster(all, size)
|
2014-11-05 00:09:24 +03:00
|
|
|
}
|
|
|
|
|
2014-11-06 09:45:01 +03:00
|
|
|
func (d *discovery) createSelf(contents string) error {
|
2014-11-01 00:12:40 +03:00
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
|
2015-01-21 23:22:16 +03:00
|
|
|
resp, err := d.c.Create(ctx, d.selfKey(), contents)
|
2014-11-01 00:12:40 +03:00
|
|
|
cancel()
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
2015-05-11 20:49:49 +03:00
|
|
|
if eerr, ok := err.(client.Error); ok && eerr.Code == client.ErrorCodeNodeExist {
|
2014-12-12 02:42:54 +03:00
|
|
|
return ErrDuplicateID
|
|
|
|
}
|
2014-09-21 22:26:59 +04:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// ensure self appears on the server we connected to
|
2015-01-30 01:27:07 +03:00
|
|
|
w := d.c.Watcher(d.selfKey(), &client.WatcherOptions{AfterIndex: resp.Node.CreatedIndex - 1})
|
2014-11-01 00:12:40 +03:00
|
|
|
_, err = w.Next(context.Background())
|
2014-09-23 02:38:19 +04:00
|
|
|
return err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2015-01-27 22:47:08 +03:00
|
|
|
func (d *discovery) checkCluster() ([]*client.Node, int, uint64, error) {
|
2014-09-23 01:25:35 +04:00
|
|
|
configKey := path.Join("/", d.cluster, "_config")
|
2014-11-01 00:12:40 +03:00
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
|
2014-09-23 00:54:21 +04:00
|
|
|
// find cluster size
|
2015-01-28 23:34:06 +03:00
|
|
|
resp, err := d.c.Get(ctx, path.Join(configKey, "size"), nil)
|
2014-11-01 00:12:40 +03:00
|
|
|
cancel()
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
2015-01-29 03:46:58 +03:00
|
|
|
if eerr, ok := err.(*client.Error); ok && eerr.Code == client.ErrorCodeKeyNotFound {
|
2014-12-11 07:02:45 +03:00
|
|
|
return nil, 0, 0, ErrSizeNotFound
|
2014-09-23 00:54:21 +04:00
|
|
|
}
|
2015-07-15 02:46:49 +03:00
|
|
|
if err == client.ErrInvalidJSON {
|
|
|
|
return nil, 0, 0, ErrBadDiscoveryEndpoint
|
|
|
|
}
|
2015-07-29 18:04:12 +03:00
|
|
|
if ce, ok := err.(*client.ClusterError); ok {
|
|
|
|
plog.Error(ce.Detail())
|
2014-10-08 00:58:58 +04:00
|
|
|
return d.checkClusterRetry()
|
|
|
|
}
|
2014-12-11 07:02:45 +03:00
|
|
|
return nil, 0, 0, err
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
2014-09-23 00:54:21 +04:00
|
|
|
size, err := strconv.Atoi(resp.Node.Value)
|
|
|
|
if err != nil {
|
2014-12-11 07:02:45 +03:00
|
|
|
return nil, 0, 0, ErrBadSizeKey
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
2014-09-23 00:54:21 +04:00
|
|
|
|
2014-11-01 00:12:40 +03:00
|
|
|
ctx, cancel = context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
|
2015-01-28 23:34:06 +03:00
|
|
|
resp, err = d.c.Get(ctx, d.cluster, nil)
|
2014-11-01 00:12:40 +03:00
|
|
|
cancel()
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
2015-07-29 18:04:12 +03:00
|
|
|
if ce, ok := err.(*client.ClusterError); ok {
|
|
|
|
plog.Error(ce.Detail())
|
2014-10-08 00:58:58 +04:00
|
|
|
return d.checkClusterRetry()
|
|
|
|
}
|
2014-12-11 07:02:45 +03:00
|
|
|
return nil, 0, 0, err
|
2014-09-23 00:54:21 +04:00
|
|
|
}
|
2015-01-27 22:47:08 +03:00
|
|
|
nodes := make([]*client.Node, 0)
|
2014-09-23 00:54:21 +04:00
|
|
|
// append non-config keys to nodes
|
|
|
|
for _, n := range resp.Node.Nodes {
|
2014-10-30 04:30:59 +03:00
|
|
|
if !(path.Base(n.Key) == path.Base(configKey)) {
|
2014-09-23 00:54:21 +04:00
|
|
|
nodes = append(nodes, n)
|
|
|
|
}
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-09-23 02:38:19 +04:00
|
|
|
snodes := sortableNodes{nodes}
|
2014-09-23 00:54:21 +04:00
|
|
|
sort.Sort(snodes)
|
2014-09-21 22:26:59 +04:00
|
|
|
|
|
|
|
// find self position
|
|
|
|
for i := range nodes {
|
2014-10-30 04:30:59 +03:00
|
|
|
if path.Base(nodes[i].Key) == path.Base(d.selfKey()) {
|
2014-09-21 22:26:59 +04:00
|
|
|
break
|
|
|
|
}
|
|
|
|
if i >= size-1 {
|
2014-12-11 07:02:45 +03:00
|
|
|
return nodes[:size], size, resp.Index, ErrFullCluster
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
}
|
2014-12-11 07:02:45 +03:00
|
|
|
return nodes, size, resp.Index, nil
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2014-10-08 00:58:58 +04:00
|
|
|
func (d *discovery) logAndBackoffForRetry(step string) {
|
|
|
|
d.retries++
|
2014-10-15 22:53:05 +04:00
|
|
|
retryTime := time.Second * (0x1 << d.retries)
|
2015-07-29 18:04:12 +03:00
|
|
|
plog.Infof("%s: error connecting to %s, retrying in %s", step, d.url, retryTime)
|
2014-10-15 22:53:05 +04:00
|
|
|
d.clock.Sleep(retryTime)
|
2014-10-08 00:58:58 +04:00
|
|
|
}
|
|
|
|
|
2015-01-27 22:47:08 +03:00
|
|
|
func (d *discovery) checkClusterRetry() ([]*client.Node, int, uint64, error) {
|
2014-10-08 00:58:58 +04:00
|
|
|
if d.retries < nRetries {
|
|
|
|
d.logAndBackoffForRetry("cluster status check")
|
|
|
|
return d.checkCluster()
|
|
|
|
}
|
2014-12-11 07:02:45 +03:00
|
|
|
return nil, 0, 0, ErrTooManyRetries
|
2014-10-08 00:58:58 +04:00
|
|
|
}
|
|
|
|
|
2015-01-27 22:47:08 +03:00
|
|
|
func (d *discovery) waitNodesRetry() ([]*client.Node, error) {
|
2014-10-08 05:20:45 +04:00
|
|
|
if d.retries < nRetries {
|
|
|
|
d.logAndBackoffForRetry("waiting for other nodes")
|
2014-12-11 07:02:45 +03:00
|
|
|
nodes, n, index, err := d.checkCluster()
|
2014-10-08 05:20:45 +04:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2014-12-11 07:02:45 +03:00
|
|
|
return d.waitNodes(nodes, n, index)
|
2014-10-08 05:20:45 +04:00
|
|
|
}
|
|
|
|
return nil, ErrTooManyRetries
|
|
|
|
}
|
|
|
|
|
2015-01-27 22:47:08 +03:00
|
|
|
func (d *discovery) waitNodes(nodes []*client.Node, size int, index uint64) ([]*client.Node, error) {
|
2014-09-21 22:26:59 +04:00
|
|
|
if len(nodes) > size {
|
|
|
|
nodes = nodes[:size]
|
|
|
|
}
|
2014-12-13 09:09:43 +03:00
|
|
|
// watch from the next index
|
2015-01-30 01:27:07 +03:00
|
|
|
w := d.c.Watcher(d.cluster, &client.WatcherOptions{AfterIndex: index, Recursive: true})
|
2015-01-27 22:47:08 +03:00
|
|
|
all := make([]*client.Node, len(nodes))
|
2014-09-21 22:26:59 +04:00
|
|
|
copy(all, nodes)
|
2014-10-30 04:30:59 +03:00
|
|
|
for _, n := range all {
|
|
|
|
if path.Base(n.Key) == path.Base(d.selfKey()) {
|
2015-06-11 19:48:10 +03:00
|
|
|
plog.Noticef("found self %s in the cluster", path.Base(d.selfKey()))
|
2014-10-30 04:30:59 +03:00
|
|
|
} else {
|
2015-06-11 19:48:10 +03:00
|
|
|
plog.Noticef("found peer %s in the cluster", path.Base(n.Key))
|
2014-10-30 04:30:59 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-21 22:26:59 +04:00
|
|
|
// wait for others
|
|
|
|
for len(all) < size {
|
2015-06-11 19:48:10 +03:00
|
|
|
plog.Noticef("found %d peer(s), waiting for %d more", len(all), size-len(all))
|
2014-11-01 00:12:40 +03:00
|
|
|
resp, err := w.Next(context.Background())
|
2014-09-21 22:26:59 +04:00
|
|
|
if err != nil {
|
2015-07-29 18:04:12 +03:00
|
|
|
if ce, ok := err.(*client.ClusterError); ok {
|
|
|
|
plog.Error(ce.Detail())
|
2014-10-08 05:20:45 +04:00
|
|
|
return d.waitNodesRetry()
|
|
|
|
}
|
2014-09-21 22:26:59 +04:00
|
|
|
return nil, err
|
|
|
|
}
|
2015-06-11 19:48:10 +03:00
|
|
|
plog.Noticef("found peer %s in the cluster", path.Base(resp.Node.Key))
|
2014-09-21 22:26:59 +04:00
|
|
|
all = append(all, resp.Node)
|
|
|
|
}
|
2015-06-11 19:48:10 +03:00
|
|
|
plog.Noticef("found %d needed peer(s)", len(all))
|
2014-09-21 22:26:59 +04:00
|
|
|
return all, nil
|
|
|
|
}
|
|
|
|
|
2014-09-22 22:13:49 +04:00
|
|
|
func (d *discovery) selfKey() string {
|
2014-10-31 03:31:14 +03:00
|
|
|
return path.Join("/", d.cluster, d.id.String())
|
2014-09-22 22:13:49 +04:00
|
|
|
}
|
|
|
|
|
2015-07-03 04:16:49 +03:00
|
|
|
func nodesToCluster(ns []*client.Node, size int) (string, error) {
|
2014-09-21 22:26:59 +04:00
|
|
|
s := make([]string, len(ns))
|
|
|
|
for i, n := range ns {
|
|
|
|
s[i] = n.Value
|
|
|
|
}
|
2015-07-03 04:16:49 +03:00
|
|
|
us := strings.Join(s, ",")
|
|
|
|
m, err := types.NewURLsMap(us)
|
|
|
|
if err != nil {
|
|
|
|
return us, ErrInvalidURL
|
|
|
|
}
|
|
|
|
if m.Len() != size {
|
|
|
|
return us, ErrDuplicateName
|
|
|
|
}
|
|
|
|
return us, nil
|
2014-09-21 22:26:59 +04:00
|
|
|
}
|
|
|
|
|
2015-01-27 22:47:08 +03:00
|
|
|
type sortableNodes struct{ Nodes []*client.Node }
|
2014-09-21 22:26:59 +04:00
|
|
|
|
2014-09-23 02:38:19 +04:00
|
|
|
func (ns sortableNodes) Len() int { return len(ns.Nodes) }
|
|
|
|
func (ns sortableNodes) Less(i, j int) bool {
|
2014-09-21 22:26:59 +04:00
|
|
|
return ns.Nodes[i].CreatedIndex < ns.Nodes[j].CreatedIndex
|
|
|
|
}
|
2014-09-23 02:38:19 +04:00
|
|
|
func (ns sortableNodes) Swap(i, j int) { ns.Nodes[i], ns.Nodes[j] = ns.Nodes[j], ns.Nodes[i] }
|