2018-03-28 03:01:18 +03:00
|
|
|
// Copyright 2018 The etcd Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package rpcpb
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2018-04-12 20:39:18 +03:00
|
|
|
"crypto/tls"
|
2018-03-28 03:01:18 +03:00
|
|
|
"fmt"
|
2018-04-06 01:08:05 +03:00
|
|
|
"net/url"
|
2018-04-11 23:06:45 +03:00
|
|
|
"os"
|
2018-03-28 03:01:18 +03:00
|
|
|
"time"
|
|
|
|
|
2018-08-29 03:13:25 +03:00
|
|
|
"go.etcd.io/etcd/clientv3"
|
|
|
|
"go.etcd.io/etcd/clientv3/snapshot"
|
|
|
|
pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
|
|
|
|
"go.etcd.io/etcd/pkg/transport"
|
2018-03-28 03:01:18 +03:00
|
|
|
|
2018-04-11 23:06:45 +03:00
|
|
|
"github.com/dustin/go-humanize"
|
|
|
|
"go.uber.org/zap"
|
2018-03-28 03:01:18 +03:00
|
|
|
grpc "google.golang.org/grpc"
|
2018-04-06 01:08:05 +03:00
|
|
|
"google.golang.org/grpc/credentials"
|
2018-03-28 03:01:18 +03:00
|
|
|
)
|
|
|
|
|
2018-04-12 03:13:18 +03:00
|
|
|
// ElectionTimeout returns an election timeout duration.
|
|
|
|
func (m *Member) ElectionTimeout() time.Duration {
|
|
|
|
return time.Duration(m.Etcd.ElectionTimeoutMs) * time.Millisecond
|
|
|
|
}
|
|
|
|
|
2018-03-28 03:01:18 +03:00
|
|
|
// DialEtcdGRPCServer creates a raw gRPC connection to an etcd member.
|
2018-04-03 20:55:47 +03:00
|
|
|
func (m *Member) DialEtcdGRPCServer(opts ...grpc.DialOption) (*grpc.ClientConn, error) {
|
2018-04-06 01:08:05 +03:00
|
|
|
dialOpts := []grpc.DialOption{
|
|
|
|
grpc.WithTimeout(5 * time.Second),
|
|
|
|
grpc.WithBlock(),
|
|
|
|
}
|
|
|
|
|
|
|
|
secure := false
|
|
|
|
for _, cu := range m.Etcd.AdvertiseClientURLs {
|
|
|
|
u, err := url.Parse(cu)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if u.Scheme == "https" { // TODO: handle unix
|
|
|
|
secure = true
|
|
|
|
}
|
2018-03-28 03:01:18 +03:00
|
|
|
}
|
2018-04-06 01:08:05 +03:00
|
|
|
|
|
|
|
if secure {
|
|
|
|
// assume save TLS assets are already stord on disk
|
|
|
|
tlsInfo := transport.TLSInfo{
|
|
|
|
CertFile: m.ClientCertPath,
|
|
|
|
KeyFile: m.ClientKeyPath,
|
|
|
|
TrustedCAFile: m.ClientTrustedCAPath,
|
2018-04-06 01:49:50 +03:00
|
|
|
|
|
|
|
// TODO: remove this with generated certs
|
|
|
|
// only need it for auto TLS
|
|
|
|
InsecureSkipVerify: true,
|
2018-04-06 01:08:05 +03:00
|
|
|
}
|
|
|
|
tlsConfig, err := tlsInfo.ClientConfig()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
creds := credentials.NewTLS(tlsConfig)
|
|
|
|
dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds))
|
|
|
|
} else {
|
|
|
|
dialOpts = append(dialOpts, grpc.WithInsecure())
|
|
|
|
}
|
|
|
|
dialOpts = append(dialOpts, opts...)
|
|
|
|
return grpc.Dial(m.EtcdClientEndpoint, dialOpts...)
|
2018-03-28 03:01:18 +03:00
|
|
|
}
|
|
|
|
|
2018-04-12 20:39:18 +03:00
|
|
|
// CreateEtcdClientConfig creates a client configuration from member.
|
|
|
|
func (m *Member) CreateEtcdClientConfig(opts ...grpc.DialOption) (cfg *clientv3.Config, err error) {
|
2018-04-06 01:08:05 +03:00
|
|
|
secure := false
|
|
|
|
for _, cu := range m.Etcd.AdvertiseClientURLs {
|
2018-04-12 20:39:18 +03:00
|
|
|
var u *url.URL
|
|
|
|
u, err = url.Parse(cu)
|
2018-04-06 01:08:05 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if u.Scheme == "https" { // TODO: handle unix
|
|
|
|
secure = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-12 20:39:18 +03:00
|
|
|
cfg = &clientv3.Config{
|
2018-04-03 21:40:36 +03:00
|
|
|
Endpoints: []string{m.EtcdClientEndpoint},
|
2018-04-12 06:18:17 +03:00
|
|
|
DialTimeout: 10 * time.Second,
|
2018-04-03 21:40:36 +03:00
|
|
|
DialOptions: opts,
|
|
|
|
}
|
2018-04-06 01:08:05 +03:00
|
|
|
if secure {
|
|
|
|
// assume save TLS assets are already stord on disk
|
|
|
|
tlsInfo := transport.TLSInfo{
|
|
|
|
CertFile: m.ClientCertPath,
|
|
|
|
KeyFile: m.ClientKeyPath,
|
|
|
|
TrustedCAFile: m.ClientTrustedCAPath,
|
2018-04-06 01:49:50 +03:00
|
|
|
|
|
|
|
// TODO: remove this with generated certs
|
|
|
|
// only need it for auto TLS
|
|
|
|
InsecureSkipVerify: true,
|
2018-04-06 01:08:05 +03:00
|
|
|
}
|
2018-04-12 20:39:18 +03:00
|
|
|
var tlsConfig *tls.Config
|
|
|
|
tlsConfig, err = tlsInfo.ClientConfig()
|
2018-04-06 01:08:05 +03:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
cfg.TLS = tlsConfig
|
2018-03-28 03:01:18 +03:00
|
|
|
}
|
2018-04-12 20:39:18 +03:00
|
|
|
return cfg, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// CreateEtcdClient creates a client from member.
|
|
|
|
func (m *Member) CreateEtcdClient(opts ...grpc.DialOption) (*clientv3.Client, error) {
|
|
|
|
cfg, err := m.CreateEtcdClientConfig(opts...)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return clientv3.New(*cfg)
|
2018-03-28 03:01:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// CheckCompact ensures that historical data before given revision has been compacted.
|
|
|
|
func (m *Member) CheckCompact(rev int64) error {
|
|
|
|
cli, err := m.CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
defer cli.Close()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
|
|
wch := cli.Watch(ctx, "\x00", clientv3.WithFromKey(), clientv3.WithRev(rev-1))
|
|
|
|
wr, ok := <-wch
|
|
|
|
cancel()
|
|
|
|
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("watch channel terminated (endpoint %q)", m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
if wr.CompactRevision != rev {
|
|
|
|
return fmt.Errorf("got compact revision %v, wanted %v (endpoint %q)", wr.CompactRevision, rev, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Defrag runs defragmentation on this member.
|
|
|
|
func (m *Member) Defrag() error {
|
|
|
|
cli, err := m.CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
defer cli.Close()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
|
|
_, err = cli.Defragment(ctx, m.EtcdClientEndpoint)
|
|
|
|
cancel()
|
2018-03-30 21:46:03 +03:00
|
|
|
return err
|
2018-03-28 03:01:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// RevHash fetches current revision and hash on this member.
|
|
|
|
func (m *Member) RevHash() (int64, int64, error) {
|
|
|
|
conn, err := m.DialEtcdGRPCServer()
|
|
|
|
if err != nil {
|
|
|
|
return 0, 0, err
|
|
|
|
}
|
|
|
|
defer conn.Close()
|
|
|
|
|
|
|
|
mt := pb.NewMaintenanceClient(conn)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
|
|
resp, err := mt.Hash(ctx, &pb.HashRequest{}, grpc.FailFast(false))
|
|
|
|
cancel()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return 0, 0, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return resp.Header.Revision, int64(resp.Hash), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Rev fetches current revision on this member.
|
|
|
|
func (m *Member) Rev(ctx context.Context) (int64, error) {
|
|
|
|
cli, err := m.CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return 0, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
defer cli.Close()
|
|
|
|
|
|
|
|
resp, err := cli.Status(ctx, m.EtcdClientEndpoint)
|
|
|
|
if err != nil {
|
|
|
|
return 0, err
|
|
|
|
}
|
|
|
|
return resp.Header.Revision, nil
|
|
|
|
}
|
|
|
|
|
2018-04-03 22:45:19 +03:00
|
|
|
// Compact compacts member storage with given revision.
|
|
|
|
// It blocks until it's physically done.
|
|
|
|
func (m *Member) Compact(rev int64, timeout time.Duration) error {
|
|
|
|
cli, err := m.CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
defer cli.Close()
|
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
|
|
_, err = cli.Compact(ctx, rev, clientv3.WithCompactPhysical())
|
|
|
|
cancel()
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-03-28 03:01:18 +03:00
|
|
|
// IsLeader returns true if this member is the current cluster leader.
|
|
|
|
func (m *Member) IsLeader() (bool, error) {
|
|
|
|
cli, err := m.CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return false, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
defer cli.Close()
|
|
|
|
|
|
|
|
resp, err := cli.Status(context.Background(), m.EtcdClientEndpoint)
|
|
|
|
if err != nil {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
return resp.Header.MemberId == resp.Leader, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// WriteHealthKey writes a health key to this member.
|
|
|
|
func (m *Member) WriteHealthKey() error {
|
|
|
|
cli, err := m.CreateEtcdClient()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
defer cli.Close()
|
|
|
|
|
|
|
|
// give enough time-out in case expensive requests (range/delete) are pending
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
|
|
_, err = cli.Put(ctx, "health", "good")
|
|
|
|
cancel()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2018-04-11 21:28:26 +03:00
|
|
|
|
2018-04-12 01:44:27 +03:00
|
|
|
// SaveSnapshot downloads a snapshot file from this member, locally.
|
|
|
|
// It's meant to requested remotely, so that local member can store
|
|
|
|
// snapshot file on local disk.
|
|
|
|
func (m *Member) SaveSnapshot(lg *zap.Logger) (err error) {
|
2018-04-11 23:06:45 +03:00
|
|
|
// remove existing snapshot first
|
2018-04-12 02:09:22 +03:00
|
|
|
if err = os.RemoveAll(m.SnapshotPath); err != nil {
|
2018-04-11 23:06:45 +03:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-04-12 20:39:18 +03:00
|
|
|
var ccfg *clientv3.Config
|
|
|
|
ccfg, err = m.CreateEtcdClientConfig()
|
2018-04-11 23:06:45 +03:00
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
|
|
|
|
}
|
|
|
|
|
2018-04-12 01:44:27 +03:00
|
|
|
lg.Info(
|
|
|
|
"snapshot save START",
|
|
|
|
zap.String("member-name", m.Etcd.Name),
|
|
|
|
zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs),
|
|
|
|
zap.String("snapshot-path", m.SnapshotPath),
|
|
|
|
)
|
2018-04-11 23:06:45 +03:00
|
|
|
now := time.Now()
|
2018-04-12 20:39:18 +03:00
|
|
|
mgr := snapshot.NewV3(lg)
|
|
|
|
if err = mgr.Save(context.Background(), *ccfg, m.SnapshotPath); err != nil {
|
2018-04-11 23:06:45 +03:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
took := time.Since(now)
|
|
|
|
|
|
|
|
var fi os.FileInfo
|
|
|
|
fi, err = os.Stat(m.SnapshotPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
var st snapshot.Status
|
|
|
|
st, err = mgr.Status(m.SnapshotPath)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-04-12 00:08:46 +03:00
|
|
|
m.SnapshotInfo = &SnapshotInfo{
|
|
|
|
MemberName: m.Etcd.Name,
|
|
|
|
MemberClientURLs: m.Etcd.AdvertiseClientURLs,
|
|
|
|
SnapshotPath: m.SnapshotPath,
|
|
|
|
SnapshotFileSize: humanize.Bytes(uint64(fi.Size())),
|
|
|
|
SnapshotTotalSize: humanize.Bytes(uint64(st.TotalSize)),
|
|
|
|
SnapshotTotalKey: int64(st.TotalKey),
|
|
|
|
SnapshotHash: int64(st.Hash),
|
|
|
|
SnapshotRevision: st.Revision,
|
|
|
|
Took: fmt.Sprintf("%v", took),
|
|
|
|
}
|
2018-04-11 23:06:45 +03:00
|
|
|
lg.Info(
|
2018-04-12 01:44:27 +03:00
|
|
|
"snapshot save END",
|
2018-04-12 00:08:46 +03:00
|
|
|
zap.String("member-name", m.SnapshotInfo.MemberName),
|
|
|
|
zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs),
|
2018-04-11 23:06:45 +03:00
|
|
|
zap.String("snapshot-path", m.SnapshotPath),
|
2018-04-12 00:08:46 +03:00
|
|
|
zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize),
|
|
|
|
zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize),
|
|
|
|
zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey),
|
|
|
|
zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash),
|
|
|
|
zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision),
|
|
|
|
zap.String("took", m.SnapshotInfo.Took),
|
2018-04-11 23:06:45 +03:00
|
|
|
)
|
2018-04-11 21:28:26 +03:00
|
|
|
return nil
|
|
|
|
}
|
2018-04-12 01:44:27 +03:00
|
|
|
|
|
|
|
// RestoreSnapshot restores a cluster from a given snapshot file on disk.
|
|
|
|
// It's meant to requested remotely, so that local member can load the
|
|
|
|
// snapshot file from local disk.
|
|
|
|
func (m *Member) RestoreSnapshot(lg *zap.Logger) (err error) {
|
|
|
|
if err = os.RemoveAll(m.EtcdOnSnapshotRestore.DataDir); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err = os.RemoveAll(m.EtcdOnSnapshotRestore.WALDir); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
lg.Info(
|
|
|
|
"snapshot restore START",
|
|
|
|
zap.String("member-name", m.Etcd.Name),
|
|
|
|
zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs),
|
|
|
|
zap.String("snapshot-path", m.SnapshotPath),
|
|
|
|
)
|
|
|
|
now := time.Now()
|
2018-04-12 20:39:18 +03:00
|
|
|
mgr := snapshot.NewV3(lg)
|
|
|
|
err = mgr.Restore(snapshot.RestoreConfig{
|
|
|
|
SnapshotPath: m.SnapshotInfo.SnapshotPath,
|
2018-04-12 01:44:27 +03:00
|
|
|
Name: m.EtcdOnSnapshotRestore.Name,
|
|
|
|
OutputDataDir: m.EtcdOnSnapshotRestore.DataDir,
|
|
|
|
OutputWALDir: m.EtcdOnSnapshotRestore.WALDir,
|
2018-04-12 20:39:18 +03:00
|
|
|
PeerURLs: m.EtcdOnSnapshotRestore.AdvertisePeerURLs,
|
|
|
|
InitialCluster: m.EtcdOnSnapshotRestore.InitialCluster,
|
2018-04-12 01:44:27 +03:00
|
|
|
InitialClusterToken: m.EtcdOnSnapshotRestore.InitialClusterToken,
|
|
|
|
SkipHashCheck: false,
|
2018-04-12 20:39:18 +03:00
|
|
|
// TODO: set SkipHashCheck it true, to recover from existing db file
|
2018-04-12 01:44:27 +03:00
|
|
|
})
|
|
|
|
took := time.Since(now)
|
|
|
|
lg.Info(
|
|
|
|
"snapshot restore END",
|
|
|
|
zap.String("member-name", m.SnapshotInfo.MemberName),
|
|
|
|
zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs),
|
|
|
|
zap.String("snapshot-path", m.SnapshotPath),
|
|
|
|
zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize),
|
|
|
|
zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize),
|
|
|
|
zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey),
|
|
|
|
zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash),
|
|
|
|
zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision),
|
|
|
|
zap.String("took", took.String()),
|
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
return err
|
|
|
|
}
|