2017-08-16 23:28:55 +03:00
|
|
|
// Copyright 2017 The etcd Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package etcdserver
|
|
|
|
|
|
|
|
import (
|
2017-09-07 00:57:25 +03:00
|
|
|
"context"
|
2017-11-23 01:57:07 +03:00
|
|
|
"fmt"
|
2017-08-16 23:28:55 +03:00
|
|
|
"time"
|
|
|
|
|
2019-04-27 00:17:16 +03:00
|
|
|
"go.etcd.io/etcd/v3/clientv3"
|
|
|
|
"go.etcd.io/etcd/v3/etcdserver/api/v3rpc/rpctypes"
|
|
|
|
pb "go.etcd.io/etcd/v3/etcdserver/etcdserverpb"
|
|
|
|
"go.etcd.io/etcd/v3/mvcc"
|
|
|
|
"go.etcd.io/etcd/v3/pkg/types"
|
2018-04-16 08:40:44 +03:00
|
|
|
|
|
|
|
"go.uber.org/zap"
|
2017-08-16 23:28:55 +03:00
|
|
|
)
|
|
|
|
|
2017-11-23 01:57:07 +03:00
|
|
|
// CheckInitialHashKV compares initial hash values with its peers
|
|
|
|
// before serving any peer/client traffic. Only mismatch when hashes
|
|
|
|
// are different at requested revision, with same compact revision.
|
|
|
|
func (s *EtcdServer) CheckInitialHashKV() error {
|
|
|
|
if !s.Cfg.InitialCorruptCheck {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-04-16 08:40:44 +03:00
|
|
|
lg := s.getLogger()
|
|
|
|
|
|
|
|
if lg != nil {
|
|
|
|
lg.Info(
|
|
|
|
"starting initial corruption check",
|
|
|
|
zap.String("local-member-id", s.ID().String()),
|
|
|
|
zap.Duration("timeout", s.Cfg.ReqTimeout()),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Infof("%s starting initial corruption check with timeout %v...", s.ID(), s.Cfg.ReqTimeout())
|
|
|
|
}
|
|
|
|
|
2017-11-23 01:57:07 +03:00
|
|
|
h, rev, crev, err := s.kv.HashByRev(0)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("%s failed to fetch hash (%v)", s.ID(), err)
|
|
|
|
}
|
|
|
|
peers := s.getPeerHashKVs(rev)
|
|
|
|
mismatch := 0
|
|
|
|
for _, p := range peers {
|
|
|
|
if p.resp != nil {
|
|
|
|
peerID := types.ID(p.resp.Header.MemberId)
|
2018-05-02 23:26:39 +03:00
|
|
|
fields := []zap.Field{
|
2018-04-16 08:40:44 +03:00
|
|
|
zap.String("local-member-id", s.ID().String()),
|
|
|
|
zap.Int64("local-member-revision", rev),
|
|
|
|
zap.Int64("local-member-compact-revision", crev),
|
|
|
|
zap.Uint32("local-member-hash", h),
|
2018-04-18 22:54:43 +03:00
|
|
|
zap.String("remote-peer-id", peerID.String()),
|
|
|
|
zap.Strings("remote-peer-endpoints", p.eps),
|
|
|
|
zap.Int64("remote-peer-revision", p.resp.Header.Revision),
|
|
|
|
zap.Int64("remote-peer-compact-revision", p.resp.CompactRevision),
|
|
|
|
zap.Uint32("remote-peer-hash", p.resp.Hash),
|
2018-04-16 08:40:44 +03:00
|
|
|
}
|
|
|
|
|
2017-11-23 01:57:07 +03:00
|
|
|
if h != p.resp.Hash {
|
|
|
|
if crev == p.resp.CompactRevision {
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn("found different hash values from remote peer", fields...)
|
|
|
|
} else {
|
|
|
|
plog.Errorf("%s's hash %d != %s's hash %d (revision %d, peer revision %d, compact revision %d)", s.ID(), h, peerID, p.resp.Hash, rev, p.resp.Header.Revision, crev)
|
|
|
|
}
|
2017-11-23 01:57:07 +03:00
|
|
|
mismatch++
|
|
|
|
} else {
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn("found different compact revision values from remote peer", fields...)
|
|
|
|
} else {
|
|
|
|
plog.Warningf("%s cannot check hash of peer(%s): peer has a different compact revision %d (revision:%d)", s.ID(), peerID, p.resp.CompactRevision, rev)
|
|
|
|
}
|
2017-11-23 01:57:07 +03:00
|
|
|
}
|
|
|
|
}
|
2018-04-16 08:40:44 +03:00
|
|
|
|
2017-11-23 01:57:07 +03:00
|
|
|
continue
|
|
|
|
}
|
2018-04-16 08:40:44 +03:00
|
|
|
|
2017-11-23 01:57:07 +03:00
|
|
|
if p.err != nil {
|
|
|
|
switch p.err {
|
|
|
|
case rpctypes.ErrFutureRev:
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn(
|
|
|
|
"cannot fetch hash from slow remote peer",
|
|
|
|
zap.String("local-member-id", s.ID().String()),
|
|
|
|
zap.Int64("local-member-revision", rev),
|
|
|
|
zap.Int64("local-member-compact-revision", crev),
|
|
|
|
zap.Uint32("local-member-hash", h),
|
2018-04-18 22:54:43 +03:00
|
|
|
zap.String("remote-peer-id", p.id.String()),
|
|
|
|
zap.Strings("remote-peer-endpoints", p.eps),
|
2018-04-16 08:40:44 +03:00
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Warningf("%s cannot check the hash of peer(%q) at revision %d: peer is lagging behind(%q)", s.ID(), p.eps, rev, p.err.Error())
|
|
|
|
}
|
2017-11-23 01:57:07 +03:00
|
|
|
case rpctypes.ErrCompacted:
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn(
|
|
|
|
"cannot fetch hash from remote peer; local member is behind",
|
|
|
|
zap.String("local-member-id", s.ID().String()),
|
|
|
|
zap.Int64("local-member-revision", rev),
|
|
|
|
zap.Int64("local-member-compact-revision", crev),
|
|
|
|
zap.Uint32("local-member-hash", h),
|
2018-04-18 22:54:43 +03:00
|
|
|
zap.String("remote-peer-id", p.id.String()),
|
|
|
|
zap.Strings("remote-peer-endpoints", p.eps),
|
2018-04-16 08:40:44 +03:00
|
|
|
zap.Error(err),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Warningf("%s cannot check the hash of peer(%q) at revision %d: local node is lagging behind(%q)", s.ID(), p.eps, rev, p.err.Error())
|
|
|
|
}
|
2017-11-23 01:57:07 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if mismatch > 0 {
|
|
|
|
return fmt.Errorf("%s found data inconsistency with peers", s.ID())
|
|
|
|
}
|
|
|
|
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Info(
|
|
|
|
"initial corruption checking passed; no corruption",
|
|
|
|
zap.String("local-member-id", s.ID().String()),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Infof("%s succeeded on initial corruption checking: no corruption", s.ID())
|
|
|
|
}
|
2017-11-23 01:57:07 +03:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-08-16 23:28:55 +03:00
|
|
|
func (s *EtcdServer) monitorKVHash() {
|
|
|
|
t := s.Cfg.CorruptCheckTime
|
|
|
|
if t == 0 {
|
|
|
|
return
|
|
|
|
}
|
2018-04-16 08:40:44 +03:00
|
|
|
|
|
|
|
lg := s.getLogger()
|
|
|
|
if lg != nil {
|
|
|
|
lg.Info(
|
|
|
|
"enabled corruption checking",
|
|
|
|
zap.String("local-member-id", s.ID().String()),
|
|
|
|
zap.Duration("interval", t),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Infof("enabled corruption checking with %s interval", t)
|
|
|
|
}
|
|
|
|
|
2017-08-16 23:28:55 +03:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-s.stopping:
|
|
|
|
return
|
|
|
|
case <-time.After(t):
|
|
|
|
}
|
|
|
|
if !s.isLeader() {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if err := s.checkHashKV(); err != nil {
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn("failed to check hash KV", zap.Error(err))
|
|
|
|
} else {
|
|
|
|
plog.Debugf("check hash kv failed %v", err)
|
|
|
|
}
|
2017-08-16 23:28:55 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *EtcdServer) checkHashKV() error {
|
2018-04-16 08:40:44 +03:00
|
|
|
lg := s.getLogger()
|
|
|
|
|
2017-08-16 23:28:55 +03:00
|
|
|
h, rev, crev, err := s.kv.HashByRev(0)
|
|
|
|
if err != nil {
|
2018-04-16 08:40:44 +03:00
|
|
|
return err
|
2017-08-16 23:28:55 +03:00
|
|
|
}
|
2017-11-23 01:57:07 +03:00
|
|
|
peers := s.getPeerHashKVs(rev)
|
2017-08-16 23:28:55 +03:00
|
|
|
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), s.Cfg.ReqTimeout())
|
|
|
|
err = s.linearizableReadNotify(ctx)
|
|
|
|
cancel()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
h2, rev2, crev2, err := s.kv.HashByRev(0)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
alarmed := false
|
|
|
|
mismatch := func(id uint64) {
|
|
|
|
if alarmed {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
alarmed = true
|
|
|
|
a := &pb.AlarmRequest{
|
2018-05-01 00:03:20 +03:00
|
|
|
MemberID: id,
|
2017-08-16 23:28:55 +03:00
|
|
|
Action: pb.AlarmRequest_ACTIVATE,
|
|
|
|
Alarm: pb.AlarmType_CORRUPT,
|
|
|
|
}
|
|
|
|
s.goAttach(func() {
|
|
|
|
s.raftRequest(s.ctx, pb.InternalRaftRequest{Alarm: a})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
if h2 != h && rev2 == rev && crev == crev2 {
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn(
|
|
|
|
"found hash mismatch",
|
|
|
|
zap.Int64("revision-1", rev),
|
|
|
|
zap.Int64("compact-revision-1", crev),
|
|
|
|
zap.Uint32("hash-1", h),
|
|
|
|
zap.Int64("revision-2", rev2),
|
|
|
|
zap.Int64("compact-revision-2", crev2),
|
|
|
|
zap.Uint32("hash-2", h2),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Warningf("mismatched hashes %d and %d for revision %d", h, h2, rev)
|
|
|
|
}
|
2017-08-16 23:28:55 +03:00
|
|
|
mismatch(uint64(s.ID()))
|
|
|
|
}
|
|
|
|
|
2017-11-23 01:57:07 +03:00
|
|
|
for _, p := range peers {
|
|
|
|
if p.resp == nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
id := p.resp.Header.MemberId
|
2017-11-20 20:34:03 +03:00
|
|
|
|
|
|
|
// leader expects follower's latest revision less than or equal to leader's
|
2017-11-23 01:57:07 +03:00
|
|
|
if p.resp.Header.Revision > rev2 {
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn(
|
|
|
|
"revision from follower must be less than or equal to leader's",
|
|
|
|
zap.Int64("leader-revision", rev2),
|
|
|
|
zap.Int64("follower-revision", p.resp.Header.Revision),
|
|
|
|
zap.String("follower-peer-id", types.ID(id).String()),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Warningf(
|
|
|
|
"revision %d from member %v, expected at most %d",
|
|
|
|
p.resp.Header.Revision,
|
|
|
|
types.ID(id),
|
|
|
|
rev2)
|
|
|
|
}
|
2017-08-16 23:28:55 +03:00
|
|
|
mismatch(id)
|
|
|
|
}
|
2017-11-20 20:34:03 +03:00
|
|
|
|
|
|
|
// leader expects follower's latest compact revision less than or equal to leader's
|
2017-11-23 01:57:07 +03:00
|
|
|
if p.resp.CompactRevision > crev2 {
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn(
|
|
|
|
"compact revision from follower must be less than or equal to leader's",
|
|
|
|
zap.Int64("leader-compact-revision", crev2),
|
|
|
|
zap.Int64("follower-compact-revision", p.resp.CompactRevision),
|
|
|
|
zap.String("follower-peer-id", types.ID(id).String()),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Warningf(
|
|
|
|
"compact revision %d from member %v, expected at most %d",
|
|
|
|
p.resp.CompactRevision,
|
|
|
|
types.ID(id),
|
|
|
|
crev2,
|
|
|
|
)
|
|
|
|
}
|
2017-08-16 23:28:55 +03:00
|
|
|
mismatch(id)
|
|
|
|
}
|
2017-11-20 20:34:03 +03:00
|
|
|
|
|
|
|
// follower's compact revision is leader's old one, then hashes must match
|
2017-11-23 01:57:07 +03:00
|
|
|
if p.resp.CompactRevision == crev && p.resp.Hash != h {
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn(
|
|
|
|
"same compact revision then hashes must match",
|
|
|
|
zap.Int64("leader-compact-revision", crev2),
|
|
|
|
zap.Uint32("leader-hash", h),
|
|
|
|
zap.Int64("follower-compact-revision", p.resp.CompactRevision),
|
|
|
|
zap.Uint32("follower-hash", p.resp.Hash),
|
|
|
|
zap.String("follower-peer-id", types.ID(id).String()),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Warningf(
|
|
|
|
"hash %d at revision %d from member %v, expected hash %d",
|
|
|
|
p.resp.Hash,
|
|
|
|
rev,
|
|
|
|
types.ID(id),
|
|
|
|
h,
|
|
|
|
)
|
|
|
|
}
|
2017-08-16 23:28:55 +03:00
|
|
|
mismatch(id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-11-23 01:57:07 +03:00
|
|
|
type peerHashKVResp struct {
|
2018-04-16 08:40:44 +03:00
|
|
|
id types.ID
|
|
|
|
eps []string
|
|
|
|
|
2017-11-23 01:57:07 +03:00
|
|
|
resp *clientv3.HashKVResponse
|
|
|
|
err error
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *EtcdServer) getPeerHashKVs(rev int64) (resps []*peerHashKVResp) {
|
|
|
|
// TODO: handle the case when "s.cluster.Members" have not
|
|
|
|
// been populated (e.g. no snapshot to load from disk)
|
|
|
|
mbs := s.cluster.Members()
|
2018-04-16 08:40:44 +03:00
|
|
|
pss := make([]peerHashKVResp, len(mbs))
|
2017-11-23 01:57:07 +03:00
|
|
|
for _, m := range mbs {
|
2017-11-20 20:33:19 +03:00
|
|
|
if m.ID == s.ID() {
|
|
|
|
continue
|
|
|
|
}
|
2018-04-16 08:40:44 +03:00
|
|
|
pss = append(pss, peerHashKVResp{id: m.ID, eps: m.PeerURLs})
|
2017-11-23 01:57:07 +03:00
|
|
|
}
|
2017-11-20 20:33:19 +03:00
|
|
|
|
2018-04-16 08:40:44 +03:00
|
|
|
lg := s.getLogger()
|
|
|
|
|
|
|
|
for _, p := range pss {
|
|
|
|
if len(p.eps) == 0 {
|
2017-11-23 01:57:07 +03:00
|
|
|
continue
|
|
|
|
}
|
2017-11-20 20:33:19 +03:00
|
|
|
cli, cerr := clientv3.New(clientv3.Config{
|
|
|
|
DialTimeout: s.Cfg.ReqTimeout(),
|
2018-04-16 08:40:44 +03:00
|
|
|
Endpoints: p.eps,
|
2017-11-20 20:33:19 +03:00
|
|
|
})
|
|
|
|
if cerr != nil {
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn(
|
|
|
|
"failed to create client to peer URL",
|
|
|
|
zap.String("local-member-id", s.ID().String()),
|
2018-04-18 22:54:43 +03:00
|
|
|
zap.String("remote-peer-id", p.id.String()),
|
|
|
|
zap.Strings("remote-peer-endpoints", p.eps),
|
2018-04-16 08:40:44 +03:00
|
|
|
zap.Error(cerr),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Warningf("%s failed to create client to peer %q for hash checking (%q)", s.ID(), p.eps, cerr.Error())
|
|
|
|
}
|
2017-11-20 20:33:19 +03:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
respsLen := len(resps)
|
|
|
|
for _, c := range cli.Endpoints() {
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), s.Cfg.ReqTimeout())
|
2017-11-23 01:57:07 +03:00
|
|
|
var resp *clientv3.HashKVResponse
|
|
|
|
resp, cerr = cli.HashKV(ctx, c, rev)
|
2017-11-20 20:33:19 +03:00
|
|
|
cancel()
|
2017-11-23 01:57:07 +03:00
|
|
|
if cerr == nil {
|
2018-04-16 08:40:44 +03:00
|
|
|
resps = append(resps, &peerHashKVResp{id: p.id, eps: p.eps, resp: resp, err: nil})
|
2017-11-20 20:33:19 +03:00
|
|
|
break
|
|
|
|
}
|
2018-04-16 08:40:44 +03:00
|
|
|
if lg != nil {
|
|
|
|
lg.Warn(
|
|
|
|
"failed hash kv request",
|
|
|
|
zap.String("local-member-id", s.ID().String()),
|
|
|
|
zap.Int64("requested-revision", rev),
|
2018-04-18 22:54:43 +03:00
|
|
|
zap.String("remote-peer-endpoint", c),
|
2018-04-16 08:40:44 +03:00
|
|
|
zap.Error(cerr),
|
|
|
|
)
|
|
|
|
} else {
|
|
|
|
plog.Warningf("%s hash-kv error %q on peer %q with revision %d", s.ID(), cerr.Error(), c, rev)
|
|
|
|
}
|
2017-11-20 20:33:19 +03:00
|
|
|
}
|
|
|
|
cli.Close()
|
|
|
|
|
|
|
|
if respsLen == len(resps) {
|
2018-04-16 08:40:44 +03:00
|
|
|
resps = append(resps, &peerHashKVResp{id: p.id, eps: p.eps, resp: nil, err: cerr})
|
2017-11-20 20:33:19 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return resps
|
|
|
|
}
|
|
|
|
|
2017-08-16 23:28:55 +03:00
|
|
|
type applierV3Corrupt struct {
|
|
|
|
applierV3
|
|
|
|
}
|
|
|
|
|
|
|
|
func newApplierV3Corrupt(a applierV3) *applierV3Corrupt { return &applierV3Corrupt{a} }
|
|
|
|
|
|
|
|
func (a *applierV3Corrupt) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error) {
|
|
|
|
return nil, ErrCorrupt
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *applierV3Corrupt) Range(txn mvcc.TxnRead, p *pb.RangeRequest) (*pb.RangeResponse, error) {
|
|
|
|
return nil, ErrCorrupt
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *applierV3Corrupt) DeleteRange(txn mvcc.TxnWrite, p *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) {
|
|
|
|
return nil, ErrCorrupt
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *applierV3Corrupt) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) {
|
|
|
|
return nil, ErrCorrupt
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *applierV3Corrupt) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, error) {
|
|
|
|
return nil, nil, ErrCorrupt
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *applierV3Corrupt) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) {
|
|
|
|
return nil, ErrCorrupt
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *applierV3Corrupt) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) {
|
|
|
|
return nil, ErrCorrupt
|
|
|
|
}
|