vitalif
/
etcd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #13687 from serathius/etcdctl 

Add downgrade commands
dependabot/go_modules/go.uber.org/atomic-1.10.0
Marek Siarkowicz 2022-02-22 17:12:23 +01:00 committed by GitHub
parent 509d8461a6 42faf9fe06
commit 6af760131e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 313 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
client/v3/maintenance.go
View File

@ -33,6 +33,14 @@ type (
HashKVResponse pb.HashKVResponse
MoveLeaderResponse pb.MoveLeaderResponse
DowngradeResponse pb.DowngradeResponse
DowngradeAction pb.DowngradeRequest_DowngradeAction
)
const (
DowngradeValidate = DowngradeAction(pb.DowngradeRequest_VALIDATE)
DowngradeEnable = DowngradeAction(pb.DowngradeRequest_ENABLE)
DowngradeCancel = DowngradeAction(pb.DowngradeRequest_CANCEL)
)
type Maintenance interface {
@ -76,12 +84,8 @@ type Maintenance interface {
// Downgrade requests downgrades, verifies feasibility or cancels downgrade
// on the cluster version.
// action is one of the following:
// VALIDATE = 0;
// ENABLE = 1;
// CANCEL = 2;
// Supported since etcd 3.5.
Downgrade(ctx context.Context, action int32, version string) (*DowngradeResponse, error)
Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error)
}
// SnapshotResponse is aggregated response from the snapshot stream.
@ -337,14 +341,14 @@ func (m *maintenance) MoveLeader(ctx context.Context, transfereeID uint64) (*Mov
return (*MoveLeaderResponse)(resp), toErr(ctx, err)
}
func (m *maintenance) Downgrade(ctx context.Context, action int32, version string) (*DowngradeResponse, error) {
actionType := pb.DowngradeRequest_VALIDATE
func (m *maintenance) Downgrade(ctx context.Context, action DowngradeAction, version string) (*DowngradeResponse, error) {
var actionType pb.DowngradeRequest_DowngradeAction
switch action {
case 0:
case DowngradeValidate:
actionType = pb.DowngradeRequest_VALIDATE
case 1:
case DowngradeEnable:
actionType = pb.DowngradeRequest_ENABLE
case 2:
case DowngradeCancel:
actionType = pb.DowngradeRequest_CANCEL
default:
return nil, errors.New("etcdclient: unknown downgrade action")

63
etcdctl/README.md
View File

@ -1083,6 +1083,69 @@ echo ${transferee_id}
# Leadership transferred from 45ddc0e800e20b93 to c89feb932daef420
```
### DOWNGRADE \<subcommand\>
NOTICE: Downgrades is an experimental feature in v3.6 and is not recommended for production clusters.
Downgrade provides commands to downgrade cluster.
Normally etcd members cannot be downgraded due to cluster version mechanism.
After initial bootstrap, cluster members agree on the cluster version. Every 5 seconds, leader checks versions of all members and picks lowers minor version.
New members will refuse joining cluster with cluster version newer than theirs, thus preventing cluster from downgrading.
Downgrade commands allow cluster administrator to force cluster version to be lowered to previous minor version, thus allowing to downgrade the cluster.
Downgrade should be is executed in stages:
1. Verify that cluster is ready be downgraded by running `etcdctl downgrade validate <TARGET_VERSION>`
2. Start the downgrade process by running `etcdctl downgrade enable <TARGET_VERSION>`
3. For each cluster member:
1. Ensure that member is ready for downgrade by confirming that it wrote `The server is ready to downgrade` log.
2. Replace member binary with one with older version.
3. Confirm that member has correctly started and joined the cluster.
4. Ensure that downgrade process has succeeded by checking leader log for `the cluster has been downgraded`
Downgrade can be canceled by running `etcdctl downgrade cancel` command.
In case of downgrade being canceled, cluster version will return to its normal behavior (pick the lowest member minor version).
If no members were downgraded, cluster version will return to original value.
If at least one member was downgraded, cluster version will stay at the `<TARGET_VALUE>` until downgraded members are upgraded back.
### DOWNGRADE VALIDATE \<TARGET_VERSION\>
DOWNGRADE VALIDATE validate downgrade capability before starting downgrade.
#### Example
```bash
./etcdctl downgrade validate 3.5
Downgrade validate success, cluster version 3.6
./etcdctl downgrade validate 3.4
Error: etcdserver: invalid downgrade target version
```
### DOWNGRADE ENABLE \<TARGET_VERSION\>
DOWNGRADE ENABLE starts a downgrade action to cluster.
#### Example
```bash
./etcdctl downgrade enable 3.5
Downgrade enable success, cluster version 3.6
```
### DOWNGRADE CANCEL \<TARGET_VERSION\>
DOWNGRADE CANCEL cancels the ongoing downgrade action to cluster.
#### Example
```bash
./etcdctl downgrade cancel
Downgrade cancel success, cluster version 3.5
```
## Concurrency commands
### LOCK [options] \<lockname\> [command arg1 arg2 ...]

136
etcdctl/ctlv3/command/downgrade_command.go Normal file
View File

@ -0,0 +1,136 @@
// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package command
import (
"errors"
"github.com/spf13/cobra"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/pkg/v3/cobrautl"
)
// NewDowngradeCommand returns the cobra command for "downgrade".
func NewDowngradeCommand() *cobra.Command {
dc := &cobra.Command{
Use: "downgrade <TARGET_VERSION>",
Short: "Downgrade related commands",
}
dc.AddCommand(NewDowngradeValidateCommand())
dc.AddCommand(NewDowngradeEnableCommand())
dc.AddCommand(NewDowngradeCancelCommand())
return dc
}
// NewDowngradeValidateCommand returns the cobra command for "downgrade validate".
func NewDowngradeValidateCommand() *cobra.Command {
cc := &cobra.Command{
Use: "validate <TARGET_VERSION>",
Short: "Validate downgrade capability before starting downgrade",
Run: downgradeValidateCommandFunc,
}
return cc
}
// NewDowngradeEnableCommand returns the cobra command for "downgrade enable".
func NewDowngradeEnableCommand() *cobra.Command {
cc := &cobra.Command{
Use: "enable <TARGET_VERSION>",
Short: "Start a downgrade action to cluster",
Run: downgradeEnableCommandFunc,
}
return cc
}
// NewDowngradeCancelCommand returns the cobra command for "downgrade cancel".
func NewDowngradeCancelCommand() *cobra.Command {
cc := &cobra.Command{
Use: "cancel",
Short: "Cancel the ongoing downgrade action to cluster",
Run: downgradeCancelCommandFunc,
}
return cc
}
// downgradeValidateCommandFunc executes the "downgrade validate" command.
func downgradeValidateCommandFunc(cmd *cobra.Command, args []string) {
if len(args) < 1 {
cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("TARGET_VERSION not provided"))
}
if len(args) > 1 {
cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("too many arguments"))
}
targetVersion := args[0]
if len(targetVersion) == 0 {
cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("target version not provided"))
}
ctx, cancel := commandCtx(cmd)
cli := mustClientFromCmd(cmd)
resp, err := cli.Downgrade(ctx, clientv3.DowngradeValidate, targetVersion)
cancel()
if err != nil {
cobrautl.ExitWithError(cobrautl.ExitError, err)
}
display.DowngradeValidate(*resp)
}
// downgradeEnableCommandFunc executes the "downgrade enable" command.
func downgradeEnableCommandFunc(cmd *cobra.Command, args []string) {
if len(args) < 1 {
cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("TARGET_VERSION not provided"))
}
if len(args) > 1 {
cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("too many arguments"))
}
targetVersion := args[0]
if len(targetVersion) == 0 {
cobrautl.ExitWithError(cobrautl.ExitBadArgs, errors.New("target version not provided"))
}
ctx, cancel := commandCtx(cmd)
cli := mustClientFromCmd(cmd)
resp, err := cli.Downgrade(ctx, clientv3.DowngradeEnable, targetVersion)
cancel()
if err != nil {
cobrautl.ExitWithError(cobrautl.ExitError, err)
}
display.DowngradeEnable(*resp)
}
// downgradeCancelCommandFunc executes the "downgrade cancel" command.
func downgradeCancelCommandFunc(cmd *cobra.Command, args []string) {
ctx, cancel := commandCtx(cmd)
cli := mustClientFromCmd(cmd)
resp, err := cli.Downgrade(ctx, clientv3.DowngradeCancel, "")
cancel()
if err != nil {
cobrautl.ExitWithError(cobrautl.ExitError, err)
}
display.DowngradeCancel(*resp)
}

10
etcdctl/ctlv3/command/printer.go
View File

@ -50,6 +50,10 @@ type printer interface {
EndpointHashKV([]epHashKV)
MoveLeader(leader, target uint64, r v3.MoveLeaderResponse)
DowngradeValidate(r v3.DowngradeResponse)
DowngradeEnable(r v3.DowngradeResponse)
DowngradeCancel(r v3.DowngradeResponse)
Alarm(v3.AlarmResponse)
RoleAdd(role string, r v3.AuthRoleAddResponse)
@ -118,6 +122,9 @@ func (p *printerRPC) Alarm(r v3.AlarmResponse) { p.p((*pb.AlarmRespons
func (p *printerRPC) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) {
p.p((*pb.MoveLeaderResponse)(&r))
}
func (p *printerRPC) DowngradeValidate(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) }
func (p *printerRPC) DowngradeEnable(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) }
func (p *printerRPC) DowngradeCancel(r v3.DowngradeResponse) { p.p((*pb.DowngradeResponse)(&r)) }
func (p *printerRPC) RoleAdd(_ string, r v3.AuthRoleAddResponse) { p.p((*pb.AuthRoleAddResponse)(&r)) }
func (p *printerRPC) RoleGet(_ string, r v3.AuthRoleGetResponse) { p.p((*pb.AuthRoleGetResponse)(&r)) }
func (p *printerRPC) RoleDelete(_ string, r v3.AuthRoleDeleteResponse) {
@ -163,6 +170,9 @@ func (p *printerUnsupported) EndpointStatus([]epStatus) { p.p(nil) }
func (p *printerUnsupported) EndpointHashKV([]epHashKV) { p.p(nil) }
func (p *printerUnsupported) MoveLeader(leader, target uint64, r v3.MoveLeaderResponse) { p.p(nil) }
func (p *printerUnsupported) DowngradeValidate(r v3.DowngradeResponse) { p.p(nil) }
func (p *printerUnsupported) DowngradeEnable(r v3.DowngradeResponse) { p.p(nil) }
func (p *printerUnsupported) DowngradeCancel(r v3.DowngradeResponse) { p.p(nil) }
func makeMemberListTable(r v3.MemberListResponse) (hdr []string, rows [][]string) {
hdr = []string{"ID", "Status", "Name", "Peer Addrs", "Client Addrs", "Is Learner"}

10
etcdctl/ctlv3/command/printer_simple.go
View File

@ -176,6 +176,16 @@ func (s *simplePrinter) MoveLeader(leader, target uint64, r v3.MoveLeaderRespons
fmt.Printf("Leadership transferred from %s to %s\n", types.ID(leader), types.ID(target))
}
func (s *simplePrinter) DowngradeValidate(r v3.DowngradeResponse) {
fmt.Printf("Downgrade validate success, cluster version %s\n", r.Version)
}
func (s *simplePrinter) DowngradeEnable(r v3.DowngradeResponse) {
fmt.Printf("Downgrade enable success, cluster version %s\n", r.Version)
}
func (s *simplePrinter) DowngradeCancel(r v3.DowngradeResponse) {
fmt.Printf("Downgrade cancel success, cluster version %s\n", r.Version)
}
func (s *simplePrinter) RoleAdd(role string, r v3.AuthRoleAddResponse) {
fmt.Printf("Role %s created\n", role)
}

1
etcdctl/ctlv3/ctl.go
View File

@ -97,6 +97,7 @@ func init() {
command.NewRoleCommand(),
command.NewCheckCommand(),
command.NewCompletionCommand(),
command.NewDowngradeCommand(),
)
}

7
server/etcdserver/v3_server.go
View File

@ -23,6 +23,7 @@ import (
"time"
pb "go.etcd.io/etcd/api/v3/etcdserverpb"
"go.etcd.io/etcd/api/v3/version"
"go.etcd.io/etcd/pkg/v3/traceutil"
"go.etcd.io/etcd/raft/v3"
"go.etcd.io/etcd/server/v3/auth"
@ -922,7 +923,7 @@ func (s *EtcdServer) downgradeValidate(ctx context.Context, v string) (*pb.Downg
if cv == nil {
return nil, ErrClusterVersionUnavailable
}
resp.Version = cv.String()
resp.Version = version.Cluster(cv.String())
err = s.Version().DowngradeValidate(ctx, targetVersion)
if err != nil {
return nil, err
@ -943,7 +944,7 @@ func (s *EtcdServer) downgradeEnable(ctx context.Context, r *pb.DowngradeRequest
lg.Warn("reject downgrade request", zap.Error(err))
return nil, err
}
resp := pb.DowngradeResponse{Version: s.ClusterVersion().String()}
resp := pb.DowngradeResponse{Version: version.Cluster(s.ClusterVersion().String())}
return &resp, nil
}
@ -952,6 +953,6 @@ func (s *EtcdServer) downgradeCancel(ctx context.Context) (*pb.DowngradeResponse
if err != nil {
s.lg.Warn("failed to cancel downgrade", zap.Error(err))
}
resp := pb.DowngradeResponse{Version: s.ClusterVersion().String()}
resp := pb.DowngradeResponse{Version: version.Cluster(s.ClusterVersion().String())}
return &resp, nil
}

21
tests/e2e/cluster_downgrade_test.go
View File

@ -15,7 +15,6 @@
package e2e
import (
"context"
"fmt"
"testing"
"time"
@ -23,7 +22,6 @@ import (
"github.com/coreos/go-semver/semver"
"go.etcd.io/etcd/api/v3/version"
"go.etcd.io/etcd/client/pkg/v3/fileutil"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/tests/v3/framework/e2e"
)
@ -79,20 +77,13 @@ func startEtcd(t *testing.T, execPath, dataDirPath string) *e2e.EtcdProcessClust
func downgradeEnable(t *testing.T, epc *e2e.EtcdProcessCluster, ver semver.Version) {
t.Log("etcdctl downgrade...")
c, err := clientv3.New(clientv3.Config{
Endpoints: epc.EndpointsV3(),
c := e2e.NewEtcdctl(epc.Cfg, epc.EndpointsV3())
e2e.ExecuteWithTimeout(t, 20*time.Second, func() {
err := c.DowngradeEnable(ver.String())
if err != nil {
t.Fatal(err)
}
})
if err != nil {
t.Fatal(err)
}
defer c.Close()
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
_, err = c.Downgrade(ctx, 1, ver.String())
if err != nil {
t.Fatal(err)
}
cancel()
}
func stopEtcd(t *testing.T, epc *e2e.EtcdProcessCluster) {

47
tests/e2e/ctl_v3_grpc_test.go
View File

@ -98,7 +98,7 @@ func TestAuthority(t *testing.T) {
defer epc.Close()
endpoints := templateEndpoints(t, tc.clientURLPattern, epc)
client := clusterEtcdctlV3(cfg, endpoints)
client := e2e.NewEtcdctl(cfg, endpoints)
err = client.Put("foo", "bar")
if err != nil {
t.Fatal(err)
@ -152,48 +152,3 @@ func firstMatch(t *testing.T, expectLine string, logs ...e2e.LogsExpect) string
}
return <-match
}
type etcdctlV3 struct {
cfg *e2e.EtcdProcessClusterConfig
endpoints []string
}
func clusterEtcdctlV3(cfg *e2e.EtcdProcessClusterConfig, endpoints []string) *etcdctlV3 {
return &etcdctlV3{
cfg: cfg,
endpoints: endpoints,
}
}
func (ctl *etcdctlV3) Put(key, value string) error {
return ctl.runCmd("put", key, value)
}
func (ctl *etcdctlV3) runCmd(args ...string) error {
cmdArgs := []string{e2e.CtlBinPath + "3"}
for k, v := range ctl.flags() {
cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v))
}
cmdArgs = append(cmdArgs, args...)
return e2e.SpawnWithExpect(cmdArgs, "OK")
}
func (ctl *etcdctlV3) flags() map[string]string {
fmap := make(map[string]string)
if ctl.cfg.ClientTLS == e2e.ClientTLS {
if ctl.cfg.IsClientAutoTLS {
fmap["insecure-transport"] = "false"
fmap["insecure-skip-tls-verify"] = "true"
} else if ctl.cfg.IsClientCRL {
fmap["cacert"] = e2e.CaPath
fmap["cert"] = e2e.RevokedCertPath
fmap["key"] = e2e.RevokedPrivateKeyPath
} else {
fmap["cacert"] = e2e.CaPath
fmap["cert"] = e2e.CertPath
fmap["key"] = e2e.PrivateKeyPath
}
}
fmap["endpoints"] = strings.Join(ctl.endpoints, ",")
return fmap
}

68
tests/framework/e2e/etcdctl.go Normal file
View File

@ -0,0 +1,68 @@
// Copyright 2022 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package e2e
import (
"fmt"
"strings"
)
type etcdctlV3 struct {
cfg *EtcdProcessClusterConfig
endpoints []string
}
func NewEtcdctl(cfg *EtcdProcessClusterConfig, endpoints []string) *etcdctlV3 {
return &etcdctlV3{
cfg: cfg,
endpoints: endpoints,
}
}
func (ctl *etcdctlV3) Put(key, value string) error {
return SpawnWithExpect(ctl.cmdArgs("put", key, value), "OK")
}
func (ctl *etcdctlV3) DowngradeEnable(version string) error {
return SpawnWithExpect(ctl.cmdArgs("downgrade", "enable", version), "Downgrade enable success")
}
func (ctl *etcdctlV3) cmdArgs(args ...string) []string {
cmdArgs := []string{CtlBinPath + "3"}
for k, v := range ctl.flags() {
cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v))
}
return append(cmdArgs, args...)
}
func (ctl *etcdctlV3) flags() map[string]string {
fmap := make(map[string]string)
if ctl.cfg.ClientTLS == ClientTLS {
if ctl.cfg.IsClientAutoTLS {
fmap["insecure-transport"] = "false"
fmap["insecure-skip-tls-verify"] = "true"
} else if ctl.cfg.IsClientCRL {
fmap["cacert"] = CaPath
fmap["cert"] = RevokedCertPath
fmap["key"] = RevokedPrivateKeyPath
} else {
fmap["cacert"] = CaPath
fmap["cert"] = CertPath
fmap["key"] = PrivateKeyPath
}
}
fmap["endpoints"] = strings.Join(ctl.endpoints, ",")
return fmap
}