From 78ea3335bfd7f5731d07d19c90bdfe9478ebf57f Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Tue, 18 Nov 2014 13:43:20 -0500 Subject: [PATCH 1/5] etcdserver: autodetect v0.4 WALs and upgrade them to v0.5 automatically --- etcdserver/server.go | 32 +++++++++++++++++++++++++----- wal/util.go | 47 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/etcdserver/server.go b/etcdserver/server.go index c746a5b48..0c30b2c53 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -36,6 +36,7 @@ import ( "github.com/coreos/etcd/etcdserver/etcdhttp/httptypes" pb "github.com/coreos/etcd/etcdserver/etcdserverpb" "github.com/coreos/etcd/etcdserver/stats" + "github.com/coreos/etcd/migrate" "github.com/coreos/etcd/pkg/pbutil" "github.com/coreos/etcd/pkg/types" "github.com/coreos/etcd/pkg/wait" @@ -190,18 +191,39 @@ type EtcdServer struct { raftLead uint64 } +// UpgradeWAL converts an older version of the EtcdServer data to the newest version. +// It must ensure that, after upgrading, the most recent version is present. +func UpgradeWAL(cfg *ServerConfig, ver wal.WalVersion) { + if ver == wal.WALv0_4 { + err := migrate.Migrate4To5(cfg.DataDir, cfg.Name) + if err != nil { + log.Fatalf("Failed migrating data-dir: %v", err) + } + } +} + // NewServer creates a new EtcdServer from the supplied configuration. The // configuration is considered static for the lifetime of the EtcdServer. func NewServer(cfg *ServerConfig) (*EtcdServer, error) { - if err := os.MkdirAll(cfg.SnapDir(), privateDirMode); err != nil { - return nil, fmt.Errorf("cannot create snapshot directory: %v", err) - } - ss := snap.New(cfg.SnapDir()) st := store.New() var w *wal.WAL var n raft.Node var id types.ID - haveWAL := wal.Exist(cfg.WALDir()) + walVersion := wal.DetectVersion(cfg.DataDir) + if walVersion == wal.UnknownWAL { + return nil, fmt.Errorf("unknown wal version in data dir %s", cfg.DataDir) + } + haveWAL := walVersion != wal.NoWAL + + if haveWAL && walVersion != wal.WALv0_5 { + UpgradeWAL(cfg, walVersion) + } + + if err := os.MkdirAll(cfg.SnapDir(), privateDirMode); err != nil { + return nil, fmt.Errorf("cannot create snapshot directory: %v", err) + } + ss := snap.New(cfg.SnapDir()) + switch { case !haveWAL && !cfg.NewCluster: us := getOtherPeerURLs(cfg.Cluster, cfg.Name) diff --git a/wal/util.go b/wal/util.go index 684df5358..34bf23322 100644 --- a/wal/util.go +++ b/wal/util.go @@ -20,8 +20,55 @@ import ( "fmt" "log" "os" + "path" ) +type StringSlice []string + +func containsStrings(source, target []string) bool { + for _, t := range target { + ok := false + for _, s := range source { + if t == s { + ok = true + } + } + if !ok { + return false + } + } + return true +} + +// WalVersion is an enum for versions of etcd logs. +type WalVersion string + +const ( + UnknownWAL WalVersion = "Unknown WAL" + NoWAL WalVersion = "No WAL" + WALv0_4 WalVersion = "0.4.x" + WALv0_5 WalVersion = "0.5.x" +) + +func DetectVersion(dirpath string) WalVersion { + names, err := readDir(dirpath) + if err != nil || len(names) == 0 { + return NoWAL + } + if containsStrings(names, []string{"snap", "wal"}) { + // .../wal cannot be empty to exist. + if Exist(path.Join(dirpath, "wal")) { + return WALv0_5 + } + return NoWAL + } + if containsStrings(names, []string{"snapshot", "conf", "log"}) { + return WALv0_4 + } + + return UnknownWAL +} + func Exist(dirpath string) bool { names, err := readDir(dirpath) if err != nil { From 59a0c64e9fc2f902d148273329bf8948a9f00b61 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Wed, 19 Nov 2014 15:33:37 -0500 Subject: [PATCH 2/5] fix import loop, add set to types, and fix comments --- etcdserver/server.go | 14 +++- migrate/log.go | 9 +-- migrate/member.go | 59 ++++++++++++++ migrate/snapshot.go | 4 +- pkg/types/set.go | 180 ++++++++++++++++++++++++++++++++++++++++++ pkg/types/set_test.go | 166 ++++++++++++++++++++++++++++++++++++++ wal/util.go | 38 +++------ 7 files changed, 433 insertions(+), 37 deletions(-) create mode 100644 migrate/member.go create mode 100644 pkg/types/set.go create mode 100644 pkg/types/set_test.go diff --git a/etcdserver/server.go b/etcdserver/server.go index 0c30b2c53..7b6435b53 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -193,13 +193,16 @@ type EtcdServer struct { // UpgradeWAL converts an older version of the EtcdServer data to the newest version. // It must ensure that, after upgrading, the most recent version is present. -func UpgradeWAL(cfg *ServerConfig, ver wal.WalVersion) { +func UpgradeWAL(cfg *ServerConfig, ver wal.WalVersion) error { if ver == wal.WALv0_4 { + log.Print("Converting v0.4 log to v0.5") err := migrate.Migrate4To5(cfg.DataDir, cfg.Name) if err != nil { log.Fatalf("Failed migrating data-dir: %v", err) + return err } } + return nil } // NewServer creates a new EtcdServer from the supplied configuration. The @@ -210,13 +213,16 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) { var n raft.Node var id types.ID walVersion := wal.DetectVersion(cfg.DataDir) - if walVersion == wal.UnknownWAL { + if walVersion == wal.WALUnknown { return nil, fmt.Errorf("unknown wal version in data dir %s", cfg.DataDir) } - haveWAL := walVersion != wal.NoWAL + haveWAL := walVersion != wal.WALNotExist if haveWAL && walVersion != wal.WALv0_5 { - UpgradeWAL(cfg, walVersion) + err := UpgradeWAL(cfg, walVersion) + if err != nil { + return nil, err + } } if err := os.MkdirAll(cfg.SnapDir(), privateDirMode); err != nil { diff --git a/migrate/log.go b/migrate/log.go index 84a0a9b0e..09bba246b 100644 --- a/migrate/log.go +++ b/migrate/log.go @@ -10,7 +10,6 @@ import ( "path" "time" - "github.com/coreos/etcd/etcdserver" etcdserverpb "github.com/coreos/etcd/etcdserver/etcdserverpb" etcd4pb "github.com/coreos/etcd/migrate/etcd4pb" "github.com/coreos/etcd/pkg/types" @@ -56,7 +55,7 @@ func (l Log4) NodeIDs() map[string]uint64 { } func StorePath(key string) string { - return path.Join(etcdserver.StoreKeysPrefix, key) + return path.Join("/1", key) } func DecodeLog4FromFile(logpath string) (Log4, error) { @@ -214,7 +213,7 @@ type JoinCommand struct { Name string `json:"name"` RaftURL string `json:"raftURL"` EtcdURL string `json:"etcdURL"` - memb etcdserver.Member + memb member } func (c *JoinCommand) Type5() raftpb.EntryType { @@ -496,13 +495,13 @@ func toEntry5(ent4 *etcd4pb.LogEntry, raftMap map[string]uint64) (*raftpb.Entry, return &ent5, nil } -func generateNodeMember(name, rafturl, etcdurl string) *etcdserver.Member { +func generateNodeMember(name, rafturl, etcdurl string) *member { pURLs, err := types.NewURLs([]string{rafturl}) if err != nil { log.Fatalf("Invalid Raft URL %s -- this log could never have worked", rafturl) } - m := etcdserver.NewMember(name, pURLs, etcdDefaultClusterName, nil) + m := NewMember(name, pURLs, etcdDefaultClusterName) m.ClientURLs = []string{etcdurl} return m } diff --git a/migrate/member.go b/migrate/member.go new file mode 100644 index 000000000..1d6a75445 --- /dev/null +++ b/migrate/member.go @@ -0,0 +1,59 @@ +/* + Copyright 2014 CoreOS, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package migrate + +import ( + "crypto/sha1" + "encoding/binary" + "sort" + + "github.com/coreos/etcd/pkg/types" +) + +type raftAttributes struct { + PeerURLs []string `json:"peerURLs"` +} + +type attributes struct { + Name string `json:"name,omitempty"` + ClientURLs []string `json:"clientURLs,omitempty"` +} + +type member struct { + ID types.ID `json:"id"` + raftAttributes + attributes +} + +func NewMember(name string, peerURLs types.URLs, clusterName string) *member { + m := &member{ + raftAttributes: raftAttributes{PeerURLs: peerURLs.StringSlice()}, + attributes: attributes{Name: name}, + } + + var b []byte + sort.Strings(m.PeerURLs) + for _, p := range m.PeerURLs { + b = append(b, []byte(p)...) + } + + b = append(b, []byte(clusterName)...) + + hash := sha1.Sum(b) + m.ID = types.ID(binary.BigEndian.Uint64(hash[:8])) + return m +} diff --git a/migrate/snapshot.go b/migrate/snapshot.go index 57dcf501f..7dfe270a6 100644 --- a/migrate/snapshot.go +++ b/migrate/snapshot.go @@ -93,11 +93,11 @@ func fixEtcd(n *node) { rafturl := q.Get("raft") m := generateNodeMember(name, rafturl, etcdurl) - attrBytes, err := json.Marshal(m.Attributes) + attrBytes, err := json.Marshal(m.attributes) if err != nil { log.Fatal("Couldn't marshal attributes") } - raftBytes, err := json.Marshal(m.RaftAttributes) + raftBytes, err := json.Marshal(m.raftAttributes) if err != nil { log.Fatal("Couldn't marshal raft attributes") } diff --git a/pkg/types/set.go b/pkg/types/set.go new file mode 100644 index 000000000..d4f323f5f --- /dev/null +++ b/pkg/types/set.go @@ -0,0 +1,180 @@ +/* + Copyright 2014 CoreOS, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package types + +import ( + "reflect" + "sort" + "sync" +) + +type Set interface { + Add(string) + Remove(string) + Contains(string) bool + Equals(Set) bool + Length() int + Values() []string + Copy() Set + Sub(Set) Set +} + +func NewUnsafeSet(values ...string) *unsafeSet { + set := &unsafeSet{make(map[string]struct{})} + for _, v := range values { + set.Add(v) + } + return set +} + +func NewThreadsafeSet(values ...string) *tsafeSet { + us := NewUnsafeSet(values...) + return &tsafeSet{us, sync.RWMutex{}} +} + +type unsafeSet struct { + d map[string]struct{} +} + +// Add adds a new value to the set (no-op if the value is already present) +func (us *unsafeSet) Add(value string) { + us.d[value] = struct{}{} +} + +// Remove removes the given value from the set +func (us *unsafeSet) Remove(value string) { + delete(us.d, value) +} + +// Contains returns whether the set contains the given value +func (us *unsafeSet) Contains(value string) (exists bool) { + _, exists = us.d[value] + return +} + +// ContainsAll returns whether the set contains all given values +func (us *unsafeSet) ContainsAll(values []string) bool { + for _, s := range values { + if !us.Contains(s) { + return false + } + } + return true +} + +// Equals returns whether the contents of two sets are identical +func (us *unsafeSet) Equals(other Set) bool { + v1 := sort.StringSlice(us.Values()) + v2 := sort.StringSlice(other.Values()) + v1.Sort() + v2.Sort() + return reflect.DeepEqual(v1, v2) +} + +// Length returns the number of elements in the set +func (us *unsafeSet) Length() int { + return len(us.d) +} + +// Values returns the values of the Set in an unspecified order. +func (us *unsafeSet) Values() (values []string) { + values = make([]string, 0) + for val, _ := range us.d { + values = append(values, val) + } + return +} + +// Copy creates a new Set containing the values of the first +func (us *unsafeSet) Copy() Set { + cp := NewUnsafeSet() + for val, _ := range us.d { + cp.Add(val) + } + + return cp +} + +// Sub removes all elements in other from the set +func (us *unsafeSet) Sub(other Set) Set { + oValues := other.Values() + result := us.Copy().(*unsafeSet) + + for _, val := range oValues { + if _, ok := result.d[val]; !ok { + continue + } + delete(result.d, val) + } + + return result +} + +type tsafeSet struct { + us *unsafeSet + m sync.RWMutex +} + +func (ts *tsafeSet) Add(value string) { + ts.m.Lock() + defer ts.m.Unlock() + ts.us.Add(value) +} + +func (ts *tsafeSet) Remove(value string) { + ts.m.Lock() + defer ts.m.Unlock() + ts.us.Remove(value) +} + +func (ts *tsafeSet) Contains(value string) (exists bool) { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Contains(value) +} + +func (ts *tsafeSet) Equals(other Set) bool { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Equals(other) +} + +func (ts *tsafeSet) Length() int { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Length() +} + +func (ts *tsafeSet) Values() (values []string) { + ts.m.RLock() + defer ts.m.RUnlock() + return ts.us.Values() +} + +func (ts *tsafeSet) Copy() Set { + ts.m.RLock() + defer ts.m.RUnlock() + usResult := ts.us.Copy().(*unsafeSet) + return &tsafeSet{usResult, sync.RWMutex{}} +} + +func (ts *tsafeSet) Sub(other Set) Set { + ts.m.RLock() + defer ts.m.RUnlock() + usResult := ts.us.Sub(other).(*unsafeSet) + return &tsafeSet{usResult, sync.RWMutex{}} +} diff --git a/pkg/types/set_test.go b/pkg/types/set_test.go new file mode 100644 index 000000000..e7f7e5fc9 --- /dev/null +++ b/pkg/types/set_test.go @@ -0,0 +1,166 @@ +/* + Copyright 2014 CoreOS, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package types + +import ( + "reflect" + "sort" + "testing" +) + +func TestUnsafeSet(t *testing.T) { + driveSetTests(t, NewUnsafeSet()) +} + +func TestThreadsafeSet(t *testing.T) { + driveSetTests(t, NewThreadsafeSet()) +} + +// Check that two slices contents are equal; order is irrelevant +func equal(a, b []string) bool { + as := sort.StringSlice(a) + bs := sort.StringSlice(b) + as.Sort() + bs.Sort() + return reflect.DeepEqual(as, bs) +} + +func driveSetTests(t *testing.T, s Set) { + // Verify operations on an empty set + eValues := []string{} + values := s.Values() + if !reflect.DeepEqual(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + if l := s.Length(); l != 0 { + t.Fatalf("Expected length=0, got %d", l) + } + for _, v := range []string{"foo", "bar", "baz"} { + if s.Contains(v) { + t.Fatalf("Expect s.Contains(%q) to be fale, got true", v) + } + } + + // Add three items, ensure they show up + s.Add("foo") + s.Add("bar") + s.Add("baz") + + eValues = []string{"foo", "bar", "baz"} + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + + for _, v := range eValues { + if !s.Contains(v) { + t.Fatalf("Expect s.Contains(%q) to be true, got false", v) + } + } + + if l := s.Length(); l != 3 { + t.Fatalf("Expected length=3, got %d", l) + } + + // Add the same item a second time, ensuring it is not duplicated + s.Add("foo") + + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + if l := s.Length(); l != 3 { + t.Fatalf("Expected length=3, got %d", l) + } + + // Remove all items, ensure they are gone + s.Remove("foo") + s.Remove("bar") + s.Remove("baz") + + eValues = []string{} + values = s.Values() + if !equal(values, eValues) { + t.Fatalf("Expect values=%v got %v", eValues, values) + } + + if l := s.Length(); l != 0 { + t.Fatalf("Expected length=0, got %d", l) + } + + // Create new copies of the set, and ensure they are unlinked to the + // original Set by making modifications + s.Add("foo") + s.Add("bar") + cp1 := s.Copy() + cp2 := s.Copy() + s.Remove("foo") + cp3 := s.Copy() + cp1.Add("baz") + + for i, tt := range []struct { + want []string + got []string + }{ + {[]string{"bar"}, s.Values()}, + {[]string{"foo", "bar", "baz"}, cp1.Values()}, + {[]string{"foo", "bar"}, cp2.Values()}, + {[]string{"bar"}, cp3.Values()}, + } { + if !equal(tt.want, tt.got) { + t.Fatalf("case %d: expect values=%v got %v", i, tt.want, tt.got) + } + } + + for i, tt := range []struct { + want bool + got bool + }{ + {true, s.Equals(cp3)}, + {true, cp3.Equals(s)}, + {false, s.Equals(cp2)}, + {false, s.Equals(cp1)}, + {false, cp1.Equals(s)}, + {false, cp2.Equals(s)}, + {false, cp2.Equals(cp1)}, + } { + if tt.got != tt.want { + t.Fatalf("case %d: want %t, got %t", i, tt.want, tt.got) + + } + } + + // Subtract values from a Set, ensuring a new Set is created and + // the original Sets are unmodified + sub1 := cp1.Sub(s) + sub2 := cp2.Sub(cp1) + + for i, tt := range []struct { + want []string + got []string + }{ + {[]string{"foo", "bar", "baz"}, cp1.Values()}, + {[]string{"foo", "bar"}, cp2.Values()}, + {[]string{"bar"}, s.Values()}, + {[]string{"foo", "baz"}, sub1.Values()}, + {[]string{}, sub2.Values()}, + } { + if !equal(tt.want, tt.got) { + t.Fatalf("case %d: expect values=%v got %v", i, tt.want, tt.got) + } + } +} diff --git a/wal/util.go b/wal/util.go index 34bf23322..b3be44e8a 100644 --- a/wal/util.go +++ b/wal/util.go @@ -21,52 +21,38 @@ import ( "log" "os" "path" + + "github.com/coreos/etcd/pkg/types" ) -type StringSlice []string - -func containsStrings(source, target []string) bool { - for _, t := range target { - ok := false - for _, s := range source { - if t == s { - ok = true - } - } - if !ok { - return false - } - } - return true -} - // WalVersion is an enum for versions of etcd logs. type WalVersion string const ( - UnknownWAL WalVersion = "Unknown WAL" - NoWAL WalVersion = "No WAL" - WALv0_4 WalVersion = "0.4.x" - WALv0_5 WalVersion = "0.5.x" + WALUnknown WalVersion = "Unknown WAL" + WALNotExist WalVersion = "No WAL" + WALv0_4 WalVersion = "0.4.x" + WALv0_5 WalVersion = "0.5.x" ) func DetectVersion(dirpath string) WalVersion { names, err := readDir(dirpath) if err != nil || len(names) == 0 { - return NoWAL + return WALNotExist } - if containsStrings(names, []string{"snap", "wal"}) { + nameSet := types.NewUnsafeSet(names...) + if nameSet.ContainsAll([]string{"snap", "wal"}) { // .../wal cannot be empty to exist. if Exist(path.Join(dirpath, "wal")) { return WALv0_5 } - return NoWAL + return WALNotExist } - if containsStrings(names, []string{"snapshot", "conf", "log"}) { + if nameSet.ContainsAll([]string{"snapshot", "conf", "log"}) { return WALv0_4 } - return UnknownWAL + return WALUnknown } func Exist(dirpath string) bool { From d1e7fee3ca501efc5d24d6dec0ef3050a2e351d1 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Thu, 20 Nov 2014 13:17:47 -0500 Subject: [PATCH 3/5] fix test import loop --- migrate/log_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/migrate/log_test.go b/migrate/log_test.go index b1db15079..7cf1482ef 100644 --- a/migrate/log_test.go +++ b/migrate/log_test.go @@ -6,8 +6,6 @@ import ( "reflect" "testing" "time" - - "github.com/coreos/etcd/etcdserver" ) func TestNewCommand(t *testing.T) { @@ -21,7 +19,7 @@ func TestNewCommand(t *testing.T) { t.Errorf("couldn't create time: %v", err) } - m := etcdserver.NewMember("alice", []url.URL{{Scheme: "http", Host: "127.0.0.1:7001"}}, etcdDefaultClusterName, nil) + m := NewMember("alice", []url.URL{{Scheme: "http", Host: "127.0.0.1:7001"}}, etcdDefaultClusterName) m.ClientURLs = []string{"http://127.0.0.1:4001"} tests := []interface{}{ From 2d5ccf12ef9afd09fca198b16011e837125d462a Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Thu, 20 Nov 2014 15:37:15 -0500 Subject: [PATCH 4/5] add snapshotted integration test --- integration/cluster_test.go | 30 +++++++++------- integration/migration_test.go | 34 ++++++++++++++++++ integration/testdata/integration046_data/conf | 1 + integration/testdata/integration046_data/log | Bin 0 -> 6742 bytes .../integration046_data/snapshot/1_90.ss | 2 ++ migrate/etcd4.go | 5 +-- 6 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 integration/migration_test.go create mode 100644 integration/testdata/integration046_data/conf create mode 100644 integration/testdata/integration046_data/log create mode 100644 integration/testdata/integration046_data/snapshot/1_90.ss diff --git a/integration/cluster_test.go b/integration/cluster_test.go index 89891a474..80d5b28d2 100644 --- a/integration/cluster_test.go +++ b/integration/cluster_test.go @@ -146,16 +146,7 @@ type cluster struct { Members []*member } -// NewCluster returns an unlaunched cluster of the given size which has been -// set to use static bootstrap. -func NewCluster(t *testing.T, size int) *cluster { - c := &cluster{} - ms := make([]*member, size) - for i := 0; i < size; i++ { - ms[i] = mustNewMember(t, c.name(i)) - } - c.Members = ms - +func fillClusterForMembers(ms []*member, cName string) error { addrs := make([]string, 0) for _, m := range ms { for _, l := range m.PeerListeners { @@ -165,11 +156,26 @@ func NewCluster(t *testing.T, size int) *cluster { clusterStr := strings.Join(addrs, ",") var err error for _, m := range ms { - m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr) + m.Cluster, err = etcdserver.NewClusterFromString(cName, clusterStr) if err != nil { - t.Fatal(err) + return err } } + return nil +} + +// NewCluster returns an unlaunched cluster of the given size which has been +// set to use static bootstrap. +func NewCluster(t *testing.T, size int) *cluster { + c := &cluster{} + ms := make([]*member, size) + for i := 0; i < size; i++ { + ms[i] = mustNewMember(t, c.name(i)) + } + c.Members = ms + if err := fillClusterForMembers(c.Members, clusterName); err != nil { + t.Fatal(err) + } return c } diff --git a/integration/migration_test.go b/integration/migration_test.go new file mode 100644 index 000000000..639ef1817 --- /dev/null +++ b/integration/migration_test.go @@ -0,0 +1,34 @@ +package integration + +import ( + "github.com/coreos/etcd/pkg/types" + "net" + "os/exec" + "testing" +) + +func TestUpgradeMember(t *testing.T) { + defer afterTest(t) + m := mustNewMember(t, "integration046") + newPeerListeners := make([]net.Listener, 0) + newPeerListeners = append(newPeerListeners, newListenerWithAddr(t, "127.0.0.1:59892")) + m.PeerListeners = newPeerListeners + urls, err := types.NewURLs([]string{"http://127.0.0.1:59892"}) + if err != nil { + t.Fatal(err) + } + m.PeerURLs = urls + m.NewCluster = true + c := &cluster{} + c.Members = []*member{m} + fillClusterForMembers(c.Members, "etcd-cluster") + cmd := exec.Command("cp", "-r", "testdata/integration046_data/conf", "testdata/integration046_data/log", "testdata/integration046_data/snapshot", m.DataDir) + err = cmd.Run() + if err != nil { + t.Fatal(err) + } + + c.Launch(t) + defer c.Terminate(t) + clusterMustProgress(t, c) +} diff --git a/integration/testdata/integration046_data/conf b/integration/testdata/integration046_data/conf new file mode 100644 index 000000000..95106f8b1 --- /dev/null +++ b/integration/testdata/integration046_data/conf @@ -0,0 +1 @@ +{"commitIndex":1,"peers":[]} \ No newline at end of file diff --git a/integration/testdata/integration046_data/log b/integration/testdata/integration046_data/log new file mode 100644 index 0000000000000000000000000000000000000000..1009d4512f5a33780968d6f55d8ea96335be037b GIT binary patch literal 6742 zcmb7|>3bVB5XLE|Yp!sVqg-LRAxs*v|P)*l&4iBc)WPNYEJ-Sm|m=&A#U*)VAZcax> z)s4|VC@P?eYIpCbx>)TD?-ix=LaY=A_r{x7Q0?SsxKotouI-3qMQIHX4 zVP&MKp93OAMtI4E=*cFiD?~Obav^XbdaB9k3XzRj1VUQLgy`vXkc)!65)c^BGdUp2 zBFuLI4CvW(kS9eNjMSW>r5q4#n#FwN(&)K#P>70@a+ce*=W{?Yjl8v7j9y3vftaU} z_1yNqm;;KtrflF8ol6JFI!3~}cCg7rUTT6mP6)_&_)S<4M4Xq?IVuX0o;@J1G&vn7 z4WeIzH$odGuU}0EMZX4Pm24Y7t?Yag)Dl06=N5bxu!noU2^u`O zqk$03$N}eRA2cx?Csx;XjV8jKj}M!ijuVR<3Xl=NHbx(%gXV6H1&!OXk8?njL|(at z#)!7;lXOsR*B}CW&{lFl6eDXa+w%?5LG%9FIJQw=%>j`jue{~bdMzDP8@80*aUhWc zqFqw}vYptpYcd^F8@5skID1q%pqNL&DGmhbpxUs5H;&6}odY68fbA>?GU=dshYg^( zT4ZxTq(~TGTN;-Rsyl3LfxW2t98k<71A)th_3Ib@jwio2%lf@M=?f0_(3 zwPgqA+3NUN4hR*2GuCrU`#c>~d$x35v~M8>i`o|m>Ya{1ifgmt?+OPR#oFD&V;rR5 zB;$~VFn15*)9QGCG^sY~5rx1>${;>Agur6RHyR~YE!>#>g_0c zrf%fXhPkxA(ByO!J#*T~duVq~B$!6G4Aqx3CpHSJ1+aIZpK?G3L0((K9h{%jK|U&S)(Y<6 zT+acaBAk60$2CGX(m^3A5 Date: Thu, 20 Nov 2014 16:49:34 -0500 Subject: [PATCH 5/5] Fix migration to allow snapshots to have the right IDs --- migrate/snapshot.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/migrate/snapshot.go b/migrate/snapshot.go index 7dfe270a6..8a457fac9 100644 --- a/migrate/snapshot.go +++ b/migrate/snapshot.go @@ -171,15 +171,17 @@ func (s *Snapshot4) Snapshot5() *raftpb.Snapshot { log.Fatal("Couldn't re-marshal new snapshot") } + nodes := s.GetNodesFromStore() + nodeList := make([]uint64, 0) + for _, v := range nodes { + nodeList = append(nodeList, v) + } + snap5 := raftpb.Snapshot{ Data: newState, Index: s.LastIndex, Term: s.LastTerm, - Nodes: make([]uint64, len(s.Peers)), - } - - for i, p := range s.Peers { - snap5.Nodes[i] = hashName(p.Name) + Nodes: nodeList, } return &snap5