vitalif
/
etcd
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2573 from yichengq/fix-detect-wal 

print out extra files in data dir instead of erroring
release-2.1
Yicheng Qin 2015-03-24 19:17:01 -07:00
parent 866a9d4e41 5e0077cc0c
commit f03ccd1120
6 changed files with 78 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
etcdserver/server.go
View File

@ -41,6 +41,7 @@ import (
"github.com/coreos/etcd/rafthttp"
"github.com/coreos/etcd/snap"
"github.com/coreos/etcd/store"
"github.com/coreos/etcd/version"
"github.com/coreos/etcd/wal"
)
@ -143,14 +144,16 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) {
var s *raft.MemoryStorage
var id types.ID
walVersion, err := wal.DetectVersion(cfg.DataDir)
// Run the migrations.
dataVer, err := version.DetectDataDir(cfg.DataDir)
if err != nil {
return nil, err
}
if walVersion == wal.WALUnknown {
return nil, fmt.Errorf("unknown wal version in data dir %s", cfg.DataDir)
if err := upgradeDataDir(cfg.DataDir, cfg.Name, dataVer); err != nil {
return nil, err
}
haveWAL := walVersion != wal.WALNotExist
haveWAL := wal.Exist(cfg.WALDir())
ss := snap.New(cfg.SnapDir())
switch {
@ -194,11 +197,6 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) {
cfg.PrintWithInitial()
id, n, s, w = startNode(cfg, cfg.Cluster.MemberIDs())
case haveWAL:
// Run the migrations.
if err := upgradeWAL(cfg.DataDir, cfg.Name, walVersion); err != nil {
return nil, err
}
if err := fileutil.IsDirWriteable(cfg.DataDir); err != nil {
return nil, fmt.Errorf("cannot write to data directory: %v", err)
}

11
etcdserver/storage.go
View File

@ -25,6 +25,7 @@ import (
"github.com/coreos/etcd/pkg/types"
"github.com/coreos/etcd/raft/raftpb"
"github.com/coreos/etcd/snap"
"github.com/coreos/etcd/version"
"github.com/coreos/etcd/wal"
"github.com/coreos/etcd/wal/walpb"
)
@ -88,9 +89,9 @@ func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID,
// upgradeWAL converts an older version of the etcdServer data to the newest version.
// It must ensure that, after upgrading, the most recent version is present.
func upgradeWAL(baseDataDir string, name string, ver wal.WalVersion) error {
func upgradeDataDir(baseDataDir string, name string, ver version.DataDirVersion) error {
switch ver {
case wal.WALv0_4:
case version.DataDir0_4:
log.Print("etcdserver: converting v0.4 log to v2.0")
err := migrate.Migrate4To2(baseDataDir, name)
if err != nil {
@ -98,16 +99,16 @@ func upgradeWAL(baseDataDir string, name string, ver wal.WalVersion) error {
return err
}
fallthrough
case wal.WALv2_0:
case version.DataDir2_0:
err := makeMemberDir(baseDataDir)
if err != nil {
return err
}
fallthrough
case wal.WALv2_0_1:
case version.DataDir2_0_1:
fallthrough
default:
log.Printf("datadir is valid for the 2.0.1 format")
log.Printf("etcdserver: datadir is valid for the 2.0.1 format")
}
return nil
}

2
test
View File

@ -15,7 +15,7 @@ COVER=${COVER:-"-cover"}
source ./build
# Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
TESTABLE_AND_FORMATTABLE="client discovery error etcdctl/command etcdmain etcdserver etcdserver/etcdhttp etcdserver/etcdhttp/httptypes migrate pkg/fileutil pkg/flags pkg/idutil pkg/ioutil pkg/netutil pkg/osutil pkg/pbutil pkg/types pkg/transport pkg/wait proxy raft snap store wal"
TESTABLE_AND_FORMATTABLE="client discovery error etcdctl/command etcdmain etcdserver etcdserver/etcdhttp etcdserver/etcdhttp/httptypes migrate pkg/fileutil pkg/flags pkg/idutil pkg/ioutil pkg/netutil pkg/osutil pkg/pbutil pkg/types pkg/transport pkg/wait proxy raft snap store version wal"
# TODO: add it to race testing when the issue is resolved
# https://github.com/golang/go/issues/9946
NO_RACE_TESTABLE="rafthttp"

55
version/version.go
View File

@ -14,7 +14,62 @@
package version
import (
"path"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/pkg/types"
)
var (
Version = "2.0.4+git"
InternalVersion = "2"
)
// WalVersion is an enum for versions of etcd logs.
type DataDirVersion string
const (
DataDirUnknown DataDirVersion = "Unknown WAL"
DataDir0_4 DataDirVersion = "0.4.x"
DataDir2_0 DataDirVersion = "2.0.0"
DataDir2_0Proxy DataDirVersion = "2.0 proxy"
DataDir2_0_1 DataDirVersion = "2.0.1"
)
func DetectDataDir(dirpath string) (DataDirVersion, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
// Error reading the directory
return DataDirUnknown, err
}
nameSet := types.NewUnsafeSet(names...)
if nameSet.Contains("member") {
ver, err := DetectDataDir(path.Join(dirpath, "member"))
if ver == DataDir2_0 {
return DataDir2_0_1, nil
} else if ver == DataDir0_4 {
// How in the blazes did it get there?
return DataDirUnknown, nil
}
return ver, err
}
if nameSet.ContainsAll([]string{"snap", "wal"}) {
// .../wal cannot be empty to exist.
walnames, err := fileutil.ReadDir(path.Join(dirpath, "wal"))
if err == nil && len(walnames) > 0 {
return DataDir2_0, nil
}
}
if nameSet.ContainsAll([]string{"proxy"}) {
return DataDir2_0Proxy, nil
}
if nameSet.ContainsAll([]string{"snapshot", "conf", "log"}) {
return DataDir0_4, nil
}
if nameSet.ContainsAll([]string{"standby_info"}) {
return DataDir0_4, nil
}
return DataDirUnknown, nil
}

28
wal/util_test.go → version/version_test.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
package version
import (
"io/ioutil"
@ -22,21 +22,20 @@ import (
"testing"
)
func TestDetectVersion(t *testing.T) {
func TestDetectDataDir(t *testing.T) {
tests := []struct {
names []string
wver WalVersion
wver DataDirVersion
}{
{[]string{}, WALNotExist},
{[]string{"member/", "member/wal/", "member/wal/1", "member/snap/"}, WALv2_0_1},
{[]string{"snap/", "wal/", "wal/1"}, WALv2_0},
{[]string{"snapshot/", "conf", "log"}, WALv0_4},
{[]string{"weird"}, WALUnknown},
{[]string{"snap/", "wal/"}, WALUnknown},
{[]string{"member/", "member/wal/", "member/wal/1", "member/snap/"}, DataDir2_0_1},
{[]string{"snap/", "wal/", "wal/1"}, DataDir2_0},
{[]string{"snapshot/", "conf", "log"}, DataDir0_4},
{[]string{"weird"}, DataDirUnknown},
{[]string{"snap/", "wal/"}, DataDirUnknown},
}
for i, tt := range tests {
p := mustMakeDir(t, tt.names...)
ver, err := DetectVersion(p)
ver, err := DetectDataDir(p)
if ver != tt.wver {
t.Errorf("#%d: version = %s, want %s", i, ver, tt.wver)
}
@ -45,15 +44,6 @@ func TestDetectVersion(t *testing.T) {
}
os.RemoveAll(p)
}
// detect on non-exist directory
v, err := DetectVersion(path.Join(os.TempDir(), "waltest", "not-exist"))
if v != WALNotExist {
t.Errorf("#non-exist: version = %s, want %s", v, WALNotExist)
}
if err != nil {
t.Errorf("#non-exist: err = %s, want %s", v, WALNotExist)
}
}
// mustMakeDir builds the directory that contains files with the given

58
wal/util.go
View File

@ -17,68 +17,10 @@ package wal
import (
"fmt"
"log"
"os"
"path"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/pkg/types"
)
// WalVersion is an enum for versions of etcd logs.
type WalVersion string
const (
WALUnknown WalVersion = "Unknown WAL"
WALNotExist WalVersion = "No WAL"
WALv0_4 WalVersion = "0.4.x"
WALv2_0 WalVersion = "2.0.0"
WALv2_0Proxy WalVersion = "2.0 proxy"
WALv2_0_1 WalVersion = "2.0.1"
)
func DetectVersion(dirpath string) (WalVersion, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
if os.IsNotExist(err) {
err = nil
}
// Error reading the directory
return WALNotExist, err
}
if len(names) == 0 {
// Empty WAL directory
return WALNotExist, nil
}
nameSet := types.NewUnsafeSet(names...)
if nameSet.Contains("member") {
ver, err := DetectVersion(path.Join(dirpath, "member"))
if ver == WALv2_0 {
return WALv2_0_1, nil
} else if ver == WALv0_4 {
// How in the blazes did it get there?
return WALUnknown, nil
}
return ver, err
}
if nameSet.ContainsAll([]string{"snap", "wal"}) {
// .../wal cannot be empty to exist.
if Exist(path.Join(dirpath, "wal")) {
return WALv2_0, nil
}
}
if nameSet.ContainsAll([]string{"proxy"}) {
return WALv2_0Proxy, nil
}
if nameSet.ContainsAll([]string{"snapshot", "conf", "log"}) {
return WALv0_4, nil
}
if nameSet.ContainsAll([]string{"standby_info"}) {
return WALv0_4, nil
}
return WALUnknown, nil
}
func Exist(dirpath string) bool {
names, err := fileutil.ReadDir(dirpath)
if err != nil {