Merge pull request #3413 from xiang90/snapshot_dir

*: support wal dir

Documentation/admin_guide.md

@@ -8,6 +8,9 @@ When first started, etcd stores its configuration into a data directory specifie
 Configuration is stored in the write ahead log and includes: the local member ID, cluster ID, and initial cluster configuration.
 The write ahead log and snapshot files are used during member operation and to recover after a restart.
 
+Having a dedicated disk to store wal files can improve the throughput and stabilize the cluster. 
+It is highly recommended to dedicate a wal disk and set `--wal-dir` to point to a directory on that device for a production cluster deployment.
+
 If a member’s data directory is ever lost or corrupted then the user should remove the etcd member from the cluster via the [members API][members-api].
 
 A user should avoid restarting an etcd member with a data directory from an out-of-date backup.
@@ -24,6 +27,8 @@ The data directory has two sub-directories in it:
 1. wal: write ahead log files are stored here. For details see the [wal package documentation][wal-pkg]
 2. snap: log snapshots are stored here. For details see the [snap package documentation][snap-pkg]
 
+If `--wal-dir` flag is set, etcd will write the write ahead log files to the specified directory instead of data directory.
+
 [wal-pkg]: http://godoc.org/github.com/coreos/etcd/wal
 [snap-pkg]: http://godoc.org/github.com/coreos/etcd/snap
 

Documentation/configuration.md

@@ -21,6 +21,11 @@ To start etcd automatically using custom settings at startup in Linux, using a [
 + default: "${name}.etcd"
 + env variable: ETCD_DATA_DIR
 
+##### -wal-dir
++ Path to the dedicated wal directory. If this flag is set, etcd will write the WAL files to the walDir rather than the dataDir. This allows a dedicated disk to be used, and helps avoid io competition between logging and other IO operations.
++ default: ""
++ env variable: ETCD_WAL_DIR
+
 ##### -snapshot-count
 + Number of committed transactions to trigger a snapshot to disk.
 + default: "10000"

etcdmain/config.go

@@ -75,6 +75,7 @@ type config struct {
 	// member
 	corsInfo       *cors.CORSInfo
 	dir            string
+	walDir         string
 	lpurls, lcurls []url.URL
 	maxSnapFiles   uint
 	maxWalFiles    uint
@@ -148,6 +149,7 @@ func NewConfig() *config {
 	// member
 	fs.Var(cfg.corsInfo, "cors", "Comma-separated white list of origins for CORS (cross-origin resource sharing).")
 	fs.StringVar(&cfg.dir, "data-dir", "", "Path to the data directory")
+	fs.StringVar(&cfg.walDir, "wal-dir", "", "Path to the dedicated wal directory")
 	fs.Var(flags.NewURLsValue("http://localhost:2380,http://localhost:7001"), "listen-peer-urls", "List of URLs to listen on for peer traffic")
 	fs.Var(flags.NewURLsValue("http://localhost:2379,http://localhost:4001"), "listen-client-urls", "List of URLs to listen on for client traffic")
 	fs.UintVar(&cfg.maxSnapFiles, "max-snapshots", defaultMaxSnapshots, "Maximum number of snapshot files to retain (0 is unlimited)")

etcdmain/etcd.go

@@ -251,6 +251,7 @@ func startEtcd(cfg *config) (<-chan struct{}, error) {
 		ClientURLs:          cfg.acurls,
 		PeerURLs:            cfg.apurls,
 		DataDir:             cfg.dir,
+		DedicatedWALDir:     cfg.walDir,
 		SnapCount:           cfg.snapCount,
 		MaxSnapFiles:        cfg.maxSnapFiles,
 		MaxWALFiles:         cfg.maxWalFiles,

etcdmain/help.go

@@ -31,6 +31,8 @@ member flags:
 		human-readable name for this member.
 	--data-dir '${name}.etcd'
 		path to the data directory.
+	--wal-dir ''
+		path to the dedicated wal directory.
 	--snapshot-count '10000'
 		number of committed transactions to trigger a snapshot to disk.
 	--heartbeat-interval '100'

etcdserver/config.go

@@ -27,12 +27,15 @@ import (
 
 // ServerConfig holds the configuration of etcd as taken from the command line or discovery.
 type ServerConfig struct {
-	Name                string
-	DiscoveryURL        string
-	DiscoveryProxy      string
-	ClientURLs          types.URLs
-	PeerURLs            types.URLs
-	DataDir             string
+	Name           string
+	DiscoveryURL   string
+	DiscoveryProxy string
+	ClientURLs     types.URLs
+	PeerURLs       types.URLs
+	DataDir        string
+	// DedicatedWALDir config will make the etcd to write the WAL to the WALDir
+	// rather than the dataDir/member/wal.
+	DedicatedWALDir     string
 	SnapCount           uint64
 	MaxSnapFiles        uint
 	MaxWALFiles         uint
@@ -105,7 +108,12 @@ func (c *ServerConfig) verifyLocalMember(strict bool) error {
 
 func (c *ServerConfig) MemberDir() string { return path.Join(c.DataDir, "member") }
 
-func (c *ServerConfig) WALDir() string { return path.Join(c.MemberDir(), "wal") }
+func (c *ServerConfig) WALDir() string {
+	if c.DedicatedWALDir != "" {
+		return c.DedicatedWALDir
+	}
+	return path.Join(c.MemberDir(), "wal")
+}
 
 func (c *ServerConfig) SnapDir() string { return path.Join(c.MemberDir(), "snap") }
 
@@ -129,6 +137,9 @@ func (c *ServerConfig) print(initial bool) {
 	}
 	plog.Infof("data dir = %s", c.DataDir)
 	plog.Infof("member dir = %s", c.MemberDir())
+	if c.DedicatedWALDir != "" {
+		plog.Infof("dedicated WAL dir = %s", c.DedicatedWALDir)
+	}
 	plog.Infof("heartbeat = %dms", c.TickMs)
 	plog.Infof("election = %dms", c.ElectionTicks*int(c.TickMs))
 	plog.Infof("snapshot count = %d", c.SnapCount)

etcdserver/server.go

@@ -189,6 +189,11 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) {
 		return nil, err
 	}
 
+	err = os.MkdirAll(cfg.MemberDir(), privateDirMode)
+	if err != nil && err != os.ErrExist {
+		return nil, err
+	}
+
 	haveWAL := wal.Exist(cfg.WALDir())
 	ss := snap.New(cfg.SnapDir())
 
@@ -258,6 +263,10 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) {
 			return nil, fmt.Errorf("cannot write to member directory: %v", err)
 		}
 
+		if err := fileutil.IsDirWriteable(cfg.WALDir()); err != nil {
+			return nil, fmt.Errorf("cannot write to WAL directory: %v", err)
+		}
+
 		if cfg.ShouldDiscover() {
 			plog.Warningf("discovery token ignored since a cluster has already been initialized. Valid log found at %q", cfg.WALDir())
 		}