From d94e712d9134a0fb92c65ea4eecfc8a69a186898 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Mon, 31 Aug 2015 15:01:18 -0700 Subject: [PATCH] *: support wal dir --- Documentation/admin_guide.md | 5 +++++ Documentation/configuration.md | 5 +++++ etcdmain/config.go | 2 ++ etcdmain/etcd.go | 1 + etcdmain/help.go | 2 ++ etcdserver/config.go | 25 ++++++++++++++++++------- etcdserver/server.go | 9 +++++++++ 7 files changed, 42 insertions(+), 7 deletions(-) diff --git a/Documentation/admin_guide.md b/Documentation/admin_guide.md index 3a172ea14..cea3d3c54 100644 --- a/Documentation/admin_guide.md +++ b/Documentation/admin_guide.md @@ -8,6 +8,9 @@ When first started, etcd stores its configuration into a data directory specifie Configuration is stored in the write ahead log and includes: the local member ID, cluster ID, and initial cluster configuration. The write ahead log and snapshot files are used during member operation and to recover after a restart. +Having a dedicated disk to store wal files can improve the throughput and stabilize the cluster. +It is highly recommended to dedicate a wal disk and set `--wal-dir` to point to a directory on that device for a production cluster deployment. + If a member’s data directory is ever lost or corrupted then the user should remove the etcd member from the cluster via the [members API][members-api]. A user should avoid restarting an etcd member with a data directory from an out-of-date backup. @@ -24,6 +27,8 @@ The data directory has two sub-directories in it: 1. wal: write ahead log files are stored here. For details see the [wal package documentation][wal-pkg] 2. snap: log snapshots are stored here. For details see the [snap package documentation][snap-pkg] +If `--wal-dir` flag is set, etcd will write the write ahead log files to the specified directory instead of data directory. + [wal-pkg]: http://godoc.org/github.com/coreos/etcd/wal [snap-pkg]: http://godoc.org/github.com/coreos/etcd/snap diff --git a/Documentation/configuration.md b/Documentation/configuration.md index 3cec96b50..c59f5d495 100644 --- a/Documentation/configuration.md +++ b/Documentation/configuration.md @@ -21,6 +21,11 @@ To start etcd automatically using custom settings at startup in Linux, using a [ + default: "${name}.etcd" + env variable: ETCD_DATA_DIR +##### -wal-dir ++ Path to the dedicated wal directory. If this flag is set, etcd will write the WAL files to the walDir rather than the dataDir. This allows a dedicated disk to be used, and helps avoid io competition between logging and other IO operations. ++ default: "" ++ env variable: ETCD_WAL_DIR + ##### -snapshot-count + Number of committed transactions to trigger a snapshot to disk. + default: "10000" diff --git a/etcdmain/config.go b/etcdmain/config.go index 55ebdb163..c8e305643 100644 --- a/etcdmain/config.go +++ b/etcdmain/config.go @@ -75,6 +75,7 @@ type config struct { // member corsInfo *cors.CORSInfo dir string + walDir string lpurls, lcurls []url.URL maxSnapFiles uint maxWalFiles uint @@ -148,6 +149,7 @@ func NewConfig() *config { // member fs.Var(cfg.corsInfo, "cors", "Comma-separated white list of origins for CORS (cross-origin resource sharing).") fs.StringVar(&cfg.dir, "data-dir", "", "Path to the data directory") + fs.StringVar(&cfg.walDir, "wal-dir", "", "Path to the dedicated wal directory") fs.Var(flags.NewURLsValue("http://localhost:2380,http://localhost:7001"), "listen-peer-urls", "List of URLs to listen on for peer traffic") fs.Var(flags.NewURLsValue("http://localhost:2379,http://localhost:4001"), "listen-client-urls", "List of URLs to listen on for client traffic") fs.UintVar(&cfg.maxSnapFiles, "max-snapshots", defaultMaxSnapshots, "Maximum number of snapshot files to retain (0 is unlimited)") diff --git a/etcdmain/etcd.go b/etcdmain/etcd.go index 28db9316c..33482202d 100644 --- a/etcdmain/etcd.go +++ b/etcdmain/etcd.go @@ -251,6 +251,7 @@ func startEtcd(cfg *config) (<-chan struct{}, error) { ClientURLs: cfg.acurls, PeerURLs: cfg.apurls, DataDir: cfg.dir, + DedicatedWALDir: cfg.walDir, SnapCount: cfg.snapCount, MaxSnapFiles: cfg.maxSnapFiles, MaxWALFiles: cfg.maxWalFiles, diff --git a/etcdmain/help.go b/etcdmain/help.go index 430f85112..312bf94f7 100644 --- a/etcdmain/help.go +++ b/etcdmain/help.go @@ -31,6 +31,8 @@ member flags: human-readable name for this member. --data-dir '${name}.etcd' path to the data directory. + --wal-dir '' + path to the dedicated wal directory. --snapshot-count '10000' number of committed transactions to trigger a snapshot to disk. --heartbeat-interval '100' diff --git a/etcdserver/config.go b/etcdserver/config.go index c32495b9e..9ed6b6d66 100644 --- a/etcdserver/config.go +++ b/etcdserver/config.go @@ -27,12 +27,15 @@ import ( // ServerConfig holds the configuration of etcd as taken from the command line or discovery. type ServerConfig struct { - Name string - DiscoveryURL string - DiscoveryProxy string - ClientURLs types.URLs - PeerURLs types.URLs - DataDir string + Name string + DiscoveryURL string + DiscoveryProxy string + ClientURLs types.URLs + PeerURLs types.URLs + DataDir string + // DedicatedWALDir config will make the etcd to write the WAL to the WALDir + // rather than the dataDir/member/wal. + DedicatedWALDir string SnapCount uint64 MaxSnapFiles uint MaxWALFiles uint @@ -105,7 +108,12 @@ func (c *ServerConfig) verifyLocalMember(strict bool) error { func (c *ServerConfig) MemberDir() string { return path.Join(c.DataDir, "member") } -func (c *ServerConfig) WALDir() string { return path.Join(c.MemberDir(), "wal") } +func (c *ServerConfig) WALDir() string { + if c.DedicatedWALDir != "" { + return c.DedicatedWALDir + } + return path.Join(c.MemberDir(), "wal") +} func (c *ServerConfig) SnapDir() string { return path.Join(c.MemberDir(), "snap") } @@ -129,6 +137,9 @@ func (c *ServerConfig) print(initial bool) { } plog.Infof("data dir = %s", c.DataDir) plog.Infof("member dir = %s", c.MemberDir()) + if c.DedicatedWALDir != "" { + plog.Infof("dedicated WAL dir = %s", c.DedicatedWALDir) + } plog.Infof("heartbeat = %dms", c.TickMs) plog.Infof("election = %dms", c.ElectionTicks*int(c.TickMs)) plog.Infof("snapshot count = %d", c.SnapCount) diff --git a/etcdserver/server.go b/etcdserver/server.go index 8e591e596..61d4c8aae 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -189,6 +189,11 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) { return nil, err } + err = os.MkdirAll(cfg.MemberDir(), privateDirMode) + if err != nil && err != os.ErrExist { + return nil, err + } + haveWAL := wal.Exist(cfg.WALDir()) ss := snap.New(cfg.SnapDir()) @@ -258,6 +263,10 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) { return nil, fmt.Errorf("cannot write to member directory: %v", err) } + if err := fileutil.IsDirWriteable(cfg.WALDir()); err != nil { + return nil, fmt.Errorf("cannot write to WAL directory: %v", err) + } + if cfg.ShouldDiscover() { plog.Warningf("discovery token ignored since a cluster has already been initialized. Valid log found at %q", cfg.WALDir()) }