version: bump to alpha.2

Add doc for backup and force-new-cluster

Dockerfile

@@ -0,0 +1,2 @@
+FROM golang:onbuild
+EXPOSE 4001 7001 2379 2380

Documentation/0.5/admin_api.md

@@ -1,72 +0,0 @@
-## Admin API
-
-### GET /v2/admin/members/
-Return an HTTP 200 OK response code and a representation of all members in the etcd cluster:
-```
-    Example Request: GET 
-                     http://localhost:2379/v2/admin/members/
-    Response formats: JSON
-    Example Response:
-```
-```json
-    {
-        "members": [
-                {
-                    "id":"272e204152",
-                    "name":"node1",
-                    "peerURLs":[
-                        "http://10.0.0.10:2379"
-                    ],
-                    "clientURLs":[
-                            "http://10.0.0.10:2380"
-                    ]
-                },
-                {
-                    "id":"2225373f43",
-                    "name":"node2",
-                    "peerURLs":[
-                        "http://127.0.0.11:2379"
-                    ],
-                    "clientURLs":[
-                        "http://127.0.0.11:2380"
-                    ]
-            },
-        ]
-    }
-```
-
-### POST /v2/admin/members/
-Add a member to the cluster.
-Returns an HTTP 201 response code and the representation of added member with a newly generated a memberID when successful. Returns a string describing the failure condition when unsuccessful. 
-
-If the POST body is malformed an HTTP 400 will be returned. If the member exists in the cluster or existed in the cluster at some point in the past an HTTP 500(TODO: fix this) will be returned. If the cluster fails to process the request within timeout an HTTP 500 will be returned, though the request may be processed later.
-```
-    Example Request: POST
-                     http://localhost:2379/v2/admin/members/
-                     Body:
-                     [{"PeerURLs":["http://10.0.0.10:2379"]}]
-    Respose formats: JSON
-    Example Response:
-```
-```json
-    [
-        {
-            "id":"3777296169",
-            "peerURLs":[
-                "http://10.0.0.10:2379"
-            ],
-        },
-    ]
-```
-
-### DELETE /v2/admin/members/:id
-Remove a member from the cluster.
-Returns empty when successful. Returns a string describing the failure condition when unsuccessful. 
-
-If the member does not exist in the cluster an HTTP 500(TODO: fix this) will be returned. If the cluster fails to process the request within timeout an HTTP 500 will be returned, though the request may be processed later.
-```
-    Response formats: JSON
-    Example Request: DELETE
-                     http://localhost:2379/v2/admin/members/272e204152
-    Example Response: Empty
-```

Documentation/0.5/admin_guide.md

@@ -0,0 +1,79 @@
+## Administration
+
+### Data Directory
+
+#### Lifecycle
+
+When first started, etcd stores its configuration into a data directory specified by the data-dir configuration parameter.
+Configuration is stored in the write ahead log and includes: the local member ID, cluster ID, and initial cluster configuration.
+The write ahead log and snapshot files are used during member operation and to recover after a restart.
+
+If a member’s data directory is ever lost or corrupted then the user should remove the etcd member from the cluster via the [members API][members-api].
+
+A user should avoid restarting an etcd member with a data directory from an out-of-date backup.
+Using an out-of-date data directory can lead to inconsistency as the member had agreed to store information via raft then re-joins saying it needs that information again.
+For maximum safety, if an etcd member suffers any sort of data corruption or loss, it must be removed from the cluster.
+Once removed the member can be re-added with an empty data directory.
+
+[members-api]: https://github.com/coreos/etcd/blob/master/Documentation/0.5/other_apis.md#members-api
+
+#### Contents
+
+The data directory has two sub-directories in it:
+
+1. wal: write ahead log files are stored here. For details see the [wal package documentation][wal-pkg]
+2. snap: log snapshots are stored here. For details see the [snap package documentation][snap-pkg]
+
+[wal-pkg]: http://godoc.org/github.com/coreos/etcd/wal
+[snap-pkg]: http://godoc.org/github.com/coreos/etcd/snap
+
+### Cluster Lifecycle
+
+If you are spinning up multiple clusters for testing it is recommended that you specify a unique initial-cluster-token for the different clusters.
+This can protect you from cluster corruption in case of mis-configuration because two members started with different cluster tokens will refuse members from each other.
+
+### Disaster Recovery
+
+etcd is designed to be resilient to machine failures. An etcd cluster can automatically recover from any number of temporary failures (for example, machine reboots), and a cluster of N members can tolerate up to _(N/2)-1_ permanent failures (where a member can no longer access the cluster, due to hardware failure or disk corruption). However, in extreme circumstances, a cluster might permanently lose enough members such that quorum is irrevocably lost. For example, if a three-node cluster suffered two simultaneous and unrecoverable machine failures, it would be normally impossible for the cluster to restore quorum and continue functioning.
+
+To recover from such scenarios, etcd provides functionality to backup and restore the datastore and recreate the cluster without data loss.
+
+#### Backing up the datastore
+
+The first step of the recovery is to backup the data directory on a functioning etcd node. To do this, use the `etcdctl backup` command, passing in the original data directory used by etcd. For example:
+
+```sh
+    etcdctl backup \
+      --data-dir /var/lib/etcd \
+      --backup-dir /tmp/etcd_backup
+```
+
+This command will rewrite some of the metadata contained in the backup (specifically, the node ID and cluster ID), which means that the node will lose its former identity. In order to recreate a cluster from the backup, you will need to start a new, single-node cluster. The metadata is rewritten to prevent the new node from inadvertently being joined onto an existing cluster.
+
+#### Restoring a backup
+
+To restore a backup using the procedure created above, start etcd with the `-force-new-cluster` option and pointing to the backup directory. This will initialize a new, single-member cluster with the default advertised peer URLs, but preserve the entire contents of the etcd data store. Continuing from the previous example:
+
+```sh
+    etcd \
+      -data-dir=/tmp/etcd_backup \
+      -force-new-cluster \
+      ...
+```
+
+Now etcd should be available on this node and serving the original datastore.
+
+Once you have verified that etcd has started successfully, shut it down and move the data back to the previous location (you may wish to make another copy as well to be safe):
+
+```sh
+    pkill etcd
+    rm -fr /var/lib/etcd
+    mv /tmp/etcd_backup /var/lib/etcd
+    etcd \
+      -data-dir=/var/lib/etcd \
+      ...
+```
+
+#### Restoring the cluster
+
+Now that the node is running successfully, you can add more nodes to the cluster and restore resiliency. See the [runtime configuration](runtime-configuration.md) guide for more details.

Documentation/0.5/api.md

@@ -401,19 +401,19 @@ curl 'http://127.0.0.1:2379/v2/keys/dir/asdf?consistent=true&wait=true'
 
 ```json
 {
-	"action": "expire",
-	"node": {
-		"createdIndex": 8,
-		"key": "/dir",
-		"modifiedIndex": 15
-	},
-	"prevNode": {
-		"createdIndex": 8,
-		"key": "/dir",
-		"dir":true,
-		"modifiedIndex": 17,
-		"expiration": "2013-12-11T10:39:35.689275857-08:00"
-	}
+    "action": "expire",
+    "node": {
+        "createdIndex": 8,
+        "key": "/dir",
+        "modifiedIndex": 15
+    },
+    "prevNode": {
+        "createdIndex": 8,
+        "key": "/dir",
+        "dir":true,
+        "modifiedIndex": 17,
+        "expiration": "2013-12-11T10:39:35.689275857-08:00"
+    }
 }
 ```
 
@@ -639,22 +639,22 @@ We should see the response as an array of items:
 ```json
 {
     "action": "get",
-    "node": {
-        "key": "/",
-        "dir": true,
-        "nodes": [
-            {
-                "key": "/foo_dir",
-                "dir": true,
-                "modifiedIndex": 2,
-                "createdIndex": 2
-            },
-            {
-                "key": "/foo",
-                "value": "two",
-                "modifiedIndex": 1,
-                "createdIndex": 1
-            }
+    "node": {
+        "key": "/",
+        "dir": true,
+        "nodes": [
+            {
+                "key": "/foo_dir",
+                "dir": true,
+                "modifiedIndex": 2,
+                "createdIndex": 2
+            },
+            {
+                "key": "/foo",
+                "value": "two",
+                "modifiedIndex": 1,
+                "createdIndex": 1
+            }
         ]
     }
 }
@@ -669,33 +669,33 @@ curl http://127.0.0.1:2379/v2/keys/?recursive=true
 
 ```json
 {
-    "action": "get",
-    "node": {
-        "key": "/",
-        "dir": true,
-        "nodes": [
-            {
-                "key": "/foo_dir",
-                "dir": true,
-                "nodes": [
-                    {
-                        "key": "/foo_dir/foo",
-                        "value": "bar",
-                        "modifiedIndex": 2,
-                        "createdIndex": 2
-                    }
-                ],
-                "modifiedIndex": 2,
-                "createdIndex": 2
-            },
-            {
-                "key": "/foo",
-                "value": "two",
-                "modifiedIndex": 1,
-                "createdIndex": 1
-            }
-        ]
-    }
+    "action": "get",
+    "node": {
+        "key": "/",
+        "dir": true,
+        "nodes": [
+            {
+                "key": "/foo_dir",
+                "dir": true,
+                "nodes": [
+                    {
+                        "key": "/foo_dir/foo",
+                        "value": "bar",
+                        "modifiedIndex": 2,
+                        "createdIndex": 2
+                    }
+                ],
+                "modifiedIndex": 2,
+                "createdIndex": 2
+            },
+            {
+                "key": "/foo",
+                "value": "two",
+                "modifiedIndex": 1,
+                "createdIndex": 1
+            }
+        ]
+    }
 }
 ```
 
@@ -896,7 +896,7 @@ curl http://127.0.0.1:2379/v2/stats/leader
         "startTime": "2014-10-24T13:15:51.184719899-07:00",
         "uptime": "7m17.859616962s"
     },
-    "name": "node1",
+    "name": "infra1",
     "recvAppendRequestCnt": 3949,
     "recvBandwidthRate": 561.5729321100841,
     "recvPkgRate": 9.008227977383449,
@@ -1005,6 +1005,6 @@ curl http://127.0.0.1:2379/v2/stats/store
 
 ## Cluster Config
 
-See the [admin API guide][admin-api] for details on the cluster management APIs.
+See the [other etcd APIs][other-apis] for details on the cluster management.
 
-[admin-api]: https://github.com/coreos/etcd/blob/master/Documentation/0.5/admin_api.md
+[other-apis]: https://github.com/coreos/etcd/blob/master/Documentation/0.5/other_apis.md

Documentation/0.5/clustering.md

@@ -1,118 +1,135 @@
 # Clustering Guide
 
-This guide will walk you through configuring a three machine etcd cluster with
-the following details:
+This guide will walk you through configuring a three machine etcd cluster with the following details:
 
 |Name	|Address	|
-|-------|---------------|
+|-------|-----------|
 |infra0	|10.0.1.10	|
 |infra1	|10.0.1.11	|
 |infra2	|10.0.1.12	|
 
 ## Static
 
-As we know the cluster members, their addresses and the size of the cluster
-before starting we can use an offline bootstrap configuration. Each machine
-will get either the following command line or environment variables:
+As we know the cluster members, their addresses and the size of the cluster before starting we can use an offline bootstrap configuration. Each machine will get either the following command line or environment variables:
 
 ```
-ETCD_INITIAL_CLUSTER=”infra0=http://10.0.1.10:2379,infra1=http://10.0.1.11:2379,infra2=http://10.0.1.12:2379”
+ETCD_INITIAL_CLUSTER="infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380"
 ETCD_INITIAL_CLUSTER_STATE=new
 ```
 
 ```
--initial-cluster infra0=http://10.0.1.10:2379,http://10.0.1.11:2379,infra2=http://10.0.1.12:2379 \
-	-initial-cluster-state new
+-initial-cluster infra0=http://10.0.1.10:2380,http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
+  -initial-cluster-state new
 ```
 
+If you are spinning up multiple clusters (or creating and destroying a single cluster) with same configuration for testing purpose, it is highly recommended that you specify a unique `initial-cluster-token` for the different clusters. By doing this, etcd can generate unique cluster IDs and member IDs for the clusters even if they otherwise have the exact same configuration. This can protect you from cross-cluster-interaction, which might corrupt your clusters.
+
 On each machine you would start etcd with these flags:
 
 ```
-$ etcd -name infra0 -initial-advertise-peer-urls https://10.0.1.10:2379 \
-	-initial-cluster infra0=http://10.0.1.10:2379,infra1=http://10.0.1.11:2379,infra2=http://10.0.1.12:2379 \
-	-initial-cluster-state new
-$ etcd -name infra1 -initial-advertise-peer-urls https://10.0.1.11:2379 \
-	-initial-cluster infra0=http://10.0.1.10:2379,infra1=http://10.0.1.11:2379,infra2=http://10.0.1.12:2379 \
-	-initial-cluster-state new
-$ etcd -name infra2 -initial-advertise-peer-urls https://10.0.1.12:2379 \
-	-initial-cluster infra0=http://10.0.1.10:2379,infra1=http://10.0.1.11:2379,infra2=http://10.0.1.12:2379 \
-	-initial-cluster-state new
+$ etcd -name infra0 -initial-advertise-peer-urls https://10.0.1.10:2380 \
+  -initial-cluster-token etcd-cluster-1 \
+  -initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
+  -initial-cluster-state new
+```
+```
+$ etcd -name infra1 -initial-advertise-peer-urls https://10.0.1.11:2380 \
+  -initial-cluster-token etcd-cluster-1 \
+  -initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
+  -initial-cluster-state new
+```
+```
+$ etcd -name infra2 -initial-advertise-peer-urls https://10.0.1.12:2380 \
+  -initial-cluster-token etcd-cluster-1 \
+  -initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
+  -initial-cluster-state new
 ```
 
-The command line parameters starting with `-initial-cluster` will be ignored on
-subsequent runs of etcd. You are free to remove the environment variables or
-command line flags after the initial bootstrap process. If you need to make
-changes to the configuration later see our guide on runtime configuration.
+The command line parameters starting with `-initial-cluster` will be ignored on subsequent runs of etcd. You are free to remove the environment variables or command line flags after the initial bootstrap process. If you need to make changes to the configuration later see our guide on [runtime configuration](runtime-configuration.md).
 
 ### Error Cases
 
-In the following case we have not included our new host in the list of
-enumerated nodes. If this is a new cluster, the node must be added to the list
-of initial cluster members.
+In the following case we have not included our new host in the list of enumerated nodes. If this is a new cluster, the node must be added to the list of initial cluster members.
+
 ```
-$ etcd -name infra1 -initial-advertise-peer-urls http://10.0.1.11:2379 \
-	-initial-cluster infra0=http://10.0.1.10:2379 \
-	-initial-cluster-state new
+$ etcd -name infra1 -initial-advertise-peer-urls http://10.0.1.11:2380 \
+  -initial-cluster infra0=http://10.0.1.10:2380 \
+  -initial-cluster-state new
 etcd: infra1 not listed in the initial cluster config
 exit 1
 ```
 
-In this case we are attempting to map a node (infra0) on a different address
-(127.0.0.1:2379) than its enumerated address in the cluster list
-(10.0.1.10:2379). If this node is to listen on multiple addresses, all
-addresses must be reflected in the “initial-cluster” configuration directive.
+In this case we are attempting to map a node (infra0) on a different address (127.0.0.1:2380) than its enumerated address in the cluster list (10.0.1.10:2380). If this node is to listen on multiple addresses, all addresses must be reflected in the "initial-cluster" configuration directive.
 
 ```
-$ etcd -name infra0 -initial-advertise-peer-urls http://127.0.0.1:2379 \
-	-initial-cluster infra0=http://10.0.1.10:2379,infra1=http://10.0.1.11:2379,infra2=http://10.0.1.12:2379 \
-	-initial-cluster-state=new
+$ etcd -name infra0 -initial-advertise-peer-urls http://127.0.0.1:2380 \
+  -initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
+  -initial-cluster-state=new
 etcd: infra0 has different advertised URLs in the cluster and advertised peer URLs list
 exit 1
 ```
 
-If you configure a peer with a different set of configuration and attempt to
-join this cluster you will get a cluster ID mismatch and etcd will exit.
+If you configure a peer with a different set of configuration and attempt to join this cluster you will get a cluster ID mismatch and etcd will exit.
 
 ```
-$ etcd -name infra3 -initial-advertise-peer-urls http://10.0.1.13:2379 \
-	-initial-cluster infra0=http://10.0.1.10:2379,infra1=http://10.0.1.11:2379,infra3=http://10.0.1.13:2379 \
-	-initial-cluster-state=new
+$ etcd -name infra3 -initial-advertise-peer-urls http://10.0.1.13:2380 \
+  -initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra3=http://10.0.1.13:2380 \
+  -initial-cluster-state=new
 etcd: conflicting cluster ID to the target cluster (c6ab534d07e8fcc4 != bc25ea2a74fb18b0). Exiting.
 exit 1
 ```
 
-
 ## Discovery
 
-In a number of cases you might not know the IPs of your cluster peers ahead of
-time. This is common when utilizing cloud providers or when your network uses
-DHCP. In these cases you can use an existing etcd cluster to bootstrap a new
-one. We call this process “discovery”.
+In a number of cases you might not know the IPs of your cluster peers ahead of time. This is common when utilizing cloud providers or when your network uses DHCP. In these cases you can use an existing etcd cluster to bootstrap a new one. We call this process "discovery".
 
-Discovery uses an existing cluster to bootstrap itself.  If you are using your
-own etcd cluster you can create a URL like so:
+### Lifetime of a Discovery URL
+
+A discovery URL identifies a unique etcd cluster. Instead of reusing a discovery URL, you should always create discovery URLs for new clusters.
+
+Moreover, discovery URLs should ONLY be used for the initial bootstrapping of a cluster. To change cluster membership after the cluster is already running, see the [runtime reconfiguration][runtime] guide.
+
+[runtime]: https://github.com/coreos/etcd/blob/master/Documentation/0.5/runtime-configuration.md
+
+### Custom etcd discovery service
+
+Discovery uses an existing cluster to bootstrap itself. If you are using your own etcd cluster you can create a URL like so:
 
 ```
-$ curl -X PUT https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83/_config/size -d value=5
+$ curl -X PUT https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83/_config/size -d value=3
 ```
 
-The URL you will use in this case will be
-`https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83`
-and the machines will use the
-`https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83`
-directory for registration as they start.
+By setting the size key to the URL, you create a discovery URL with expected-cluster-size of 3.
 
-If you do not have access to an existing cluster you can use the hosted
-discovery.etcd.io service.  You can create a private discovery URL using the
-"new" endpoint like so:
+The URL you will use in this case will be `https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83` and the etcd members will use the `https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83` directory for registration as they start.
+
+Now we start etcd with those relevant flags for each member:
+
+```
+$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
+  -discovery https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83
+```
+```
+$ etcd -name infra1 -initial-advertise-peer-urls http://10.0.1.11:2380 \
+  -discovery https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83
+```
+```
+$ etcd -name infra2 -initial-advertise-peer-urls http://10.0.1.12:2380 \
+  -discovery https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83
+```
+
+This will cause each member to register itself with the custom etcd discovery service and begin the cluster once all machines have been registered.
+
+### Public discovery service
+
+If you do not have access to an existing cluster you can use the public discovery service hosted at discovery.etcd.io.  You can create a private discovery URL using the "new" endpoint like so:
 
 ```
 $ curl https://discovery.etcd.io/new?size=3
 https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
 ```
 
-This will create the cluster with an initial expected size of 3 members. If you
-do not specify a size a default of 3 will be used.
+This will create the cluster with an initial expected size of 3 members. If you do not specify a size a default of 3 will be used.
 
 ```
 ETCD_DISCOVERY=https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
@@ -122,16 +139,22 @@ ETCD_DISCOVERY=https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573d
 -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
 ```
 
-Now we start etcd with those relevant flags on each machine:
+Now we start etcd with those relevant flags for each member:
 
 ```
-$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2379 -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
-$ etcd -name infra1 -initial-advertise-peer-urls http://10.0.1.11:2379 -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
-$ etcd -name infra2 -initial-advertise-peer-urls http://10.0.1.12:2379 -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
+$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
+  -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
+```
+```
+$ etcd -name infra1 -initial-advertise-peer-urls http://10.0.1.11:2380 \
+  -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
+```
+```
+$ etcd -name infra2 -initial-advertise-peer-urls http://10.0.1.12:2380 \
+  -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
 ```
 
-This will cause each machine to register itself with the etcd service and begin
-the cluster once all machines have been registered.
+This will cause each member to register itself with the discovery service and begin the cluster once all members have been registered.
 
 You can use the environment variable `ETCD_DISCOVERY_PROXY` to cause etcd to use an HTTP proxy to connect to the discovery service.
 
@@ -140,7 +163,8 @@ You can use the environment variable `ETCD_DISCOVERY_PROXY` to cause etcd to use
 #### Discovery Server Errors
 
 ```
-$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2379 -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
+$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
+  -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
 etcd: error: the cluster doesn’t have a size configuration value in https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de/_config
 exit 1
 ```
@@ -148,7 +172,8 @@ exit 1
 #### User Errors
 
 ```
-$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2379 -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
+$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
+  -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
 etcd: error: the cluster using discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de has already started with all 5 members
 exit 1
 ```
@@ -159,20 +184,16 @@ This is a harmless warning notifying you that the discovery URL will be
 ignored on this machine.
 
 ```
-$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2379 -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
+$ etcd -name infra0 -initial-advertise-peer-urls http://10.0.1.10:2380 \
+  -discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
 etcd: warn: ignoring discovery URL: etcd has already been initialized and has a valid log in /var/lib/etcd
 ```
 
 # 0.4 to 0.5+ Migration Guide
 
-In etcd 0.5 we introduced the ability to listen on more than one address and to
-advertise multiple addresses. This makes using etcd easier when you have
-complex networking, such as private and public networks on various cloud
-providers.
+In etcd 0.5 we introduced the ability to listen on more than one address and to advertise multiple addresses. This makes using etcd easier when you have complex networking, such as private and public networks on various cloud providers.
 
-To make understanding this feature easier, we changed the naming of some flags,
-but we support the old flags to make the migration from the old to new version
-easier.
+To make understanding this feature easier, we changed the naming of some flags, but we support the old flags to make the migration from the old to new version easier.
 
 |Old Flag		|New Flag		|Migration Behavior									|
 |-----------------------|-----------------------|---------------------------------------------------------------------------------------|

Documentation/0.5/other_apis.md

@@ -0,0 +1,96 @@
+## Members API
+
+* [List members](#list-members)
+* [Add a member](#add-a-member)
+* [Delete a member](#delete-a-member)
+
+## List members
+
+Return an HTTP 200 OK response code and a representation of all members in the etcd cluster.
+
+### Request
+
+```
+GET /v2/members HTTP/1.1
+```
+
+### Example
+
+```
+curl http://10.0.0.10:2379/v2/members
+```
+
+```json
+{
+    "members": [
+        {
+            "id": "272e204152",
+            "name": "infra1",
+            "peerURLs": [
+                "http://10.0.0.10:2380"
+            ],
+            "clientURLs": [
+                "http://10.0.0.10:2379"
+            ]
+        },
+        {
+            "id": "2225373f43",
+            "name": "infra2",
+            "peerURLs": [
+                "http://10.0.0.11:2380"
+            ],
+            "clientURLs": [
+                "http://10.0.0.11:2379"
+            ]
+        },
+    ]
+}
+```
+
+## Add a member
+
+Returns an HTTP 201 response code and the representation of added member with a newly generated a memberID when successful. Returns a string describing the failure condition when unsuccessful. 
+
+If the POST body is malformed an HTTP 400 will be returned. If the member exists in the cluster or existed in the cluster at some point in the past an HTTP 409 will be returned. If any of the given peerURLs exists in the cluster an HTTP 409 will be returned. If the cluster fails to process the request within timeout an HTTP 500 will be returned, though the request may be processed later.
+
+### Request
+
+```
+POST /v2/members HTTP/1.1
+
+{"peerURLs": ["http://10.0.0.10:2379"]}
+```
+
+### Example
+
+```
+curl http://10.0.0.10:2379/v2/members -XPOST -H "Content-Type: application/json" -d '{"peerURLs":["http://10.0.0.10:2379"]}'
+```
+
+```json
+{
+    "id": "3777296169",
+    "peerURLs": [
+        "http://10.0.0.10:2379"
+    ]
+}
+```
+
+## Delete a member
+
+Remove a member from the cluster. The member ID must be a hex-encoded uint64.
+Returns empty when successful. Returns a string describing the failure condition when unsuccessful. 
+
+If the member does not exist in the cluster an HTTP 500(TODO: fix this) will be returned. If the cluster fails to process the request within timeout an HTTP 500 will be returned, though the request may be processed later.
+
+### Request
+
+```
+DELETE /v2/members/<id> HTTP/1.1
+```
+
+### Example
+
+```
+curl http://10.0.0.10:2379/v2/members/272e204152 -XDELETE
+```

Documentation/0.5/runtime-configuration.md

@@ -0,0 +1,143 @@
+## Runtime Reconfiguration
+
+etcd comes with support for incremental runtime reconfiguration, which allows users to update the membership of the cluster at run time.
+
+## Reconfiguration Use Cases
+
+Let us walk through the four use cases for re-configuring a cluster: replacing a member, increasing or decreasing cluster size, and restarting a cluster from a majority failure.
+
+### Replace a Member
+
+The most common use case of cluster reconfiguration is to replace a member because of a permanent failure of the existing member: for example, hardware failure, loss of network address, or data directory corruption.
+It is important to replace failed members as soon as the failure is detected.
+If etcd falls below a simple majority of members it can no longer accept writes: e.g. in a 3 member cluster the loss of two members will cause writes to fail and the cluster to stop operating.
+
+### Increase Cluster Size
+
+To make your cluster more resilient to machine failure you can increase the size of the cluster.
+For example, if the cluster consists of three machines, it can tolerate one failure.
+If we increase the cluster size to five, it can tolerate two machine failures.
+
+Increasing the cluster size can also provide better read performance.
+When a client accesses etcd, the normal read gets the data from the local copy of each member (members always shares the same view of the cluster at the same index, which is guaranteed by the sequential consistency of etcd).
+Since clients can read from any member, increasing the number of members thus increases overall read throughput.
+
+### Decrease Cluster Size
+
+To improve the write performance of a cluster, you might want to trade off resilience by removing members.
+etcd replicates the data to the majority of members of the cluster before committing the write.
+Decreasing the cluster size means the etcd cluster has to do less work for each write, thus increasing the write performance.
+
+### Restart Cluster from Majority Failure
+
+If the majority of your cluster is lost, then you need to take manual action in order to recover safely.
+The basic steps in the recovery process include creating a new cluster using the old data, forcing a single member to act as the leader, and finally using runtime configuration to add members to this new cluster.
+
+TODO: https://github.com/coreos/etcd/issues/1242
+
+## Cluster Reconfiguration Operations
+
+Now that we have the use cases in mind, let us lay out the operations involved in each.
+
+Before making any change, the simple majority (quorum) of etcd members must be available.
+This is essentially the same requirement as for any other write to etcd.
+
+All changes to the cluster are done one at a time:
+
+To replace a single member you will make an add then a remove operation
+To increase from 3 to 5 members you will make two add operations
+To decrease from 5 to 3 you will make two remove operations
+
+All of these examples will use the `etcdctl` command line tool that ships with etcd.
+If you want to use the member API directly you can find the documentation [here](https://github.com/coreos/etcd/blob/master/Documentation/0.5/other_apis.md).
+
+### Remove a Member
+
+First, we need to find the target member:
+
+```
+$ etcdctl member list
+6e3bd23ae5f1eae0: name=node2 peerURLs=http://localhost:7002 clientURLs=http://127.0.0.1:4002
+924e2e83e93f2560: name=node3 peerURLs=http://localhost:7003 clientURLs=http://127.0.0.1:4003
+a8266ecf031671f3: name=node1 peerURLs=http://localhost:7001 clientURLs=http://127.0.0.1:4001
+```
+
+Let us say the member ID we want to remove is a8266ecf031671f3.
+We then use the `remove` command to perform the removal:
+
+```
+$ etcdctl member remove a8266ecf031671f3
+Removed member a8266ecf031671f3 from cluster
+```
+
+The target member will stop itself at this point and print out the removal in the log:
+
+```
+etcd: this member has been permanently removed from the cluster. Exiting.
+```
+
+Removal of the leader is safe, but the cluster will be out of progress for a period of election timeout because it needs to elect the new leader.
+
+### Add a Member
+
+Adding a member is a two step process:
+
+ * Add the new member to the cluster via the [members API](https://github.com/coreos/etcd/blob/master/Documentation/0.5/other_apis.md#post-v2members) or the `etcdctl member add` command.
+ * Start the member with the correct configuration.
+
+Using `etcdctl` let's add the new member to the cluster:
+
+```
+$ etcdctl member add infra3 http://10.0.1.13:2380
+added member 9bf1b35fc7761a23 to cluster
+ETCD_NAME="infra3"
+ETCD_INITIAL_CLUSTER="infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380,infra3=http://10.0.1.13:2380"
+ETCD_INITIAL_CLUSTER_STATE=existing
+```
+
+> Notice that infra3 was added to the cluster using its advertised peer URL.
+
+Now start the new etcd process with the relevant flags for the new member:
+
+```
+$ export ETCD_NAME="infra3"
+$ export ETCD_INITIAL_CLUSTER="infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380,infra3=http://10.0.1.13:2380"
+$ export ETCD_INITIAL_CLUSTER_STATE=existing
+$ etcd -listen-client-urls http://10.0.1.13:2379 -advertise-client-urls http://10.0.1.13:2379  -listen-peer-urls http://10.0.1.13:2380 -initial-advertise-peer-urls http://10.0.1.13:2380
+```
+
+The new member will run as a part of the cluster and immediately begin catching up with the rest of the cluster.
+
+If you are adding multiple members the best practice is to configure the new member, then start the process, then configure the next, and so on.
+A common case is increasing a cluster from 1 to 3: if you add one member to a 1-node cluster, the cluster cannot make progress before the new member starts because it needs two members as majority to agree on the consensus.
+
+#### Error Cases
+
+In the following case we have not included our new host in the list of enumerated nodes.
+If this is a new cluster, the node must be added to the list of initial cluster members.
+
+```
+$ etcd -name infra3 \
+  -initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380 \
+  -initial-cluster-state existing
+etcdserver: assign ids error: the member count is unequal
+exit 1
+```
+
+In this case we give a different address (10.0.1.14:2380) to the one that we used to join the cluster (10.0.1.13:2380).
+
+```
+$ etcd -name infra4 \
+  -initial-cluster infra0=http://10.0.1.10:2380,infra1=http://10.0.1.11:2380,infra2=http://10.0.1.12:2380,infra4=http://10.0.1.14:2380 \
+  -initial-cluster-state existing
+etcdserver: assign ids error: unmatched member while checking PeerURLs
+exit 1
+```
+
+When we start etcd using the data directory of a removed member, etcd will exit automatically if it connects to any alive member in the cluster:
+
+```
+$ etcd
+etcd: this member has been permanently removed from the cluster. Exiting.
+exit 1
+```

Documentation/libraries-and-tools.md

@@ -101,3 +101,6 @@ A detailed recap of client functionalities can be found in the [clients compatib
 - [GoogleCloudPlatform/kubernetes](https://github.com/GoogleCloudPlatform/kubernetes) - Container cluster manager.
 - [mailgun/vulcand](https://github.com/mailgun/vulcand) - HTTP proxy that uses etcd as a configuration backend.
 - [duedil-ltd/discodns](https://github.com/duedil-ltd/discodns) - Simple DNS nameserver using etcd as a database for names and records.
+- [skynetservices/skydns](https://github.com/skynetservices/skydns) - RFC compliant DNS server
+- [xordataexchange/crypt](https://github.com/xordataexchange/crypt) - Securely store values in etcd using GPG encryption
+- [spf13/viper](https://github.com/spf13/viper) - Go configuration library, reads values from ENV, pflags, files, and etcd with optional encryption

Documentation/security.md

@@ -124,7 +124,7 @@ DISCOVERY_URL=... # from https://discovery.etcd.io/new
 etcd -name node1 -data-dir node1 -ca-file=/path/to/ca.crt -cert-file=/path/to/node1.crt -key-file=/path/to/node1.key -peer-addr ${node1_public_ip}:7001 -discovery ${DISCOVERY_URL}
 
 # Node2
-etcd -name node1 -data-dir node2 -ca-file=/path/to/ca.crt -cert-file=/path/to/node2.crt -key-file=/path/to/node2.key -peer-addr ${node2_public_ip}:7001 -discovery ${DISCOVERY_URL}
+etcd -name node2 -data-dir node2 -ca-file=/path/to/ca.crt -cert-file=/path/to/node2.crt -key-file=/path/to/node2.key -peer-addr ${node2_public_ip}:7001 -discovery ${DISCOVERY_URL}
 ```
 
 The etcd nodes will form a cluster and all communication between nodes in the cluster will be encrypted and authenticated using the client certificates. You will see in the output of etcd that the addresses it connects to use HTTPS.

Godeps/Godeps.json

@@ -16,8 +16,8 @@
 		},
 		{
 			"ImportPath": "github.com/codegangsta/cli",
-			"Comment": "1.0.0-72-gbb91895",
-			"Rev": "bb9189510af1f49580c073c9e59e8bf288f0df27"
+			"Comment": "1.2.0-26-gf7ebb76",
+			"Rev": "f7ebb761e83e21225d1d8954fde853bf8edd46c4"
 		},
 		{
 			"ImportPath": "github.com/coreos/go-etcd/etcd",

Godeps/_workspace/src/github.com/codegangsta/cli/.travis.yml

@@ -1,2 +1,6 @@
 language: go
 go: 1.1
+
+script:
+- go vet ./...
+- go test -v ./...

Godeps/_workspace/src/github.com/codegangsta/cli/README.md

@@ -9,17 +9,17 @@ http://godoc.org/github.com/codegangsta/cli
 ## Overview
 Command line apps are usually so tiny that there is absolutely no reason why your code should *not* be self-documenting. Things like generating help text and parsing command flags/options should not hinder productivity when writing a command line app.
 
-This is where cli.go comes into play. cli.go makes command line programming fun, organized, and expressive!
+**This is where cli.go comes into play.** cli.go makes command line programming fun, organized, and expressive!
 
 ## Installation
 Make sure you have a working Go environment (go 1.1 is *required*). [See the install instructions](http://golang.org/doc/install.html).
 
-To install cli.go, simply run:
+To install `cli.go`, simply run:
 ```
 $ go get github.com/codegangsta/cli
 ```
 
-Make sure your PATH includes to the `$GOPATH/bin` directory so your commands can be easily used:
+Make sure your `PATH` includes to the `$GOPATH/bin` directory so your commands can be easily used:
 ```
 export PATH=$PATH:$GOPATH/bin
 ```
@@ -122,7 +122,7 @@ GLOBAL OPTIONS
 ```
 
 ### Arguments
-You can lookup arguments by calling the `Args` function on cli.Context.
+You can lookup arguments by calling the `Args` function on `cli.Context`.
 
 ``` go
 ...
@@ -137,7 +137,11 @@ Setting and querying flags is simple.
 ``` go
 ...
 app.Flags = []cli.Flag {
-  cli.StringFlag{"lang", "english", "language for the greeting"},
+  cli.StringFlag{
+    Name: "lang",
+    Value: "english",
+    Usage: "language for the greeting",
+  },
 }
 app.Action = func(c *cli.Context) {
   name := "someone"
@@ -155,11 +159,30 @@ app.Action = func(c *cli.Context) {
 
 #### Alternate Names
 
-You can set alternate (or short) names for flags by providing a comma-delimited list for the Name. e.g.
+You can set alternate (or short) names for flags by providing a comma-delimited list for the `Name`. e.g.
 
 ``` go
 app.Flags = []cli.Flag {
-  cli.StringFlag{"lang, l", "english", "language for the greeting"},
+  cli.StringFlag{
+    Name: "lang, l",
+    Value: "english",
+    Usage: "language for the greeting",
+  },
+}
+```
+
+#### Values from the Environment
+
+You can also have the default value set from the environment via `EnvVar`.  e.g.
+
+``` go
+app.Flags = []cli.Flag {
+  cli.StringFlag{
+    Name: "lang, l",
+    Value: "english",
+    Usage: "language for the greeting",
+    EnvVar: "APP_LANG",
+  },
 }
 ```
 
@@ -214,8 +237,8 @@ app.Commands = []cli.Command{
 
 ### Bash Completion
 
-You can enable completion commands by setting the EnableBashCompletion
-flag on the App object.  By default, this setting will only auto-complete to
+You can enable completion commands by setting the `EnableBashCompletion`
+flag on the `App` object.  By default, this setting will only auto-complete to
 show an app's subcommands, but you can write your own completion methods for
 the App or its subcommands.
 ```go
@@ -237,7 +260,7 @@ app.Commands = []cli.Command{
         return
       }
       for _, t := range tasks {
-        println(t)
+        fmt.Println(t)
       }
     },
   }
@@ -247,11 +270,18 @@ app.Commands = []cli.Command{
 
 #### To Enable
 
-Source the autocomplete/bash_autocomplete file in your .bashrc file while
-setting the PROG variable to the name of your program:
+Source the `autocomplete/bash_autocomplete` file in your `.bashrc` file while
+setting the `PROG` variable to the name of your program:
 
 `PROG=myprogram source /.../cli/autocomplete/bash_autocomplete`
 
 
+## Contribution Guidelines
+Feel free to put up a pull request to fix a bug or maybe add a feature. I will give it a code review and make sure that it does not break backwards compatibility. If I or any other collaborators agree that it is in line with the vision of the project, we will work with you to get the code into a mergeable state and merge it into the master branch.
+
+If you are have contributed something significant to the project, I will most likely add you as a collaborator. As a collaborator you are given the ability to merge others pull requests. It is very important that new code does not break existing code, so be careful about what code you do choose to merge. If you have any questions feel free to link @codegangsta to the issue in question and we can review it together.
+
+If you feel like you have contributed to the project but have not yet been added as a collaborator, I probably forgot to add you. Hit @codegangsta up over email and we will get it figured out.
+
 ## About
 cli.go is written by none other than the [Code Gangsta](http://codegangsta.io)

Godeps/_workspace/src/github.com/codegangsta/cli/app.go

@@ -22,6 +22,8 @@ type App struct {
 	Flags []Flag
 	// Boolean to enable bash completion commands
 	EnableBashCompletion bool
+	// Boolean to hide built-in help command
+	HideHelp bool
 	// An action to execute when the bash-completion flag is set
 	BashComplete func(context *Context)
 	// An action to execute before any subcommands are run, but after the context is ready
@@ -58,16 +60,15 @@ func NewApp() *App {
 		BashComplete: DefaultAppComplete,
 		Action:       helpCommand.Action,
 		Compiled:     compileTime(),
-		Author:       "Author",
-		Email:        "unknown@email",
 	}
 }
 
 // Entry point to the cli app. Parses the arguments slice and routes to the proper flag/args combination
 func (a *App) Run(arguments []string) error {
 	// append help to commands
-	if a.Command(helpCommand.Name) == nil {
+	if a.Command(helpCommand.Name) == nil && !a.HideHelp {
 		a.Commands = append(a.Commands, helpCommand)
+		a.appendFlag(HelpFlag)
 	}
 
 	//append version/help flags
@@ -75,7 +76,6 @@ func (a *App) Run(arguments []string) error {
 		a.appendFlag(BashCompletionFlag)
 	}
 	a.appendFlag(VersionFlag)
-	a.appendFlag(HelpFlag)
 
 	// parse flags
 	set := flagSet(a.Name, a.Flags)
@@ -131,12 +131,21 @@ func (a *App) Run(arguments []string) error {
 	return nil
 }
 
+// Another entry point to the cli app, takes care of passing arguments and error handling
+func (a *App) RunAndExitOnError() {
+	if err := a.Run(os.Args); err != nil {
+		os.Stderr.WriteString(fmt.Sprintln(err))
+		os.Exit(1)
+	}
+}
+
 // Invokes the subcommand given the context, parses ctx.Args() to generate command-specific flags
 func (a *App) RunAsSubcommand(ctx *Context) error {
 	// append help to commands
 	if len(a.Commands) > 0 {
-		if a.Command(helpCommand.Name) == nil {
+		if a.Command(helpCommand.Name) == nil && !a.HideHelp {
 			a.Commands = append(a.Commands, helpCommand)
+			a.appendFlag(HelpFlag)
 		}
 	}
 
@@ -144,14 +153,13 @@ func (a *App) RunAsSubcommand(ctx *Context) error {
 	if a.EnableBashCompletion {
 		a.appendFlag(BashCompletionFlag)
 	}
-	a.appendFlag(HelpFlag)
 
 	// parse flags
 	set := flagSet(a.Name, a.Flags)
 	set.SetOutput(ioutil.Discard)
 	err := set.Parse(ctx.Args().Tail())
 	nerr := normalizeFlags(a.Flags, set)
-	context := NewContext(a, set, set)
+	context := NewContext(a, set, ctx.globalSet)
 
 	if nerr != nil {
 		fmt.Println(nerr)

Godeps/_workspace/src/github.com/codegangsta/cli/app_test.go

@@ -2,9 +2,10 @@ package cli_test
 
 import (
 	"fmt"
-	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
 	"os"
 	"testing"
+
+	"github.com/codegangsta/cli"
 )
 
 func ExampleApp() {
@@ -42,7 +43,11 @@ func ExampleAppSubcommand() {
 					Usage:       "sends a greeting in english",
 					Description: "greets someone in english",
 					Flags: []cli.Flag{
-						cli.StringFlag{"name", "Bob", "Name of the person to greet"},
+						cli.StringFlag{
+							Name:  "name",
+							Value: "Bob",
+							Usage: "Name of the person to greet",
+						},
 					},
 					Action: func(c *cli.Context) {
 						fmt.Println("Hello,", c.String("name"))
@@ -83,12 +88,10 @@ func ExampleAppHelp() {
 	//    describeit - use it to see a description
 	//
 	// USAGE:
-	//    command describeit [command options] [arguments...]
+	//    command describeit [arguments...]
 	//
 	// DESCRIPTION:
 	//    This is how we describe describeit the function
-	//
-	// OPTIONS:
 }
 
 func ExampleAppBashComplete() {
@@ -254,11 +257,11 @@ func TestApp_ParseSliceFlags(t *testing.T) {
 	var expectedStringSlice = []string{"8.8.8.8", "8.8.4.4"}
 
 	if !IntsEquals(parsedIntSlice, expectedIntSlice) {
-		t.Errorf("%s does not match %s", parsedIntSlice, expectedIntSlice)
+		t.Errorf("%v does not match %v", parsedIntSlice, expectedIntSlice)
 	}
 
 	if !StrsEquals(parsedStringSlice, expectedStringSlice) {
-		t.Errorf("%s does not match %s", parsedStringSlice, expectedStringSlice)
+		t.Errorf("%v does not match %v", parsedStringSlice, expectedStringSlice)
 	}
 }
 
@@ -347,6 +350,26 @@ func TestAppHelpPrinter(t *testing.T) {
 	}
 }
 
+func TestAppVersionPrinter(t *testing.T) {
+	oldPrinter := cli.VersionPrinter
+	defer func() {
+		cli.VersionPrinter = oldPrinter
+	}()
+
+	var wasCalled = false
+	cli.VersionPrinter = func(c *cli.Context) {
+		wasCalled = true
+	}
+
+	app := cli.NewApp()
+	ctx := cli.NewContext(app, nil, nil)
+	cli.ShowVersion(ctx)
+
+	if wasCalled == false {
+		t.Errorf("Version printer expected to be called, but was not")
+	}
+}
+
 func TestAppCommandNotFound(t *testing.T) {
 	beforeRun, subcommandRun := false, false
 	app := cli.NewApp()
@@ -369,3 +392,32 @@ func TestAppCommandNotFound(t *testing.T) {
 	expect(t, beforeRun, true)
 	expect(t, subcommandRun, false)
 }
+
+func TestGlobalFlagsInSubcommands(t *testing.T) {
+	subcommandRun := false
+	app := cli.NewApp()
+
+	app.Flags = []cli.Flag{
+		cli.BoolFlag{Name: "debug, d", Usage: "Enable debugging"},
+	}
+
+	app.Commands = []cli.Command{
+		cli.Command{
+			Name: "foo",
+			Subcommands: []cli.Command{
+				{
+					Name: "bar",
+					Action: func(c *cli.Context) {
+						if c.GlobalBool("debug") {
+							subcommandRun = true
+						}
+					},
+				},
+			},
+		},
+	}
+
+	app.Run([]string{"command", "-d", "foo", "bar"})
+
+	expect(t, subcommandRun, true)
+}

Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete

@@ -5,7 +5,7 @@ _cli_bash_autocomplete() {
      COMPREPLY=()
      cur="${COMP_WORDS[COMP_CWORD]}"
      prev="${COMP_WORDS[COMP_CWORD-1]}"
-     opts=$( ${COMP_WORDS[@]:0:COMP_CWORD} --generate-bash-completion )
+     opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion )
      COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
      return 0
  }

Godeps/_workspace/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete

@@ -0,0 +1,5 @@
+autoload -U compinit && compinit
+autoload -U bashcompinit && bashcompinit
+
+script_dir=$(dirname $0)
+source ${script_dir}/bash_autocomplete

Godeps/_workspace/src/github.com/codegangsta/cli/cli_test.go

@@ -1,8 +1,9 @@
 package cli_test
 
 import (
-	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
 	"os"
+
+	"github.com/codegangsta/cli"
 )
 
 func Example() {
@@ -47,7 +48,11 @@ func ExampleSubcommand() {
 					Usage:       "sends a greeting in english",
 					Description: "greets someone in english",
 					Flags: []cli.Flag{
-						cli.StringFlag{"name", "Bob", "Name of the person to greet"},
+						cli.StringFlag{
+							Name:  "name",
+							Value: "Bob",
+							Usage: "Name of the person to greet",
+						},
 					},
 					Action: func(c *cli.Context) {
 						println("Hello, ", c.String("name"))
@@ -57,7 +62,11 @@ func ExampleSubcommand() {
 					ShortName: "sp",
 					Usage:     "sends a greeting in spanish",
 					Flags: []cli.Flag{
-						cli.StringFlag{"surname", "Jones", "Surname of the person to greet"},
+						cli.StringFlag{
+							Name:  "surname",
+							Value: "Jones",
+							Usage: "Surname of the person to greet",
+						},
 					},
 					Action: func(c *cli.Context) {
 						println("Hola, ", c.String("surname"))
@@ -67,7 +76,11 @@ func ExampleSubcommand() {
 					ShortName: "fr",
 					Usage:     "sends a greeting in french",
 					Flags: []cli.Flag{
-						cli.StringFlag{"nickname", "Stevie", "Nickname of the person to greet"},
+						cli.StringFlag{
+							Name:  "nickname",
+							Value: "Stevie",
+							Usage: "Nickname of the person to greet",
+						},
 					},
 					Action: func(c *cli.Context) {
 						println("Bonjour, ", c.String("nickname"))

Godeps/_workspace/src/github.com/codegangsta/cli/command.go

@@ -29,6 +29,8 @@ type Command struct {
 	Flags []Flag
 	// Treat all flags as normal arguments if true
 	SkipFlagParsing bool
+	// Boolean to hide built-in help command
+	HideHelp bool
 }
 
 // Invokes the command given the context, parses ctx.Args() to generate command-specific flags
@@ -38,11 +40,13 @@ func (c Command) Run(ctx *Context) error {
 		return c.startApp(ctx)
 	}
 
-	// append help to flags
-	c.Flags = append(
-		c.Flags,
-		HelpFlag,
-	)
+	if !c.HideHelp {
+		// append help to flags
+		c.Flags = append(
+			c.Flags,
+			HelpFlag,
+		)
+	}
 
 	if ctx.App.EnableBashCompletion {
 		c.Flags = append(c.Flags, BashCompletionFlag)
@@ -114,9 +118,13 @@ func (c Command) startApp(ctx *Context) error {
 		app.Usage = c.Usage
 	}
 
+	// set CommandNotFound
+	app.CommandNotFound = ctx.App.CommandNotFound
+
 	// set the flags and commands
 	app.Commands = c.Subcommands
 	app.Flags = c.Flags
+	app.HideHelp = c.HideHelp
 
 	// bash completion
 	app.EnableBashCompletion = ctx.App.EnableBashCompletion

Godeps/_workspace/src/github.com/codegangsta/cli/command_test.go

@@ -2,8 +2,9 @@ package cli_test
 
 import (
 	"flag"
-	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
 	"testing"
+
+	"github.com/codegangsta/cli"
 )
 
 func TestCommandDoNotIgnoreFlags(t *testing.T) {

Godeps/_workspace/src/github.com/codegangsta/cli/context.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // Context is a type that is passed through to
@@ -29,6 +30,11 @@ func (c *Context) Int(name string) int {
 	return lookupInt(name, c.flagSet)
 }
 
+// Looks up the value of a local time.Duration flag, returns 0 if no time.Duration flag exists
+func (c *Context) Duration(name string) time.Duration {
+	return lookupDuration(name, c.flagSet)
+}
+
 // Looks up the value of a local float64 flag, returns 0 if no float64 flag exists
 func (c *Context) Float64(name string) float64 {
 	return lookupFloat64(name, c.flagSet)
@@ -69,6 +75,11 @@ func (c *Context) GlobalInt(name string) int {
 	return lookupInt(name, c.globalSet)
 }
 
+// Looks up the value of a global time.Duration flag, returns 0 if no time.Duration flag exists
+func (c *Context) GlobalDuration(name string) time.Duration {
+	return lookupDuration(name, c.globalSet)
+}
+
 // Looks up the value of a global bool flag, returns false if no bool flag exists
 func (c *Context) GlobalBool(name string) bool {
 	return lookupBool(name, c.globalSet)
@@ -105,6 +116,18 @@ func (c *Context) IsSet(name string) bool {
 	return c.setFlags[name] == true
 }
 
+// Returns a slice of flag names used in this context.
+func (c *Context) FlagNames() (names []string) {
+	for _, flag := range c.Command.Flags {
+		name := strings.Split(flag.getName(), ",")[0]
+		if name == "help" {
+			continue
+		}
+		names = append(names, name)
+	}
+	return
+}
+
 type Args []string
 
 // Returns the command line arguments associated with the context.
@@ -140,6 +163,15 @@ func (a Args) Present() bool {
 	return len(a) != 0
 }
 
+// Swaps arguments at the given indexes
+func (a Args) Swap(from, to int) error {
+	if from >= len(a) || to >= len(a) {
+		return errors.New("index out of range")
+	}
+	a[from], a[to] = a[to], a[from]
+	return nil
+}
+
 func lookupInt(name string, set *flag.FlagSet) int {
 	f := set.Lookup(name)
 	if f != nil {
@@ -153,6 +185,18 @@ func lookupInt(name string, set *flag.FlagSet) int {
 	return 0
 }
 
+func lookupDuration(name string, set *flag.FlagSet) time.Duration {
+	f := set.Lookup(name)
+	if f != nil {
+		val, err := time.ParseDuration(f.Value.String())
+		if err == nil {
+			return val
+		}
+	}
+
+	return 0
+}
+
 func lookupFloat64(name string, set *flag.FlagSet) float64 {
 	f := set.Lookup(name)
 	if f != nil {

Godeps/_workspace/src/github.com/codegangsta/cli/context_test.go

@@ -2,8 +2,10 @@ package cli_test
 
 import (
 	"flag"
-	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
 	"testing"
+	"time"
+
+	"github.com/codegangsta/cli"
 )
 
 func TestNewContext(t *testing.T) {
@@ -26,6 +28,13 @@ func TestContext_Int(t *testing.T) {
 	expect(t, c.Int("myflag"), 12)
 }
 
+func TestContext_Duration(t *testing.T) {
+	set := flag.NewFlagSet("test", 0)
+	set.Duration("myflag", time.Duration(12*time.Second), "doc")
+	c := cli.NewContext(nil, set, set)
+	expect(t, c.Duration("myflag"), time.Duration(12*time.Second))
+}
+
 func TestContext_String(t *testing.T) {
 	set := flag.NewFlagSet("test", 0)
 	set.String("myflag", "hello world", "doc")

Godeps/_workspace/src/github.com/codegangsta/cli/flag.go

@@ -3,18 +3,28 @@ package cli
 import (
 	"flag"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // This flag enables bash-completion for all commands and subcommands
-var BashCompletionFlag = BoolFlag{"generate-bash-completion", ""}
+var BashCompletionFlag = BoolFlag{
+	Name: "generate-bash-completion",
+}
 
 // This flag prints the version for the application
-var VersionFlag = BoolFlag{"version, v", "print the version"}
+var VersionFlag = BoolFlag{
+	Name:  "version, v",
+	Usage: "print the version",
+}
 
 // This flag prints the help for all commands and subcommands
-var HelpFlag = BoolFlag{"help, h", "show help"}
+var HelpFlag = BoolFlag{
+	Name:  "help, h",
+	Usage: "show help",
+}
 
 // Flag is a common interface related to parsing flags in cli.
 // For more advanced flag parsing techniques, it is recomended that
@@ -51,16 +61,24 @@ type Generic interface {
 
 // GenericFlag is the flag type for types implementing Generic
 type GenericFlag struct {
-	Name  string
-	Value Generic
-	Usage string
+	Name   string
+	Value  Generic
+	Usage  string
+	EnvVar string
 }
 
 func (f GenericFlag) String() string {
-	return fmt.Sprintf("%s%s %v\t`%v` %s", prefixFor(f.Name), f.Name, f.Value, "-"+f.Name+" option -"+f.Name+" option", f.Usage)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s%s %v\t`%v` %s", prefixFor(f.Name), f.Name, f.Value, "-"+f.Name+" option -"+f.Name+" option", f.Usage))
 }
 
 func (f GenericFlag) Apply(set *flag.FlagSet) {
+	val := f.Value
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			val.Set(envVal)
+		}
+	}
+
 	eachName(f.Name, func(name string) {
 		set.Var(f.Value, name, f.Usage)
 	})
@@ -86,18 +104,29 @@ func (f *StringSlice) Value() []string {
 }
 
 type StringSliceFlag struct {
-	Name  string
-	Value *StringSlice
-	Usage string
+	Name   string
+	Value  *StringSlice
+	Usage  string
+	EnvVar string
 }
 
 func (f StringSliceFlag) String() string {
 	firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
 	pref := prefixFor(firstName)
-	return fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
 }
 
 func (f StringSliceFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			newVal := &StringSlice{}
+			for _, s := range strings.Split(envVal, ",") {
+				newVal.Set(s)
+			}
+			f.Value = newVal
+		}
+	}
+
 	eachName(f.Name, func(name string) {
 		set.Var(f.Value, name, f.Usage)
 	})
@@ -129,18 +158,32 @@ func (f *IntSlice) Value() []int {
 }
 
 type IntSliceFlag struct {
-	Name  string
-	Value *IntSlice
-	Usage string
+	Name   string
+	Value  *IntSlice
+	Usage  string
+	EnvVar string
 }
 
 func (f IntSliceFlag) String() string {
 	firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
 	pref := prefixFor(firstName)
-	return fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
 }
 
 func (f IntSliceFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			newVal := &IntSlice{}
+			for _, s := range strings.Split(envVal, ",") {
+				err := newVal.Set(s)
+				if err != nil {
+					fmt.Fprintf(os.Stderr, err.Error())
+				}
+			}
+			f.Value = newVal
+		}
+	}
+
 	eachName(f.Name, func(name string) {
 		set.Var(f.Value, name, f.Usage)
 	})
@@ -151,17 +194,28 @@ func (f IntSliceFlag) getName() string {
 }
 
 type BoolFlag struct {
-	Name  string
-	Usage string
+	Name   string
+	Usage  string
+	EnvVar string
 }
 
 func (f BoolFlag) String() string {
-	return fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
 }
 
 func (f BoolFlag) Apply(set *flag.FlagSet) {
+	val := false
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			envValBool, err := strconv.ParseBool(envVal)
+			if err == nil {
+				val = envValBool
+			}
+		}
+	}
+
 	eachName(f.Name, func(name string) {
-		set.Bool(name, false, f.Usage)
+		set.Bool(name, val, f.Usage)
 	})
 }
 
@@ -170,17 +224,28 @@ func (f BoolFlag) getName() string {
 }
 
 type BoolTFlag struct {
-	Name  string
-	Usage string
+	Name   string
+	Usage  string
+	EnvVar string
 }
 
 func (f BoolTFlag) String() string {
-	return fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
 }
 
 func (f BoolTFlag) Apply(set *flag.FlagSet) {
+	val := true
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			envValBool, err := strconv.ParseBool(envVal)
+			if err == nil {
+				val = envValBool
+			}
+		}
+	}
+
 	eachName(f.Name, func(name string) {
-		set.Bool(name, true, f.Usage)
+		set.Bool(name, val, f.Usage)
 	})
 }
 
@@ -189,9 +254,10 @@ func (f BoolTFlag) getName() string {
 }
 
 type StringFlag struct {
-	Name  string
-	Value string
-	Usage string
+	Name   string
+	Value  string
+	Usage  string
+	EnvVar string
 }
 
 func (f StringFlag) String() string {
@@ -204,10 +270,16 @@ func (f StringFlag) String() string {
 		fmtString = "%s %v\t%v"
 	}
 
-	return fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage)
+	return withEnvHint(f.EnvVar, fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage))
 }
 
 func (f StringFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			f.Value = envVal
+		}
+	}
+
 	eachName(f.Name, func(name string) {
 		set.String(name, f.Value, f.Usage)
 	})
@@ -218,16 +290,26 @@ func (f StringFlag) getName() string {
 }
 
 type IntFlag struct {
-	Name  string
-	Value int
-	Usage string
+	Name   string
+	Value  int
+	Usage  string
+	EnvVar string
 }
 
 func (f IntFlag) String() string {
-	return fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
 }
 
 func (f IntFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			envValInt, err := strconv.ParseUint(envVal, 10, 64)
+			if err == nil {
+				f.Value = int(envValInt)
+			}
+		}
+	}
+
 	eachName(f.Name, func(name string) {
 		set.Int(name, f.Value, f.Usage)
 	})
@@ -237,17 +319,57 @@ func (f IntFlag) getName() string {
 	return f.Name
 }
 
+type DurationFlag struct {
+	Name   string
+	Value  time.Duration
+	Usage  string
+	EnvVar string
+}
+
+func (f DurationFlag) String() string {
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
+}
+
+func (f DurationFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			envValDuration, err := time.ParseDuration(envVal)
+			if err == nil {
+				f.Value = envValDuration
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		set.Duration(name, f.Value, f.Usage)
+	})
+}
+
+func (f DurationFlag) getName() string {
+	return f.Name
+}
+
 type Float64Flag struct {
-	Name  string
-	Value float64
-	Usage string
+	Name   string
+	Value  float64
+	Usage  string
+	EnvVar string
 }
 
 func (f Float64Flag) String() string {
-	return fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s '%v'\t%v", prefixedNames(f.Name), f.Value, f.Usage))
 }
 
 func (f Float64Flag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		if envVal := os.Getenv(f.EnvVar); envVal != "" {
+			envValFloat, err := strconv.ParseFloat(envVal, 10)
+			if err == nil {
+				f.Value = float64(envValFloat)
+			}
+		}
+	}
+
 	eachName(f.Name, func(name string) {
 		set.Float64(name, f.Value, f.Usage)
 	})
@@ -278,3 +400,11 @@ func prefixedNames(fullName string) (prefixed string) {
 	}
 	return
 }
+
+func withEnvHint(envVar, str string) string {
+	envText := ""
+	if envVar != "" {
+		envText = fmt.Sprintf(" [$%s]", envVar)
+	}
+	return str + envText
+}

Godeps/_workspace/src/github.com/codegangsta/cli/flag_test.go

@@ -1,12 +1,13 @@
 package cli_test
 
 import (
-	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
-
 	"fmt"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/codegangsta/cli"
 )
 
 var boolFlagTests = []struct {
@@ -52,6 +53,71 @@ func TestStringFlagHelpOutput(t *testing.T) {
 	}
 }
 
+func TestStringFlagWithEnvVarHelpOutput(t *testing.T) {
+
+	os.Setenv("APP_FOO", "derp")
+	for _, test := range stringFlagTests {
+		flag := cli.StringFlag{Name: test.name, Value: test.value, EnvVar: "APP_FOO"}
+		output := flag.String()
+
+		if !strings.HasSuffix(output, " [$APP_FOO]") {
+			t.Errorf("%s does not end with [$APP_FOO]", output)
+		}
+	}
+}
+
+var stringSliceFlagTests = []struct {
+	name     string
+	value    *cli.StringSlice
+	expected string
+}{
+	{"help", func() *cli.StringSlice {
+		s := &cli.StringSlice{}
+		s.Set("")
+		return s
+	}(), "--help '--help option --help option'\t"},
+	{"h", func() *cli.StringSlice {
+		s := &cli.StringSlice{}
+		s.Set("")
+		return s
+	}(), "-h '-h option -h option'\t"},
+	{"h", func() *cli.StringSlice {
+		s := &cli.StringSlice{}
+		s.Set("")
+		return s
+	}(), "-h '-h option -h option'\t"},
+	{"test", func() *cli.StringSlice {
+		s := &cli.StringSlice{}
+		s.Set("Something")
+		return s
+	}(), "--test '--test option --test option'\t"},
+}
+
+func TestStringSliceFlagHelpOutput(t *testing.T) {
+
+	for _, test := range stringSliceFlagTests {
+		flag := cli.StringSliceFlag{Name: test.name, Value: test.value}
+		output := flag.String()
+
+		if output != test.expected {
+			t.Errorf("%q does not match %q", output, test.expected)
+		}
+	}
+}
+
+func TestStringSliceFlagWithEnvVarHelpOutput(t *testing.T) {
+
+	os.Setenv("APP_QWWX", "11,4")
+	for _, test := range stringSliceFlagTests {
+		flag := cli.StringSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_QWWX"}
+		output := flag.String()
+
+		if !strings.HasSuffix(output, " [$APP_QWWX]") {
+			t.Errorf("%q does not end with [$APP_QWWX]", output)
+		}
+	}
+}
+
 var intFlagTests = []struct {
 	name     string
 	expected string
@@ -72,6 +138,92 @@ func TestIntFlagHelpOutput(t *testing.T) {
 	}
 }
 
+func TestIntFlagWithEnvVarHelpOutput(t *testing.T) {
+
+	os.Setenv("APP_BAR", "2")
+	for _, test := range intFlagTests {
+		flag := cli.IntFlag{Name: test.name, EnvVar: "APP_BAR"}
+		output := flag.String()
+
+		if !strings.HasSuffix(output, " [$APP_BAR]") {
+			t.Errorf("%s does not end with [$APP_BAR]", output)
+		}
+	}
+}
+
+var durationFlagTests = []struct {
+	name     string
+	expected string
+}{
+	{"help", "--help '0'\t"},
+	{"h", "-h '0'\t"},
+}
+
+func TestDurationFlagHelpOutput(t *testing.T) {
+
+	for _, test := range durationFlagTests {
+		flag := cli.DurationFlag{Name: test.name}
+		output := flag.String()
+
+		if output != test.expected {
+			t.Errorf("%s does not match %s", output, test.expected)
+		}
+	}
+}
+
+func TestDurationFlagWithEnvVarHelpOutput(t *testing.T) {
+
+	os.Setenv("APP_BAR", "2h3m6s")
+	for _, test := range durationFlagTests {
+		flag := cli.DurationFlag{Name: test.name, EnvVar: "APP_BAR"}
+		output := flag.String()
+
+		if !strings.HasSuffix(output, " [$APP_BAR]") {
+			t.Errorf("%s does not end with [$APP_BAR]", output)
+		}
+	}
+}
+
+var intSliceFlagTests = []struct {
+	name     string
+	value    *cli.IntSlice
+	expected string
+}{
+	{"help", &cli.IntSlice{}, "--help '--help option --help option'\t"},
+	{"h", &cli.IntSlice{}, "-h '-h option -h option'\t"},
+	{"h", &cli.IntSlice{}, "-h '-h option -h option'\t"},
+	{"test", func() *cli.IntSlice {
+		i := &cli.IntSlice{}
+		i.Set("9")
+		return i
+	}(), "--test '--test option --test option'\t"},
+}
+
+func TestIntSliceFlagHelpOutput(t *testing.T) {
+
+	for _, test := range intSliceFlagTests {
+		flag := cli.IntSliceFlag{Name: test.name, Value: test.value}
+		output := flag.String()
+
+		if output != test.expected {
+			t.Errorf("%q does not match %q", output, test.expected)
+		}
+	}
+}
+
+func TestIntSliceFlagWithEnvVarHelpOutput(t *testing.T) {
+
+	os.Setenv("APP_SMURF", "42,3")
+	for _, test := range intSliceFlagTests {
+		flag := cli.IntSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_SMURF"}
+		output := flag.String()
+
+		if !strings.HasSuffix(output, " [$APP_SMURF]") {
+			t.Errorf("%q does not end with [$APP_SMURF]", output)
+		}
+	}
+}
+
 var float64FlagTests = []struct {
 	name     string
 	expected string
@@ -92,6 +244,54 @@ func TestFloat64FlagHelpOutput(t *testing.T) {
 	}
 }
 
+func TestFloat64FlagWithEnvVarHelpOutput(t *testing.T) {
+
+	os.Setenv("APP_BAZ", "99.4")
+	for _, test := range float64FlagTests {
+		flag := cli.Float64Flag{Name: test.name, EnvVar: "APP_BAZ"}
+		output := flag.String()
+
+		if !strings.HasSuffix(output, " [$APP_BAZ]") {
+			t.Errorf("%s does not end with [$APP_BAZ]", output)
+		}
+	}
+}
+
+var genericFlagTests = []struct {
+	name     string
+	value    cli.Generic
+	expected string
+}{
+	{"help", &Parser{}, "--help <nil>\t`-help option -help option` "},
+	{"h", &Parser{}, "-h <nil>\t`-h option -h option` "},
+	{"test", &Parser{}, "--test <nil>\t`-test option -test option` "},
+}
+
+func TestGenericFlagHelpOutput(t *testing.T) {
+
+	for _, test := range genericFlagTests {
+		flag := cli.GenericFlag{Name: test.name}
+		output := flag.String()
+
+		if output != test.expected {
+			t.Errorf("%q does not match %q", output, test.expected)
+		}
+	}
+}
+
+func TestGenericFlagWithEnvVarHelpOutput(t *testing.T) {
+
+	os.Setenv("APP_ZAP", "3")
+	for _, test := range genericFlagTests {
+		flag := cli.GenericFlag{Name: test.name, EnvVar: "APP_ZAP"}
+		output := flag.String()
+
+		if !strings.HasSuffix(output, " [$APP_ZAP]") {
+			t.Errorf("%s does not end with [$APP_ZAP]", output)
+		}
+	}
+}
+
 func TestParseMultiString(t *testing.T) {
 	(&cli.App{
 		Flags: []cli.Flag{
@@ -108,6 +308,23 @@ func TestParseMultiString(t *testing.T) {
 	}).Run([]string{"run", "-s", "10"})
 }
 
+func TestParseMultiStringFromEnv(t *testing.T) {
+	os.Setenv("APP_COUNT", "20")
+	(&cli.App{
+		Flags: []cli.Flag{
+			cli.StringFlag{Name: "count, c", EnvVar: "APP_COUNT"},
+		},
+		Action: func(ctx *cli.Context) {
+			if ctx.String("count") != "20" {
+				t.Errorf("main name not set")
+			}
+			if ctx.String("c") != "20" {
+				t.Errorf("short name not set")
+			}
+		},
+	}).Run([]string{"run"})
+}
+
 func TestParseMultiStringSlice(t *testing.T) {
 	(&cli.App{
 		Flags: []cli.Flag{
@@ -124,6 +341,24 @@ func TestParseMultiStringSlice(t *testing.T) {
 	}).Run([]string{"run", "-s", "10", "-s", "20"})
 }
 
+func TestParseMultiStringSliceFromEnv(t *testing.T) {
+	os.Setenv("APP_INTERVALS", "20,30,40")
+
+	(&cli.App{
+		Flags: []cli.Flag{
+			cli.StringSliceFlag{Name: "intervals, i", Value: &cli.StringSlice{}, EnvVar: "APP_INTERVALS"},
+		},
+		Action: func(ctx *cli.Context) {
+			if !reflect.DeepEqual(ctx.StringSlice("intervals"), []string{"20", "30", "40"}) {
+				t.Errorf("main name not set from env")
+			}
+			if !reflect.DeepEqual(ctx.StringSlice("i"), []string{"20", "30", "40"}) {
+				t.Errorf("short name not set from env")
+			}
+		},
+	}).Run([]string{"run"})
+}
+
 func TestParseMultiInt(t *testing.T) {
 	a := cli.App{
 		Flags: []cli.Flag{
@@ -141,6 +376,93 @@ func TestParseMultiInt(t *testing.T) {
 	a.Run([]string{"run", "-s", "10"})
 }
 
+func TestParseMultiIntFromEnv(t *testing.T) {
+	os.Setenv("APP_TIMEOUT_SECONDS", "10")
+	a := cli.App{
+		Flags: []cli.Flag{
+			cli.IntFlag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"},
+		},
+		Action: func(ctx *cli.Context) {
+			if ctx.Int("timeout") != 10 {
+				t.Errorf("main name not set")
+			}
+			if ctx.Int("t") != 10 {
+				t.Errorf("short name not set")
+			}
+		},
+	}
+	a.Run([]string{"run"})
+}
+
+func TestParseMultiIntSlice(t *testing.T) {
+	(&cli.App{
+		Flags: []cli.Flag{
+			cli.IntSliceFlag{Name: "serve, s", Value: &cli.IntSlice{}},
+		},
+		Action: func(ctx *cli.Context) {
+			if !reflect.DeepEqual(ctx.IntSlice("serve"), []int{10, 20}) {
+				t.Errorf("main name not set")
+			}
+			if !reflect.DeepEqual(ctx.IntSlice("s"), []int{10, 20}) {
+				t.Errorf("short name not set")
+			}
+		},
+	}).Run([]string{"run", "-s", "10", "-s", "20"})
+}
+
+func TestParseMultiIntSliceFromEnv(t *testing.T) {
+	os.Setenv("APP_INTERVALS", "20,30,40")
+
+	(&cli.App{
+		Flags: []cli.Flag{
+			cli.IntSliceFlag{Name: "intervals, i", Value: &cli.IntSlice{}, EnvVar: "APP_INTERVALS"},
+		},
+		Action: func(ctx *cli.Context) {
+			if !reflect.DeepEqual(ctx.IntSlice("intervals"), []int{20, 30, 40}) {
+				t.Errorf("main name not set from env")
+			}
+			if !reflect.DeepEqual(ctx.IntSlice("i"), []int{20, 30, 40}) {
+				t.Errorf("short name not set from env")
+			}
+		},
+	}).Run([]string{"run"})
+}
+
+func TestParseMultiFloat64(t *testing.T) {
+	a := cli.App{
+		Flags: []cli.Flag{
+			cli.Float64Flag{Name: "serve, s"},
+		},
+		Action: func(ctx *cli.Context) {
+			if ctx.Float64("serve") != 10.2 {
+				t.Errorf("main name not set")
+			}
+			if ctx.Float64("s") != 10.2 {
+				t.Errorf("short name not set")
+			}
+		},
+	}
+	a.Run([]string{"run", "-s", "10.2"})
+}
+
+func TestParseMultiFloat64FromEnv(t *testing.T) {
+	os.Setenv("APP_TIMEOUT_SECONDS", "15.5")
+	a := cli.App{
+		Flags: []cli.Flag{
+			cli.Float64Flag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"},
+		},
+		Action: func(ctx *cli.Context) {
+			if ctx.Float64("timeout") != 15.5 {
+				t.Errorf("main name not set")
+			}
+			if ctx.Float64("t") != 15.5 {
+				t.Errorf("short name not set")
+			}
+		},
+	}
+	a.Run([]string{"run"})
+}
+
 func TestParseMultiBool(t *testing.T) {
 	a := cli.App{
 		Flags: []cli.Flag{
@@ -158,6 +480,59 @@ func TestParseMultiBool(t *testing.T) {
 	a.Run([]string{"run", "--serve"})
 }
 
+func TestParseMultiBoolFromEnv(t *testing.T) {
+	os.Setenv("APP_DEBUG", "1")
+	a := cli.App{
+		Flags: []cli.Flag{
+			cli.BoolFlag{Name: "debug, d", EnvVar: "APP_DEBUG"},
+		},
+		Action: func(ctx *cli.Context) {
+			if ctx.Bool("debug") != true {
+				t.Errorf("main name not set from env")
+			}
+			if ctx.Bool("d") != true {
+				t.Errorf("short name not set from env")
+			}
+		},
+	}
+	a.Run([]string{"run"})
+}
+
+func TestParseMultiBoolT(t *testing.T) {
+	a := cli.App{
+		Flags: []cli.Flag{
+			cli.BoolTFlag{Name: "serve, s"},
+		},
+		Action: func(ctx *cli.Context) {
+			if ctx.BoolT("serve") != true {
+				t.Errorf("main name not set")
+			}
+			if ctx.BoolT("s") != true {
+				t.Errorf("short name not set")
+			}
+		},
+	}
+	a.Run([]string{"run", "--serve"})
+}
+
+func TestParseMultiBoolTFromEnv(t *testing.T) {
+	os.Setenv("APP_DEBUG", "0")
+	a := cli.App{
+		Flags: []cli.Flag{
+			cli.BoolTFlag{Name: "debug, d", EnvVar: "APP_DEBUG"},
+		},
+		Action: func(ctx *cli.Context) {
+			if ctx.BoolT("debug") != false {
+				t.Errorf("main name not set from env")
+			}
+			if ctx.BoolT("d") != false {
+				t.Errorf("short name not set from env")
+			}
+		},
+	}
+	a.Run([]string{"run"})
+}
+
 type Parser [2]string
 
 func (p *Parser) Set(value string) error {
@@ -192,3 +567,21 @@ func TestParseGeneric(t *testing.T) {
 	}
 	a.Run([]string{"run", "-s", "10,20"})
 }
+
+func TestParseGenericFromEnv(t *testing.T) {
+	os.Setenv("APP_SERVE", "20,30")
+	a := cli.App{
+		Flags: []cli.Flag{
+			cli.GenericFlag{Name: "serve, s", Value: &Parser{}, EnvVar: "APP_SERVE"},
+		},
+		Action: func(ctx *cli.Context) {
+			if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"20", "30"}) {
+				t.Errorf("main name not set from env")
+			}
+			if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"20", "30"}) {
+				t.Errorf("short name not set from env")
+			}
+		},
+	}
+	a.Run([]string{"run"})
+}

Godeps/_workspace/src/github.com/codegangsta/cli/help.go

@@ -14,17 +14,21 @@ var AppHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}
 
 USAGE:
-   {{.Name}} [global options] command [command options] [arguments...]
+   {{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...]
 
 VERSION:
-   {{.Version}}
+   {{.Version}}{{if or .Author .Email}}
+
+AUTHOR:{{if .Author}}
+  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
+  {{.Email}}{{end}}{{end}}
 
 COMMANDS:
    {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
-   {{end}}
+   {{end}}{{if .Flags}}
 GLOBAL OPTIONS:
    {{range .Flags}}{{.}}
-   {{end}}
+   {{end}}{{end}}
 `
 
 // The text template for the command help topic.
@@ -34,14 +38,14 @@ var CommandHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}
 
 USAGE:
-   command {{.Name}} [command options] [arguments...]
+   command {{.Name}}{{if .Flags}} [command options]{{end}} [arguments...]{{if .Description}}
 
 DESCRIPTION:
-   {{.Description}}
+   {{.Description}}{{end}}{{if .Flags}}
 
 OPTIONS:
    {{range .Flags}}{{.}}
-   {{end}}
+   {{end}}{{ end }}
 `
 
 // The text template for the subcommand help topic.
@@ -51,14 +55,14 @@ var SubcommandHelpTemplate = `NAME:
    {{.Name}} - {{.Usage}}
 
 USAGE:
-   {{.Name}} [global options] command [command options] [arguments...]
+   {{.Name}} command{{if .Flags}} [command options]{{end}} [arguments...]
 
 COMMANDS:
    {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
-   {{end}}
+   {{end}}{{if .Flags}}
 OPTIONS:
    {{range .Flags}}{{.}}
-   {{end}}
+   {{end}}{{end}}
 `
 
 var helpCommand = Command{
@@ -92,6 +96,9 @@ var helpSubcommand = Command{
 // Prints help for the App
 var HelpPrinter = printHelp
 
+// Prints version for the App
+var VersionPrinter = printVersion
+
 func ShowAppHelp(c *Context) {
 	HelpPrinter(AppHelpTemplate, c.App)
 }
@@ -129,6 +136,10 @@ func ShowSubcommandHelp(c *Context) {
 
 // Prints the version number of the App
 func ShowVersion(c *Context) {
+	VersionPrinter(c)
+}
+
+func printVersion(c *Context) {
 	fmt.Printf("%v version %v\n", c.App.Name, c.App.Version)
 }
 

Procfile

@@ -1,5 +1,5 @@
 # Use goreman to run `go get github.com/mattn/goreman`
-etcd1: bin/etcd -name node1 -listen-client-urls http://127.0.0.1:4001 -advertise-client-urls http://127.0.0.1:4001 -listen-peer-urls http://127.0.0.1:7001 -initial-advertise-peer-urls http://127.0.0.1:7001 -initial-cluster 'node1=http://localhost:7001,node2=http://localhost:7002,node3=http://localhost:7003' -initial-cluster-state new
-etcd2: bin/etcd -name node2 -listen-client-urls http://127.0.0.1:4002 -advertise-client-urls http://127.0.0.1:4002 -listen-peer-urls http://127.0.0.1:7002 -initial-advertise-peer-urls http://127.0.0.1:7002 -initial-cluster 'node1=http://localhost:7001,node2=http://localhost:7002,node3=http://localhost:7003' -initial-cluster-state new
-etcd3: bin/etcd -name node3 -listen-client-urls http://127.0.0.1:4003 -advertise-client-urls http://127.0.0.1:4003 -listen-peer-urls http://127.0.0.1:7003 -initial-advertise-peer-urls http://127.0.0.1:7003 -initial-cluster 'node1=http://localhost:7001,node2=http://localhost:7002,node3=http://localhost:7003' -initial-cluster-state new
-proxy: bin/etcd -proxy=on -bind-addr 127.0.0.1:8080 -initial-cluster 'node1=http://localhost:7001,node2=http://localhost:7002,node3=http://localhost:7003'
+etcd1: bin/etcd -name infra1 -listen-client-urls http://127.0.0.1:4001 -advertise-client-urls http://127.0.0.1:4001 -listen-peer-urls http://127.0.0.1:7001 -initial-advertise-peer-urls http://127.0.0.1:7001 -initial-cluster-token etcd-cluster-1 -initial-cluster 'infra1=http://localhost:7001,infra2=http://localhost:7002,infra3=http://localhost:7003' -initial-cluster-state new
+etcd2: bin/etcd -name infra2 -listen-client-urls http://127.0.0.1:4002 -advertise-client-urls http://127.0.0.1:4002 -listen-peer-urls http://127.0.0.1:7002 -initial-advertise-peer-urls http://127.0.0.1:7002 -initial-cluster-token etcd-cluster-1 -initial-cluster 'infra1=http://localhost:7001,infra2=http://localhost:7002,infra3=http://localhost:7003' -initial-cluster-state new
+etcd3: bin/etcd -name infra3 -listen-client-urls http://127.0.0.1:4003 -advertise-client-urls http://127.0.0.1:4003 -listen-peer-urls http://127.0.0.1:7003 -initial-advertise-peer-urls http://127.0.0.1:7003 -initial-cluster-token etcd-cluster-1 -initial-cluster 'infra1=http://localhost:7001,infra2=http://localhost:7002,infra3=http://localhost:7003' -initial-cluster-state new
+proxy: bin/etcd -proxy=on -bind-addr 127.0.0.1:8080 -initial-cluster 'infra1=http://localhost:7001,infra2=http://localhost:7002,infra3=http://localhost:7003'

README.md

@@ -1,6 +1,7 @@
 # etcd
 
 [![Build Status](https://travis-ci.org/coreos/etcd.png?branch=master)](https://travis-ci.org/coreos/etcd)
+[![Docker Repository on Quay.io](https://quay.io/repository/coreos/etcd-git/status "Docker Repository on Quay.io")](https://quay.io/repository/coreos/etcd-git)
 
 ### WARNING ###
 

client/http.go

@@ -17,6 +17,8 @@
 package client
 
 import (
+	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -26,20 +28,117 @@ import (
 )
 
 var (
-	ErrTimeout            = context.DeadlineExceeded
+	ErrTimeout          = context.DeadlineExceeded
+	ErrCanceled         = context.Canceled
+	ErrNoEndpoints      = errors.New("no endpoints available")
+	ErrTooManyRedirects = errors.New("too many redirects")
+
 	DefaultRequestTimeout = 5 * time.Second
+	DefaultMaxRedirects   = 10
 )
 
-// transport mimics http.Transport to provide an interface which can be
+type SyncableHTTPClient interface {
+	HTTPClient
+	Sync(context.Context) error
+	Endpoints() []string
+}
+
+type HTTPClient interface {
+	Do(context.Context, HTTPAction) (*http.Response, []byte, error)
+}
+
+type HTTPAction interface {
+	HTTPRequest(url.URL) *http.Request
+}
+
+// CancelableTransport mimics http.Transport to provide an interface which can be
 // substituted for testing (since the RoundTripper interface alone does not
 // require the CancelRequest method)
-type transport interface {
+type CancelableTransport interface {
 	http.RoundTripper
 	CancelRequest(req *http.Request)
 }
 
-type httpAction interface {
-	httpRequest(url.URL) *http.Request
+func NewHTTPClient(tr CancelableTransport, eps []string) (SyncableHTTPClient, error) {
+	return newHTTPClusterClient(tr, eps)
+}
+
+func newHTTPClusterClient(tr CancelableTransport, eps []string) (*httpClusterClient, error) {
+	c := httpClusterClient{
+		transport: tr,
+		endpoints: eps,
+		clients:   make([]HTTPClient, len(eps)),
+	}
+
+	for i, ep := range eps {
+		u, err := url.Parse(ep)
+		if err != nil {
+			return nil, err
+		}
+
+		c.clients[i] = &redirectFollowingHTTPClient{
+			max: DefaultMaxRedirects,
+			client: &httpClient{
+				transport: tr,
+				endpoint:  *u,
+			},
+		}
+	}
+
+	return &c, nil
+}
+
+type httpClusterClient struct {
+	transport CancelableTransport
+	endpoints []string
+	clients   []HTTPClient
+}
+
+func (c *httpClusterClient) Do(ctx context.Context, act HTTPAction) (resp *http.Response, body []byte, err error) {
+	if len(c.clients) == 0 {
+		return nil, nil, ErrNoEndpoints
+	}
+	for _, hc := range c.clients {
+		resp, body, err = hc.Do(ctx, act)
+		if err != nil {
+			if err == ErrTimeout || err == ErrCanceled {
+				return nil, nil, err
+			}
+			continue
+		}
+		if resp.StatusCode/100 == 5 {
+			continue
+		}
+		break
+	}
+	return
+}
+
+func (c *httpClusterClient) Endpoints() []string {
+	return c.endpoints
+}
+
+func (c *httpClusterClient) Sync(ctx context.Context) error {
+	mAPI := NewMembersAPI(c)
+	ms, err := mAPI.List(ctx)
+	if err != nil {
+		return err
+	}
+
+	eps := make([]string, 0)
+	for _, m := range ms {
+		eps = append(eps, m.ClientURLs...)
+	}
+	if len(eps) == 0 {
+		return ErrNoEndpoints
+	}
+	nc, err := newHTTPClusterClient(c.transport, eps)
+	if err != nil {
+		return err
+	}
+
+	*c = *nc
+	return nil
 }
 
 type roundTripResponse struct {
@@ -48,34 +147,12 @@ type roundTripResponse struct {
 }
 
 type httpClient struct {
-	transport transport
+	transport CancelableTransport
 	endpoint  url.URL
-	timeout   time.Duration
 }
 
-func newHTTPClient(tr *http.Transport, ep string, to time.Duration) (*httpClient, error) {
-	u, err := url.Parse(ep)
-	if err != nil {
-		return nil, err
-	}
-
-	c := &httpClient{
-		transport: tr,
-		endpoint:  *u,
-		timeout:   to,
-	}
-
-	return c, nil
-}
-
-func (c *httpClient) doWithTimeout(act httpAction) (*http.Response, []byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
-	defer cancel()
-	return c.do(ctx, act)
-}
-
-func (c *httpClient) do(ctx context.Context, act httpAction) (*http.Response, []byte, error) {
-	req := act.httpRequest(c.endpoint)
+func (c *httpClient) Do(ctx context.Context, act HTTPAction) (*http.Response, []byte, error) {
+	req := act.HTTPRequest(c.endpoint)
 
 	rtchan := make(chan roundTripResponse, 1)
 	go func() {
@@ -112,3 +189,45 @@ func (c *httpClient) do(ctx context.Context, act httpAction) (*http.Response, []
 	body, err := ioutil.ReadAll(resp.Body)
 	return resp, body, err
 }
+
+type redirectFollowingHTTPClient struct {
+	client HTTPClient
+	max    int
+}
+
+func (r *redirectFollowingHTTPClient) Do(ctx context.Context, act HTTPAction) (*http.Response, []byte, error) {
+	for i := 0; i <= r.max; i++ {
+		resp, body, err := r.client.Do(ctx, act)
+		if err != nil {
+			return nil, nil, err
+		}
+		if resp.StatusCode/100 == 3 {
+			hdr := resp.Header.Get("Location")
+			if hdr == "" {
+				return nil, nil, fmt.Errorf("Location header not set")
+			}
+			loc, err := url.Parse(hdr)
+			if err != nil {
+				return nil, nil, fmt.Errorf("Location header not valid URL: %s", hdr)
+			}
+			act = &redirectedHTTPAction{
+				action:   act,
+				location: *loc,
+			}
+			continue
+		}
+		return resp, body, nil
+	}
+	return nil, nil, ErrTooManyRedirects
+}
+
+type redirectedHTTPAction struct {
+	action   HTTPAction
+	location url.URL
+}
+
+func (r *redirectedHTTPAction) HTTPRequest(ep url.URL) *http.Request {
+	orig := r.action.HTTPRequest(ep)
+	orig.URL = &r.location
+	return orig
+}

client/http_test.go

@@ -29,6 +29,39 @@ import (
 	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
 )
 
+type staticHTTPClient struct {
+	resp http.Response
+	err  error
+}
+
+func (s *staticHTTPClient) Do(context.Context, HTTPAction) (*http.Response, []byte, error) {
+	return &s.resp, nil, s.err
+}
+
+type staticHTTPAction struct {
+	request http.Request
+}
+
+type staticHTTPResponse struct {
+	resp http.Response
+	err  error
+}
+
+func (s *staticHTTPAction) HTTPRequest(url.URL) *http.Request {
+	return &s.request
+}
+
+type multiStaticHTTPClient struct {
+	responses []staticHTTPResponse
+	cur       int
+}
+
+func (s *multiStaticHTTPClient) Do(context.Context, HTTPAction) (*http.Response, []byte, error) {
+	r := s.responses[s.cur]
+	s.cur++
+	return &r.resp, nil, r.err
+}
+
 type fakeTransport struct {
 	respchan     chan *http.Response
 	errchan      chan error
@@ -65,7 +98,7 @@ func (t *fakeTransport) CancelRequest(*http.Request) {
 
 type fakeAction struct{}
 
-func (a *fakeAction) httpRequest(url.URL) *http.Request {
+func (a *fakeAction) HTTPRequest(url.URL) *http.Request {
 	return &http.Request{}
 }
 
@@ -78,7 +111,7 @@ func TestHTTPClientDoSuccess(t *testing.T) {
 		Body:       ioutil.NopCloser(strings.NewReader("foo")),
 	}
 
-	resp, body, err := c.do(context.Background(), &fakeAction{})
+	resp, body, err := c.Do(context.Background(), &fakeAction{})
 	if err != nil {
 		t.Fatalf("incorrect error value: want=nil got=%v", err)
 	}
@@ -100,7 +133,7 @@ func TestHTTPClientDoError(t *testing.T) {
 
 	tr.errchan <- errors.New("fixture")
 
-	_, _, err := c.do(context.Background(), &fakeAction{})
+	_, _, err := c.Do(context.Background(), &fakeAction{})
 	if err == nil {
 		t.Fatalf("expected non-nil error, got nil")
 	}
@@ -113,7 +146,7 @@ func TestHTTPClientDoCancelContext(t *testing.T) {
 	tr.startCancel <- struct{}{}
 	tr.finishCancel <- struct{}{}
 
-	_, _, err := c.do(context.Background(), &fakeAction{})
+	_, _, err := c.Do(context.Background(), &fakeAction{})
 	if err == nil {
 		t.Fatalf("expected non-nil error, got nil")
 	}
@@ -126,7 +159,7 @@ func TestHTTPClientDoCancelContextWaitForRoundTrip(t *testing.T) {
 	donechan := make(chan struct{})
 	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
-		c.do(ctx, &fakeAction{})
+		c.Do(ctx, &fakeAction{})
 		close(donechan)
 	}()
 
@@ -149,3 +182,304 @@ func TestHTTPClientDoCancelContextWaitForRoundTrip(t *testing.T) {
 		t.Fatalf("httpClient.do did not exit within 1s")
 	}
 }
+
+func TestHTTPClusterClientDo(t *testing.T) {
+	fakeErr := errors.New("fake!")
+	tests := []struct {
+		client   *httpClusterClient
+		wantCode int
+		wantErr  error
+	}{
+		// first good response short-circuits Do
+		{
+			client: &httpClusterClient{
+				clients: []HTTPClient{
+					&staticHTTPClient{resp: http.Response{StatusCode: http.StatusTeapot}},
+					&staticHTTPClient{err: fakeErr},
+				},
+			},
+			wantCode: http.StatusTeapot,
+		},
+
+		// fall through to good endpoint if err is arbitrary
+		{
+			client: &httpClusterClient{
+				clients: []HTTPClient{
+					&staticHTTPClient{err: fakeErr},
+					&staticHTTPClient{resp: http.Response{StatusCode: http.StatusTeapot}},
+				},
+			},
+			wantCode: http.StatusTeapot,
+		},
+
+		// ErrTimeout short-circuits Do
+		{
+			client: &httpClusterClient{
+				clients: []HTTPClient{
+					&staticHTTPClient{err: ErrTimeout},
+					&staticHTTPClient{resp: http.Response{StatusCode: http.StatusTeapot}},
+				},
+			},
+			wantErr: ErrTimeout,
+		},
+
+		// ErrCanceled short-circuits Do
+		{
+			client: &httpClusterClient{
+				clients: []HTTPClient{
+					&staticHTTPClient{err: ErrCanceled},
+					&staticHTTPClient{resp: http.Response{StatusCode: http.StatusTeapot}},
+				},
+			},
+			wantErr: ErrCanceled,
+		},
+
+		// return err if there are no endpoints
+		{
+			client: &httpClusterClient{
+				clients: []HTTPClient{},
+			},
+			wantErr: ErrNoEndpoints,
+		},
+
+		// return err if all endpoints return arbitrary errors
+		{
+			client: &httpClusterClient{
+				clients: []HTTPClient{
+					&staticHTTPClient{err: fakeErr},
+					&staticHTTPClient{err: fakeErr},
+				},
+			},
+			wantErr: fakeErr,
+		},
+
+		// 500-level errors cause Do to fallthrough to next endpoint
+		{
+			client: &httpClusterClient{
+				clients: []HTTPClient{
+					&staticHTTPClient{resp: http.Response{StatusCode: http.StatusBadGateway}},
+					&staticHTTPClient{resp: http.Response{StatusCode: http.StatusTeapot}},
+				},
+			},
+			wantCode: http.StatusTeapot,
+		},
+	}
+
+	for i, tt := range tests {
+		resp, _, err := tt.client.Do(context.Background(), nil)
+		if !reflect.DeepEqual(tt.wantErr, err) {
+			t.Errorf("#%d: got err=%v, want=%v", i, err, tt.wantErr)
+			continue
+		}
+
+		if resp == nil {
+			if tt.wantCode != 0 {
+				t.Errorf("#%d: resp is nil, want=%d", i, tt.wantCode)
+			}
+			continue
+		}
+
+		if resp.StatusCode != tt.wantCode {
+			t.Errorf("#%d: resp code=%d, want=%d", i, resp.StatusCode, tt.wantCode)
+			continue
+		}
+	}
+}
+
+func TestRedirectedHTTPAction(t *testing.T) {
+	act := &redirectedHTTPAction{
+		action: &staticHTTPAction{
+			request: http.Request{
+				Method: "DELETE",
+				URL: &url.URL{
+					Scheme: "https",
+					Host:   "foo.example.com",
+					Path:   "/ping",
+				},
+			},
+		},
+		location: url.URL{
+			Scheme: "https",
+			Host:   "bar.example.com",
+			Path:   "/pong",
+		},
+	}
+
+	want := &http.Request{
+		Method: "DELETE",
+		URL: &url.URL{
+			Scheme: "https",
+			Host:   "bar.example.com",
+			Path:   "/pong",
+		},
+	}
+	got := act.HTTPRequest(url.URL{Scheme: "http", Host: "baz.example.com", Path: "/pang"})
+
+	if !reflect.DeepEqual(want, got) {
+		t.Fatalf("HTTPRequest is %#v, want %#v", want, got)
+	}
+}
+
+func TestRedirectFollowingHTTPClient(t *testing.T) {
+	tests := []struct {
+		max      int
+		client   HTTPClient
+		wantCode int
+		wantErr  error
+	}{
+		// errors bubbled up
+		{
+			max: 2,
+			client: &multiStaticHTTPClient{
+				responses: []staticHTTPResponse{
+					staticHTTPResponse{
+						err: errors.New("fail!"),
+					},
+				},
+			},
+			wantErr: errors.New("fail!"),
+		},
+
+		// no need to follow redirect if none given
+		{
+			max: 2,
+			client: &multiStaticHTTPClient{
+				responses: []staticHTTPResponse{
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTeapot,
+						},
+					},
+				},
+			},
+			wantCode: http.StatusTeapot,
+		},
+
+		// redirects if less than max
+		{
+			max: 2,
+			client: &multiStaticHTTPClient{
+				responses: []staticHTTPResponse{
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTemporaryRedirect,
+							Header:     http.Header{"Location": []string{"http://example.com"}},
+						},
+					},
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTeapot,
+						},
+					},
+				},
+			},
+			wantCode: http.StatusTeapot,
+		},
+
+		// succeed after reaching max redirects
+		{
+			max: 2,
+			client: &multiStaticHTTPClient{
+				responses: []staticHTTPResponse{
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTemporaryRedirect,
+							Header:     http.Header{"Location": []string{"http://example.com"}},
+						},
+					},
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTemporaryRedirect,
+							Header:     http.Header{"Location": []string{"http://example.com"}},
+						},
+					},
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTeapot,
+						},
+					},
+				},
+			},
+			wantCode: http.StatusTeapot,
+		},
+
+		// fail at max+1 redirects
+		{
+			max: 1,
+			client: &multiStaticHTTPClient{
+				responses: []staticHTTPResponse{
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTemporaryRedirect,
+							Header:     http.Header{"Location": []string{"http://example.com"}},
+						},
+					},
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTemporaryRedirect,
+							Header:     http.Header{"Location": []string{"http://example.com"}},
+						},
+					},
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTeapot,
+						},
+					},
+				},
+			},
+			wantErr: ErrTooManyRedirects,
+		},
+
+		// fail if Location header not set
+		{
+			max: 1,
+			client: &multiStaticHTTPClient{
+				responses: []staticHTTPResponse{
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTemporaryRedirect,
+						},
+					},
+				},
+			},
+			wantErr: errors.New("Location header not set"),
+		},
+
+		// fail if Location header is invalid
+		{
+			max: 1,
+			client: &multiStaticHTTPClient{
+				responses: []staticHTTPResponse{
+					staticHTTPResponse{
+						resp: http.Response{
+							StatusCode: http.StatusTemporaryRedirect,
+							Header:     http.Header{"Location": []string{":"}},
+						},
+					},
+				},
+			},
+			wantErr: errors.New("Location header not valid URL: :"),
+		},
+	}
+
+	for i, tt := range tests {
+		client := &redirectFollowingHTTPClient{client: tt.client, max: tt.max}
+		resp, _, err := client.Do(context.Background(), nil)
+		if !reflect.DeepEqual(tt.wantErr, err) {
+			t.Errorf("#%d: got err=%v, want=%v", i, err, tt.wantErr)
+			continue
+		}
+
+		if resp == nil {
+			if tt.wantCode != 0 {
+				t.Errorf("#%d: resp is nil, want=%d", i, tt.wantCode)
+			}
+			continue
+		}
+
+		if resp.StatusCode != tt.wantCode {
+			t.Errorf("#%d: resp code=%d, want=%d", i, resp.StatusCode, tt.wantCode)
+			continue
+		}
+	}
+}

client/keys.go

@@ -41,36 +41,30 @@ var (
 	ErrKeyExists   = errors.New("client: key already exists")
 )
 
-func NewKeysAPI(tr *http.Transport, ep string, to time.Duration) (KeysAPI, error) {
-	return newHTTPKeysAPIWithPrefix(tr, ep, to, DefaultV2KeysPrefix)
-}
-
-func NewDiscoveryKeysAPI(tr *http.Transport, ep string, to time.Duration) (KeysAPI, error) {
-	return newHTTPKeysAPIWithPrefix(tr, ep, to, "")
-}
-
-func newHTTPKeysAPIWithPrefix(tr *http.Transport, ep string, to time.Duration, prefix string) (*HTTPKeysAPI, error) {
-	c, err := newHTTPClient(tr, ep, to)
-	if err != nil {
-		return nil, err
-	}
-
-	kAPI := HTTPKeysAPI{
+func NewKeysAPI(c HTTPClient) KeysAPI {
+	return &httpKeysAPI{
 		client: c,
+		prefix: DefaultV2KeysPrefix,
 	}
+}
 
-	return &kAPI, nil
+func NewDiscoveryKeysAPI(c HTTPClient) KeysAPI {
+	return &httpKeysAPI{
+		client: c,
+		prefix: "",
+	}
 }
 
 type KeysAPI interface {
-	Create(key, value string, ttl time.Duration) (*Response, error)
-	Get(key string) (*Response, error)
+	Create(ctx context.Context, key, value string, ttl time.Duration) (*Response, error)
+	Get(ctx context.Context, key string) (*Response, error)
+
 	Watch(key string, idx uint64) Watcher
 	RecursiveWatch(key string, idx uint64) Watcher
 }
 
 type Watcher interface {
-	Next() (*Response, error)
+	Next(context.Context) (*Response, error)
 }
 
 type Response struct {
@@ -92,46 +86,50 @@ func (n *Node) String() string {
 	return fmt.Sprintf("{Key: %s, CreatedIndex: %d, ModifiedIndex: %d}", n.Key, n.CreatedIndex, n.ModifiedIndex)
 }
 
-type HTTPKeysAPI struct {
-	client *httpClient
+type httpKeysAPI struct {
+	client HTTPClient
+	prefix string
 }
 
-func (k *HTTPKeysAPI) Create(key, val string, ttl time.Duration) (*Response, error) {
+func (k *httpKeysAPI) Create(ctx context.Context, key, val string, ttl time.Duration) (*Response, error) {
 	create := &createAction{
-		Key:   key,
-		Value: val,
+		Prefix: k.prefix,
+		Key:    key,
+		Value:  val,
 	}
 	if ttl >= 0 {
 		uttl := uint64(ttl.Seconds())
 		create.TTL = &uttl
 	}
 
-	httpresp, body, err := k.client.doWithTimeout(create)
+	resp, body, err := k.client.Do(ctx, create)
 	if err != nil {
 		return nil, err
 	}
 
-	return unmarshalHTTPResponse(httpresp.StatusCode, body)
+	return unmarshalHTTPResponse(resp.StatusCode, body)
 }
 
-func (k *HTTPKeysAPI) Get(key string) (*Response, error) {
+func (k *httpKeysAPI) Get(ctx context.Context, key string) (*Response, error) {
 	get := &getAction{
+		Prefix:    k.prefix,
 		Key:       key,
 		Recursive: false,
 	}
 
-	httpresp, body, err := k.client.doWithTimeout(get)
+	resp, body, err := k.client.Do(ctx, get)
 	if err != nil {
 		return nil, err
 	}
 
-	return unmarshalHTTPResponse(httpresp.StatusCode, body)
+	return unmarshalHTTPResponse(resp.StatusCode, body)
 }
 
-func (k *HTTPKeysAPI) Watch(key string, idx uint64) Watcher {
+func (k *httpKeysAPI) Watch(key string, idx uint64) Watcher {
 	return &httpWatcher{
 		client: k.client,
 		nextWait: waitAction{
+			Prefix:    k.prefix,
 			Key:       key,
 			WaitIndex: idx,
 			Recursive: false,
@@ -139,10 +137,11 @@ func (k *HTTPKeysAPI) Watch(key string, idx uint64) Watcher {
 	}
 }
 
-func (k *HTTPKeysAPI) RecursiveWatch(key string, idx uint64) Watcher {
+func (k *httpKeysAPI) RecursiveWatch(key string, idx uint64) Watcher {
 	return &httpWatcher{
 		client: k.client,
 		nextWait: waitAction{
+			Prefix:    k.prefix,
 			Key:       key,
 			WaitIndex: idx,
 			Recursive: true,
@@ -151,13 +150,12 @@ func (k *HTTPKeysAPI) RecursiveWatch(key string, idx uint64) Watcher {
 }
 
 type httpWatcher struct {
-	client   *httpClient
+	client   HTTPClient
 	nextWait waitAction
 }
 
-func (hw *httpWatcher) Next() (*Response, error) {
-	//TODO(bcwaldon): This needs to be cancellable by the calling user
-	httpresp, body, err := hw.client.do(context.Background(), &hw.nextWait)
+func (hw *httpWatcher) Next(ctx context.Context) (*Response, error) {
+	httpresp, body, err := hw.client.Do(ctx, &hw.nextWait)
 	if err != nil {
 		return nil, err
 	}
@@ -171,21 +169,24 @@ func (hw *httpWatcher) Next() (*Response, error) {
 	return resp, nil
 }
 
-// v2KeysURL forms a URL representing the location of a key. The provided
-// endpoint must be the root of the etcd keys API. For example, a valid
-// endpoint probably has the path "/v2/keys".
-func v2KeysURL(ep url.URL, key string) *url.URL {
-	ep.Path = path.Join(ep.Path, key)
+// v2KeysURL forms a URL representing the location of a key.
+// The endpoint argument represents the base URL of an etcd
+// server. The prefix is the path needed to route from the
+// provided endpoint's path to the root of the keys API
+// (typically "/v2/keys").
+func v2KeysURL(ep url.URL, prefix, key string) *url.URL {
+	ep.Path = path.Join(ep.Path, prefix, key)
 	return &ep
 }
 
 type getAction struct {
+	Prefix    string
 	Key       string
 	Recursive bool
 }
 
-func (g *getAction) httpRequest(ep url.URL) *http.Request {
-	u := v2KeysURL(ep, g.Key)
+func (g *getAction) HTTPRequest(ep url.URL) *http.Request {
+	u := v2KeysURL(ep, g.Prefix, g.Key)
 
 	params := u.Query()
 	params.Set("recursive", strconv.FormatBool(g.Recursive))
@@ -196,13 +197,14 @@ func (g *getAction) httpRequest(ep url.URL) *http.Request {
 }
 
 type waitAction struct {
+	Prefix    string
 	Key       string
 	WaitIndex uint64
 	Recursive bool
 }
 
-func (w *waitAction) httpRequest(ep url.URL) *http.Request {
-	u := v2KeysURL(ep, w.Key)
+func (w *waitAction) HTTPRequest(ep url.URL) *http.Request {
+	u := v2KeysURL(ep, w.Prefix, w.Key)
 
 	params := u.Query()
 	params.Set("wait", "true")
@@ -215,13 +217,14 @@ func (w *waitAction) httpRequest(ep url.URL) *http.Request {
 }
 
 type createAction struct {
-	Key   string
-	Value string
-	TTL   *uint64
+	Prefix string
+	Key    string
+	Value  string
+	TTL    *uint64
 }
 
-func (c *createAction) httpRequest(ep url.URL) *http.Request {
-	u := v2KeysURL(ep, c.Key)
+func (c *createAction) HTTPRequest(ep url.URL) *http.Request {
+	u := v2KeysURL(ep, c.Prefix, c.Key)
 
 	params := u.Query()
 	params.Set("prevExist", "false")

client/keys_test.go

@@ -29,12 +29,14 @@ import (
 func TestV2KeysURLHelper(t *testing.T) {
 	tests := []struct {
 		endpoint url.URL
+		prefix   string
 		key      string
 		want     url.URL
 	}{
 		// key is empty, no problem
 		{
 			endpoint: url.URL{Scheme: "http", Host: "example.com", Path: "/v2/keys"},
+			prefix:   "",
 			key:      "",
 			want:     url.URL{Scheme: "http", Host: "example.com", Path: "/v2/keys"},
 		},
@@ -42,6 +44,7 @@ func TestV2KeysURLHelper(t *testing.T) {
 		// key is joined to path
 		{
 			endpoint: url.URL{Scheme: "http", Host: "example.com", Path: "/v2/keys"},
+			prefix:   "",
 			key:      "/foo/bar",
 			want:     url.URL{Scheme: "http", Host: "example.com", Path: "/v2/keys/foo/bar"},
 		},
@@ -49,6 +52,7 @@ func TestV2KeysURLHelper(t *testing.T) {
 		// key is joined to path when path is empty
 		{
 			endpoint: url.URL{Scheme: "http", Host: "example.com", Path: ""},
+			prefix:   "",
 			key:      "/foo/bar",
 			want:     url.URL{Scheme: "http", Host: "example.com", Path: "/foo/bar"},
 		},
@@ -56,6 +60,7 @@ func TestV2KeysURLHelper(t *testing.T) {
 		// Host field carries through with port
 		{
 			endpoint: url.URL{Scheme: "http", Host: "example.com:8080", Path: "/v2/keys"},
+			prefix:   "",
 			key:      "",
 			want:     url.URL{Scheme: "http", Host: "example.com:8080", Path: "/v2/keys"},
 		},
@@ -63,13 +68,21 @@ func TestV2KeysURLHelper(t *testing.T) {
 		// Scheme carries through
 		{
 			endpoint: url.URL{Scheme: "https", Host: "example.com", Path: "/v2/keys"},
+			prefix:   "",
 			key:      "",
 			want:     url.URL{Scheme: "https", Host: "example.com", Path: "/v2/keys"},
 		},
+		// Prefix is applied
+		{
+			endpoint: url.URL{Scheme: "https", Host: "example.com", Path: "/foo"},
+			prefix:   "/bar",
+			key:      "/baz",
+			want:     url.URL{Scheme: "https", Host: "example.com", Path: "/foo/bar/baz"},
+		},
 	}
 
 	for i, tt := range tests {
-		got := v2KeysURL(tt.endpoint, tt.key)
+		got := v2KeysURL(tt.endpoint, tt.prefix, tt.key)
 		if tt.want != *got {
 			t.Errorf("#%d: want=%#v, got=%#v", i, tt.want, *got)
 		}
@@ -104,7 +117,7 @@ func TestGetAction(t *testing.T) {
 			Key:       "/foo/bar",
 			Recursive: tt.recursive,
 		}
-		got := *f.httpRequest(ep)
+		got := *f.HTTPRequest(ep)
 
 		wantURL := wantURL
 		wantURL.RawQuery = tt.wantQuery
@@ -153,7 +166,7 @@ func TestWaitAction(t *testing.T) {
 			WaitIndex: tt.waitIndex,
 			Recursive: tt.recursive,
 		}
-		got := *f.httpRequest(ep)
+		got := *f.HTTPRequest(ep)
 
 		wantURL := wantURL
 		wantURL.RawQuery = tt.wantQuery
@@ -200,7 +213,7 @@ func TestCreateAction(t *testing.T) {
 			Value: tt.value,
 			TTL:   tt.ttl,
 		}
-		got := *f.httpRequest(ep)
+		got := *f.HTTPRequest(ep)
 
 		err := assertResponse(got, wantURL, wantHeader, []byte(tt.wantBody))
 		if err != nil {
@@ -224,7 +237,7 @@ func assertResponse(got http.Request, wantURL *url.URL, wantHeader http.Header,
 		}
 	} else {
 		if wantBody == nil {
-			return fmt.Errorf("want.Body=%v got.Body=%v", wantBody, got.Body)
+			return fmt.Errorf("want.Body=%v got.Body=%s", wantBody, got.Body)
 		} else {
 			gotBytes, err := ioutil.ReadAll(got.Body)
 			if err != nil {
@@ -232,7 +245,7 @@ func assertResponse(got http.Request, wantURL *url.URL, wantHeader http.Header,
 			}
 
 			if !reflect.DeepEqual(wantBody, gotBytes) {
-				return fmt.Errorf("want.Body=%v got.Body=%v", wantBody, gotBytes)
+				return fmt.Errorf("want.Body=%s got.Body=%s", wantBody, gotBytes)
 			}
 		}
 	}

client/members.go

@@ -0,0 +1,159 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
+	"github.com/coreos/etcd/etcdserver/etcdhttp/httptypes"
+	"github.com/coreos/etcd/pkg/types"
+)
+
+var (
+	DefaultV2MembersPrefix = "/v2/members"
+)
+
+func NewMembersAPI(c HTTPClient) MembersAPI {
+	return &httpMembersAPI{
+		client: c,
+	}
+}
+
+type MembersAPI interface {
+	List(ctx context.Context) ([]httptypes.Member, error)
+	Add(ctx context.Context, peerURL string) (*httptypes.Member, error)
+	Remove(ctx context.Context, mID string) error
+}
+
+type httpMembersAPI struct {
+	client HTTPClient
+}
+
+func (m *httpMembersAPI) List(ctx context.Context) ([]httptypes.Member, error) {
+	req := &membersAPIActionList{}
+	resp, body, err := m.client.Do(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := assertStatusCode(resp.StatusCode, http.StatusOK); err != nil {
+		return nil, err
+	}
+
+	var mCollection httptypes.MemberCollection
+	if err := json.Unmarshal(body, &mCollection); err != nil {
+		return nil, err
+	}
+
+	return []httptypes.Member(mCollection), nil
+}
+
+func (m *httpMembersAPI) Add(ctx context.Context, peerURL string) (*httptypes.Member, error) {
+	urls, err := types.NewURLs([]string{peerURL})
+	if err != nil {
+		return nil, err
+	}
+
+	req := &membersAPIActionAdd{peerURLs: urls}
+	resp, body, err := m.client.Do(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := assertStatusCode(resp.StatusCode, http.StatusCreated, http.StatusConflict); err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusCreated {
+		var httperr httptypes.HTTPError
+		if err := json.Unmarshal(body, &httperr); err != nil {
+			return nil, err
+		}
+		return nil, httperr
+	}
+
+	var memb httptypes.Member
+	if err := json.Unmarshal(body, &memb); err != nil {
+		return nil, err
+	}
+
+	return &memb, nil
+}
+
+func (m *httpMembersAPI) Remove(ctx context.Context, memberID string) error {
+	req := &membersAPIActionRemove{memberID: memberID}
+	resp, _, err := m.client.Do(ctx, req)
+	if err != nil {
+		return err
+	}
+
+	return assertStatusCode(resp.StatusCode, http.StatusNoContent)
+}
+
+type membersAPIActionList struct{}
+
+func (l *membersAPIActionList) HTTPRequest(ep url.URL) *http.Request {
+	u := v2MembersURL(ep)
+	req, _ := http.NewRequest("GET", u.String(), nil)
+	return req
+}
+
+type membersAPIActionRemove struct {
+	memberID string
+}
+
+func (d *membersAPIActionRemove) HTTPRequest(ep url.URL) *http.Request {
+	u := v2MembersURL(ep)
+	u.Path = path.Join(u.Path, d.memberID)
+	req, _ := http.NewRequest("DELETE", u.String(), nil)
+	return req
+}
+
+type membersAPIActionAdd struct {
+	peerURLs types.URLs
+}
+
+func (a *membersAPIActionAdd) HTTPRequest(ep url.URL) *http.Request {
+	u := v2MembersURL(ep)
+	m := httptypes.MemberCreateRequest{PeerURLs: a.peerURLs}
+	b, _ := json.Marshal(&m)
+	req, _ := http.NewRequest("POST", u.String(), bytes.NewReader(b))
+	req.Header.Set("Content-Type", "application/json")
+	return req
+}
+
+func assertStatusCode(got int, want ...int) (err error) {
+	for _, w := range want {
+		if w == got {
+			return nil
+		}
+	}
+	return fmt.Errorf("unexpected status code %d", got)
+}
+
+// v2MembersURL add the necessary path to the provided endpoint
+// to route requests to the default v2 members API.
+func v2MembersURL(ep url.URL) *url.URL {
+	ep.Path = path.Join(ep.Path, DefaultV2MembersPrefix)
+	return &ep
+}

client/members_test.go

@@ -0,0 +1,113 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package client
+
+import (
+	"net/http"
+	"net/url"
+	"reflect"
+	"testing"
+
+	"github.com/coreos/etcd/pkg/types"
+)
+
+func TestMembersAPIActionList(t *testing.T) {
+	ep := url.URL{Scheme: "http", Host: "example.com"}
+	act := &membersAPIActionList{}
+
+	wantURL := &url.URL{
+		Scheme: "http",
+		Host:   "example.com",
+		Path:   "/v2/members",
+	}
+
+	got := *act.HTTPRequest(ep)
+	err := assertResponse(got, wantURL, http.Header{}, nil)
+	if err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestMembersAPIActionAdd(t *testing.T) {
+	ep := url.URL{Scheme: "http", Host: "example.com"}
+	act := &membersAPIActionAdd{
+		peerURLs: types.URLs([]url.URL{
+			url.URL{Scheme: "https", Host: "127.0.0.1:8081"},
+			url.URL{Scheme: "http", Host: "127.0.0.1:8080"},
+		}),
+	}
+
+	wantURL := &url.URL{
+		Scheme: "http",
+		Host:   "example.com",
+		Path:   "/v2/members",
+	}
+	wantHeader := http.Header{
+		"Content-Type": []string{"application/json"},
+	}
+	wantBody := []byte(`{"peerURLs":["https://127.0.0.1:8081","http://127.0.0.1:8080"]}`)
+
+	got := *act.HTTPRequest(ep)
+	err := assertResponse(got, wantURL, wantHeader, wantBody)
+	if err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestMembersAPIActionRemove(t *testing.T) {
+	ep := url.URL{Scheme: "http", Host: "example.com"}
+	act := &membersAPIActionRemove{memberID: "XXX"}
+
+	wantURL := &url.URL{
+		Scheme: "http",
+		Host:   "example.com",
+		Path:   "/v2/members/XXX",
+	}
+
+	got := *act.HTTPRequest(ep)
+	err := assertResponse(got, wantURL, http.Header{}, nil)
+	if err != nil {
+		t.Error(err.Error())
+	}
+}
+
+func TestAssertStatusCode(t *testing.T) {
+	if err := assertStatusCode(404, 400); err == nil {
+		t.Errorf("assertStatusCode failed to detect conflict in 400 vs 404")
+	}
+
+	if err := assertStatusCode(404, 400, 404); err != nil {
+		t.Errorf("assertStatusCode found conflict in (404,400) vs 400: %v", err)
+	}
+}
+
+func TestV2MembersURL(t *testing.T) {
+	got := v2MembersURL(url.URL{
+		Scheme: "http",
+		Host:   "foo.example.com:4002",
+		Path:   "/pants",
+	})
+	want := &url.URL{
+		Scheme: "http",
+		Host:   "foo.example.com:4002",
+		Path:   "/pants/v2/members",
+	}
+
+	if !reflect.DeepEqual(want, got) {
+		t.Fatalf("v2MembersURL got %#v, want %#v", got, want)
+	}
+}

discovery/discovery.go

@@ -22,15 +22,16 @@ import (
 	"log"
 	"net/http"
 	"net/url"
-	"os"
 	"path"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
 	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/jonboulle/clockwork"
 	"github.com/coreos/etcd/client"
+	"github.com/coreos/etcd/pkg/types"
 )
 
 var (
@@ -44,20 +45,33 @@ var (
 )
 
 const (
-	// Environment variable used to configure an HTTP proxy for discovery
-	DiscoveryProxyEnv = "ETCD_DISCOVERY_PROXY"
 	// Number of retries discovery will attempt before giving up and erroring out.
 	nRetries = uint(3)
 )
 
-type Discoverer interface {
-	Discover() (string, error)
+// JoinCluster will connect to the discovery service at the given url, and
+// register the server represented by the given id and config to the cluster
+func JoinCluster(durl, dproxyurl string, id types.ID, config string) (string, error) {
+	d, err := newDiscovery(durl, dproxyurl, id)
+	if err != nil {
+		return "", err
+	}
+	return d.joinCluster(config)
+}
+
+// GetCluster will connect to the discovery service at the given url and
+// retrieve a string describing the cluster
+func GetCluster(durl, dproxyurl string) (string, error) {
+	d, err := newDiscovery(durl, dproxyurl, 0)
+	if err != nil {
+		return "", err
+	}
+	return d.getCluster()
 }
 
 type discovery struct {
 	cluster string
-	id      uint64
-	config  string
+	id      types.ID
 	c       client.KeysAPI
 	retries uint
 	url     *url.URL
@@ -65,11 +79,10 @@ type discovery struct {
 	clock clockwork.Clock
 }
 
-// proxyFuncFromEnv builds a proxy function if the appropriate environment
-// variable is set. It performs basic sanitization of the environment variable
-// and returns any error encountered.
-func proxyFuncFromEnv() (func(*http.Request) (*url.URL, error), error) {
-	proxy := os.Getenv(DiscoveryProxyEnv)
+// newProxyFunc builds a proxy function from the given string, which should
+// represent a URL that can be used as a proxy. It performs basic
+// sanitization of the URL and returns any error encountered.
+func newProxyFunc(proxy string) (func(*http.Request) (*url.URL, error), error) {
 	if proxy == "" {
 		return nil, nil
 	}
@@ -94,40 +107,39 @@ func proxyFuncFromEnv() (func(*http.Request) (*url.URL, error), error) {
 	return http.ProxyURL(proxyURL), nil
 }
 
-func New(durl string, id uint64, config string) (Discoverer, error) {
+func newDiscovery(durl, dproxyurl string, id types.ID) (*discovery, error) {
 	u, err := url.Parse(durl)
 	if err != nil {
 		return nil, err
 	}
 	token := u.Path
 	u.Path = ""
-	pf, err := proxyFuncFromEnv()
+	pf, err := newProxyFunc(dproxyurl)
 	if err != nil {
 		return nil, err
 	}
-	c, err := client.NewDiscoveryKeysAPI(&http.Transport{Proxy: pf}, u.String(), client.DefaultRequestTimeout)
+	c, err := client.NewHTTPClient(&http.Transport{Proxy: pf}, []string{u.String()})
 	if err != nil {
 		return nil, err
 	}
+	dc := client.NewDiscoveryKeysAPI(c)
 	return &discovery{
 		cluster: token,
+		c:       dc,
 		id:      id,
-		config:  config,
-		c:       c,
 		url:     u,
 		clock:   clockwork.NewRealClock(),
 	}, nil
 }
 
-func (d *discovery) Discover() (string, error) {
-	// fast path: if the cluster is full, returns the error
-	// do not need to register itself to the cluster in this
-	// case.
+func (d *discovery) joinCluster(config string) (string, error) {
+	// fast path: if the cluster is full, return the error
+	// do not need to register to the cluster in this case.
 	if _, _, err := d.checkCluster(); err != nil {
 		return "", err
 	}
 
-	if err := d.createSelf(); err != nil {
+	if err := d.createSelf(config); err != nil {
 		// Fails, even on a timeout, if createSelf times out.
 		// TODO(barakmich): Retrying the same node might want to succeed here
 		// (ie, createSelf should be idempotent for discovery).
@@ -147,22 +159,42 @@ func (d *discovery) Discover() (string, error) {
 	return nodesToCluster(all), nil
 }
 
-func (d *discovery) createSelf() error {
-	resp, err := d.c.Create(d.selfKey(), d.config, -1)
+func (d *discovery) getCluster() (string, error) {
+	nodes, size, err := d.checkCluster()
+	if err != nil {
+		if err == ErrFullCluster {
+			return nodesToCluster(nodes), nil
+		}
+		return "", err
+	}
+
+	all, err := d.waitNodes(nodes, size)
+	if err != nil {
+		return "", err
+	}
+	return nodesToCluster(all), nil
+}
+
+func (d *discovery) createSelf(contents string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
+	resp, err := d.c.Create(ctx, d.selfKey(), contents, -1)
+	cancel()
 	if err != nil {
 		return err
 	}
 
 	// ensure self appears on the server we connected to
 	w := d.c.Watch(d.selfKey(), resp.Node.CreatedIndex)
-	_, err = w.Next()
+	_, err = w.Next(context.Background())
 	return err
 }
 
 func (d *discovery) checkCluster() (client.Nodes, int, error) {
 	configKey := path.Join("/", d.cluster, "_config")
+	ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
 	// find cluster size
-	resp, err := d.c.Get(path.Join(configKey, "size"))
+	resp, err := d.c.Get(ctx, path.Join(configKey, "size"))
+	cancel()
 	if err != nil {
 		if err == client.ErrKeyNoExist {
 			return nil, 0, ErrSizeNotFound
@@ -177,7 +209,9 @@ func (d *discovery) checkCluster() (client.Nodes, int, error) {
 		return nil, 0, ErrBadSizeKey
 	}
 
-	resp, err = d.c.Get(d.cluster)
+	ctx, cancel = context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
+	resp, err = d.c.Get(ctx, d.cluster)
+	cancel()
 	if err != nil {
 		if err == client.ErrTimeout {
 			return d.checkClusterRetry()
@@ -187,7 +221,7 @@ func (d *discovery) checkCluster() (client.Nodes, int, error) {
 	nodes := make(client.Nodes, 0)
 	// append non-config keys to nodes
 	for _, n := range resp.Node.Nodes {
-		if !strings.Contains(n.Key, configKey) {
+		if !(path.Base(n.Key) == path.Base(configKey)) {
 			nodes = append(nodes, n)
 		}
 	}
@@ -197,11 +231,11 @@ func (d *discovery) checkCluster() (client.Nodes, int, error) {
 
 	// find self position
 	for i := range nodes {
-		if strings.Contains(nodes[i].Key, d.selfKey()) {
+		if path.Base(nodes[i].Key) == path.Base(d.selfKey()) {
 			break
 		}
 		if i >= size-1 {
-			return nil, size, ErrFullCluster
+			return nodes[:size], size, ErrFullCluster
 		}
 	}
 	return nodes, size, nil
@@ -241,22 +275,33 @@ func (d *discovery) waitNodes(nodes client.Nodes, size int) (client.Nodes, error
 	w := d.c.RecursiveWatch(d.cluster, nodes[len(nodes)-1].ModifiedIndex+1)
 	all := make(client.Nodes, len(nodes))
 	copy(all, nodes)
+	for _, n := range all {
+		if path.Base(n.Key) == path.Base(d.selfKey()) {
+			log.Printf("discovery: found self %s in the cluster", path.Base(d.selfKey()))
+		} else {
+			log.Printf("discovery: found peer %s in the cluster", path.Base(n.Key))
+		}
+	}
+
 	// wait for others
 	for len(all) < size {
-		resp, err := w.Next()
+		log.Printf("discovery: found %d peer(s), waiting for %d more", len(all), size-len(all))
+		resp, err := w.Next(context.Background())
 		if err != nil {
 			if err == client.ErrTimeout {
 				return d.waitNodesRetry()
 			}
 			return nil, err
 		}
+		log.Printf("discovery: found peer %s in the cluster", path.Base(resp.Node.Key))
 		all = append(all, resp.Node)
 	}
+	log.Printf("discovery: found %d needed peer(s)", len(all))
 	return all, nil
 }
 
 func (d *discovery) selfKey() string {
-	return path.Join("/", d.cluster, fmt.Sprintf("%d", d.id))
+	return path.Join("/", d.cluster, d.id.String())
 }
 
 func nodesToCluster(ns client.Nodes) string {

discovery/discovery_test.go

@@ -20,21 +20,19 @@ import (
 	"errors"
 	"math/rand"
 	"net/http"
-	"os"
+	"reflect"
 	"sort"
 	"strconv"
-
-	"reflect"
 	"testing"
 	"time"
 
+	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
 	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/jonboulle/clockwork"
 	"github.com/coreos/etcd/client"
 )
 
-func TestProxyFuncFromEnvUnset(t *testing.T) {
-	os.Setenv(DiscoveryProxyEnv, "")
-	pf, err := proxyFuncFromEnv()
+func TestNewProxyFuncUnset(t *testing.T) {
+	pf, err := newProxyFunc("")
 	if pf != nil {
 		t.Fatal("unexpected non-nil proxyFunc")
 	}
@@ -43,14 +41,13 @@ func TestProxyFuncFromEnvUnset(t *testing.T) {
 	}
 }
 
-func TestProxyFuncFromEnvBad(t *testing.T) {
+func TestNewProxyFuncBad(t *testing.T) {
 	tests := []string{
 		"%%",
 		"http://foo.com/%1",
 	}
 	for i, in := range tests {
-		os.Setenv(DiscoveryProxyEnv, in)
-		pf, err := proxyFuncFromEnv()
+		pf, err := newProxyFunc(in)
 		if pf != nil {
 			t.Errorf("#%d: unexpected non-nil proxyFunc", i)
 		}
@@ -60,14 +57,13 @@ func TestProxyFuncFromEnvBad(t *testing.T) {
 	}
 }
 
-func TestProxyFuncFromEnv(t *testing.T) {
+func TestNewProxyFunc(t *testing.T) {
 	tests := map[string]string{
 		"bar.com":              "http://bar.com",
 		"http://disco.foo.bar": "http://disco.foo.bar",
 	}
 	for in, w := range tests {
-		os.Setenv(DiscoveryProxyEnv, in)
-		pf, err := proxyFuncFromEnv()
+		pf, err := newProxyFunc(in)
 		if pf == nil {
 			t.Errorf("%s: unexpected nil proxyFunc", in)
 			continue
@@ -88,7 +84,7 @@ func TestProxyFuncFromEnv(t *testing.T) {
 }
 
 func TestCheckCluster(t *testing.T) {
-	cluster := "1000"
+	cluster := "/prefix/1000"
 	self := "/1000/1"
 
 	tests := []struct {
@@ -100,6 +96,7 @@ func TestCheckCluster(t *testing.T) {
 			// self is in the size range
 			[]*client.Node{
 				{Key: "/1000/_config/size", Value: "3", CreatedIndex: 1},
+				{Key: "/1000/_config/"},
 				{Key: self, CreatedIndex: 2},
 				{Key: "/1000/2", CreatedIndex: 3},
 				{Key: "/1000/3", CreatedIndex: 4},
@@ -112,6 +109,7 @@ func TestCheckCluster(t *testing.T) {
 			// self is in the size range
 			[]*client.Node{
 				{Key: "/1000/_config/size", Value: "3", CreatedIndex: 1},
+				{Key: "/1000/_config/"},
 				{Key: "/1000/2", CreatedIndex: 2},
 				{Key: "/1000/3", CreatedIndex: 3},
 				{Key: self, CreatedIndex: 4},
@@ -124,6 +122,7 @@ func TestCheckCluster(t *testing.T) {
 			// self is out of the size range
 			[]*client.Node{
 				{Key: "/1000/_config/size", Value: "3", CreatedIndex: 1},
+				{Key: "/1000/_config/"},
 				{Key: "/1000/2", CreatedIndex: 2},
 				{Key: "/1000/3", CreatedIndex: 3},
 				{Key: "/1000/4", CreatedIndex: 4},
@@ -136,6 +135,7 @@ func TestCheckCluster(t *testing.T) {
 			// self is not in the cluster
 			[]*client.Node{
 				{Key: "/1000/_config/size", Value: "3", CreatedIndex: 1},
+				{Key: "/1000/_config/"},
 				{Key: "/1000/2", CreatedIndex: 2},
 				{Key: "/1000/3", CreatedIndex: 3},
 			},
@@ -145,6 +145,7 @@ func TestCheckCluster(t *testing.T) {
 		{
 			[]*client.Node{
 				{Key: "/1000/_config/size", Value: "3", CreatedIndex: 1},
+				{Key: "/1000/_config/"},
 				{Key: "/1000/2", CreatedIndex: 2},
 				{Key: "/1000/3", CreatedIndex: 3},
 				{Key: "/1000/4", CreatedIndex: 4},
@@ -175,7 +176,7 @@ func TestCheckCluster(t *testing.T) {
 			rs = append(rs, &client.Response{
 				Node: &client.Node{
 					Key:   cluster,
-					Nodes: tt.nodes,
+					Nodes: tt.nodes[1:],
 				},
 			})
 		}
@@ -317,7 +318,7 @@ func TestCreateSelf(t *testing.T) {
 
 	for i, tt := range tests {
 		d := discovery{cluster: "1000", c: tt.c}
-		if err := d.createSelf(); err != tt.werr {
+		if err := d.createSelf(""); err != tt.werr {
 			t.Errorf("#%d: err = %v, want %v", i, err, nil)
 		}
 	}
@@ -392,7 +393,7 @@ type clientWithResp struct {
 	w  client.Watcher
 }
 
-func (c *clientWithResp) Create(key string, value string, ttl time.Duration) (*client.Response, error) {
+func (c *clientWithResp) Create(ctx context.Context, key string, value string, ttl time.Duration) (*client.Response, error) {
 	if len(c.rs) == 0 {
 		return &client.Response{}, nil
 	}
@@ -401,7 +402,7 @@ func (c *clientWithResp) Create(key string, value string, ttl time.Duration) (*c
 	return r, nil
 }
 
-func (c *clientWithResp) Get(key string) (*client.Response, error) {
+func (c *clientWithResp) Get(ctx context.Context, key string) (*client.Response, error) {
 	if len(c.rs) == 0 {
 		return &client.Response{}, client.ErrKeyNoExist
 	}
@@ -423,11 +424,11 @@ type clientWithErr struct {
 	w   client.Watcher
 }
 
-func (c *clientWithErr) Create(key string, value string, ttl time.Duration) (*client.Response, error) {
+func (c *clientWithErr) Create(ctx context.Context, key string, value string, ttl time.Duration) (*client.Response, error) {
 	return &client.Response{}, c.err
 }
 
-func (c *clientWithErr) Get(key string) (*client.Response, error) {
+func (c *clientWithErr) Get(ctx context.Context, key string) (*client.Response, error) {
 	return &client.Response{}, c.err
 }
 
@@ -443,7 +444,7 @@ type watcherWithResp struct {
 	rs []*client.Response
 }
 
-func (w *watcherWithResp) Next() (*client.Response, error) {
+func (w *watcherWithResp) Next(context.Context) (*client.Response, error) {
 	if len(w.rs) == 0 {
 		return &client.Response{}, nil
 	}
@@ -456,7 +457,7 @@ type watcherWithErr struct {
 	err error
 }
 
-func (w *watcherWithErr) Next() (*client.Response, error) {
+func (w *watcherWithErr) Next(context.Context) (*client.Response, error) {
 	return &client.Response{}, w.err
 }
 
@@ -467,20 +468,20 @@ type clientWithRetry struct {
 	failTimes int
 }
 
-func (c *clientWithRetry) Create(key string, value string, ttl time.Duration) (*client.Response, error) {
+func (c *clientWithRetry) Create(ctx context.Context, key string, value string, ttl time.Duration) (*client.Response, error) {
 	if c.failCount < c.failTimes {
 		c.failCount++
 		return nil, client.ErrTimeout
 	}
-	return c.clientWithResp.Create(key, value, ttl)
+	return c.clientWithResp.Create(ctx, key, value, ttl)
 }
 
-func (c *clientWithRetry) Get(key string) (*client.Response, error) {
+func (c *clientWithRetry) Get(ctx context.Context, key string) (*client.Response, error) {
 	if c.failCount < c.failTimes {
 		c.failCount++
 		return nil, client.ErrTimeout
 	}
-	return c.clientWithResp.Get(key)
+	return c.clientWithResp.Get(ctx, key)
 }
 
 // watcherWithRetry will timeout all requests up to failTimes
@@ -490,7 +491,7 @@ type watcherWithRetry struct {
 	failTimes int
 }
 
-func (w *watcherWithRetry) Next() (*client.Response, error) {
+func (w *watcherWithRetry) Next(context.Context) (*client.Response, error) {
 	if w.failCount < w.failTimes {
 		w.failCount++
 		return nil, client.ErrTimeout

error/error.go

@@ -63,6 +63,16 @@ var errors = map[int]string{
 	EcodeClientInternal: "Client Internal Error",
 }
 
+var errorStatus = map[int]int{
+	EcodeKeyNotFound:  http.StatusNotFound,
+	EcodeNotFile:      http.StatusForbidden,
+	EcodeDirNotEmpty:  http.StatusForbidden,
+	EcodeTestFailed:   http.StatusPreconditionFailed,
+	EcodeNodeExist:    http.StatusPreconditionFailed,
+	EcodeRaftInternal: http.StatusInternalServerError,
+	EcodeLeaderElect:  http.StatusInternalServerError,
+}
+
 const (
 	EcodeKeyNotFound      = 100
 	EcodeTestFailed       = 101
@@ -133,22 +143,17 @@ func (e Error) toJsonString() string {
 	return string(b)
 }
 
-func (e Error) Write(w http.ResponseWriter) {
-	w.Header().Add("X-Etcd-Index", fmt.Sprint(e.Index))
-	// 3xx is raft internal error
-	status := http.StatusBadRequest
-	switch e.ErrorCode {
-	case EcodeKeyNotFound:
-		status = http.StatusNotFound
-	case EcodeNotFile, EcodeDirNotEmpty:
-		status = http.StatusForbidden
-	case EcodeTestFailed, EcodeNodeExist:
-		status = http.StatusPreconditionFailed
-	default:
-		if e.ErrorCode/100 == 3 {
-			status = http.StatusInternalServerError
-		}
+func (e Error) statusCode() int {
+	status, ok := errorStatus[e.ErrorCode]
+	if !ok {
+		status = http.StatusBadRequest
 	}
-	w.WriteHeader(status)
+	return status
+}
+
+func (e Error) WriteTo(w http.ResponseWriter) {
+	w.Header().Add("X-Etcd-Index", fmt.Sprint(e.Index))
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(e.statusCode())
 	fmt.Fprintln(w, e.toJsonString())
 }

error/error_test.go

@@ -0,0 +1,52 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package error
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestErrorWriteTo(t *testing.T) {
+	for k, _ := range errors {
+		err := NewError(k, "", 1)
+		rr := httptest.NewRecorder()
+		err.WriteTo(rr)
+
+		if err.statusCode() != rr.Code {
+			t.Errorf("HTTP status code %d, want %d", rr.Code, err.statusCode())
+		}
+
+		gbody := strings.TrimSuffix(rr.Body.String(), "\n")
+		if err.toJsonString() != gbody {
+			t.Errorf("HTTP body %q, want %q", gbody, err.toJsonString())
+		}
+
+		wheader := http.Header(map[string][]string{
+			"Content-Type": []string{"application/json"},
+			"X-Etcd-Index": []string{"1"},
+		})
+
+		if !reflect.DeepEqual(wheader, rr.HeaderMap) {
+			t.Errorf("HTTP headers %v, want %v", rr.HeaderMap, wheader)
+		}
+	}
+
+}

etcdctl/README.md

@@ -127,6 +127,13 @@ $ etcdctl ls --recursive
 /adir/key2
 ```
 
+Directories can also have a trailing `/` added to output using `-p`.
+
+```
+$ etcdctl ls -p
+/akey
+/adir/
+```
 
 ### Deleting a key
 

etcdctl/command/backup_command.go

@@ -0,0 +1,87 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package command
+
+import (
+	"log"
+	"math/rand"
+	"path"
+	"time"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
+	"github.com/coreos/etcd/etcdserver"
+	"github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"github.com/coreos/etcd/pkg/pbutil"
+	"github.com/coreos/etcd/snap"
+	"github.com/coreos/etcd/wal"
+)
+
+func NewBackupCommand() cli.Command {
+	return cli.Command{
+		Name:  "backup",
+		Usage: "backup an etcd directory",
+		Flags: []cli.Flag{
+			cli.StringFlag{Name: "data-dir", Value: "", Usage: "Path to the etcd data dir"},
+			cli.StringFlag{Name: "backup-dir", Value: "", Usage: "Path to the backup dir"},
+		},
+		Action: handleBackup,
+	}
+}
+
+// handleBackup handles a request that intends to do a backup.
+func handleBackup(c *cli.Context) {
+	srcSnap := path.Join(c.String("data-dir"), "snap")
+	destSnap := path.Join(c.String("backup-dir"), "snap")
+	srcWAL := path.Join(c.String("data-dir"), "wal")
+	destWAL := path.Join(c.String("backup-dir"), "wal")
+
+	ss := snap.New(srcSnap)
+	snapshot, err := ss.Load()
+	if err != nil && err != snap.ErrNoSnapshot {
+		log.Fatal(err)
+	}
+	var index uint64
+	if snapshot != nil {
+		index = snapshot.Index
+		newss := snap.New(destSnap)
+		newss.SaveSnap(*snapshot)
+	}
+
+	w, err := wal.OpenAtIndex(srcWAL, index)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer w.Close()
+	wmetadata, state, ents, err := w.ReadAll()
+	if err != nil {
+		log.Fatal(err)
+	}
+	var metadata etcdserverpb.Metadata
+	pbutil.MustUnmarshal(&metadata, wmetadata)
+	rand.Seed(time.Now().UnixNano())
+	metadata.NodeID = etcdserver.GenID()
+	metadata.ClusterID = etcdserver.GenID()
+
+	neww, err := wal.Create(destWAL, pbutil.MustMarshal(&metadata))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer neww.Close()
+	if err := neww.Save(state, ents); err != nil {
+		log.Fatal(err)
+	}
+}

etcdctl/command/error.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/exec_watch_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (
@@ -65,7 +81,6 @@ func execWatchCommandFunc(c *cli.Context, client *etcd.Client) (*etcd.Response,
 	}()
 
 	receiver := make(chan *etcd.Response)
-	client.SetConsistency(etcd.WEAK_CONSISTENCY)
 	go client.Watch(key, uint64(index), recursive, receiver, stop)
 
 	for {

etcdctl/command/get_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/handle.go

@@ -1,10 +1,25 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net/url"
 	"os"
 	"strings"
 
@@ -14,6 +29,7 @@ import (
 
 type handlerFunc func(*cli.Context, *etcd.Client) (*etcd.Response, error)
 type printFunc func(*etcd.Response, string)
+type contextualPrintFunc func(*cli.Context, *etcd.Response, string)
 
 // dumpCURL blindly dumps all curl output to os.Stderr
 func dumpCURL(client *etcd.Client) {
@@ -23,68 +39,35 @@ func dumpCURL(client *etcd.Client) {
 	}
 }
 
-// createHttpPath attaches http scheme to the given address if needed
-func createHttpPath(addr string) (string, error) {
-	u, err := url.Parse(addr)
-	if err != nil {
-		return "", err
-	}
-
-	if u.Scheme == "" {
-		u.Scheme = "http"
-	}
-	return u.String(), nil
-}
-
 // rawhandle wraps the command function handlers and sets up the
 // environment but performs no output formatting.
 func rawhandle(c *cli.Context, fn handlerFunc) (*etcd.Response, error) {
-	sync := !c.GlobalBool("no-sync")
-
-	peerstr := c.GlobalString("peers")
-
-	// Use an environment variable if nothing was supplied on the
-	// command line
-	if peerstr == "" {
-		peerstr = os.Getenv("ETCDCTL_PEERS")
+	endpoints, err := getEndpoints(c)
+	if err != nil {
+		return nil, err
 	}
 
-	// If we still don't have peers, use a default
-	if peerstr == "" {
-		peerstr = "127.0.0.1:4001"
+	tr, err := getTransport(c)
+	if err != nil {
+		return nil, err
 	}
 
-	peers := strings.Split(peerstr, ",")
-
-	// If no sync, create http path for each peer address
-	if !sync {
-		revisedPeers := make([]string, 0)
-		for _, peer := range peers {
-			if revisedPeer, err := createHttpPath(peer); err != nil {
-				fmt.Fprintf(os.Stderr, "Unsupported url %v: %v\n", peer, err)
-			} else {
-				revisedPeers = append(revisedPeers, revisedPeer)
-			}
-		}
-		peers = revisedPeers
-	}
-
-	client := etcd.NewClient(peers)
+	client := etcd.NewClient(endpoints)
+	client.SetTransport(tr)
 
 	if c.GlobalBool("debug") {
 		go dumpCURL(client)
 	}
 
 	// Sync cluster.
-	if sync {
+	if !c.GlobalBool("no-sync") {
 		if ok := client.SyncCluster(); !ok {
-			handleError(FailedToConnectToHost, errors.New("Cannot sync with the cluster using peers "+strings.Join(peers, ", ")))
+			handleError(FailedToConnectToHost, errors.New("cannot sync with the cluster using endpoints "+strings.Join(endpoints, ", ")))
 		}
 	}
 
 	if c.GlobalBool("debug") {
-		fmt.Fprintf(os.Stderr, "Cluster-Peers: %s\n",
-			strings.Join(client.GetCluster(), " "))
+		fmt.Fprintf(os.Stderr, "Cluster-Endpoints: %s\n", strings.Join(client.GetCluster(), ", "))
 	}
 
 	// Execute handler function.
@@ -106,6 +89,19 @@ func handlePrint(c *cli.Context, fn handlerFunc, pFn printFunc) {
 	}
 }
 
+// Just like handlePrint but also passed the context of the command
+func handleContextualPrint(c *cli.Context, fn handlerFunc, pFn contextualPrintFunc) {
+	resp, err := rawhandle(c, fn)
+
+	if err != nil {
+		handleError(ErrorFromEtcd, err)
+	}
+
+	if resp != nil && pFn != nil {
+		pFn(c, resp, c.GlobalString("output"))
+	}
+}
+
 // handleDir handles a request that wants to do operations on a single dir.
 // Dir cannot be printed out, so we set NIL print function here.
 func handleDir(c *cli.Context, fn handlerFunc) {

etcdctl/command/ls_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (
@@ -12,7 +28,9 @@ func NewLsCommand() cli.Command {
 		Name:  "ls",
 		Usage: "retrieve a directory",
 		Flags: []cli.Flag{
+			cli.BoolFlag{Name: "sort", Usage: "returns result in sorted order"},
 			cli.BoolFlag{Name: "recursive", Usage: "returns all values for key and child keys"},
+			cli.BoolFlag{Name: "p", Usage: "append slash (/) to directories"},
 		},
 		Action: func(c *cli.Context) {
 			handleLs(c, lsCommandFunc)
@@ -22,17 +40,17 @@ func NewLsCommand() cli.Command {
 
 // handleLs handles a request that intends to do ls-like operations.
 func handleLs(c *cli.Context, fn handlerFunc) {
-	handlePrint(c, fn, printLs)
+	handleContextualPrint(c, fn, printLs)
 }
 
 // printLs writes a response out in a manner similar to the `ls` command in unix.
 // Non-empty directories list their contents and files list their name.
-func printLs(resp *etcd.Response, format string) {
+func printLs(c *cli.Context, resp *etcd.Response, format string) {
 	if !resp.Node.Dir {
 		fmt.Println(resp.Node.Key)
 	}
 	for _, node := range resp.Node.Nodes {
-		rPrint(node)
+		rPrint(c, node)
 	}
 }
 
@@ -43,15 +61,22 @@ func lsCommandFunc(c *cli.Context, client *etcd.Client) (*etcd.Response, error)
 		key = c.Args()[0]
 	}
 	recursive := c.Bool("recursive")
+	sort := c.Bool("sort")
 
 	// Retrieve the value from the server.
-	return client.Get(key, false, recursive)
+	return client.Get(key, sort, recursive)
 }
 
 // rPrint recursively prints out the nodes in the node structure.
-func rPrint(n *etcd.Node) {
-	fmt.Println(n.Key)
+func rPrint(c *cli.Context, n *etcd.Node) {
+
+	if n.Dir && c.Bool("p") {
+		fmt.Println(fmt.Sprintf("%v/", n.Key))
+	} else {
+		fmt.Println(n.Key)
+	}
+
 	for _, node := range n.Nodes {
-		rPrint(node)
+		rPrint(c, node)
 	}
 }

etcdctl/command/member_commands.go

@@ -0,0 +1,173 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package command
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
+	"github.com/coreos/etcd/client"
+)
+
+func NewMemberCommand() cli.Command {
+	return cli.Command{
+		Name:  "member",
+		Usage: "member add, remove and list subcommands",
+		Subcommands: []cli.Command{
+			cli.Command{
+				Name:   "list",
+				Usage:  "enumerate existing cluster members",
+				Action: actionMemberList,
+			},
+			cli.Command{
+				Name:   "add",
+				Usage:  "add a new member to the etcd cluster",
+				Action: actionMemberAdd,
+			},
+			cli.Command{
+				Name:   "remove",
+				Usage:  "remove an existing member from the etcd cluster",
+				Action: actionMemberRemove,
+			},
+		},
+	}
+}
+
+func mustNewMembersAPI(c *cli.Context) client.MembersAPI {
+	eps, err := getEndpoints(c)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	tr, err := getTransport(c)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	hc, err := client.NewHTTPClient(tr, eps)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	if !c.GlobalBool("no-sync") {
+		ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
+		err := hc.Sync(ctx)
+		cancel()
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err.Error())
+			os.Exit(1)
+		}
+	}
+
+	if c.GlobalBool("debug") {
+		fmt.Fprintf(os.Stderr, "Cluster-Endpoints: %s\n", strings.Join(hc.Endpoints(), ", "))
+	}
+
+	return client.NewMembersAPI(hc)
+}
+
+func actionMemberList(c *cli.Context) {
+	if len(c.Args()) != 0 {
+		fmt.Fprintln(os.Stderr, "No arguments accepted")
+		os.Exit(1)
+	}
+	mAPI := mustNewMembersAPI(c)
+	ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
+	members, err := mAPI.List(ctx)
+	cancel()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	for _, m := range members {
+		fmt.Printf("%s: name=%s peerURLs=%s clientURLs=%s\n", m.ID, m.Name, strings.Join(m.PeerURLs, ","), strings.Join(m.ClientURLs, ","))
+	}
+}
+
+func actionMemberAdd(c *cli.Context) {
+	args := c.Args()
+	if len(args) != 2 {
+		fmt.Fprintln(os.Stderr, "Provide a name and a single member peerURL")
+		os.Exit(1)
+	}
+
+	mAPI := mustNewMembersAPI(c)
+
+	url := args[1]
+	ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
+	m, err := mAPI.Add(ctx, url)
+	cancel()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	newID := m.ID
+	newName := args[0]
+	fmt.Printf("Added member named %s with ID %s to cluster\n", newName, newID)
+
+	ctx, cancel = context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
+	members, err := mAPI.List(ctx)
+	cancel()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	conf := []string{}
+	for _, m := range members {
+		for _, u := range m.PeerURLs {
+			n := m.Name
+			if m.ID == newID {
+				n = newName
+			}
+			conf = append(conf, fmt.Sprintf("%s=%s", n, u))
+		}
+	}
+
+	fmt.Print("\n")
+	fmt.Printf("ETCD_NAME=%q\n", newName)
+	fmt.Printf("ETCD_INITIAL_CLUSTER=%q\n", strings.Join(conf, ","))
+	fmt.Printf("ETCD_INITIAL_CLUSTER_STATE=\"existing\"\n")
+}
+
+func actionMemberRemove(c *cli.Context) {
+	args := c.Args()
+	if len(args) != 1 {
+		fmt.Fprintln(os.Stderr, "Provide a single member ID")
+		os.Exit(1)
+	}
+
+	mAPI := mustNewMembersAPI(c)
+	mID := args[0]
+	ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
+	err := mAPI.Remove(ctx, mID)
+	cancel()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	fmt.Printf("Removed member %s from cluster\n", mID)
+}

etcdctl/command/mk_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/mkdir_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/rm_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/rmdir_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/set_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/set_dir_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/update_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/update_dir_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/util.go

@@ -1,10 +1,32 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (
 	"errors"
 	"io"
 	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
 	"strings"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
+	"github.com/coreos/etcd/pkg/transport"
 )
 
 var (
@@ -33,3 +55,47 @@ func argOrStdin(args []string, stdin io.Reader, i int) (string, error) {
 	}
 	return string(bytes), nil
 }
+
+func getPeersFlagValue(c *cli.Context) []string {
+	peerstr := c.GlobalString("peers")
+
+	// Use an environment variable if nothing was supplied on the
+	// command line
+	if peerstr == "" {
+		peerstr = os.Getenv("ETCDCTL_PEERS")
+	}
+
+	// If we still don't have peers, use a default
+	if peerstr == "" {
+		peerstr = "127.0.0.1:4001"
+	}
+
+	return strings.Split(peerstr, ",")
+}
+
+func getEndpoints(c *cli.Context) ([]string, error) {
+	eps := getPeersFlagValue(c)
+	for i, ep := range eps {
+		u, err := url.Parse(ep)
+		if err != nil {
+			return nil, err
+		}
+
+		if u.Scheme == "" {
+			u.Scheme = "http"
+		}
+
+		eps[i] = u.String()
+	}
+	return eps, nil
+}
+
+func getTransport(c *cli.Context) (*http.Transport, error) {
+	tls := transport.TLSInfo{
+		CAFile:   c.GlobalString("ca-file"),
+		CertFile: c.GlobalString("cert-file"),
+		KeyFile:  c.GlobalString("key-file"),
+	}
+	return transport.NewTransport(tls)
+
+}

etcdctl/command/util_test.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/command/watch_command.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package command
 
 import (

etcdctl/main.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package main
 
 import (
@@ -19,8 +35,12 @@ func main() {
 		cli.BoolFlag{Name: "no-sync", Usage: "don't synchronize cluster information before sending request"},
 		cli.StringFlag{Name: "output, o", Value: "simple", Usage: "output response in the given format (`simple` or `json`)"},
 		cli.StringFlag{Name: "peers, C", Value: "", Usage: "a comma-delimited list of machine addresses in the cluster (default: \"127.0.0.1:4001\")"},
+		cli.StringFlag{Name: "cert-file", Value: "", Usage: "identify HTTPS client using this SSL certificate file"},
+		cli.StringFlag{Name: "key-file", Value: "", Usage: "identify HTTPS client using this SSL key file"},
+		cli.StringFlag{Name: "ca-file", Value: "", Usage: "verify certificates of HTTPS-enabled servers using this CA bundle"},
 	}
 	app.Commands = []cli.Command{
+		command.NewBackupCommand(),
 		command.NewMakeCommand(),
 		command.NewMakeDirCommand(),
 		command.NewRemoveCommand(),
@@ -33,6 +53,8 @@ func main() {
 		command.NewUpdateDirCommand(),
 		command.NewWatchCommand(),
 		command.NewExecWatchCommand(),
+		command.NewMemberCommand(),
 	}
+
 	app.Run(os.Args)
 }

etcdmain/doc.go

@@ -0,0 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/* Package etcd contains the main entry point for the etcd binary. */
+
+package etcdmain

etcdmain/etcd.go

@@ -0,0 +1,401 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package etcdmain
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/coreos/etcd/discovery"
+	"github.com/coreos/etcd/etcdserver"
+	"github.com/coreos/etcd/etcdserver/etcdhttp"
+	"github.com/coreos/etcd/pkg/cors"
+	"github.com/coreos/etcd/pkg/fileutil"
+	"github.com/coreos/etcd/pkg/flags"
+	"github.com/coreos/etcd/pkg/transport"
+	"github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/etcd/proxy"
+	"github.com/coreos/etcd/version"
+)
+
+const (
+	// the owner can make/remove files inside the directory
+	privateDirMode = 0700
+
+	proxyFlagOff      = "off"
+	proxyFlagReadonly = "readonly"
+	proxyFlagOn       = "on"
+
+	fallbackFlagExit  = "exit"
+	fallbackFlagProxy = "proxy"
+
+	clusterStateFlagNew      = "new"
+	clusterStateFlagExisting = "existing"
+)
+
+var (
+	fs              = flag.NewFlagSet("etcd", flag.ContinueOnError)
+	name            = fs.String("name", "default", "Unique human-readable name for this node")
+	dir             = fs.String("data-dir", "", "Path to the data directory")
+	durl            = fs.String("discovery", "", "Discovery service used to bootstrap the cluster")
+	dproxy          = fs.String("discovery-proxy", "", "HTTP proxy to use for traffic to discovery service")
+	snapCount       = fs.Uint64("snapshot-count", etcdserver.DefaultSnapCount, "Number of committed transactions to trigger a snapshot")
+	printVersion    = fs.Bool("version", false, "Print the version and exit")
+	forceNewCluster = fs.Bool("force-new-cluster", false, "Force to create a new one member cluster")
+
+	initialCluster      = fs.String("initial-cluster", "default=http://localhost:2380,default=http://localhost:7001", "Initial cluster configuration for bootstrapping")
+	initialClusterToken = fs.String("initial-cluster-token", "etcd-cluster", "Initial cluster token for the etcd cluster during bootstrap")
+
+	corsInfo      = &cors.CORSInfo{}
+	clientTLSInfo = transport.TLSInfo{}
+	peerTLSInfo   = transport.TLSInfo{}
+
+	proxyFlag = flags.NewStringsFlag(
+		proxyFlagOff,
+		proxyFlagReadonly,
+		proxyFlagOn,
+	)
+	fallbackFlag = flags.NewStringsFlag(
+		fallbackFlagExit,
+		fallbackFlagProxy,
+	)
+	clusterStateFlag = flags.NewStringsFlag(
+		clusterStateFlagNew,
+		clusterStateFlagExisting,
+	)
+
+	ignored = []string{
+		"cluster-active-size",
+		"cluster-remove-delay",
+		"cluster-sync-interval",
+		"config",
+		"force",
+		"max-result-buffer",
+		"max-retry-attempts",
+		"peer-heartbeat-interval",
+		"peer-election-timeout",
+		"retry-interval",
+		"snapshot",
+		"v",
+		"vv",
+	}
+)
+
+func init() {
+	fs.Var(clusterStateFlag, "initial-cluster-state", "Initial cluster configuration for bootstrapping")
+	if err := clusterStateFlag.Set(clusterStateFlagNew); err != nil {
+		// Should never happen.
+		log.Panicf("unexpected error setting up clusterStateFlag: %v", err)
+	}
+
+	fs.Var(flags.NewURLsValue("http://localhost:2380,http://localhost:7001"), "initial-advertise-peer-urls", "List of this member's peer URLs to advertise to the rest of the cluster")
+	fs.Var(flags.NewURLsValue("http://localhost:2379,http://localhost:4001"), "advertise-client-urls", "List of this member's client URLs to advertise to the rest of the cluster")
+	fs.Var(flags.NewURLsValue("http://localhost:2380,http://localhost:7001"), "listen-peer-urls", "List of URLs to listen on for peer traffic")
+	fs.Var(flags.NewURLsValue("http://localhost:2379,http://localhost:4001"), "listen-client-urls", "List of URLs to listen on for client traffic")
+
+	fs.Var(corsInfo, "cors", "Comma-separated white list of origins for CORS (cross-origin resource sharing).")
+
+	fs.Var(proxyFlag, "proxy", fmt.Sprintf("Valid values include %s", strings.Join(proxyFlag.Values, ", ")))
+	if err := proxyFlag.Set(proxyFlagOff); err != nil {
+		// Should never happen.
+		log.Panicf("unexpected error setting up proxyFlag: %v", err)
+	}
+	fs.Var(fallbackFlag, "discovery-fallback", fmt.Sprintf("Valid values include %s", strings.Join(fallbackFlag.Values, ", ")))
+	if err := fallbackFlag.Set(fallbackFlagProxy); err != nil {
+		// Should never happen.
+		log.Panicf("unexpected error setting up discovery-fallback flag: %v", err)
+	}
+
+	fs.StringVar(&clientTLSInfo.CAFile, "ca-file", "", "Path to the client server TLS CA file.")
+	fs.StringVar(&clientTLSInfo.CertFile, "cert-file", "", "Path to the client server TLS cert file.")
+	fs.StringVar(&clientTLSInfo.KeyFile, "key-file", "", "Path to the client server TLS key file.")
+
+	fs.StringVar(&peerTLSInfo.CAFile, "peer-ca-file", "", "Path to the peer server TLS CA file.")
+	fs.StringVar(&peerTLSInfo.CertFile, "peer-cert-file", "", "Path to the peer server TLS cert file.")
+	fs.StringVar(&peerTLSInfo.KeyFile, "peer-key-file", "", "Path to the peer server TLS key file.")
+
+	// backwards-compatibility with v0.4.6
+	fs.Var(&flags.IPAddressPort{}, "addr", "DEPRECATED: Use -advertise-client-urls instead.")
+	fs.Var(&flags.IPAddressPort{}, "bind-addr", "DEPRECATED: Use -listen-client-urls instead.")
+	fs.Var(&flags.IPAddressPort{}, "peer-addr", "DEPRECATED: Use -initial-advertise-peer-urls instead.")
+	fs.Var(&flags.IPAddressPort{}, "peer-bind-addr", "DEPRECATED: Use -listen-peer-urls instead.")
+
+	for _, f := range ignored {
+		fs.Var(&flags.IgnoredFlag{Name: f}, f, "")
+	}
+
+	fs.Var(&flags.DeprecatedFlag{Name: "peers"}, "peers", "DEPRECATED: Use -initial-cluster instead")
+	fs.Var(&flags.DeprecatedFlag{Name: "peers-file"}, "peers-file", "DEPRECATED: Use -initial-cluster instead")
+}
+
+func Main() {
+	fs.Usage = flags.UsageWithIgnoredFlagsFunc(fs, ignored)
+	perr := fs.Parse(os.Args[1:])
+	switch perr {
+	case nil:
+	case flag.ErrHelp:
+		os.Exit(0)
+	default:
+		os.Exit(2)
+	}
+
+	if *printVersion {
+		fmt.Println("etcd version", version.Version)
+		os.Exit(0)
+	}
+
+	err := flags.SetFlagsFromEnv(fs)
+	if err != nil {
+		log.Fatalf("etcd: %v", err)
+	}
+
+	shouldProxy := proxyFlag.String() != proxyFlagOff
+	if !shouldProxy {
+		err = startEtcd()
+		if err == discovery.ErrFullCluster && fallbackFlag.String() == fallbackFlagProxy {
+			log.Printf("etcd: discovery cluster full, falling back to %s", fallbackFlagProxy)
+			shouldProxy = true
+		}
+	}
+	if shouldProxy {
+		err = startProxy()
+	}
+	if err != nil {
+		log.Fatalf("etcd: %v", err)
+	}
+	// Block indefinitely
+	<-make(chan struct{})
+}
+
+// startEtcd launches the etcd server and HTTP handlers for client/server communication.
+func startEtcd() error {
+	cls, err := setupCluster()
+	if err != nil {
+		fmt.Errorf("error setting up initial cluster: %v", err)
+	}
+
+	if *dir == "" {
+		*dir = fmt.Sprintf("%v.etcd", *name)
+		fmt.Errorf("no data-dir provided, using default data-dir ./%s", *dir)
+	}
+	if err := os.MkdirAll(*dir, privateDirMode); err != nil {
+		fmt.Errorf("cannot create data directory: %v", err)
+	}
+	if err := fileutil.IsDirWriteable(*dir); err != nil {
+		fmt.Errorf("cannot write to data directory: %v", err)
+	}
+
+	pt, err := transport.NewTransport(peerTLSInfo)
+	if err != nil {
+		return err
+	}
+
+	acurls, err := flags.URLsFromFlags(fs, "advertise-client-urls", "addr", clientTLSInfo)
+	if err != nil {
+		return err
+	}
+
+	lpurls, err := flags.URLsFromFlags(fs, "listen-peer-urls", "peer-bind-addr", peerTLSInfo)
+	if err != nil {
+		return err
+	}
+
+	plns := make([]net.Listener, 0)
+	for _, u := range lpurls {
+		var l net.Listener
+		l, err = transport.NewListener(u.Host, peerTLSInfo)
+		if err != nil {
+			return err
+		}
+
+		urlStr := u.String()
+		log.Print("etcd: listening for peers on ", urlStr)
+		defer func() {
+			if err != nil {
+				l.Close()
+				log.Print("etcd: stopping listening for peers on ", urlStr)
+			}
+		}()
+		plns = append(plns, l)
+	}
+
+	lcurls, err := flags.URLsFromFlags(fs, "listen-client-urls", "bind-addr", clientTLSInfo)
+	if err != nil {
+		return err
+	}
+
+	clns := make([]net.Listener, 0)
+	for _, u := range lcurls {
+		var l net.Listener
+		l, err = transport.NewListener(u.Host, clientTLSInfo)
+		if err != nil {
+			return err
+		}
+
+		urlStr := u.String()
+		log.Print("etcd: listening for client requests on ", urlStr)
+		defer func() {
+			if err != nil {
+				l.Close()
+				log.Print("etcd: stopping listening for client requests on ", urlStr)
+			}
+		}()
+		clns = append(clns, l)
+	}
+
+	cfg := &etcdserver.ServerConfig{
+		Name:            *name,
+		ClientURLs:      acurls,
+		DataDir:         *dir,
+		SnapCount:       *snapCount,
+		Cluster:         cls,
+		DiscoveryURL:    *durl,
+		DiscoveryProxy:  *dproxy,
+		NewCluster:      clusterStateFlag.String() == clusterStateFlagNew,
+		ForceNewCluster: *forceNewCluster,
+		Transport:       pt,
+	}
+	var s *etcdserver.EtcdServer
+	s, err = etcdserver.NewServer(cfg)
+	if err != nil {
+		return err
+	}
+	s.Start()
+
+	ch := &cors.CORSHandler{
+		Handler: etcdhttp.NewClientHandler(s),
+		Info:    corsInfo,
+	}
+	ph := etcdhttp.NewPeerHandler(s)
+	// Start the peer server in a goroutine
+	for _, l := range plns {
+		go func(l net.Listener) {
+			log.Fatal(http.Serve(l, ph))
+		}(l)
+	}
+	// Start a client server goroutine for each listen address
+	for _, l := range clns {
+		go func(l net.Listener) {
+			log.Fatal(http.Serve(l, ch))
+		}(l)
+	}
+	return nil
+}
+
+// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
+func startProxy() error {
+	cls, err := setupCluster()
+	if err != nil {
+		return fmt.Errorf("error setting up initial cluster: %v", err)
+	}
+
+	if *durl != "" {
+		s, err := discovery.GetCluster(*durl, *dproxy)
+		if err != nil {
+			return err
+		}
+		if cls, err = etcdserver.NewClusterFromString(*durl, s); err != nil {
+			return err
+		}
+	}
+
+	pt, err := transport.NewTransport(clientTLSInfo)
+	if err != nil {
+		return err
+	}
+
+	// TODO(jonboulle): update peerURLs dynamically (i.e. when updating
+	// clientURLs) instead of just using the initial fixed list here
+	peerURLs := cls.PeerURLs()
+	uf := func() []string {
+		cls, err := etcdserver.GetClusterFromPeers(peerURLs)
+		if err != nil {
+			log.Printf("proxy: %v", err)
+			return []string{}
+		}
+		return cls.ClientURLs()
+	}
+	ph := proxy.NewHandler(pt, uf)
+	ph = &cors.CORSHandler{
+		Handler: ph,
+		Info:    corsInfo,
+	}
+
+	if proxyFlag.String() == proxyFlagReadonly {
+		ph = proxy.NewReadonlyHandler(ph)
+	}
+	lcurls, err := flags.URLsFromFlags(fs, "listen-client-urls", "bind-addr", clientTLSInfo)
+	if err != nil {
+		return err
+	}
+	// Start a proxy server goroutine for each listen address
+	for _, u := range lcurls {
+		l, err := transport.NewListener(u.Host, clientTLSInfo)
+		if err != nil {
+			return err
+		}
+
+		host := u.Host
+		go func() {
+			log.Print("proxy: listening for client requests on ", host)
+			log.Fatal(http.Serve(l, ph))
+		}()
+	}
+	return nil
+}
+
+// setupCluster sets up the cluster definition for bootstrap or discovery.
+func setupCluster() (*etcdserver.Cluster, error) {
+	set := make(map[string]bool)
+	fs.Visit(func(f *flag.Flag) {
+		set[f.Name] = true
+	})
+	if set["discovery"] && set["initial-cluster"] {
+		return nil, fmt.Errorf("both discovery and bootstrap-config are set")
+	}
+	apurls, err := flags.URLsFromFlags(fs, "initial-advertise-peer-urls", "addr", peerTLSInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	var cls *etcdserver.Cluster
+	switch {
+	case set["discovery"]:
+		clusterStr := genClusterString(*name, apurls)
+		cls, err = etcdserver.NewClusterFromString(*durl, clusterStr)
+	case set["initial-cluster"]:
+		fallthrough
+	default:
+		// We're statically configured, and cluster has appropriately been set.
+		// Try to configure by indexing the static cluster by name.
+		cls, err = etcdserver.NewClusterFromString(*initialClusterToken, *initialCluster)
+	}
+	return cls, err
+}
+
+func genClusterString(name string, urls types.URLs) string {
+	addrs := make([]string, 0)
+	for _, u := range urls {
+		addrs = append(addrs, fmt.Sprintf("%v=%v", name, u.String()))
+	}
+	return strings.Join(addrs, ",")
+}

etcdmain/etcd_test.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package main
+package etcdmain
 
 import (
 	"testing"
@@ -24,9 +24,9 @@ import (
 
 func TestGenClusterString(t *testing.T) {
 	tests := []struct {
-		name string
-		urls []string
-		wstr string
+		token string
+		urls  []string
+		wstr  string
 	}{
 		{
 			"default", []string{"http://127.0.0.1:4001"},
@@ -42,7 +42,7 @@ func TestGenClusterString(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected new urls error: %v", err)
 		}
-		str := genClusterString(tt.name, urls)
+		str := genClusterString(tt.token, urls)
 		if str != tt.wstr {
 			t.Errorf("#%d: cluster = %s, want %s", i, str, tt.wstr)
 		}

etcdserver/cluster.go

@@ -27,10 +27,12 @@ import (
 	"reflect"
 	"sort"
 	"strings"
+	"sync"
 
 	etcdErr "github.com/coreos/etcd/error"
 	"github.com/coreos/etcd/pkg/flags"
 	"github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/etcd/raft/raftpb"
 	"github.com/coreos/etcd/store"
 )
 
@@ -40,29 +42,31 @@ const (
 )
 
 type ClusterInfo interface {
-	ID() uint64
+	ID() types.ID
 	ClientURLs() []string
 	// Members returns a slice of members sorted by their ID
 	Members() []*Member
-	Member(id uint64) *Member
+	Member(id types.ID) *Member
+	IsIDRemoved(id types.ID) bool
 }
 
 // Cluster is a list of Members that belong to the same raft cluster
 type Cluster struct {
-	id      uint64
-	name    string
-	members map[uint64]*Member
+	id      types.ID
+	token   string
+	members map[types.ID]*Member
 	// removed contains the ids of removed members in the cluster.
 	// removed id cannot be reused.
-	removed map[uint64]bool
+	removed map[types.ID]bool
 	store   store.Store
+	sync.Mutex
 }
 
-// NewClusterFromString returns Cluster through given clusterName and parsing
+// NewClusterFromString returns Cluster through given cluster token and parsing
 // members from a sets of names to IPs discovery formatted like:
 // mach0=http://1.1.1.1,mach0=http://2.2.2.2,mach1=http://3.3.3.3,mach2=http://4.4.4.4
-func NewClusterFromString(name string, cluster string) (*Cluster, error) {
-	c := newCluster(name)
+func NewClusterFromString(token string, cluster string) (*Cluster, error) {
+	c := newCluster(token)
 
 	v, err := url.ParseQuery(strings.Replace(cluster, ",", "&", -1))
 	if err != nil {
@@ -76,7 +80,7 @@ func NewClusterFromString(name string, cluster string) (*Cluster, error) {
 		if err := purls.Set(strings.Join(urls, ",")); err != nil {
 			return nil, err
 		}
-		m := NewMember(name, types.URLs(*purls), c.name, nil)
+		m := NewMember(name, types.URLs(*purls), c.token, nil)
 		if _, ok := c.members[m.ID]; ok {
 			return nil, fmt.Errorf("Member exists with identical ID %v", m)
 		}
@@ -86,41 +90,15 @@ func NewClusterFromString(name string, cluster string) (*Cluster, error) {
 	return c, nil
 }
 
-func NewClusterFromStore(name string, st store.Store) *Cluster {
-	c := newCluster(name)
+func NewClusterFromStore(token string, st store.Store) *Cluster {
+	c := newCluster(token)
 	c.store = st
-
-	e, err := c.store.Get(storeMembersPrefix, true, true)
-	if err != nil {
-		if isKeyNotFound(err) {
-			return c
-		}
-		log.Panicf("get storeMembers should never fail: %v", err)
-	}
-	for _, n := range e.Node.Nodes {
-		m, err := nodeToMember(n)
-		if err != nil {
-			log.Panicf("nodeToMember should never fail: %v", err)
-		}
-		c.members[m.ID] = m
-	}
-
-	e, err = c.store.Get(storeRemovedMembersPrefix, true, true)
-	if err != nil {
-		if isKeyNotFound(err) {
-			return c
-		}
-		log.Panicf("get storeRemovedMembers should never fail: %v", err)
-	}
-	for _, n := range e.Node.Nodes {
-		c.removed[parseMemberID(n.Key)] = true
-	}
-
+	c.members, c.removed = membersFromStore(c.store)
 	return c
 }
 
-func NewClusterFromMembers(name string, id uint64, membs []*Member) *Cluster {
-	c := newCluster(name)
+func NewClusterFromMembers(token string, id types.ID, membs []*Member) *Cluster {
+	c := newCluster(token)
 	c.id = id
 	for _, m := range membs {
 		c.members[m.ID] = m
@@ -128,20 +106,22 @@ func NewClusterFromMembers(name string, id uint64, membs []*Member) *Cluster {
 	return c
 }
 
-func newCluster(name string) *Cluster {
+func newCluster(token string) *Cluster {
 	return &Cluster{
-		name:    name,
-		members: make(map[uint64]*Member),
-		removed: make(map[uint64]bool),
+		token:   token,
+		members: make(map[types.ID]*Member),
+		removed: make(map[types.ID]bool),
 	}
 }
 
-func (c Cluster) ID() uint64 { return c.id }
+func (c *Cluster) ID() types.ID { return c.id }
 
-func (c Cluster) Members() []*Member {
+func (c *Cluster) Members() []*Member {
+	c.Lock()
+	defer c.Unlock()
 	var sms SortableMemberSlice
 	for _, m := range c.members {
-		sms = append(sms, m)
+		sms = append(sms, m.Clone())
 	}
 	sort.Sort(sms)
 	return []*Member(sms)
@@ -153,42 +133,52 @@ func (s SortableMemberSlice) Len() int           { return len(s) }
 func (s SortableMemberSlice) Less(i, j int) bool { return s[i].ID < s[j].ID }
 func (s SortableMemberSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 
-func (c *Cluster) Member(id uint64) *Member {
-	return c.members[id]
+func (c *Cluster) Member(id types.ID) *Member {
+	c.Lock()
+	defer c.Unlock()
+	return c.members[id].Clone()
 }
 
 // MemberByName returns a Member with the given name if exists.
 // If more than one member has the given name, it will panic.
 func (c *Cluster) MemberByName(name string) *Member {
+	c.Lock()
+	defer c.Unlock()
 	var memb *Member
 	for _, m := range c.members {
 		if m.Name == name {
 			if memb != nil {
-				panic("two members with the given name exist in the cluster")
+				log.Panicf("two members with the given name %q exist", name)
 			}
 			memb = m
 		}
 	}
-	return memb
+	return memb.Clone()
 }
 
-func (c Cluster) MemberIDs() []uint64 {
-	var ids []uint64
+func (c *Cluster) MemberIDs() []types.ID {
+	c.Lock()
+	defer c.Unlock()
+	var ids []types.ID
 	for _, m := range c.members {
 		ids = append(ids, m.ID)
 	}
-	sort.Sort(types.Uint64Slice(ids))
+	sort.Sort(types.IDSlice(ids))
 	return ids
 }
 
-func (c *Cluster) IsIDRemoved(id uint64) bool {
+func (c *Cluster) IsIDRemoved(id types.ID) bool {
+	c.Lock()
+	defer c.Unlock()
 	return c.removed[id]
 }
 
 // PeerURLs returns a list of all peer addresses. Each address is prefixed
 // with the scheme (currently "http://"). The returned list is sorted in
 // ascending lexicographical order.
-func (c Cluster) PeerURLs() []string {
+func (c *Cluster) PeerURLs() []string {
+	c.Lock()
+	defer c.Unlock()
 	endpoints := make([]string, 0)
 	for _, p := range c.members {
 		for _, addr := range p.PeerURLs {
@@ -202,7 +192,9 @@ func (c Cluster) PeerURLs() []string {
 // ClientURLs returns a list of all client addresses. Each address is prefixed
 // with the scheme (currently "http://"). The returned list is sorted in
 // ascending lexicographical order.
-func (c Cluster) ClientURLs() []string {
+func (c *Cluster) ClientURLs() []string {
+	c.Lock()
+	defer c.Unlock()
 	urls := make([]string, 0)
 	for _, p := range c.members {
 		for _, url := range p.ClientURLs {
@@ -213,7 +205,9 @@ func (c Cluster) ClientURLs() []string {
 	return urls
 }
 
-func (c Cluster) String() string {
+func (c *Cluster) String() string {
+	c.Lock()
+	defer c.Unlock()
 	sl := []string{}
 	for _, m := range c.members {
 		for _, u := range m.PeerURLs {
@@ -230,7 +224,7 @@ func (c Cluster) String() string {
 // cluster. If the validation fails, an error will be returned.
 func (c *Cluster) ValidateAndAssignIDs(membs []*Member) error {
 	if len(c.members) != len(membs) {
-		return fmt.Errorf("cannot update %v from %v because the member count is unequal", c.members, membs)
+		return fmt.Errorf("member count is unequal")
 	}
 	omembs := make([]*Member, 0)
 	for _, m := range c.members {
@@ -244,7 +238,7 @@ func (c *Cluster) ValidateAndAssignIDs(membs []*Member) error {
 		}
 		omembs[i].ID = membs[i].ID
 	}
-	c.members = make(map[uint64]*Member)
+	c.members = make(map[types.ID]*Member)
 	for _, m := range omembs {
 		c.members[m.ID] = m
 	}
@@ -255,55 +249,102 @@ func (c *Cluster) genID() {
 	mIDs := c.MemberIDs()
 	b := make([]byte, 8*len(mIDs))
 	for i, id := range mIDs {
-		binary.BigEndian.PutUint64(b[8*i:], id)
+		binary.BigEndian.PutUint64(b[8*i:], uint64(id))
 	}
 	hash := sha1.Sum(b)
-	c.id = binary.BigEndian.Uint64(hash[:8])
+	c.id = types.ID(binary.BigEndian.Uint64(hash[:8]))
 }
 
-func (c *Cluster) SetID(id uint64) { c.id = id }
+func (c *Cluster) SetID(id types.ID) { c.id = id }
 
 func (c *Cluster) SetStore(st store.Store) { c.store = st }
 
+// ValidateConfigurationChange takes a proposed ConfChange and
+// ensures that it is still valid.
+func (c *Cluster) ValidateConfigurationChange(cc raftpb.ConfChange) error {
+	members, removed := membersFromStore(c.store)
+	if removed[types.ID(cc.NodeID)] {
+		return ErrIDRemoved
+	}
+	switch cc.Type {
+	case raftpb.ConfChangeAddNode:
+		if members[types.ID(cc.NodeID)] != nil {
+			return ErrIDExists
+		}
+		urls := make(map[string]bool)
+		for _, m := range members {
+			for _, u := range m.PeerURLs {
+				urls[u] = true
+			}
+		}
+		m := new(Member)
+		if err := json.Unmarshal(cc.Context, m); err != nil {
+			log.Panicf("unmarshal member should never fail: %v", err)
+		}
+		for _, u := range m.PeerURLs {
+			if urls[u] {
+				return ErrPeerURLexists
+			}
+		}
+	case raftpb.ConfChangeRemoveNode:
+		if members[types.ID(cc.NodeID)] == nil {
+			return ErrIDNotFound
+		}
+	default:
+		log.Panicf("ConfChange type should be either AddNode or RemoveNode")
+	}
+	return nil
+}
+
 // AddMember puts a new Member into the store.
 // A Member with a matching id must not exist.
 func (c *Cluster) AddMember(m *Member) {
+	c.Lock()
+	defer c.Unlock()
 	b, err := json.Marshal(m.RaftAttributes)
 	if err != nil {
-		log.Panicf("marshal error: %v", err)
+		log.Panicf("marshal raftAttributes should never fail: %v", err)
 	}
 	p := path.Join(memberStoreKey(m.ID), raftAttributesSuffix)
 	if _, err := c.store.Create(p, false, string(b), false, store.Permanent); err != nil {
-		log.Panicf("add raftAttributes should never fail: %v", err)
+		log.Panicf("create raftAttributes should never fail: %v", err)
 	}
 	b, err = json.Marshal(m.Attributes)
 	if err != nil {
-		log.Panicf("marshal error: %v", err)
+		log.Panicf("marshal attributes should never fail: %v", err)
 	}
 	p = path.Join(memberStoreKey(m.ID), attributesSuffix)
 	if _, err := c.store.Create(p, false, string(b), false, store.Permanent); err != nil {
-		log.Panicf("add attributes should never fail: %v", err)
+		log.Panicf("create attributes should never fail: %v", err)
 	}
 	c.members[m.ID] = m
 }
 
 // RemoveMember removes a member from the store.
 // The given id MUST exist, or the function panics.
-func (c *Cluster) RemoveMember(id uint64) {
+func (c *Cluster) RemoveMember(id types.ID) {
+	c.Lock()
+	defer c.Unlock()
 	if _, err := c.store.Delete(memberStoreKey(id), true, true); err != nil {
-		log.Panicf("delete peer should never fail: %v", err)
+		log.Panicf("delete member should never fail: %v", err)
 	}
 	delete(c.members, id)
 	if _, err := c.store.Create(removedMemberStoreKey(id), false, "", false, store.Permanent); err != nil {
-		log.Panicf("creating RemovedMember should never fail: %v", err)
+		log.Panicf("create removedMember should never fail: %v", err)
 	}
 	c.removed[id] = true
 }
 
+func (c *Cluster) UpdateMemberAttributes(id types.ID, attr Attributes) {
+	c.Lock()
+	defer c.Unlock()
+	c.members[id].Attributes = attr
+}
+
 // nodeToMember builds member through a store node.
 // the child nodes of the given node should be sorted by key.
 func nodeToMember(n *store.NodeExtern) (*Member, error) {
-	m := &Member{ID: parseMemberID(n.Key)}
+	m := &Member{ID: mustParseMemberIDFromKey(n.Key)}
 	if len(n.Nodes) != 2 {
 		return m, fmt.Errorf("len(nodes) = %d, want 2", len(n.Nodes))
 	}
@@ -322,6 +363,37 @@ func nodeToMember(n *store.NodeExtern) (*Member, error) {
 	return m, nil
 }
 
+func membersFromStore(st store.Store) (map[types.ID]*Member, map[types.ID]bool) {
+	members := make(map[types.ID]*Member)
+	removed := make(map[types.ID]bool)
+	e, err := st.Get(storeMembersPrefix, true, true)
+	if err != nil {
+		if isKeyNotFound(err) {
+			return members, removed
+		}
+		log.Panicf("get storeMembers should never fail: %v", err)
+	}
+	for _, n := range e.Node.Nodes {
+		m, err := nodeToMember(n)
+		if err != nil {
+			log.Panicf("nodeToMember should never fail: %v", err)
+		}
+		members[m.ID] = m
+	}
+
+	e, err = st.Get(storeRemovedMembersPrefix, true, true)
+	if err != nil {
+		if isKeyNotFound(err) {
+			return members, removed
+		}
+		log.Panicf("get storeRemovedMembers should never fail: %v", err)
+	}
+	for _, n := range e.Node.Nodes {
+		removed[mustParseMemberIDFromKey(n.Key)] = true
+	}
+	return members, removed
+}
+
 func isKeyNotFound(err error) bool {
 	e, ok := err.(*etcdErr.Error)
 	return ok && e.ErrorCode == etcdErr.EcodeKeyNotFound

etcdserver/cluster_state.go

@@ -1,53 +0,0 @@
-/*
-   Copyright 2014 CoreOS, Inc.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package etcdserver
-
-import (
-	"errors"
-)
-
-const (
-	ClusterStateValueNew      = "new"
-	ClusterStateValueExisting = "existing"
-)
-
-var (
-	ClusterStateValues = []string{
-		ClusterStateValueNew,
-		ClusterStateValueExisting,
-	}
-)
-
-// ClusterState implements the flag.Value interface.
-type ClusterState string
-
-// Set verifies the argument to be a valid member of ClusterStateFlagValues
-// before setting the underlying flag value.
-func (cs *ClusterState) Set(s string) error {
-	for _, v := range ClusterStateValues {
-		if s == v {
-			*cs = ClusterState(s)
-			return nil
-		}
-	}
-
-	return errors.New("invalid value")
-}
-
-func (cs *ClusterState) String() string {
-	return string(*cs)
-}

etcdserver/cluster_test.go

@@ -17,10 +17,14 @@
 package etcdserver
 
 import (
+	"encoding/json"
+	"fmt"
 	"path"
 	"reflect"
 	"testing"
 
+	"github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/etcd/raft/raftpb"
 	"github.com/coreos/etcd/store"
 )
 
@@ -43,8 +47,8 @@ func TestClusterFromString(t *testing.T) {
 		if err != nil {
 			t.Fatalf("#%d: unexpected new error: %v", i, err)
 		}
-		if c.name != "abc" {
-			t.Errorf("#%d: name = %v, want abc", i, c.name)
+		if c.token != "abc" {
+			t.Errorf("#%d: token = %v, want abc", i, c.token)
 		}
 		wc := newTestCluster(tt.mems)
 		if !reflect.DeepEqual(c.members, wc.members) {
@@ -92,15 +96,13 @@ func TestClusterFromStore(t *testing.T) {
 		},
 	}
 	for i, tt := range tests {
-		st := store.New()
 		hc := newTestCluster(nil)
-		hc.SetStore(st)
 		for _, m := range tt.mems {
 			hc.AddMember(&m)
 		}
-		c := NewClusterFromStore("abc", st)
-		if c.name != "abc" {
-			t.Errorf("#%d: name = %v, want %v", i, c.name, "abc")
+		c := NewClusterFromStore("abc", hc.store)
+		if c.token != "abc" {
+			t.Errorf("#%d: token = %v, want %v", i, c.token, "abc")
 		}
 		wc := newTestCluster(tt.mems)
 		if !reflect.DeepEqual(c.members, wc.members) {
@@ -115,7 +117,7 @@ func TestClusterMember(t *testing.T) {
 		newTestMember(2, nil, "node2", nil),
 	}
 	tests := []struct {
-		id    uint64
+		id    types.ID
 		match bool
 	}{
 		{1, true},
@@ -165,7 +167,7 @@ func TestClusterMemberIDs(t *testing.T) {
 		newTestMember(4, nil, "", nil),
 		newTestMember(100, nil, "", nil),
 	})
-	w := []uint64{1, 4, 100}
+	w := []types.ID{1, 4, 100}
 	g := c.MemberIDs()
 	if !reflect.DeepEqual(w, g) {
 		t.Errorf("IDs = %+v, want %+v", g, w)
@@ -327,7 +329,7 @@ func TestClusterValidateAndAssignIDs(t *testing.T) {
 	tests := []struct {
 		clmembs []Member
 		membs   []*Member
-		wids    []uint64
+		wids    []types.ID
 	}{
 		{
 			[]Member{
@@ -338,7 +340,7 @@ func TestClusterValidateAndAssignIDs(t *testing.T) {
 				newTestMemberp(3, []string{"http://127.0.0.1:2379"}, "", nil),
 				newTestMemberp(4, []string{"http://127.0.0.2:2379"}, "", nil),
 			},
-			[]uint64{3, 4},
+			[]types.ID{3, 4},
 		},
 	}
 	for i, tt := range tests {
@@ -352,6 +354,77 @@ func TestClusterValidateAndAssignIDs(t *testing.T) {
 	}
 }
 
+func TestClusterValidateConfigurationChange(t *testing.T) {
+	cl := newCluster("")
+	cl.SetStore(store.New())
+	for i := 1; i <= 4; i++ {
+		attr := RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", i)}}
+		cl.AddMember(&Member{ID: types.ID(i), RaftAttributes: attr})
+	}
+	cl.RemoveMember(4)
+
+	attr := RaftAttributes{PeerURLs: []string{fmt.Sprintf("http://127.0.0.1:%d", 1)}}
+	cxt, err := json.Marshal(&Member{ID: types.ID(5), RaftAttributes: attr})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		cc   raftpb.ConfChange
+		werr error
+	}{
+		{
+			raftpb.ConfChange{
+				Type:   raftpb.ConfChangeRemoveNode,
+				NodeID: 3,
+			},
+			nil,
+		},
+		{
+			raftpb.ConfChange{
+				Type:   raftpb.ConfChangeAddNode,
+				NodeID: 4,
+			},
+			ErrIDRemoved,
+		},
+		{
+			raftpb.ConfChange{
+				Type:   raftpb.ConfChangeRemoveNode,
+				NodeID: 4,
+			},
+			ErrIDRemoved,
+		},
+		{
+			raftpb.ConfChange{
+				Type:   raftpb.ConfChangeAddNode,
+				NodeID: 1,
+			},
+			ErrIDExists,
+		},
+		{
+			raftpb.ConfChange{
+				Type:    raftpb.ConfChangeAddNode,
+				NodeID:  5,
+				Context: cxt,
+			},
+			ErrPeerURLexists,
+		},
+		{
+			raftpb.ConfChange{
+				Type:   raftpb.ConfChangeRemoveNode,
+				NodeID: 5,
+			},
+			ErrIDNotFound,
+		},
+	}
+	for i, tt := range tests {
+		err := cl.ValidateConfigurationChange(tt.cc)
+		if err != tt.werr {
+			t.Errorf("#%d: validateConfigurationChange error = %v, want %v", i, err, tt.werr)
+		}
+	}
+}
+
 func TestClusterGenID(t *testing.T) {
 	cs := newTestCluster([]Member{
 		newTestMember(1, nil, "", nil),
@@ -439,7 +512,7 @@ func TestClusterAddMember(t *testing.T) {
 
 func TestClusterMembers(t *testing.T) {
 	cls := &Cluster{
-		members: map[uint64]*Member{
+		members: map[types.ID]*Member{
 			1:   &Member{ID: 1},
 			20:  &Member{ID: 20},
 			100: &Member{ID: 100},
@@ -461,7 +534,7 @@ func TestClusterMembers(t *testing.T) {
 
 func TestClusterString(t *testing.T) {
 	cls := &Cluster{
-		members: map[uint64]*Member{
+		members: map[types.ID]*Member{
 			1: newTestMemberp(
 				1,
 				[]string{"http://1.1.1.1:1111", "http://0.0.0.0:0000"},
@@ -533,16 +606,17 @@ func TestNodeToMember(t *testing.T) {
 }
 
 func newTestCluster(membs []Member) *Cluster {
-	c := &Cluster{members: make(map[uint64]*Member), removed: make(map[uint64]bool)}
-	for i, m := range membs {
-		c.members[m.ID] = &membs[i]
+	c := &Cluster{members: make(map[types.ID]*Member), removed: make(map[types.ID]bool)}
+	c.store = store.New()
+	for i := range membs {
+		c.AddMember(&membs[i])
 	}
 	return c
 }
 
 func newTestMember(id uint64, peerURLs []string, name string, clientURLs []string) Member {
 	return Member{
-		ID:             id,
+		ID:             types.ID(id),
 		RaftAttributes: RaftAttributes{PeerURLs: peerURLs},
 		Attributes:     Attributes{Name: name, ClientURLs: clientURLs},
 	}

etcdserver/config.go

@@ -27,14 +27,16 @@ import (
 
 // ServerConfig holds the configuration of etcd as taken from the command line or discovery.
 type ServerConfig struct {
-	Name         string
-	DiscoveryURL string
-	ClientURLs   types.URLs
-	DataDir      string
-	SnapCount    uint64
-	Cluster      *Cluster
-	ClusterState ClusterState
-	Transport    *http.Transport
+	Name            string
+	DiscoveryURL    string
+	DiscoveryProxy  string
+	ClientURLs      types.URLs
+	DataDir         string
+	SnapCount       uint64
+	Cluster         *Cluster
+	NewCluster      bool
+	ForceNewCluster bool
+	Transport       *http.Transport
 }
 
 // VerifyBootstrapConfig sanity-checks the initial config and returns an error
@@ -45,11 +47,11 @@ func (c *ServerConfig) VerifyBootstrapConfig() error {
 	if m == nil {
 		return fmt.Errorf("couldn't find local name %s in the initial cluster configuration", c.Name)
 	}
-	if m.ID == raft.None {
+	if uint64(m.ID) == raft.None {
 		return fmt.Errorf("cannot use %x as member id", raft.None)
 	}
 
-	if c.DiscoveryURL == "" && c.ClusterState != ClusterStateValueNew {
+	if c.DiscoveryURL == "" && !c.NewCluster {
 		return fmt.Errorf("initial cluster state unset and no wal or discovery URL found")
 	}
 

etcdserver/config_test.go

@@ -21,44 +21,42 @@ import "testing"
 func TestBootstrapConfigVerify(t *testing.T) {
 	tests := []struct {
 		clusterSetting string
-		clst           ClusterState
+		newclst        bool
 		disc           string
 		shouldError    bool
 	}{
 		{
 			// Node must exist in cluster
 			"",
-			ClusterStateValueNew,
+			true,
 			"",
 			true,
 		},
 		{
 			// Cannot have duplicate URLs in cluster config
 			"node1=http://localhost:7001,node2=http://localhost:7001,node2=http://localhost:7002",
-			ClusterStateValueNew,
+			true,
 			"",
 			true,
 		},
 		{
 			// Node defined, ClusterState OK
 			"node1=http://localhost:7001,node2=http://localhost:7002",
-			ClusterStateValueNew,
+			true,
 			"",
 			false,
 		},
 		{
 			// Node defined, discovery OK
 			"node1=http://localhost:7001",
-			// TODO(jonboulle): replace with ClusterStateExisting once it exists
-			"",
+			false,
 			"http://discovery",
 			false,
 		},
 		{
 			// Cannot have ClusterState!=new && !discovery
 			"node1=http://localhost:7001",
-			// TODO(jonboulle): replace with ClusterStateExisting once it exists
-			ClusterState("foo"),
+			false,
 			"",
 			true,
 		},
@@ -74,7 +72,7 @@ func TestBootstrapConfigVerify(t *testing.T) {
 			Name:         "node1",
 			DiscoveryURL: tt.disc,
 			Cluster:      cluster,
-			ClusterState: tt.clst,
+			NewCluster:   tt.newclst,
 		}
 		err = cfg.VerifyBootstrapConfig()
 		if (err == nil) && tt.shouldError {

etcdserver/etcdhttp/client.go

@@ -0,0 +1,561 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package etcdhttp
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/jonboulle/clockwork"
+	etcdErr "github.com/coreos/etcd/error"
+	"github.com/coreos/etcd/etcdserver"
+	"github.com/coreos/etcd/etcdserver/etcdhttp/httptypes"
+	"github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/etcd/store"
+	"github.com/coreos/etcd/version"
+)
+
+const (
+	keysPrefix               = "/v2/keys"
+	deprecatedMachinesPrefix = "/v2/machines"
+	membersPrefix            = "/v2/members"
+	statsPrefix              = "/v2/stats"
+	versionPrefix            = "/version"
+)
+
+// NewClientHandler generates a muxed http.Handler with the given parameters to serve etcd client requests.
+func NewClientHandler(server *etcdserver.EtcdServer) http.Handler {
+	kh := &keysHandler{
+		server:      server,
+		clusterInfo: server.Cluster,
+		timer:       server,
+		timeout:     defaultServerTimeout,
+	}
+
+	sh := &statsHandler{
+		stats: server,
+	}
+
+	mh := &membersHandler{
+		server:      server,
+		clusterInfo: server.Cluster,
+		clock:       clockwork.NewRealClock(),
+	}
+
+	dmh := &deprecatedMachinesHandler{
+		clusterInfo: server.Cluster,
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", http.NotFound)
+	mux.HandleFunc(versionPrefix, serveVersion)
+	mux.Handle(keysPrefix, kh)
+	mux.Handle(keysPrefix+"/", kh)
+	mux.HandleFunc(statsPrefix+"/store", sh.serveStore)
+	mux.HandleFunc(statsPrefix+"/self", sh.serveSelf)
+	mux.HandleFunc(statsPrefix+"/leader", sh.serveLeader)
+	mux.Handle(membersPrefix, mh)
+	mux.Handle(membersPrefix+"/", mh)
+	mux.Handle(deprecatedMachinesPrefix, dmh)
+	return mux
+}
+
+type keysHandler struct {
+	server      etcdserver.Server
+	clusterInfo etcdserver.ClusterInfo
+	timer       etcdserver.RaftTimer
+	timeout     time.Duration
+}
+
+func (h *keysHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "HEAD", "GET", "PUT", "POST", "DELETE") {
+		return
+	}
+	w.Header().Set("X-Etcd-Cluster-ID", h.clusterInfo.ID().String())
+
+	ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
+	defer cancel()
+
+	rr, err := parseKeyRequest(r, etcdserver.GenID(), clockwork.NewRealClock())
+	if err != nil {
+		writeError(w, err)
+		return
+	}
+
+	resp, err := h.server.Do(ctx, rr)
+	if err != nil {
+		err = trimErrorPrefix(err, etcdserver.StoreKeysPrefix)
+		writeError(w, err)
+		return
+	}
+
+	switch {
+	case resp.Event != nil:
+		if err := writeKeyEvent(w, resp.Event, h.timer); err != nil {
+			// Should never be reached
+			log.Printf("error writing event: %v", err)
+		}
+	case resp.Watcher != nil:
+		ctx, cancel := context.WithTimeout(context.Background(), defaultWatchTimeout)
+		defer cancel()
+		handleKeyWatch(ctx, w, resp.Watcher, rr.Stream, h.timer)
+	default:
+		writeError(w, errors.New("received response with no Event/Watcher!"))
+	}
+}
+
+type deprecatedMachinesHandler struct {
+	clusterInfo etcdserver.ClusterInfo
+}
+
+func (h *deprecatedMachinesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET", "HEAD") {
+		return
+	}
+	endpoints := h.clusterInfo.ClientURLs()
+	w.Write([]byte(strings.Join(endpoints, ", ")))
+}
+
+type membersHandler struct {
+	server      etcdserver.Server
+	clusterInfo etcdserver.ClusterInfo
+	clock       clockwork.Clock
+}
+
+func (h *membersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET", "POST", "DELETE") {
+		return
+	}
+	w.Header().Set("X-Etcd-Cluster-ID", h.clusterInfo.ID().String())
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultServerTimeout)
+	defer cancel()
+
+	switch r.Method {
+	case "GET":
+		if trimPrefix(r.URL.Path, membersPrefix) != "" {
+			writeError(w, httptypes.NewHTTPError(http.StatusNotFound, "Not found"))
+			return
+		}
+		mc := newMemberCollection(h.clusterInfo.Members())
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(mc); err != nil {
+			log.Printf("etcdhttp: %v", err)
+		}
+	case "POST":
+		ctype := r.Header.Get("Content-Type")
+		if ctype != "application/json" {
+			writeError(w, httptypes.NewHTTPError(http.StatusUnsupportedMediaType, fmt.Sprintf("Bad Content-Type %s, accept application/json", ctype)))
+			return
+		}
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			writeError(w, httptypes.NewHTTPError(http.StatusBadRequest, err.Error()))
+			return
+		}
+		req := httptypes.MemberCreateRequest{}
+		if err := json.Unmarshal(b, &req); err != nil {
+			writeError(w, httptypes.NewHTTPError(http.StatusBadRequest, err.Error()))
+			return
+		}
+
+		now := h.clock.Now()
+		m := etcdserver.NewMember("", req.PeerURLs, "", &now)
+		err = h.server.AddMember(ctx, *m)
+		switch {
+		case err == etcdserver.ErrIDExists || err == etcdserver.ErrPeerURLexists:
+			writeError(w, httptypes.NewHTTPError(http.StatusConflict, err.Error()))
+			return
+		case err != nil:
+			log.Printf("etcdhttp: error adding node %s: %v", m.ID, err)
+			writeError(w, err)
+			return
+		}
+		res := newMember(m)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusCreated)
+		if err := json.NewEncoder(w).Encode(res); err != nil {
+			log.Printf("etcdhttp: %v", err)
+		}
+	case "DELETE":
+		idStr := trimPrefix(r.URL.Path, membersPrefix)
+		if idStr == "" {
+			http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+			return
+		}
+		id, err := types.IDFromString(idStr)
+		if err != nil {
+			writeError(w, httptypes.NewHTTPError(http.StatusNotFound, fmt.Sprintf("No such member: %s", idStr)))
+			return
+		}
+		err = h.server.RemoveMember(ctx, uint64(id))
+		switch {
+		case err == etcdserver.ErrIDRemoved:
+			writeError(w, httptypes.NewHTTPError(http.StatusGone, fmt.Sprintf("Member permanently removed: %s", idStr)))
+		case err == etcdserver.ErrIDNotFound:
+			writeError(w, httptypes.NewHTTPError(http.StatusNotFound, fmt.Sprintf("No such member: %s", idStr)))
+		case err != nil:
+			log.Printf("etcdhttp: error removing node %s: %v", id, err)
+			writeError(w, err)
+		default:
+			w.WriteHeader(http.StatusNoContent)
+		}
+	}
+}
+
+type statsHandler struct {
+	stats etcdserver.Stats
+}
+
+func (h *statsHandler) serveStore(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET") {
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(h.stats.StoreStats())
+}
+
+func (h *statsHandler) serveSelf(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET") {
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(h.stats.SelfStats())
+}
+
+func (h *statsHandler) serveLeader(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET") {
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(h.stats.LeaderStats())
+}
+
+func serveVersion(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET") {
+		return
+	}
+	w.Write([]byte("etcd " + version.Version))
+}
+
+// parseKeyRequest converts a received http.Request on keysPrefix to
+// a server Request, performing validation of supplied fields as appropriate.
+// If any validation fails, an empty Request and non-nil error is returned.
+func parseKeyRequest(r *http.Request, id uint64, clock clockwork.Clock) (etcdserverpb.Request, error) {
+	emptyReq := etcdserverpb.Request{}
+
+	err := r.ParseForm()
+	if err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidForm,
+			err.Error(),
+		)
+	}
+
+	if !strings.HasPrefix(r.URL.Path, keysPrefix) {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidForm,
+			"incorrect key prefix",
+		)
+	}
+	p := path.Join(etcdserver.StoreKeysPrefix, r.URL.Path[len(keysPrefix):])
+
+	var pIdx, wIdx uint64
+	if pIdx, err = getUint64(r.Form, "prevIndex"); err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeIndexNaN,
+			`invalid value for "prevIndex"`,
+		)
+	}
+	if wIdx, err = getUint64(r.Form, "waitIndex"); err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeIndexNaN,
+			`invalid value for "waitIndex"`,
+		)
+	}
+
+	var rec, sort, wait, dir, quorum, stream bool
+	if rec, err = getBool(r.Form, "recursive"); err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidField,
+			`invalid value for "recursive"`,
+		)
+	}
+	if sort, err = getBool(r.Form, "sorted"); err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidField,
+			`invalid value for "sorted"`,
+		)
+	}
+	if wait, err = getBool(r.Form, "wait"); err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidField,
+			`invalid value for "wait"`,
+		)
+	}
+	// TODO(jonboulle): define what parameters dir is/isn't compatible with?
+	if dir, err = getBool(r.Form, "dir"); err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidField,
+			`invalid value for "dir"`,
+		)
+	}
+	if quorum, err = getBool(r.Form, "quorum"); err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidField,
+			`invalid value for "quorum"`,
+		)
+	}
+	if stream, err = getBool(r.Form, "stream"); err != nil {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidField,
+			`invalid value for "stream"`,
+		)
+	}
+
+	if wait && r.Method != "GET" {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodeInvalidField,
+			`"wait" can only be used with GET requests`,
+		)
+	}
+
+	pV := r.FormValue("prevValue")
+	if _, ok := r.Form["prevValue"]; ok && pV == "" {
+		return emptyReq, etcdErr.NewRequestError(
+			etcdErr.EcodePrevValueRequired,
+			`"prevValue" cannot be empty`,
+		)
+	}
+
+	// TTL is nullable, so leave it null if not specified
+	// or an empty string
+	var ttl *uint64
+	if len(r.FormValue("ttl")) > 0 {
+		i, err := getUint64(r.Form, "ttl")
+		if err != nil {
+			return emptyReq, etcdErr.NewRequestError(
+				etcdErr.EcodeTTLNaN,
+				`invalid value for "ttl"`,
+			)
+		}
+		ttl = &i
+	}
+
+	// prevExist is nullable, so leave it null if not specified
+	var pe *bool
+	if _, ok := r.Form["prevExist"]; ok {
+		bv, err := getBool(r.Form, "prevExist")
+		if err != nil {
+			return emptyReq, etcdErr.NewRequestError(
+				etcdErr.EcodeInvalidField,
+				"invalid value for prevExist",
+			)
+		}
+		pe = &bv
+	}
+
+	rr := etcdserverpb.Request{
+		ID:        id,
+		Method:    r.Method,
+		Path:      p,
+		Val:       r.FormValue("value"),
+		Dir:       dir,
+		PrevValue: pV,
+		PrevIndex: pIdx,
+		PrevExist: pe,
+		Wait:      wait,
+		Since:     wIdx,
+		Recursive: rec,
+		Sorted:    sort,
+		Quorum:    quorum,
+		Stream:    stream,
+	}
+
+	if pe != nil {
+		rr.PrevExist = pe
+	}
+
+	// Null TTL is equivalent to unset Expiration
+	if ttl != nil {
+		expr := time.Duration(*ttl) * time.Second
+		rr.Expiration = clock.Now().Add(expr).UnixNano()
+	}
+
+	return rr, nil
+}
+
+// writeKeyEvent trims the prefix of key path in a single Event under
+// StoreKeysPrefix, serializes it and writes the resulting JSON to the given
+// ResponseWriter, along with the appropriate headers.
+func writeKeyEvent(w http.ResponseWriter, ev *store.Event, rt etcdserver.RaftTimer) error {
+	if ev == nil {
+		return errors.New("cannot write empty Event!")
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("X-Etcd-Index", fmt.Sprint(ev.EtcdIndex))
+	w.Header().Set("X-Raft-Index", fmt.Sprint(rt.Index()))
+	w.Header().Set("X-Raft-Term", fmt.Sprint(rt.Term()))
+
+	if ev.IsCreated() {
+		w.WriteHeader(http.StatusCreated)
+	}
+
+	ev = trimEventPrefix(ev, etcdserver.StoreKeysPrefix)
+	return json.NewEncoder(w).Encode(ev)
+}
+
+func handleKeyWatch(ctx context.Context, w http.ResponseWriter, wa store.Watcher, stream bool, rt etcdserver.RaftTimer) {
+	defer wa.Remove()
+	ech := wa.EventChan()
+	var nch <-chan bool
+	if x, ok := w.(http.CloseNotifier); ok {
+		nch = x.CloseNotify()
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("X-Etcd-Index", fmt.Sprint(wa.StartIndex()))
+	w.Header().Set("X-Raft-Index", fmt.Sprint(rt.Index()))
+	w.Header().Set("X-Raft-Term", fmt.Sprint(rt.Term()))
+	w.WriteHeader(http.StatusOK)
+
+	// Ensure headers are flushed early, in case of long polling
+	w.(http.Flusher).Flush()
+
+	for {
+		select {
+		case <-nch:
+			// Client closed connection. Nothing to do.
+			return
+		case <-ctx.Done():
+			// Timed out. net/http will close the connection for us, so nothing to do.
+			return
+		case ev, ok := <-ech:
+			if !ok {
+				// If the channel is closed this may be an indication of
+				// that notifications are much more than we are able to
+				// send to the client in time. Then we simply end streaming.
+				return
+			}
+			ev = trimEventPrefix(ev, etcdserver.StoreKeysPrefix)
+			if err := json.NewEncoder(w).Encode(ev); err != nil {
+				// Should never be reached
+				log.Printf("error writing event: %v\n", err)
+				return
+			}
+			if !stream {
+				return
+			}
+			w.(http.Flusher).Flush()
+		}
+	}
+}
+
+func trimEventPrefix(ev *store.Event, prefix string) *store.Event {
+	if ev == nil {
+		return nil
+	}
+	// Since the *Event may reference one in the store history
+	// history, we must copy it before modifying
+	e := ev.Clone()
+	e.Node = trimNodeExternPrefix(e.Node, prefix)
+	e.PrevNode = trimNodeExternPrefix(e.PrevNode, prefix)
+	return e
+}
+
+func trimNodeExternPrefix(n *store.NodeExtern, prefix string) *store.NodeExtern {
+	if n == nil {
+		return nil
+	}
+	n.Key = strings.TrimPrefix(n.Key, prefix)
+	for _, nn := range n.Nodes {
+		nn = trimNodeExternPrefix(nn, prefix)
+	}
+	return n
+}
+
+func trimErrorPrefix(err error, prefix string) error {
+	if e, ok := err.(*etcdErr.Error); ok {
+		e.Cause = strings.TrimPrefix(e.Cause, prefix)
+	}
+	return err
+}
+
+// getUint64 extracts a uint64 by the given key from a Form. If the key does
+// not exist in the form, 0 is returned. If the key exists but the value is
+// badly formed, an error is returned. If multiple values are present only the
+// first is considered.
+func getUint64(form url.Values, key string) (i uint64, err error) {
+	if vals, ok := form[key]; ok {
+		i, err = strconv.ParseUint(vals[0], 10, 64)
+	}
+	return
+}
+
+// getBool extracts a bool by the given key from a Form. If the key does not
+// exist in the form, false is returned. If the key exists but the value is
+// badly formed, an error is returned. If multiple values are present only the
+// first is considered.
+func getBool(form url.Values, key string) (b bool, err error) {
+	if vals, ok := form[key]; ok {
+		b, err = strconv.ParseBool(vals[0])
+	}
+	return
+}
+
+// trimPrefix removes a given prefix and any slash following the prefix
+// e.g.: trimPrefix("foo", "foo") == trimPrefix("foo/", "foo") == ""
+func trimPrefix(p, prefix string) (s string) {
+	s = strings.TrimPrefix(p, prefix)
+	s = strings.TrimPrefix(s, "/")
+	return
+}
+
+func newMemberCollection(ms []*etcdserver.Member) *httptypes.MemberCollection {
+	c := httptypes.MemberCollection(make([]httptypes.Member, len(ms)))
+
+	for i, m := range ms {
+		c[i] = newMember(m)
+	}
+
+	return &c
+}
+
+func newMember(m *etcdserver.Member) httptypes.Member {
+	tm := httptypes.Member{
+		ID:         m.ID.String(),
+		Name:       m.Name,
+		PeerURLs:   make([]string, len(m.PeerURLs)),
+		ClientURLs: make([]string, len(m.ClientURLs)),
+	}
+
+	copy(tm.PeerURLs, m.PeerURLs)
+	copy(tm.ClientURLs, m.ClientURLs)
+
+	return tm
+}

etcdserver/etcdhttp/http.go

@@ -17,42 +17,19 @@
 package etcdhttp
 
 import (
-	"encoding/json"
 	"errors"
-	"fmt"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"net/url"
-	"path"
-	"strconv"
 	"strings"
 	"time"
 
-	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
-	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/jonboulle/clockwork"
 	etcdErr "github.com/coreos/etcd/error"
-	"github.com/coreos/etcd/etcdserver"
-	"github.com/coreos/etcd/etcdserver/etcdserverpb"
-	"github.com/coreos/etcd/pkg/types"
-	"github.com/coreos/etcd/raft/raftpb"
-	"github.com/coreos/etcd/store"
-	"github.com/coreos/etcd/version"
+	"github.com/coreos/etcd/etcdserver/etcdhttp/httptypes"
 )
 
 const (
-	// prefixes of client endpoint
-	keysPrefix               = "/v2/keys"
-	deprecatedMachinesPrefix = "/v2/machines"
-	adminMembersPrefix       = "/v2/admin/members/"
-	statsPrefix              = "/v2/stats"
-	versionPrefix            = "/version"
-	// prefixes of peer endpoint
-	raftPrefix    = "/raft"
-	membersPrefix = "/members"
-
 	// time to wait for response from EtcdServer requests
-	defaultServerTimeout = 500 * time.Millisecond
+	defaultServerTimeout = 5 * time.Minute
 
 	// time to wait for a Watch request
 	defaultWatchTimeout = 5 * time.Minute
@@ -60,433 +37,6 @@ const (
 
 var errClosed = errors.New("etcdhttp: client closed connection")
 
-// NewClientHandler generates a muxed http.Handler with the given parameters to serve etcd client requests.
-func NewClientHandler(server *etcdserver.EtcdServer) http.Handler {
-	sh := &serverHandler{
-		server:      server,
-		clusterInfo: server.Cluster,
-		stats:       server,
-		timer:       server,
-		timeout:     defaultServerTimeout,
-		clock:       clockwork.NewRealClock(),
-	}
-	mux := http.NewServeMux()
-	mux.HandleFunc(keysPrefix, sh.serveKeys)
-	mux.HandleFunc(keysPrefix+"/", sh.serveKeys)
-	mux.HandleFunc(statsPrefix+"/store", sh.serveStoreStats)
-	mux.HandleFunc(statsPrefix+"/self", sh.serveSelfStats)
-	mux.HandleFunc(statsPrefix+"/leader", sh.serveLeaderStats)
-	mux.HandleFunc(deprecatedMachinesPrefix, sh.serveMachines)
-	mux.HandleFunc(adminMembersPrefix, sh.serveAdminMembers)
-	mux.HandleFunc(versionPrefix, sh.serveVersion)
-	mux.HandleFunc("/", http.NotFound)
-	return mux
-}
-
-// NewPeerHandler generates an http.Handler to handle etcd peer (raft) requests.
-func NewPeerHandler(server *etcdserver.EtcdServer) http.Handler {
-	sh := &serverHandler{
-		server:      server,
-		stats:       server,
-		clusterInfo: server.Cluster,
-		clock:       clockwork.NewRealClock(),
-	}
-	mux := http.NewServeMux()
-	mux.HandleFunc(raftPrefix, sh.serveRaft)
-	mux.HandleFunc(membersPrefix, sh.serveMembers)
-	mux.HandleFunc("/", http.NotFound)
-	return mux
-}
-
-// serverHandler provides http.Handlers for etcd client and raft communication.
-type serverHandler struct {
-	timeout     time.Duration
-	server      etcdserver.Server
-	stats       etcdserver.Stats
-	timer       etcdserver.RaftTimer
-	clusterInfo etcdserver.ClusterInfo
-	clock       clockwork.Clock
-}
-
-func (h serverHandler) serveKeys(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "GET", "PUT", "POST", "DELETE") {
-		return
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
-	defer cancel()
-
-	rr, err := parseKeyRequest(r, etcdserver.GenID(), clockwork.NewRealClock())
-	if err != nil {
-		writeError(w, err)
-		return
-	}
-
-	resp, err := h.server.Do(ctx, rr)
-	if err != nil {
-		err = trimErrorPrefix(err, etcdserver.StoreKeysPrefix)
-		writeError(w, err)
-		return
-	}
-
-	switch {
-	case resp.Event != nil:
-		if err := writeKeyEvent(w, resp.Event, h.timer); err != nil {
-			// Should never be reached
-			log.Printf("error writing event: %v", err)
-		}
-	case resp.Watcher != nil:
-		ctx, cancel := context.WithTimeout(context.Background(), defaultWatchTimeout)
-		defer cancel()
-		handleKeyWatch(ctx, w, resp.Watcher, rr.Stream, h.timer)
-	default:
-		writeError(w, errors.New("received response with no Event/Watcher!"))
-	}
-}
-
-// serveMachines responds address list in the format '0.0.0.0, 1.1.1.1'.
-func (h serverHandler) serveMachines(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "GET", "HEAD") {
-		return
-	}
-	endpoints := h.clusterInfo.ClientURLs()
-	w.Write([]byte(strings.Join(endpoints, ", ")))
-}
-
-func (h serverHandler) serveAdminMembers(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "GET", "POST", "DELETE") {
-		return
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), defaultServerTimeout)
-	defer cancel()
-
-	switch r.Method {
-	case "GET":
-		if s := strings.TrimPrefix(r.URL.Path, adminMembersPrefix); s != "" {
-			http.NotFound(w, r)
-			return
-		}
-		ms := struct {
-			Members []*etcdserver.Member `json:"members"`
-		}{
-			Members: h.clusterInfo.Members(),
-		}
-		w.Header().Set("Content-Type", "application/json")
-		if err := json.NewEncoder(w).Encode(ms); err != nil {
-			log.Printf("etcdhttp: %v", err)
-		}
-	case "POST":
-		ctype := r.Header.Get("Content-Type")
-		if ctype != "application/json" {
-			http.Error(w, fmt.Sprintf("bad Content-Type %s, accept application/json", ctype), http.StatusBadRequest)
-			return
-		}
-		b, err := ioutil.ReadAll(r.Body)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-		raftAttr := etcdserver.RaftAttributes{}
-		if err := json.Unmarshal(b, &raftAttr); err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-		validURLs, err := types.NewURLs(raftAttr.PeerURLs)
-		if err != nil {
-			http.Error(w, "bad peer urls", http.StatusBadRequest)
-			return
-		}
-		now := h.clock.Now()
-		m := etcdserver.NewMember("", validURLs, "", &now)
-		if err := h.server.AddMember(ctx, *m); err != nil {
-			log.Printf("etcdhttp: error adding node %x: %v", m.ID, err)
-			writeError(w, err)
-			return
-		}
-		log.Printf("etcdhttp: added node %x with peer urls %v", m.ID, raftAttr.PeerURLs)
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(http.StatusCreated)
-		if err := json.NewEncoder(w).Encode(m); err != nil {
-			log.Printf("etcdhttp: %v", err)
-		}
-	case "DELETE":
-		idStr := strings.TrimPrefix(r.URL.Path, adminMembersPrefix)
-		id, err := strconv.ParseUint(idStr, 16, 64)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-		log.Printf("etcdhttp: remove node %x", id)
-		if err := h.server.RemoveMember(ctx, id); err != nil {
-			log.Printf("etcdhttp: error removing node %x: %v", id, err)
-			writeError(w, err)
-			return
-		}
-		w.WriteHeader(http.StatusNoContent)
-	}
-}
-
-func (h serverHandler) serveStoreStats(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "GET") {
-		return
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(h.stats.StoreStats())
-}
-
-func (h serverHandler) serveSelfStats(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "GET") {
-		return
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(h.stats.SelfStats())
-}
-
-func (h serverHandler) serveLeaderStats(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "GET") {
-		return
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Write(h.stats.LeaderStats())
-}
-
-func (h serverHandler) serveVersion(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "GET") {
-		return
-	}
-	w.Write([]byte("etcd " + version.Version))
-}
-
-func (h serverHandler) serveRaft(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "POST") {
-		return
-	}
-
-	wcid := strconv.FormatUint(h.clusterInfo.ID(), 16)
-	w.Header().Set("X-Etcd-Cluster-ID", wcid)
-
-	gcid := r.Header.Get("X-Etcd-Cluster-ID")
-	if gcid != wcid {
-		log.Printf("etcdhttp: request ignored due to cluster ID mismatch got %s want %s", gcid, wcid)
-		http.Error(w, "clusterID mismatch", http.StatusPreconditionFailed)
-		return
-	}
-
-	b, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		log.Println("etcdhttp: error reading raft message:", err)
-		http.Error(w, "error reading raft message", http.StatusBadRequest)
-		return
-	}
-	var m raftpb.Message
-	if err := m.Unmarshal(b); err != nil {
-		log.Println("etcdhttp: error unmarshaling raft message:", err)
-		http.Error(w, "error unmarshaling raft message", http.StatusBadRequest)
-		return
-	}
-	if err := h.server.Process(context.TODO(), m); err != nil {
-		log.Println("etcdhttp: error processing raft message:", err)
-		switch err {
-		case etcdserver.ErrRemoved:
-			http.Error(w, "cannot process message from removed node", http.StatusForbidden)
-		default:
-			writeError(w, err)
-		}
-		return
-	}
-	if m.Type == raftpb.MsgApp {
-		h.stats.UpdateRecvApp(m.From, r.ContentLength)
-	}
-	w.WriteHeader(http.StatusNoContent)
-}
-
-func (h serverHandler) serveMembers(w http.ResponseWriter, r *http.Request) {
-	if !allowMethod(w, r.Method, "GET") {
-		return
-	}
-	cid := strconv.FormatUint(h.clusterInfo.ID(), 16)
-	w.Header().Set("X-Etcd-Cluster-ID", cid)
-
-	if r.URL.Path != membersPrefix {
-		http.Error(w, "bad path", http.StatusBadRequest)
-		return
-	}
-	ms := h.clusterInfo.Members()
-	w.Header().Set("Content-Type", "application/json")
-	if err := json.NewEncoder(w).Encode(ms); err != nil {
-		log.Printf("etcdhttp: %v", err)
-	}
-}
-
-// parseKeyRequest converts a received http.Request on keysPrefix to
-// a server Request, performing validation of supplied fields as appropriate.
-// If any validation fails, an empty Request and non-nil error is returned.
-func parseKeyRequest(r *http.Request, id uint64, clock clockwork.Clock) (etcdserverpb.Request, error) {
-	emptyReq := etcdserverpb.Request{}
-
-	err := r.ParseForm()
-	if err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidForm,
-			err.Error(),
-		)
-	}
-
-	if !strings.HasPrefix(r.URL.Path, keysPrefix) {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidForm,
-			"incorrect key prefix",
-		)
-	}
-	p := path.Join(etcdserver.StoreKeysPrefix, r.URL.Path[len(keysPrefix):])
-
-	var pIdx, wIdx uint64
-	if pIdx, err = getUint64(r.Form, "prevIndex"); err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeIndexNaN,
-			`invalid value for "prevIndex"`,
-		)
-	}
-	if wIdx, err = getUint64(r.Form, "waitIndex"); err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeIndexNaN,
-			`invalid value for "waitIndex"`,
-		)
-	}
-
-	var rec, sort, wait, dir, quorum, stream bool
-	if rec, err = getBool(r.Form, "recursive"); err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidField,
-			`invalid value for "recursive"`,
-		)
-	}
-	if sort, err = getBool(r.Form, "sorted"); err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidField,
-			`invalid value for "sorted"`,
-		)
-	}
-	if wait, err = getBool(r.Form, "wait"); err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidField,
-			`invalid value for "wait"`,
-		)
-	}
-	// TODO(jonboulle): define what parameters dir is/isn't compatible with?
-	if dir, err = getBool(r.Form, "dir"); err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidField,
-			`invalid value for "dir"`,
-		)
-	}
-	if quorum, err = getBool(r.Form, "quorum"); err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidField,
-			`invalid value for "quorum"`,
-		)
-	}
-	if stream, err = getBool(r.Form, "stream"); err != nil {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidField,
-			`invalid value for "stream"`,
-		)
-	}
-
-	if wait && r.Method != "GET" {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidField,
-			`"wait" can only be used with GET requests`,
-		)
-	}
-
-	pV := r.FormValue("prevValue")
-	if _, ok := r.Form["prevValue"]; ok && pV == "" {
-		return emptyReq, etcdErr.NewRequestError(
-			etcdErr.EcodeInvalidField,
-			`"prevValue" cannot be empty`,
-		)
-	}
-
-	// TTL is nullable, so leave it null if not specified
-	// or an empty string
-	var ttl *uint64
-	if len(r.FormValue("ttl")) > 0 {
-		i, err := getUint64(r.Form, "ttl")
-		if err != nil {
-			return emptyReq, etcdErr.NewRequestError(
-				etcdErr.EcodeTTLNaN,
-				`invalid value for "ttl"`,
-			)
-		}
-		ttl = &i
-	}
-
-	// prevExist is nullable, so leave it null if not specified
-	var pe *bool
-	if _, ok := r.Form["prevExist"]; ok {
-		bv, err := getBool(r.Form, "prevExist")
-		if err != nil {
-			return emptyReq, etcdErr.NewRequestError(
-				etcdErr.EcodeInvalidField,
-				"invalid value for prevExist",
-			)
-		}
-		pe = &bv
-	}
-
-	rr := etcdserverpb.Request{
-		ID:        id,
-		Method:    r.Method,
-		Path:      p,
-		Val:       r.FormValue("value"),
-		Dir:       dir,
-		PrevValue: pV,
-		PrevIndex: pIdx,
-		PrevExist: pe,
-		Wait:      wait,
-		Since:     wIdx,
-		Recursive: rec,
-		Sorted:    sort,
-		Quorum:    quorum,
-		Stream:    stream,
-	}
-
-	if pe != nil {
-		rr.PrevExist = pe
-	}
-
-	// Null TTL is equivalent to unset Expiration
-	if ttl != nil {
-		expr := time.Duration(*ttl) * time.Second
-		rr.Expiration = clock.Now().Add(expr).UnixNano()
-	}
-
-	return rr, nil
-}
-
-// getUint64 extracts a uint64 by the given key from a Form. If the key does
-// not exist in the form, 0 is returned. If the key exists but the value is
-// badly formed, an error is returned. If multiple values are present only the
-// first is considered.
-func getUint64(form url.Values, key string) (i uint64, err error) {
-	if vals, ok := form[key]; ok {
-		i, err = strconv.ParseUint(vals[0], 10, 64)
-	}
-	return
-}
-
-// getBool extracts a bool by the given key from a Form. If the key does not
-// exist in the form, false is returned. If the key exists but the value is
-// badly formed, an error is returned. If multiple values are present only the
-// first is considered.
-func getBool(form url.Values, key string) (b bool, err error) {
-	if vals, ok := form[key]; ok {
-		b, err = strconv.ParseBool(vals[0])
-	}
-	return
-}
-
 // writeError logs and writes the given Error to the ResponseWriter
 // If Error is an etcdErr, it is rendered to the ResponseWriter
 // Otherwise, it is assumed to be an InternalServerError
@@ -494,77 +44,15 @@ func writeError(w http.ResponseWriter, err error) {
 	if err == nil {
 		return
 	}
-	log.Println(err)
-	if e, ok := err.(*etcdErr.Error); ok {
-		e.Write(w)
-	} else {
-		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
-	}
-}
-
-// writeKeyEvent trims the prefix of key path in a single Event under
-// StoreKeysPrefix, serializes it and writes the resulting JSON to the given
-// ResponseWriter, along with the appropriate headers.
-func writeKeyEvent(w http.ResponseWriter, ev *store.Event, rt etcdserver.RaftTimer) error {
-	if ev == nil {
-		return errors.New("cannot write empty Event!")
-	}
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("X-Etcd-Index", fmt.Sprint(ev.EtcdIndex))
-	w.Header().Set("X-Raft-Index", fmt.Sprint(rt.Index()))
-	w.Header().Set("X-Raft-Term", fmt.Sprint(rt.Term()))
-
-	if ev.IsCreated() {
-		w.WriteHeader(http.StatusCreated)
-	}
-
-	ev = trimEventPrefix(ev, etcdserver.StoreKeysPrefix)
-	return json.NewEncoder(w).Encode(ev)
-}
-
-func handleKeyWatch(ctx context.Context, w http.ResponseWriter, wa store.Watcher, stream bool, rt etcdserver.RaftTimer) {
-	defer wa.Remove()
-	ech := wa.EventChan()
-	var nch <-chan bool
-	if x, ok := w.(http.CloseNotifier); ok {
-		nch = x.CloseNotify()
-	}
-
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("X-Etcd-Index", fmt.Sprint(wa.StartIndex()))
-	w.Header().Set("X-Raft-Index", fmt.Sprint(rt.Index()))
-	w.Header().Set("X-Raft-Term", fmt.Sprint(rt.Term()))
-	w.WriteHeader(http.StatusOK)
-
-	// Ensure headers are flushed early, in case of long polling
-	w.(http.Flusher).Flush()
-
-	for {
-		select {
-		case <-nch:
-			// Client closed connection. Nothing to do.
-			return
-		case <-ctx.Done():
-			// Timed out. net/http will close the connection for us, so nothing to do.
-			return
-		case ev, ok := <-ech:
-			if !ok {
-				// If the channel is closed this may be an indication of
-				// that notifications are much more than we are able to
-				// send to the client in time. Then we simply end streaming.
-				return
-			}
-			ev = trimEventPrefix(ev, etcdserver.StoreKeysPrefix)
-			if err := json.NewEncoder(w).Encode(ev); err != nil {
-				// Should never be reached
-				log.Printf("error writing event: %v\n", err)
-				return
-			}
-			if !stream {
-				return
-			}
-			w.(http.Flusher).Flush()
-		}
+	switch e := err.(type) {
+	case *etcdErr.Error:
+		e.WriteTo(w)
+	case *httptypes.HTTPError:
+		e.WriteTo(w)
+	default:
+		log.Printf("etcdhttp: unexpected error: %v", err)
+		herr := httptypes.NewHTTPError(http.StatusInternalServerError, "Internal Server Error")
+		herr.WriteTo(w)
 	}
 }
 
@@ -581,30 +69,3 @@ func allowMethod(w http.ResponseWriter, m string, ms ...string) bool {
 	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
 	return false
 }
-
-func trimEventPrefix(ev *store.Event, prefix string) *store.Event {
-	if ev == nil {
-		return nil
-	}
-	ev.Node = trimNodeExternPrefix(ev.Node, prefix)
-	ev.PrevNode = trimNodeExternPrefix(ev.PrevNode, prefix)
-	return ev
-}
-
-func trimNodeExternPrefix(n *store.NodeExtern, prefix string) *store.NodeExtern {
-	if n == nil {
-		return nil
-	}
-	n.Key = strings.TrimPrefix(n.Key, prefix)
-	for _, nn := range n.Nodes {
-		nn = trimNodeExternPrefix(nn, prefix)
-	}
-	return n
-}
-
-func trimErrorPrefix(err error, prefix string) error {
-	if e, ok := err.(*etcdErr.Error); ok {
-		e.Cause = strings.TrimPrefix(e.Cause, prefix)
-	}
-	return err
-}

etcdserver/etcdhttp/httptypes/doc.go

@@ -0,0 +1,21 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+/*
+Package httptypes defines how etcd's HTTP API entities are serialized to and deserialized from JSON.
+*/
+
+package httptypes

etcdserver/etcdhttp/httptypes/errors.go

@@ -0,0 +1,51 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package httptypes
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+)
+
+type HTTPError struct {
+	Message string `json:"message"`
+	// HTTP return code
+	Code int `json:"-"`
+}
+
+func (e HTTPError) Error() string {
+	return e.Message
+}
+
+// TODO(xiangli): handle http write errors
+func (e HTTPError) WriteTo(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(e.Code)
+	b, err := json.Marshal(e)
+	if err != nil {
+		log.Panicf("marshal HTTPError should never fail: %v", err)
+	}
+	w.Write(b)
+}
+
+func NewHTTPError(code int, m string) *HTTPError {
+	return &HTTPError{
+		Message: m,
+		Code:    code,
+	}
+}

etcdserver/etcdhttp/httptypes/errors_test.go

@@ -0,0 +1,49 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package httptypes
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+)
+
+func TestHTTPErrorWriteTo(t *testing.T) {
+	err := NewHTTPError(http.StatusBadRequest, "what a bad request you made!")
+	rr := httptest.NewRecorder()
+	err.WriteTo(rr)
+
+	wcode := http.StatusBadRequest
+	wheader := http.Header(map[string][]string{
+		"Content-Type": []string{"application/json"},
+	})
+	wbody := `{"message":"what a bad request you made!"}`
+
+	if wcode != rr.Code {
+		t.Errorf("HTTP status code %d, want %d", rr.Code, wcode)
+	}
+
+	if !reflect.DeepEqual(wheader, rr.HeaderMap) {
+		t.Errorf("HTTP headers %v, want %v", rr.HeaderMap, wheader)
+	}
+
+	gbody := rr.Body.String()
+	if wbody != gbody {
+		t.Errorf("HTTP body %q, want %q", gbody, wbody)
+	}
+}

etcdserver/etcdhttp/httptypes/member.go

@@ -0,0 +1,97 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package httptypes
+
+import (
+	"encoding/json"
+
+	"github.com/coreos/etcd/pkg/types"
+)
+
+type Member struct {
+	ID         string   `json:"id"`
+	Name       string   `json:"name"`
+	PeerURLs   []string `json:"peerURLs"`
+	ClientURLs []string `json:"clientURLs"`
+}
+
+type MemberCreateRequest struct {
+	PeerURLs types.URLs
+}
+
+func (m *MemberCreateRequest) MarshalJSON() ([]byte, error) {
+	s := struct {
+		PeerURLs []string `json:"peerURLs"`
+	}{
+		PeerURLs: make([]string, len(m.PeerURLs)),
+	}
+
+	for i, u := range m.PeerURLs {
+		s.PeerURLs[i] = u.String()
+	}
+
+	return json.Marshal(&s)
+}
+
+func (m *MemberCreateRequest) UnmarshalJSON(data []byte) error {
+	s := struct {
+		PeerURLs []string `json:"peerURLs"`
+	}{}
+
+	err := json.Unmarshal(data, &s)
+	if err != nil {
+		return err
+	}
+
+	urls, err := types.NewURLs(s.PeerURLs)
+	if err != nil {
+		return err
+	}
+
+	m.PeerURLs = urls
+	return nil
+}
+
+type MemberCollection []Member
+
+func (c *MemberCollection) MarshalJSON() ([]byte, error) {
+	d := struct {
+		Members []Member `json:"members"`
+	}{
+		Members: []Member(*c),
+	}
+
+	return json.Marshal(d)
+}
+
+func (c *MemberCollection) UnmarshalJSON(data []byte) error {
+	d := struct {
+		Members []Member
+	}{}
+
+	if err := json.Unmarshal(data, &d); err != nil {
+		return err
+	}
+
+	if d.Members == nil {
+		*c = make([]Member, 0)
+		return nil
+	}
+
+	*c = d.Members
+	return nil
+}

etcdserver/etcdhttp/httptypes/member_test.go

@@ -0,0 +1,220 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package httptypes
+
+import (
+	"encoding/json"
+	"net/url"
+	"reflect"
+	"testing"
+
+	"github.com/coreos/etcd/pkg/types"
+)
+
+func TestMemberUnmarshal(t *testing.T) {
+	tests := []struct {
+		body       []byte
+		wantMember Member
+		wantError  bool
+	}{
+		// no URLs, just check ID & Name
+		{
+			body:       []byte(`{"id": "c", "name": "dungarees"}`),
+			wantMember: Member{ID: "c", Name: "dungarees", PeerURLs: nil, ClientURLs: nil},
+		},
+
+		// both client and peer URLs
+		{
+			body: []byte(`{"peerURLs": ["http://127.0.0.1:4001"], "clientURLs": ["http://127.0.0.1:4001"]}`),
+			wantMember: Member{
+				PeerURLs: []string{
+					"http://127.0.0.1:4001",
+				},
+				ClientURLs: []string{
+					"http://127.0.0.1:4001",
+				},
+			},
+		},
+
+		// multiple peer URLs
+		{
+			body: []byte(`{"peerURLs": ["http://127.0.0.1:4001", "https://example.com"]}`),
+			wantMember: Member{
+				PeerURLs: []string{
+					"http://127.0.0.1:4001",
+					"https://example.com",
+				},
+				ClientURLs: nil,
+			},
+		},
+
+		// multiple client URLs
+		{
+			body: []byte(`{"clientURLs": ["http://127.0.0.1:4001", "https://example.com"]}`),
+			wantMember: Member{
+				PeerURLs: nil,
+				ClientURLs: []string{
+					"http://127.0.0.1:4001",
+					"https://example.com",
+				},
+			},
+		},
+
+		// invalid JSON
+		{
+			body:      []byte(`{"peerU`),
+			wantError: true,
+		},
+	}
+
+	for i, tt := range tests {
+		got := Member{}
+		err := json.Unmarshal(tt.body, &got)
+		if tt.wantError != (err != nil) {
+			t.Errorf("#%d: want error %t, got %v", i, tt.wantError, err)
+			continue
+		}
+
+		if !reflect.DeepEqual(tt.wantMember, got) {
+			t.Errorf("#%d: incorrect output: want=%#v, got=%#v", i, tt.wantMember, got)
+		}
+	}
+}
+
+func TestMemberCollectionUnmarshal(t *testing.T) {
+	tests := []struct {
+		body []byte
+		want MemberCollection
+	}{
+		{
+			body: []byte(`{"members":[]}`),
+			want: MemberCollection([]Member{}),
+		},
+		{
+			body: []byte(`{"members":[{"id":"2745e2525fce8fe","peerURLs":["http://127.0.0.1:7003"],"name":"node3","clientURLs":["http://127.0.0.1:4003"]},{"id":"42134f434382925","peerURLs":["http://127.0.0.1:2380","http://127.0.0.1:7001"],"name":"node1","clientURLs":["http://127.0.0.1:2379","http://127.0.0.1:4001"]},{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}]}`),
+			want: MemberCollection(
+				[]Member{
+					{
+						ID:   "2745e2525fce8fe",
+						Name: "node3",
+						PeerURLs: []string{
+							"http://127.0.0.1:7003",
+						},
+						ClientURLs: []string{
+							"http://127.0.0.1:4003",
+						},
+					},
+					{
+						ID:   "42134f434382925",
+						Name: "node1",
+						PeerURLs: []string{
+							"http://127.0.0.1:2380",
+							"http://127.0.0.1:7001",
+						},
+						ClientURLs: []string{
+							"http://127.0.0.1:2379",
+							"http://127.0.0.1:4001",
+						},
+					},
+					{
+						ID:   "94088180e21eb87b",
+						Name: "node2",
+						PeerURLs: []string{
+							"http://127.0.0.1:7002",
+						},
+						ClientURLs: []string{
+							"http://127.0.0.1:4002",
+						},
+					},
+				},
+			),
+		},
+	}
+
+	for i, tt := range tests {
+		var got MemberCollection
+		err := json.Unmarshal(tt.body, &got)
+		if err != nil {
+			t.Errorf("#%d: unexpected error: %v", i, err)
+			continue
+		}
+
+		if !reflect.DeepEqual(tt.want, got) {
+			t.Errorf("#%d: incorrect output: want=%#v, got=%#v", i, tt.want, got)
+		}
+	}
+}
+
+func TestMemberCreateRequestUnmarshal(t *testing.T) {
+	body := []byte(`{"peerURLs": ["http://127.0.0.1:8081", "https://127.0.0.1:8080"]}`)
+	want := MemberCreateRequest{
+		PeerURLs: types.URLs([]url.URL{
+			url.URL{Scheme: "http", Host: "127.0.0.1:8081"},
+			url.URL{Scheme: "https", Host: "127.0.0.1:8080"},
+		}),
+	}
+
+	var req MemberCreateRequest
+	if err := json.Unmarshal(body, &req); err != nil {
+		t.Fatalf("Unmarshal returned unexpected err=%v", err)
+	}
+
+	if !reflect.DeepEqual(want, req) {
+		t.Fatalf("Failed to unmarshal MemberCreateRequest: want=%#v, got=%#v", want, req)
+	}
+}
+
+func TestMemberCreateRequestUnmarshalFail(t *testing.T) {
+	tests := [][]byte{
+		// invalid JSON
+		[]byte(``),
+		[]byte(`{`),
+
+		// spot-check validation done in types.NewURLs
+		[]byte(`{"peerURLs": "foo"}`),
+		[]byte(`{"peerURLs": ["."]}`),
+		[]byte(`{"peerURLs": []}`),
+		[]byte(`{"peerURLs": ["http://127.0.0.1:4001/foo"]}`),
+		[]byte(`{"peerURLs": ["http://127.0.0.1"]}`),
+	}
+
+	for i, tt := range tests {
+		var req MemberCreateRequest
+		if err := json.Unmarshal(tt, &req); err == nil {
+			t.Errorf("#%d: expected err, got nil", i)
+		}
+	}
+}
+
+func TestMemberCreateRequestMarshal(t *testing.T) {
+	req := MemberCreateRequest{
+		PeerURLs: types.URLs([]url.URL{
+			url.URL{Scheme: "http", Host: "127.0.0.1:8081"},
+			url.URL{Scheme: "https", Host: "127.0.0.1:8080"},
+		}),
+	}
+	want := []byte(`{"peerURLs":["http://127.0.0.1:8081","https://127.0.0.1:8080"]}`)
+
+	got, err := json.Marshal(&req)
+	if err != nil {
+		t.Fatalf("Marshal returned unexpected err=%v", err)
+	}
+
+	if !reflect.DeepEqual(want, got) {
+		t.Fatalf("Failed to marshal MemberCreateRequest: want=%s, got=%s", want, got)
+	}
+}

etcdserver/etcdhttp/peer.go

@@ -0,0 +1,123 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package etcdhttp
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
+	"github.com/coreos/etcd/etcdserver"
+	"github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/etcd/raft/raftpb"
+)
+
+const (
+	raftPrefix        = "/raft"
+	peerMembersPrefix = "/members"
+)
+
+// NewPeerHandler generates an http.Handler to handle etcd peer (raft) requests.
+func NewPeerHandler(server *etcdserver.EtcdServer) http.Handler {
+	rh := &raftHandler{
+		stats:       server,
+		server:      server,
+		clusterInfo: server.Cluster,
+	}
+
+	mh := &peerMembersHandler{
+		clusterInfo: server.Cluster,
+	}
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", http.NotFound)
+	mux.Handle(raftPrefix, rh)
+	mux.Handle(peerMembersPrefix, mh)
+	return mux
+}
+
+type raftHandler struct {
+	stats       etcdserver.Stats
+	server      etcdserver.Server
+	clusterInfo etcdserver.ClusterInfo
+}
+
+func (h *raftHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "POST") {
+		return
+	}
+
+	wcid := h.clusterInfo.ID().String()
+	w.Header().Set("X-Etcd-Cluster-ID", wcid)
+
+	gcid := r.Header.Get("X-Etcd-Cluster-ID")
+	if gcid != wcid {
+		log.Printf("etcdhttp: request ignored due to cluster ID mismatch got %s want %s", gcid, wcid)
+		http.Error(w, "clusterID mismatch", http.StatusPreconditionFailed)
+		return
+	}
+
+	b, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		log.Println("etcdhttp: error reading raft message:", err)
+		http.Error(w, "error reading raft message", http.StatusBadRequest)
+		return
+	}
+	var m raftpb.Message
+	if err := m.Unmarshal(b); err != nil {
+		log.Println("etcdhttp: error unmarshaling raft message:", err)
+		http.Error(w, "error unmarshaling raft message", http.StatusBadRequest)
+		return
+	}
+	if err := h.server.Process(context.TODO(), m); err != nil {
+		switch err {
+		case etcdserver.ErrRemoved:
+			log.Printf("etcdhttp: reject message from removed member %s", types.ID(m.From).String())
+			http.Error(w, "cannot process message from removed member", http.StatusForbidden)
+		default:
+			writeError(w, err)
+		}
+		return
+	}
+	if m.Type == raftpb.MsgApp {
+		h.stats.UpdateRecvApp(types.ID(m.From), r.ContentLength)
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+type peerMembersHandler struct {
+	clusterInfo etcdserver.ClusterInfo
+}
+
+func (h *peerMembersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if !allowMethod(w, r.Method, "GET") {
+		return
+	}
+	w.Header().Set("X-Etcd-Cluster-ID", h.clusterInfo.ID().String())
+
+	if r.URL.Path != peerMembersPrefix {
+		http.Error(w, "bad path", http.StatusBadRequest)
+		return
+	}
+	ms := h.clusterInfo.Members()
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(ms); err != nil {
+		log.Printf("etcdhttp: %v", err)
+	}
+}

etcdserver/etcdhttp/peer_test.go

@@ -0,0 +1,254 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package etcdhttp
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/coreos/etcd/etcdserver"
+	"github.com/coreos/etcd/raft/raftpb"
+)
+
+func mustMarshalMsg(t *testing.T, m raftpb.Message) []byte {
+	json, err := m.Marshal()
+	if err != nil {
+		t.Fatalf("error marshalling raft Message: %#v", err)
+	}
+	return json
+}
+
+// errReader implements io.Reader to facilitate a broken request.
+type errReader struct{}
+
+func (er *errReader) Read(_ []byte) (int, error) { return 0, errors.New("some error") }
+
+func TestServeRaft(t *testing.T) {
+	testCases := []struct {
+		method    string
+		body      io.Reader
+		serverErr error
+		clusterID string
+
+		wcode int
+	}{
+		{
+			// bad method
+			"GET",
+			bytes.NewReader(
+				mustMarshalMsg(
+					t,
+					raftpb.Message{},
+				),
+			),
+			nil,
+			"0",
+			http.StatusMethodNotAllowed,
+		},
+		{
+			// bad method
+			"PUT",
+			bytes.NewReader(
+				mustMarshalMsg(
+					t,
+					raftpb.Message{},
+				),
+			),
+			nil,
+			"0",
+			http.StatusMethodNotAllowed,
+		},
+		{
+			// bad method
+			"DELETE",
+			bytes.NewReader(
+				mustMarshalMsg(
+					t,
+					raftpb.Message{},
+				),
+			),
+			nil,
+			"0",
+			http.StatusMethodNotAllowed,
+		},
+		{
+			// bad request body
+			"POST",
+			&errReader{},
+			nil,
+			"0",
+			http.StatusBadRequest,
+		},
+		{
+			// bad request protobuf
+			"POST",
+			strings.NewReader("malformed garbage"),
+			nil,
+			"0",
+			http.StatusBadRequest,
+		},
+		{
+			// good request, etcdserver.Server internal error
+			"POST",
+			bytes.NewReader(
+				mustMarshalMsg(
+					t,
+					raftpb.Message{},
+				),
+			),
+			errors.New("some error"),
+			"0",
+			http.StatusInternalServerError,
+		},
+		{
+			// good request from removed member
+			"POST",
+			bytes.NewReader(
+				mustMarshalMsg(
+					t,
+					raftpb.Message{},
+				),
+			),
+			etcdserver.ErrRemoved,
+			"0",
+			http.StatusForbidden,
+		},
+		{
+			// good request
+			"POST",
+			bytes.NewReader(
+				mustMarshalMsg(
+					t,
+					raftpb.Message{},
+				),
+			),
+			nil,
+			"1",
+			http.StatusPreconditionFailed,
+		},
+		{
+			// good request
+			"POST",
+			bytes.NewReader(
+				mustMarshalMsg(
+					t,
+					raftpb.Message{},
+				),
+			),
+			nil,
+			"0",
+			http.StatusNoContent,
+		},
+	}
+	for i, tt := range testCases {
+		req, err := http.NewRequest(tt.method, "foo", tt.body)
+		if err != nil {
+			t.Fatalf("#%d: could not create request: %#v", i, err)
+		}
+		req.Header.Set("X-Etcd-Cluster-ID", tt.clusterID)
+		rw := httptest.NewRecorder()
+		h := &raftHandler{stats: nil, server: &errServer{tt.serverErr}, clusterInfo: &fakeCluster{id: 0}}
+		h.ServeHTTP(rw, req)
+		if rw.Code != tt.wcode {
+			t.Errorf("#%d: got code=%d, want %d", i, rw.Code, tt.wcode)
+		}
+	}
+}
+
+func TestServeMembersFails(t *testing.T) {
+	tests := []struct {
+		method string
+		wcode  int
+	}{
+		{
+			"POST",
+			http.StatusMethodNotAllowed,
+		},
+		{
+			"DELETE",
+			http.StatusMethodNotAllowed,
+		},
+		{
+			"BAD",
+			http.StatusMethodNotAllowed,
+		},
+	}
+	for i, tt := range tests {
+		rw := httptest.NewRecorder()
+		h := &peerMembersHandler{clusterInfo: nil}
+		h.ServeHTTP(rw, &http.Request{Method: tt.method})
+		if rw.Code != tt.wcode {
+			t.Errorf("#%d: code=%d, want %d", i, rw.Code, tt.wcode)
+		}
+	}
+}
+
+func TestServeMembersGet(t *testing.T) {
+	memb1 := etcdserver.Member{ID: 1, Attributes: etcdserver.Attributes{ClientURLs: []string{"http://localhost:8080"}}}
+	memb2 := etcdserver.Member{ID: 2, Attributes: etcdserver.Attributes{ClientURLs: []string{"http://localhost:8081"}}}
+	cluster := &fakeCluster{
+		id:      1,
+		members: map[uint64]*etcdserver.Member{1: &memb1, 2: &memb2},
+	}
+	h := &peerMembersHandler{clusterInfo: cluster}
+	msb, err := json.Marshal([]etcdserver.Member{memb1, memb2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	wms := string(msb) + "\n"
+
+	tests := []struct {
+		path  string
+		wcode int
+		wct   string
+		wbody string
+	}{
+		{peerMembersPrefix, http.StatusOK, "application/json", wms},
+		{path.Join(peerMembersPrefix, "bad"), http.StatusBadRequest, "text/plain; charset=utf-8", "bad path\n"},
+	}
+
+	for i, tt := range tests {
+		req, err := http.NewRequest("GET", mustNewURL(t, tt.path).String(), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		rw := httptest.NewRecorder()
+		h.ServeHTTP(rw, req)
+
+		if rw.Code != tt.wcode {
+			t.Errorf("#%d: code=%d, want %d", i, rw.Code, tt.wcode)
+		}
+		if gct := rw.Header().Get("Content-Type"); gct != tt.wct {
+			t.Errorf("#%d: content-type = %s, want %s", i, gct, tt.wct)
+		}
+		if rw.Body.String() != tt.wbody {
+			t.Errorf("#%d: body = %s, want %s", i, rw.Body.String(), tt.wbody)
+		}
+		gcid := rw.Header().Get("X-Etcd-Cluster-ID")
+		wcid := cluster.ID().String()
+		if gcid != wcid {
+			t.Errorf("#%d: cid = %s, want %s", i, gcid, wcid)
+		}
+	}
+}

etcdserver/force_cluster.go

@@ -0,0 +1,152 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package etcdserver
+
+import (
+	"encoding/json"
+	"log"
+	"sort"
+
+	"github.com/coreos/etcd/pkg/pbutil"
+	"github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/etcd/raft"
+	"github.com/coreos/etcd/raft/raftpb"
+	"github.com/coreos/etcd/wal"
+)
+
+func restartAsStandaloneNode(cfg *ServerConfig, index uint64, snapshot *raftpb.Snapshot) (id types.ID, n raft.Node, w *wal.WAL) {
+	var err error
+	if w, err = wal.OpenAtIndex(cfg.WALDir(), index); err != nil {
+		log.Fatalf("etcdserver: open wal error: %v", err)
+	}
+	id, cid, st, ents, err := readWAL(w, index)
+	if err != nil {
+		log.Fatalf("etcdserver: read wal error: %v", err)
+	}
+	cfg.Cluster.SetID(cid)
+
+	// discard the previously uncommitted entries
+	if len(ents) != 0 {
+		ents = ents[:st.Commit+1]
+	}
+
+	// force append the configuration change entries
+	toAppEnts := createConfigChangeEnts(getIDs(snapshot, ents), uint64(id), st.Term, st.Commit)
+	ents = append(ents, toAppEnts...)
+
+	// force commit newly appended entries
+	for _, e := range toAppEnts {
+		err := w.SaveEntry(&e)
+		if err != nil {
+			log.Fatalf("etcdserver: %v", err)
+		}
+	}
+	if len(ents) != 0 {
+		st.Commit = ents[len(ents)-1].Index
+	}
+
+	log.Printf("etcdserver: forcing restart of member %s in cluster %s at commit index %d", id, cfg.Cluster.ID(), st.Commit)
+	n = raft.RestartNode(uint64(id), 10, 1, snapshot, st, ents)
+	return
+}
+
+// getIDs returns an ordered set of IDs included in the given snapshot and
+// the entries. The given snapshot/entries can contain two kinds of
+// ID-related entry:
+// - ConfChangeAddNode, in which case the contained ID will be added into the set.
+// - ConfChangeAddRemove, in which case the contained ID will be removed from the set.
+func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 {
+	ids := make(map[uint64]bool)
+	if snap != nil {
+		for _, id := range snap.Nodes {
+			ids[id] = true
+		}
+	}
+	for _, e := range ents {
+		if e.Type != raftpb.EntryConfChange {
+			continue
+		}
+		var cc raftpb.ConfChange
+		pbutil.MustUnmarshal(&cc, e.Data)
+		switch cc.Type {
+		case raftpb.ConfChangeAddNode:
+			ids[cc.NodeID] = true
+		case raftpb.ConfChangeRemoveNode:
+			delete(ids, cc.NodeID)
+		default:
+			log.Panicf("ConfChange Type should be either ConfChangeAddNode or ConfChangeRemoveNode!")
+		}
+	}
+	sids := make(types.Uint64Slice, 0)
+	for id := range ids {
+		sids = append(sids, id)
+	}
+	sort.Sort(sids)
+	return []uint64(sids)
+}
+
+// createConfigChangeEnts creates a series of Raft entries (i.e.
+// EntryConfChange) to remove the set of given IDs from the cluster. The ID
+// `self` is _not_ removed, even if present in the set.
+// If `self` is not inside the given ids, it creates a Raft entry to add a
+// default member with the given `self`.
+func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raftpb.Entry {
+	ents := make([]raftpb.Entry, 0)
+	next := index + 1
+	found := false
+	for _, id := range ids {
+		if id == self {
+			found = true
+			continue
+		}
+		cc := &raftpb.ConfChange{
+			Type:   raftpb.ConfChangeRemoveNode,
+			NodeID: id,
+		}
+		e := raftpb.Entry{
+			Type:  raftpb.EntryConfChange,
+			Data:  pbutil.MustMarshal(cc),
+			Term:  term,
+			Index: next,
+		}
+		ents = append(ents, e)
+		next++
+	}
+	if !found {
+		m := Member{
+			ID:             types.ID(self),
+			RaftAttributes: RaftAttributes{PeerURLs: []string{"http://localhost:7001", "http://localhost:2380"}},
+		}
+		ctx, err := json.Marshal(m)
+		if err != nil {
+			log.Panicf("marshal member should never fail: %v", err)
+		}
+		cc := &raftpb.ConfChange{
+			Type:    raftpb.ConfChangeAddNode,
+			NodeID:  self,
+			Context: ctx,
+		}
+		e := raftpb.Entry{
+			Type:  raftpb.EntryConfChange,
+			Data:  pbutil.MustMarshal(cc),
+			Term:  term,
+			Index: next,
+		}
+		ents = append(ents, e)
+	}
+	return ents
+}

etcdserver/force_cluster_test.go

@@ -0,0 +1,136 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package etcdserver
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/coreos/etcd/pkg/pbutil"
+	"github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/etcd/raft/raftpb"
+)
+
+func TestGetIDs(t *testing.T) {
+	addcc := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 2}
+	addEntry := raftpb.Entry{Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(addcc)}
+	removecc := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 2}
+	removeEntry := raftpb.Entry{Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc)}
+	normalEntry := raftpb.Entry{Type: raftpb.EntryNormal}
+
+	tests := []struct {
+		snap *raftpb.Snapshot
+		ents []raftpb.Entry
+
+		widSet []uint64
+	}{
+		{nil, []raftpb.Entry{}, []uint64{}},
+		{&raftpb.Snapshot{Nodes: []uint64{1}}, []raftpb.Entry{}, []uint64{1}},
+		{&raftpb.Snapshot{Nodes: []uint64{1}}, []raftpb.Entry{addEntry}, []uint64{1, 2}},
+		{&raftpb.Snapshot{Nodes: []uint64{1}}, []raftpb.Entry{addEntry, removeEntry}, []uint64{1}},
+		{&raftpb.Snapshot{Nodes: []uint64{1}}, []raftpb.Entry{addEntry, normalEntry}, []uint64{1, 2}},
+		{&raftpb.Snapshot{Nodes: []uint64{1}}, []raftpb.Entry{addEntry, removeEntry, normalEntry}, []uint64{1}},
+	}
+
+	for i, tt := range tests {
+		idSet := getIDs(tt.snap, tt.ents)
+		if !reflect.DeepEqual(idSet, tt.widSet) {
+			t.Errorf("#%d: idset = %#v, want %#v", i, idSet, tt.widSet)
+		}
+	}
+}
+
+func TestCreateConfigChangeEnts(t *testing.T) {
+	m := Member{
+		ID:             types.ID(1),
+		RaftAttributes: RaftAttributes{PeerURLs: []string{"http://localhost:7001", "http://localhost:2380"}},
+	}
+	ctx, err := json.Marshal(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	addcc1 := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1, Context: ctx}
+	removecc2 := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 2}
+	removecc3 := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 3}
+	tests := []struct {
+		ids         []uint64
+		self        uint64
+		term, index uint64
+
+		wents []raftpb.Entry
+	}{
+		{
+			[]uint64{1},
+			1,
+			1, 1,
+
+			[]raftpb.Entry{},
+		},
+		{
+			[]uint64{1, 2},
+			1,
+			1, 1,
+
+			[]raftpb.Entry{{Term: 1, Index: 2, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)}},
+		},
+		{
+			[]uint64{1, 2},
+			1,
+			2, 2,
+
+			[]raftpb.Entry{{Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)}},
+		},
+		{
+			[]uint64{1, 2, 3},
+			1,
+			2, 2,
+
+			[]raftpb.Entry{
+				{Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)},
+				{Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)},
+			},
+		},
+		{
+			[]uint64{2, 3},
+			2,
+			2, 2,
+
+			[]raftpb.Entry{
+				{Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)},
+			},
+		},
+		{
+			[]uint64{2, 3},
+			1,
+			2, 2,
+
+			[]raftpb.Entry{
+				{Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)},
+				{Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)},
+				{Term: 2, Index: 5, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(addcc1)},
+			},
+		},
+	}
+
+	for i, tt := range tests {
+		gents := createConfigChangeEnts(tt.ids, tt.self, tt.term, tt.index)
+		if !reflect.DeepEqual(gents, tt.wents) {
+			t.Errorf("#%d: ents = %v, want %v", i, gents, tt.wents)
+		}
+	}
+}

etcdserver/member.go

@@ -24,7 +24,6 @@ import (
 	"math/rand"
 	"path"
 	"sort"
-	"strconv"
 	"time"
 
 	"github.com/coreos/etcd/pkg/types"
@@ -43,7 +42,7 @@ type Attributes struct {
 }
 
 type Member struct {
-	ID uint64 `json:"id"`
+	ID types.ID `json:"id"`
 	RaftAttributes
 	Attributes
 }
@@ -68,7 +67,7 @@ func NewMember(name string, peerURLs types.URLs, clusterName string, now *time.T
 	}
 
 	hash := sha1.Sum(b)
-	m.ID = binary.BigEndian.Uint64(hash[:8])
+	m.ID = types.ID(binary.BigEndian.Uint64(hash[:8]))
 	return m
 }
 
@@ -76,25 +75,46 @@ func NewMember(name string, peerURLs types.URLs, clusterName string, now *time.T
 // It will panic if there is no PeerURLs available in Member.
 func (m *Member) PickPeerURL() string {
 	if len(m.PeerURLs) == 0 {
-		panic("member should always have some peer url")
+		log.Panicf("member should always have some peer url")
 	}
 	return m.PeerURLs[rand.Intn(len(m.PeerURLs))]
 }
 
-func memberStoreKey(id uint64) string {
-	return path.Join(storeMembersPrefix, idAsHex(id))
+func (m *Member) Clone() *Member {
+	if m == nil {
+		return nil
+	}
+	mm := &Member{
+		ID: m.ID,
+		Attributes: Attributes{
+			Name: m.Name,
+		},
+	}
+	if m.PeerURLs != nil {
+		mm.PeerURLs = make([]string, len(m.PeerURLs))
+		copy(mm.PeerURLs, m.PeerURLs)
+	}
+	if m.ClientURLs != nil {
+		mm.ClientURLs = make([]string, len(m.ClientURLs))
+		copy(mm.ClientURLs, m.ClientURLs)
+	}
+	return mm
 }
 
-func parseMemberID(key string) uint64 {
-	id, err := strconv.ParseUint(path.Base(key), 16, 64)
+func memberStoreKey(id types.ID) string {
+	return path.Join(storeMembersPrefix, id.String())
+}
+
+func mustParseMemberIDFromKey(key string) types.ID {
+	id, err := types.IDFromString(path.Base(key))
 	if err != nil {
 		log.Panicf("unexpected parse member id error: %v", err)
 	}
 	return id
 }
 
-func removedMemberStoreKey(id uint64) string {
-	return path.Join(storeRemovedMembersPrefix, idAsHex(id))
+func removedMemberStoreKey(id types.ID) string {
+	return path.Join(storeRemovedMembersPrefix, id.String())
 }
 
 type SortableMemberSliceByPeerURLs []*Member

etcdserver/member_test.go

@@ -18,8 +18,11 @@ package etcdserver
 
 import (
 	"net/url"
+	"reflect"
 	"testing"
 	"time"
+
+	"github.com/coreos/etcd/pkg/types"
 )
 
 func timeParse(value string) *time.Time {
@@ -33,7 +36,7 @@ func timeParse(value string) *time.Time {
 func TestMemberTime(t *testing.T) {
 	tests := []struct {
 		mem *Member
-		id  uint64
+		id  types.ID
 	}{
 		{NewMember("mem1", []url.URL{{Scheme: "http", Host: "10.0.0.8:2379"}}, "", nil), 14544069596553697298},
 		// Same ID, different name (names shouldn't matter)
@@ -86,3 +89,21 @@ func TestMemberPick(t *testing.T) {
 		}
 	}
 }
+
+func TestMemberClone(t *testing.T) {
+	tests := []*Member{
+		newTestMemberp(1, nil, "abc", nil),
+		newTestMemberp(1, []string{"http://a"}, "abc", nil),
+		newTestMemberp(1, nil, "abc", []string{"http://b"}),
+		newTestMemberp(1, []string{"http://a"}, "abc", []string{"http://b"}),
+	}
+	for i, tt := range tests {
+		nm := tt.Clone()
+		if nm == tt {
+			t.Errorf("#%d: the pointers are the same, and clone doesn't happen", i)
+		}
+		if !reflect.DeepEqual(nm, tt) {
+			t.Errorf("#%d: member = %+v, want %+v", i, nm, tt)
+		}
+	}
+}

etcdserver/sender.go

@@ -21,101 +21,169 @@ import (
 	"fmt"
 	"log"
 	"net/http"
-	"strconv"
 	"time"
 
 	"github.com/coreos/etcd/etcdserver/stats"
+	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft/raftpb"
 )
 
-const raftPrefix = "/raft"
+const (
+	raftPrefix    = "/raft"
+	connPerSender = 4
+)
 
-// Sender creates the default production sender used to transport raft messages
-// in the cluster. The returned sender will update the given ServerStats and
-// LeaderStats appropriately.
-func Sender(t *http.Transport, cl *Cluster, ss *stats.ServerStats, ls *stats.LeaderStats) func(msgs []raftpb.Message) {
-	c := &http.Client{Transport: t}
-
-	return func(msgs []raftpb.Message) {
-		for _, m := range msgs {
-			// TODO: reuse go routines
-			// limit the number of outgoing connections for the same receiver
-			go send(c, cl, m, ss, ls)
-		}
-	}
+type sendHub struct {
+	tr      *http.Transport
+	cl      ClusterInfo
+	ss      *stats.ServerStats
+	ls      *stats.LeaderStats
+	senders map[types.ID]*sender
 }
 
-// send uses the given client to send a message to a member in the given
-// ClusterStore, retrying up to 3 times for each message. The given
-// ServerStats and LeaderStats are updated appropriately
-func send(c *http.Client, cl *Cluster, m raftpb.Message, ss *stats.ServerStats, ls *stats.LeaderStats) {
-	cid := cl.ID()
-	// TODO (xiangli): reasonable retry logic
-	for i := 0; i < 3; i++ {
-		memb := cl.Member(m.To)
-		if memb == nil {
-			// TODO: unknown peer id.. what do we do? I
-			// don't think his should ever happen, need to
-			// look into this further.
-			log.Printf("etcdhttp: no member for %d", m.To)
-			return
+// newSendHub creates the default send hub used to transport raft messages
+// to other members. The returned sendHub will update the given ServerStats and
+// LeaderStats appropriately.
+func newSendHub(t *http.Transport, cl ClusterInfo, ss *stats.ServerStats, ls *stats.LeaderStats) *sendHub {
+	h := &sendHub{
+		tr:      t,
+		cl:      cl,
+		ss:      ss,
+		ls:      ls,
+		senders: make(map[types.ID]*sender),
+	}
+	for _, m := range cl.Members() {
+		h.Add(m)
+	}
+	return h
+}
+
+func (h *sendHub) Send(msgs []raftpb.Message) {
+	for _, m := range msgs {
+		to := types.ID(m.To)
+		s, ok := h.senders[to]
+		if !ok {
+			if !h.cl.IsIDRemoved(to) {
+				log.Printf("etcdserver: send message to unknown receiver %s", to)
+			}
+			continue
 		}
-		u := fmt.Sprintf("%s%s", memb.PickPeerURL(), raftPrefix)
 
 		// TODO: don't block. we should be able to have 1000s
 		// of messages out at a time.
 		data, err := m.Marshal()
 		if err != nil {
-			log.Println("etcdhttp: dropping message:", err)
+			log.Println("sender: dropping message:", err)
 			return // drop bad message
 		}
 		if m.Type == raftpb.MsgApp {
-			ss.SendAppendReq(len(data))
+			h.ss.SendAppendReq(len(data))
 		}
-		to := idAsHex(m.To)
-		fs := ls.Follower(to)
 
-		start := time.Now()
-		sent := httpPost(c, u, cid, data)
-		end := time.Now()
-		if sent {
-			fs.Succ(end.Sub(start))
-			return
-		}
-		fs.Fail()
-		// TODO: backoff
+		// TODO (xiangli): reasonable retry logic
+		s.send(data)
 	}
 }
 
-// httpPost POSTs a data payload to a url using the given client. Returns true
-// if the POST succeeds, false on any failure.
-func httpPost(c *http.Client, url string, cid uint64, data []byte) bool {
-	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+func (h *sendHub) Stop() {
+	for _, s := range h.senders {
+		s.stop()
+	}
+}
+
+func (h *sendHub) Add(m *Member) {
+	if _, ok := h.senders[m.ID]; ok {
+		return
+	}
+	// TODO: considering how to switch between all available peer urls
+	u := fmt.Sprintf("%s%s", m.PickPeerURL(), raftPrefix)
+	c := &http.Client{Transport: h.tr}
+	fs := h.ls.Follower(m.ID.String())
+	s := newSender(u, h.cl.ID(), c, fs)
+	h.senders[m.ID] = s
+}
+
+func (h *sendHub) Remove(id types.ID) {
+	h.senders[id].stop()
+	delete(h.senders, id)
+}
+
+type sender struct {
+	u   string
+	cid types.ID
+	c   *http.Client
+	fs  *stats.FollowerStats
+	q   chan []byte
+}
+
+func newSender(u string, cid types.ID, c *http.Client, fs *stats.FollowerStats) *sender {
+	s := &sender{
+		u:   u,
+		cid: cid,
+		c:   c,
+		fs:  fs,
+		q:   make(chan []byte),
+	}
+	for i := 0; i < connPerSender; i++ {
+		go s.handle()
+	}
+	return s
+}
+
+func (s *sender) send(data []byte) {
+	select {
+	case s.q <- data:
+	default:
+		log.Printf("sender: reach the maximal serving to %s", s.u)
+	}
+}
+
+func (s *sender) stop() {
+	close(s.q)
+}
+
+func (s *sender) handle() {
+	for d := range s.q {
+		start := time.Now()
+		err := s.post(d)
+		end := time.Now()
+		if err != nil {
+			s.fs.Fail()
+			log.Printf("sender: %v", err)
+			continue
+		}
+		s.fs.Succ(end.Sub(start))
+	}
+}
+
+// post POSTs a data payload to a url. Returns nil if the POST succeeds,
+// error on any failure.
+func (s *sender) post(data []byte) error {
+	req, err := http.NewRequest("POST", s.u, bytes.NewBuffer(data))
 	if err != nil {
-		// TODO: log the error?
-		return false
+		return fmt.Errorf("new request to %s error: %v", s.u, err)
 	}
 	req.Header.Set("Content-Type", "application/protobuf")
-	req.Header.Set("X-Etcd-Cluster-ID", strconv.FormatUint(cid, 16))
-	resp, err := c.Do(req)
+	req.Header.Set("X-Etcd-Cluster-ID", s.cid.String())
+	resp, err := s.c.Do(req)
 	if err != nil {
-		// TODO: log the error?
-		return false
+		return fmt.Errorf("error posting to %q: %v", req.URL.String(), err)
 	}
 	resp.Body.Close()
 
 	switch resp.StatusCode {
 	case http.StatusPreconditionFailed:
 		// TODO: shutdown the etcdserver gracefully?
-		log.Panicf("clusterID mismatch")
-		return false
+		log.Fatalf("etcd: conflicting cluster ID with the target cluster (%s != %s)", resp.Header.Get("X-Etcd-Cluster-ID"), s.cid)
+		return nil
 	case http.StatusForbidden:
 		// TODO: stop the server
-		log.Panicf("the member has been removed")
-		return false
+		log.Println("etcd: this member has been permanently removed from the cluster")
+		log.Fatalln("etcd: the data-dir used by this member must be removed so that this host can be re-added with a new member ID")
+		return nil
 	case http.StatusNoContent:
-		return true
+		return nil
 	default:
-		return false
+		return fmt.Errorf("unhandled status %s", http.StatusText(resp.StatusCode))
 	}
 }

etcdserver/sender_test.go

@@ -0,0 +1,84 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package etcdserver
+
+import (
+	"testing"
+
+	"github.com/coreos/etcd/etcdserver/stats"
+	"github.com/coreos/etcd/pkg/types"
+)
+
+func TestSendHubInitSenders(t *testing.T) {
+	membs := []Member{
+		newTestMember(1, []string{"http://a"}, "", nil),
+		newTestMember(2, []string{"http://b"}, "", nil),
+		newTestMember(3, []string{"http://c"}, "", nil),
+	}
+	cl := newTestCluster(membs)
+	ls := stats.NewLeaderStats("")
+	h := newSendHub(nil, cl, nil, ls)
+
+	ids := cl.MemberIDs()
+	if len(h.senders) != len(ids) {
+		t.Errorf("len(ids) = %d, want %d", len(h.senders), len(ids))
+	}
+	for _, id := range ids {
+		if _, ok := h.senders[id]; !ok {
+			t.Errorf("senders[%s] is nil, want exists", id)
+		}
+	}
+}
+
+func TestSendHubAdd(t *testing.T) {
+	cl := newTestCluster(nil)
+	ls := stats.NewLeaderStats("")
+	h := newSendHub(nil, cl, nil, ls)
+	m := newTestMemberp(1, []string{"http://a"}, "", nil)
+	h.Add(m)
+
+	if _, ok := ls.Followers["1"]; !ok {
+		t.Errorf("FollowerStats[1] is nil, want exists")
+	}
+	s, ok := h.senders[types.ID(1)]
+	if !ok {
+		t.Fatalf("senders[1] is nil, want exists")
+	}
+	if s.u != "http://a/raft" {
+		t.Errorf("url = %s, want %s", s.u, "http://a/raft")
+	}
+
+	h.Add(m)
+	ns := h.senders[types.ID(1)]
+	if s != ns {
+		t.Errorf("sender = %p, want %p", ns, s)
+	}
+}
+
+func TestSendHubRemove(t *testing.T) {
+	membs := []Member{
+		newTestMember(1, []string{"http://a"}, "", nil),
+	}
+	cl := newTestCluster(membs)
+	ls := stats.NewLeaderStats("")
+	h := newSendHub(nil, cl, nil, ls)
+	h.Remove(types.ID(1))
+
+	if _, ok := h.senders[types.ID(1)]; ok {
+		t.Fatalf("senders[1] exists, want removed")
+	}
+}

etcdserver/server.go

@@ -27,7 +27,6 @@ import (
 	"os"
 	"path"
 	"regexp"
-	"strconv"
 	"sync/atomic"
 	"time"
 
@@ -36,6 +35,7 @@ import (
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/etcdserver/stats"
 	"github.com/coreos/etcd/pkg/pbutil"
+	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft"
 	"github.com/coreos/etcd/raft/raftpb"
 	"github.com/coreos/etcd/snap"
@@ -64,6 +64,9 @@ var (
 	ErrIDRemoved     = errors.New("etcdserver: ID removed")
 	ErrIDExists      = errors.New("etcdserver: ID exists")
 	ErrIDNotFound    = errors.New("etcdserver: ID not found")
+	ErrPeerURLexists = errors.New("etcdserver: peerURL exists")
+	ErrCanceled      = errors.New("etcdserver: request cancelled")
+	ErrTimeout       = errors.New("etcdserver: request timed out")
 
 	storeMembersPrefix        = path.Join(StoreAdminPrefix, "members")
 	storeRemovedMembersPrefix = path.Join(StoreAdminPrefix, "removed_members")
@@ -75,20 +78,25 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-type sendFunc func(m []raftpb.Message)
-
 type Response struct {
 	Event   *store.Event
 	Watcher store.Watcher
 	err     error
 }
 
+type Sender interface {
+	Send(m []raftpb.Message)
+	Add(m *Member)
+	Remove(id types.ID)
+	Stop()
+}
+
 type Storage interface {
 	// Save function saves ents and state to the underlying stable storage.
 	// Save MUST block until st and ents are on stable storage.
-	Save(st raftpb.HardState, ents []raftpb.Entry)
+	Save(st raftpb.HardState, ents []raftpb.Entry) error
 	// SaveSnap function saves snapshot to the underlying stable storage.
-	SaveSnap(snap raftpb.Snapshot)
+	SaveSnap(snap raftpb.Snapshot) error
 
 	// TODO: WAL should be able to control cut itself. After implement self-controled cut,
 	// remove it in this interface.
@@ -129,7 +137,7 @@ type Stats interface {
 	// StoreStats returns statistics of the store backing this EtcdServer
 	StoreStats() []byte
 	// UpdateRecvApp updates the underlying statistics in response to a receiving an Append request
-	UpdateRecvApp(from uint64, length int64)
+	UpdateRecvApp(from types.ID, length int64)
 }
 
 type RaftTimer interface {
@@ -142,7 +150,7 @@ type EtcdServer struct {
 	w          wait.Wait
 	done       chan struct{}
 	stopped    chan struct{}
-	id         uint64
+	id         types.ID
 	attributes Attributes
 
 	Cluster *Cluster
@@ -153,11 +161,11 @@ type EtcdServer struct {
 	stats  *stats.ServerStats
 	lstats *stats.LeaderStats
 
-	// send specifies the send function for sending msgs to members. send
+	// sender specifies the sender to send msgs to members. sending msgs
 	// MUST NOT block. It is okay to drop messages, since clients should
 	// timeout and reissue their messages.  If send is nil, server will
 	// panic.
-	send sendFunc
+	sender Sender
 
 	storage Storage
 
@@ -173,47 +181,44 @@ type EtcdServer struct {
 
 // NewServer creates a new EtcdServer from the supplied configuration. The
 // configuration is considered static for the lifetime of the EtcdServer.
-func NewServer(cfg *ServerConfig) *EtcdServer {
+func NewServer(cfg *ServerConfig) (*EtcdServer, error) {
 	if err := os.MkdirAll(cfg.SnapDir(), privateDirMode); err != nil {
-		log.Fatalf("etcdserver: cannot create snapshot directory: %v", err)
+		return nil, fmt.Errorf("cannot create snapshot directory: %v", err)
 	}
 	ss := snap.New(cfg.SnapDir())
 	st := store.New()
 	var w *wal.WAL
 	var n raft.Node
-	var id uint64
+	var id types.ID
 	haveWAL := wal.Exist(cfg.WALDir())
 	switch {
-	case !haveWAL && cfg.ClusterState == ClusterStateValueExisting:
+	case !haveWAL && !cfg.NewCluster:
 		cl, err := GetClusterFromPeers(cfg.Cluster.PeerURLs())
 		if err != nil {
-			log.Fatal(err)
+			return nil, fmt.Errorf("cannot fetch cluster info from peer urls: %v", err)
 		}
 		if err := cfg.Cluster.ValidateAndAssignIDs(cl.Members()); err != nil {
-			log.Fatalf("etcdserver: %v", err)
+			return nil, fmt.Errorf("error validating IDs from cluster %s: %v", cl, err)
 		}
 		cfg.Cluster.SetID(cl.id)
 		cfg.Cluster.SetStore(st)
 		id, n, w = startNode(cfg, nil)
-	case !haveWAL && cfg.ClusterState == ClusterStateValueNew:
+	case !haveWAL && cfg.NewCluster:
 		if err := cfg.VerifyBootstrapConfig(); err != nil {
-			log.Fatalf("etcdserver: %v", err)
+			return nil, err
 		}
 		m := cfg.Cluster.MemberByName(cfg.Name)
 		if cfg.ShouldDiscover() {
-			d, err := discovery.New(cfg.DiscoveryURL, m.ID, cfg.Cluster.String())
+			s, err := discovery.JoinCluster(cfg.DiscoveryURL, cfg.DiscoveryProxy, m.ID, cfg.Cluster.String())
 			if err != nil {
-				log.Fatalf("etcdserver: cannot init discovery %v", err)
+				return nil, err
 			}
-			s, err := d.Discover()
-			if err != nil {
-				log.Fatalf("etcdserver: %v", err)
-			}
-			if cfg.Cluster, err = NewClusterFromString(cfg.Cluster.name, s); err != nil {
-				log.Fatalf("etcdserver: %v", err)
+			if cfg.Cluster, err = NewClusterFromString(cfg.Cluster.token, s); err != nil {
+				return nil, err
 			}
 		}
 		cfg.Cluster.SetStore(st)
+		log.Printf("etcdserver: initial cluster members: %s", cfg.Cluster)
 		id, n, w = startNode(cfg, cfg.Cluster.MemberIDs())
 	case haveWAL:
 		if cfg.ShouldDiscover() {
@@ -222,25 +227,33 @@ func NewServer(cfg *ServerConfig) *EtcdServer {
 		var index uint64
 		snapshot, err := ss.Load()
 		if err != nil && err != snap.ErrNoSnapshot {
-			log.Fatal(err)
+			return nil, err
 		}
 		if snapshot != nil {
 			log.Printf("etcdserver: recovering from snapshot at index %d", snapshot.Index)
 			st.Recovery(snapshot.Data)
 			index = snapshot.Index
 		}
-		cfg.Cluster = NewClusterFromStore(cfg.Cluster.name, st)
-		id, n, w = restartNode(cfg, index, snapshot)
+		cfg.Cluster = NewClusterFromStore(cfg.Cluster.token, st)
+		if snapshot != nil {
+			log.Printf("etcdserver: loaded peers from snapshot: %s", cfg.Cluster)
+		}
+		if !cfg.ForceNewCluster {
+			id, n, w = restartNode(cfg, index, snapshot)
+		} else {
+			id, n, w = restartAsStandaloneNode(cfg, index, snapshot)
+		}
 	default:
-		log.Fatalf("etcdserver: unsupported bootstrap config")
+		return nil, fmt.Errorf("unsupported bootstrap config")
 	}
 
 	sstats := &stats.ServerStats{
 		Name: cfg.Name,
-		ID:   idAsHex(id),
+		ID:   id.String(),
 	}
-	lstats := stats.NewLeaderStats(idAsHex(id))
+	lstats := stats.NewLeaderStats(id.String())
 
+	shub := newSendHub(cfg.Transport, cfg.Cluster, sstats, lstats)
 	s := &EtcdServer{
 		store:      st,
 		node:       n,
@@ -253,12 +266,12 @@ func NewServer(cfg *ServerConfig) *EtcdServer {
 		}{w, ss},
 		stats:      sstats,
 		lstats:     lstats,
-		send:       Sender(cfg.Transport, cfg.Cluster, sstats, lstats),
+		sender:     shub,
 		Ticker:     time.Tick(100 * time.Millisecond),
 		SyncTicker: time.Tick(500 * time.Millisecond),
 		snapCount:  cfg.SnapCount,
 	}
-	return s
+	return s, nil
 }
 
 // Start prepares and starts server in a new goroutine. It is no longer safe to
@@ -287,7 +300,7 @@ func (s *EtcdServer) start() {
 }
 
 func (s *EtcdServer) Process(ctx context.Context, m raftpb.Message) error {
-	if s.Cluster.IsIDRemoved(m.From) {
+	if s.Cluster.IsIDRemoved(types.ID(m.From)) {
 		return ErrRemoved
 	}
 	return s.node.Step(ctx, m)
@@ -312,16 +325,19 @@ func (s *EtcdServer) run() {
 				}
 			}
 
-			s.storage.Save(rd.HardState, rd.Entries)
-			s.storage.SaveSnap(rd.Snapshot)
-			s.send(rd.Messages)
+			if err := s.storage.Save(rd.HardState, rd.Entries); err != nil {
+				log.Fatalf("etcdserver: save state and entries error: %v", err)
+			}
+			if err := s.storage.SaveSnap(rd.Snapshot); err != nil {
+				log.Fatalf("etcdserver: create snapshot error: %v", err)
+			}
+			s.sender.Send(rd.Messages)
 
 			// TODO(bmizerany): do this in the background, but take
 			// care to apply entries in a single goroutine, and not
 			// race them.
-			// TODO: apply configuration change into ClusterStore.
 			if len(rd.CommittedEntries) != 0 {
-				appliedi = s.apply(rd.CommittedEntries, nodes)
+				appliedi = s.apply(rd.CommittedEntries)
 			}
 
 			if rd.Snapshot.Index > snapi {
@@ -331,11 +347,13 @@ func (s *EtcdServer) run() {
 			// recover from snapshot if it is more updated than current applied
 			if rd.Snapshot.Index > appliedi {
 				if err := s.store.Recovery(rd.Snapshot.Data); err != nil {
-					panic("TODO: this is bad, what do we do about it?")
+					log.Panicf("recovery store error: %v", err)
 				}
 				appliedi = rd.Snapshot.Index
 			}
 
+			s.node.Advance()
+
 			if appliedi-snapi > s.snapCount {
 				s.snapshot(appliedi, nodes)
 				snapi = appliedi
@@ -355,6 +373,7 @@ func (s *EtcdServer) Stop() {
 	s.node.Stop()
 	close(s.done)
 	<-s.stopped
+	s.sender.Stop()
 }
 
 // Do interprets r and performs an operation on s.store according to r.Method
@@ -364,7 +383,7 @@ func (s *EtcdServer) Stop() {
 // an error.
 func (s *EtcdServer) Do(ctx context.Context, r pb.Request) (Response, error) {
 	if r.ID == 0 {
-		panic("r.ID cannot be 0")
+		log.Panicf("request ID should never be 0")
 	}
 	if r.Method == "GET" && r.Quorum {
 		r.Method = "QGET"
@@ -383,7 +402,7 @@ func (s *EtcdServer) Do(ctx context.Context, r pb.Request) (Response, error) {
 			return resp, resp.err
 		case <-ctx.Done():
 			s.w.Trigger(r.ID, nil) // GC wait
-			return Response{}, ctx.Err()
+			return Response{}, parseCtxErr(ctx.Err())
 		case <-s.done:
 			return Response{}, ErrStopped
 		}
@@ -402,6 +421,12 @@ func (s *EtcdServer) Do(ctx context.Context, r pb.Request) (Response, error) {
 			}
 			return Response{Event: ev}, nil
 		}
+	case "HEAD":
+		ev, err := s.store.Get(r.Path, r.Recursive, r.Sorted)
+		if err != nil {
+			return Response{}, err
+		}
+		return Response{Event: ev}, nil
 	default:
 		return Response{}, ErrUnknownMethod
 	}
@@ -420,8 +445,8 @@ func (s *EtcdServer) StoreStats() []byte {
 	return s.store.JsonStats()
 }
 
-func (s *EtcdServer) UpdateRecvApp(from uint64, length int64) {
-	s.stats.RecvAppendReq(idAsHex(from), int(length))
+func (s *EtcdServer) UpdateRecvApp(from types.ID, length int64) {
+	s.stats.RecvAppendReq(from.String(), int(length))
 }
 
 func (s *EtcdServer) AddMember(ctx context.Context, memb Member) error {
@@ -433,7 +458,7 @@ func (s *EtcdServer) AddMember(ctx context.Context, memb Member) error {
 	cc := raftpb.ConfChange{
 		ID:      GenID(),
 		Type:    raftpb.ConfChangeAddNode,
-		NodeID:  memb.ID,
+		NodeID:  uint64(memb.ID),
 		Context: b,
 	}
 	return s.configure(ctx, cc)
@@ -457,12 +482,12 @@ func (s *EtcdServer) Term() uint64 {
 	return atomic.LoadUint64(&s.raftTerm)
 }
 
-// configure sends configuration change through consensus then performs it.
-// It will block until the change is performed or there is an error.
+// configure sends a configuration change through consensus and
+// then waits for it to be applied to the server. It
+// will block until the change is performed or there is an error.
 func (s *EtcdServer) configure(ctx context.Context, cc raftpb.ConfChange) error {
 	ch := s.w.Register(cc.ID)
 	if err := s.node.ProposeConfChange(ctx, cc); err != nil {
-		log.Printf("configure error: %v", err)
 		s.w.Trigger(cc.ID, nil)
 		return err
 	}
@@ -472,12 +497,12 @@ func (s *EtcdServer) configure(ctx context.Context, cc raftpb.ConfChange) error
 			return err
 		}
 		if x != nil {
-			log.Panicf("unexpected return type")
+			log.Panicf("return type should always be error")
 		}
 		return nil
 	case <-ctx.Done():
 		s.w.Trigger(cc.ID, nil) // GC wait
-		return ctx.Err()
+		return parseCtxErr(ctx.Err())
 	case <-s.done:
 		return ErrStopped
 	}
@@ -526,7 +551,7 @@ func (s *EtcdServer) publish(retryInterval time.Duration) {
 		cancel()
 		switch err {
 		case nil:
-			log.Printf("etcdserver: published %+v to the cluster", s.attributes)
+			log.Printf("etcdserver: published %+v to cluster %s", s.attributes, s.Cluster.ID())
 			return
 		case ErrStopped:
 			log.Printf("etcdserver: aborting publish because server is stopped")
@@ -545,7 +570,9 @@ func getExpirationTime(r *pb.Request) time.Time {
 	return t
 }
 
-func (s *EtcdServer) apply(es []raftpb.Entry, nodes []uint64) uint64 {
+// apply takes an Entry received from Raft (after it has been committed) and
+// applies it to the current state of the EtcdServer
+func (s *EtcdServer) apply(es []raftpb.Entry) uint64 {
 	var applied uint64
 	for i := range es {
 		e := es[i]
@@ -557,9 +584,9 @@ func (s *EtcdServer) apply(es []raftpb.Entry, nodes []uint64) uint64 {
 		case raftpb.EntryConfChange:
 			var cc raftpb.ConfChange
 			pbutil.MustUnmarshal(&cc, e.Data)
-			s.w.Trigger(cc.ID, s.applyConfChange(cc, nodes))
+			s.w.Trigger(cc.ID, s.applyConfChange(cc))
 		default:
-			panic("unexpected entry type")
+			log.Panicf("entry type should be either EntryNormal or EntryConfChange")
 		}
 		atomic.StoreUint64(&s.raftIndex, e.Index)
 		atomic.StoreUint64(&s.raftTerm, e.Term)
@@ -590,14 +617,12 @@ func (s *EtcdServer) applyRequest(r pb.Request) Response {
 			return f(s.store.CompareAndSwap(r.Path, r.PrevValue, r.PrevIndex, r.Val, expr))
 		default:
 			if storeMemberAttributeRegexp.MatchString(r.Path) {
-				id := parseMemberID(path.Dir(r.Path))
-				m := s.Cluster.Member(id)
-				if m == nil {
-					log.Fatalf("fetch member %x should never fail", id)
-				}
-				if err := json.Unmarshal([]byte(r.Val), &m.Attributes); err != nil {
-					log.Fatalf("unmarshal %s should never fail: %v", r.Val, err)
+				id := mustParseMemberIDFromKey(path.Dir(r.Path))
+				var attr Attributes
+				if err := json.Unmarshal([]byte(r.Val), &attr); err != nil {
+					log.Panicf("unmarshal %s should never fail: %v", r.Val, err)
 				}
+				s.Cluster.UpdateMemberAttributes(id, attr)
 			}
 			return f(s.store.Set(r.Path, r.Dir, r.Val, expr))
 		}
@@ -619,8 +644,10 @@ func (s *EtcdServer) applyRequest(r pb.Request) Response {
 	}
 }
 
-func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange, nodes []uint64) error {
-	if err := s.checkConfChange(cc, nodes); err != nil {
+// applyConfChange applies a ConfChange to the server. It is only
+// invoked with a ConfChange that has already passed through Raft
+func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange) error {
+	if err := s.Cluster.ValidateConfigurationChange(cc); err != nil {
 		cc.NodeID = raft.None
 		s.node.ApplyConfChange(cc)
 		return err
@@ -630,33 +657,19 @@ func (s *EtcdServer) applyConfChange(cc raftpb.ConfChange, nodes []uint64) error
 	case raftpb.ConfChangeAddNode:
 		m := new(Member)
 		if err := json.Unmarshal(cc.Context, m); err != nil {
-			panic("unexpected unmarshal error")
+			log.Panicf("unmarshal member should never fail: %v", err)
 		}
-		if cc.NodeID != m.ID {
-			panic("unexpected nodeID mismatch")
+		if cc.NodeID != uint64(m.ID) {
+			log.Panicf("nodeID should always be equal to member ID")
 		}
 		s.Cluster.AddMember(m)
+		s.sender.Add(m)
+		log.Printf("etcdserver: added node %s %v to cluster %s", types.ID(cc.NodeID), m.PeerURLs, s.Cluster.ID())
 	case raftpb.ConfChangeRemoveNode:
-		s.Cluster.RemoveMember(cc.NodeID)
-	}
-	return nil
-}
-
-func (s *EtcdServer) checkConfChange(cc raftpb.ConfChange, nodes []uint64) error {
-	if s.Cluster.IsIDRemoved(cc.NodeID) {
-		return ErrIDRemoved
-	}
-	switch cc.Type {
-	case raftpb.ConfChangeAddNode:
-		if containsUint64(nodes, cc.NodeID) {
-			return ErrIDExists
-		}
-	case raftpb.ConfChangeRemoveNode:
-		if !containsUint64(nodes, cc.NodeID) {
-			return ErrIDNotFound
-		}
-	default:
-		panic("unexpected ConfChange type")
+		id := types.ID(cc.NodeID)
+		s.Cluster.RemoveMember(id)
+		s.sender.Remove(id)
+		log.Printf("etcdserver: removed node %s from cluster %s", id, s.Cluster.ID())
 	}
 	return nil
 }
@@ -667,32 +680,39 @@ func (s *EtcdServer) snapshot(snapi uint64, snapnodes []uint64) {
 	// TODO: current store will never fail to do a snapshot
 	// what should we do if the store might fail?
 	if err != nil {
-		panic("TODO: this is bad, what do we do about it?")
+		log.Panicf("store save should never fail: %v", err)
 	}
 	s.node.Compact(snapi, snapnodes, d)
-	s.storage.Cut()
+	if err := s.storage.Cut(); err != nil {
+		log.Panicf("rotate wal file should never fail: %v", err)
+	}
 }
 
+// GetClusterFromPeers takes a set of URLs representing etcd peers, and
+// attempts to construct a Cluster by accessing the members endpoint on one of
+// these URLs. The first URL to provide a response is used. If no URLs provide
+// a response, or a Cluster cannot be successfully created from a received
+// response, an error is returned.
 func GetClusterFromPeers(urls []string) (*Cluster, error) {
 	for _, u := range urls {
 		resp, err := http.Get(u + "/members")
 		if err != nil {
-			log.Printf("etcdserver: get /members on %s: %v", u, err)
+			log.Printf("etcdserver: could not get cluster response from %s: %v", u, err)
 			continue
 		}
 		b, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			log.Printf("etcdserver: read body error: %v", err)
+			log.Printf("etcdserver: could not read the body of cluster response: %v", err)
 			continue
 		}
 		var membs []*Member
 		if err := json.Unmarshal(b, &membs); err != nil {
-			log.Printf("etcdserver: unmarshal body error: %v", err)
+			log.Printf("etcdserver: could not unmarshal cluster response: %v", err)
 			continue
 		}
-		id, err := strconv.ParseUint(resp.Header.Get("X-Etcd-Cluster-ID"), 16, 64)
+		id, err := types.IDFromString(resp.Header.Get("X-Etcd-Cluster-ID"))
 		if err != nil {
-			log.Printf("etcdserver: parse uint error: %v", err)
+			log.Printf("etcdserver: could not parse the cluster ID from cluster res: %v", err)
 			continue
 		}
 		return NewClusterFromMembers("", id, membs), nil
@@ -700,46 +720,61 @@ func GetClusterFromPeers(urls []string) (*Cluster, error) {
 	return nil, fmt.Errorf("etcdserver: could not retrieve cluster information from the given urls")
 }
 
-func startNode(cfg *ServerConfig, ids []uint64) (id uint64, n raft.Node, w *wal.WAL) {
+func startNode(cfg *ServerConfig, ids []types.ID) (id types.ID, n raft.Node, w *wal.WAL) {
 	var err error
 	// TODO: remove the discoveryURL when it becomes part of the source for
 	// generating nodeID.
 	member := cfg.Cluster.MemberByName(cfg.Name)
-	metadata := pbutil.MustMarshal(&pb.Metadata{NodeID: member.ID, ClusterID: cfg.Cluster.ID()})
+	metadata := pbutil.MustMarshal(
+		&pb.Metadata{
+			NodeID:    uint64(member.ID),
+			ClusterID: uint64(cfg.Cluster.ID()),
+		},
+	)
 	if w, err = wal.Create(cfg.WALDir(), metadata); err != nil {
-		log.Fatal(err)
+		log.Fatalf("etcdserver: create wal error: %v", err)
 	}
 	peers := make([]raft.Peer, len(ids))
 	for i, id := range ids {
 		ctx, err := json.Marshal((*cfg.Cluster).Member(id))
 		if err != nil {
-			log.Fatal(err)
+			log.Panicf("marshal member should never fail: %v", err)
 		}
-		peers[i] = raft.Peer{ID: id, Context: ctx}
+		peers[i] = raft.Peer{ID: uint64(id), Context: ctx}
 	}
 	id = member.ID
-	log.Printf("etcdserver: start node %x in cluster %x", id, cfg.Cluster.ID())
-	n = raft.StartNode(id, peers, 10, 1)
+	log.Printf("etcdserver: start node %s in cluster %s", id, cfg.Cluster.ID())
+	n = raft.StartNode(uint64(id), peers, 10, 1)
 	return
 }
 
-func restartNode(cfg *ServerConfig, index uint64, snapshot *raftpb.Snapshot) (id uint64, n raft.Node, w *wal.WAL) {
+func restartNode(cfg *ServerConfig, index uint64, snapshot *raftpb.Snapshot) (id types.ID, n raft.Node, w *wal.WAL) {
 	var err error
 	// restart a node from previous wal
 	if w, err = wal.OpenAtIndex(cfg.WALDir(), index); err != nil {
-		log.Fatal(err)
+		log.Fatalf("etcdserver: open wal error: %v", err)
 	}
-	wmetadata, st, ents, err := w.ReadAll()
+	id, clusterID, st, ents, err := readWAL(w, index)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("etcdserver: read wal error: %v", err)
+	}
+	cfg.Cluster.SetID(clusterID)
+	log.Printf("etcdserver: restart member %s in cluster %s at commit index %d", id, cfg.Cluster.ID(), st.Commit)
+	n = raft.RestartNode(uint64(id), 10, 1, snapshot, st, ents)
+	return
+}
+
+func readWAL(w *wal.WAL, index uint64) (id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry, err error) {
+	var wmetadata []byte
+	wmetadata, st, ents, err = w.ReadAll()
+	if err != nil {
+		return
 	}
 
 	var metadata pb.Metadata
 	pbutil.MustUnmarshal(&metadata, wmetadata)
-	id = metadata.NodeID
-	cfg.Cluster.SetID(metadata.ClusterID)
-	log.Printf("etcdserver: restart member %x in cluster %x at commit index %d", id, cfg.Cluster.ID(), st.Commit)
-	n = raft.RestartNode(id, 10, 1, snapshot, st, ents)
+	id = types.ID(metadata.NodeID)
+	cid = types.ID(metadata.ClusterID)
 	return
 }
 
@@ -752,6 +787,17 @@ func GenID() (n uint64) {
 	return
 }
 
+func parseCtxErr(err error) error {
+	switch err {
+	case context.Canceled:
+		return ErrCanceled
+	case context.DeadlineExceeded:
+		return ErrTimeout
+	default:
+		return err
+	}
+}
+
 func getBool(v *bool) (vv bool, set bool) {
 	if v == nil {
 		return false, false
@@ -767,7 +813,3 @@ func containsUint64(a []uint64, x uint64) bool {
 	}
 	return false
 }
-
-func idAsHex(id uint64) string {
-	return strconv.FormatUint(id, 16)
-}

etcdserver/server_test.go

@@ -19,6 +19,8 @@ package etcdserver
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"log"
 	"math/rand"
 	"path"
 	"reflect"
@@ -29,12 +31,17 @@ import (
 
 	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
-	"github.com/coreos/etcd/pkg"
+	"github.com/coreos/etcd/pkg/testutil"
+	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft"
 	"github.com/coreos/etcd/raft/raftpb"
 	"github.com/coreos/etcd/store"
 )
 
+func init() {
+	log.SetOutput(ioutil.Discard)
+}
+
 func TestGetExpirationTime(t *testing.T) {
 	tests := []struct {
 		r    pb.Request
@@ -86,6 +93,16 @@ func TestDoLocalAction(t *testing.T) {
 				},
 			},
 		},
+		{
+			pb.Request{Method: "HEAD", ID: 1},
+			Response{Event: &store.Event{}}, nil,
+			[]action{
+				action{
+					name:   "Get",
+					params: []interface{}{"", false, false},
+				},
+			},
+		},
 		{
 			pb.Request{Method: "BADMETHOD", ID: 1},
 			Response{}, ErrUnknownMethod, []action{},
@@ -126,6 +143,10 @@ func TestDoBadLocalAction(t *testing.T) {
 			pb.Request{Method: "GET", ID: 1},
 			[]action{action{name: "Get"}},
 		},
+		{
+			pb.Request{Method: "HEAD", ID: 1},
+			[]action{action{name: "Get"}},
+		},
 	}
 	for i, tt := range tests {
 		st := &errStoreRecorder{err: storeErr}
@@ -406,8 +427,13 @@ func TestApplyRequestOnAdminMemberAttributes(t *testing.T) {
 
 // TODO: test ErrIDRemoved
 func TestApplyConfChangeError(t *testing.T) {
-	nodes := []uint64{1, 2, 3}
-	removed := map[uint64]bool{4: true}
+	cl := newCluster("")
+	cl.SetStore(store.New())
+	for i := 1; i <= 4; i++ {
+		cl.AddMember(&Member{ID: types.ID(i)})
+	}
+	cl.RemoveMember(4)
+
 	tests := []struct {
 		cc   raftpb.ConfChange
 		werr error
@@ -443,12 +469,11 @@ func TestApplyConfChangeError(t *testing.T) {
 	}
 	for i, tt := range tests {
 		n := &nodeRecorder{}
-		cl := &Cluster{removed: removed}
 		srv := &EtcdServer{
 			node:    n,
 			Cluster: cl,
 		}
-		err := srv.applyConfChange(tt.cc, nodes)
+		err := srv.applyConfChange(tt.cc)
 		if err != tt.werr {
 			t.Errorf("#%d: applyConfChange error = %v, want %v", i, err, tt.werr)
 		}
@@ -468,19 +493,25 @@ func TestApplyConfChangeError(t *testing.T) {
 func TestClusterOf1(t *testing.T) { testServer(t, 1) }
 func TestClusterOf3(t *testing.T) { testServer(t, 3) }
 
+type fakeSender struct {
+	ss []*EtcdServer
+}
+
+func (s *fakeSender) Send(msgs []raftpb.Message) {
+	for _, m := range msgs {
+		s.ss[m.To-1].node.Step(context.TODO(), m)
+	}
+}
+func (s *fakeSender) Add(m *Member)      {}
+func (s *fakeSender) Remove(id types.ID) {}
+func (s *fakeSender) Stop()              {}
+
 func testServer(t *testing.T, ns uint64) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	ss := make([]*EtcdServer, ns)
 
-	send := func(msgs []raftpb.Message) {
-		for _, m := range msgs {
-			t.Logf("m = %+v\n", m)
-			ss[m.To-1].node.Step(ctx, m)
-		}
-	}
-
 	ids := make([]uint64, ns)
 	for i := uint64(0); i < ns; i++ {
 		ids[i] = i + 1
@@ -491,12 +522,13 @@ func testServer(t *testing.T, ns uint64) {
 		n := raft.StartNode(id, members, 10, 1)
 		tk := time.NewTicker(10 * time.Millisecond)
 		defer tk.Stop()
+		st := store.New()
 		cl := newCluster("abc")
-		cl.SetStore(&storeRecorder{})
+		cl.SetStore(st)
 		srv := &EtcdServer{
 			node:    n,
-			store:   store.New(),
-			send:    send,
+			store:   st,
+			sender:  &fakeSender{ss},
 			storage: &storageRecorder{},
 			Ticker:  tk.C,
 			Cluster: cl,
@@ -521,8 +553,8 @@ func testServer(t *testing.T, ns uint64) {
 
 		g, w := resp.Event.Node, &store.NodeExtern{
 			Key:           "/foo",
-			ModifiedIndex: uint64(i),
-			CreatedIndex:  uint64(i),
+			ModifiedIndex: uint64(i) + 2*ns,
+			CreatedIndex:  uint64(i) + 2*ns,
 			Value:         stringp("bar"),
 		}
 
@@ -561,11 +593,11 @@ func TestDoProposal(t *testing.T) {
 		// this makes <-tk always successful, which accelerates internal clock
 		close(tk)
 		cl := newCluster("abc")
-		cl.SetStore(&storeRecorder{})
+		cl.SetStore(store.New())
 		srv := &EtcdServer{
 			node:    n,
 			store:   st,
-			send:    func(_ []raftpb.Message) {},
+			sender:  &nopSender{},
 			storage: &storageRecorder{},
 			Ticker:  tk,
 			Cluster: cl,
@@ -614,8 +646,8 @@ func TestDoProposalCancelled(t *testing.T) {
 	if len(gaction) != 0 {
 		t.Errorf("len(action) = %v, want 0", len(gaction))
 	}
-	if err != context.Canceled {
-		t.Fatalf("err = %v, want %v", err, context.Canceled)
+	if err != ErrCanceled {
+		t.Fatalf("err = %v, want %v", err, ErrCanceled)
 	}
 	w := []action{action{name: "Register1"}, action{name: "Trigger1"}}
 	if !reflect.DeepEqual(wait.action, w) {
@@ -623,6 +655,18 @@ func TestDoProposalCancelled(t *testing.T) {
 	}
 }
 
+func TestDoProposalTimeout(t *testing.T) {
+	ctx, _ := context.WithTimeout(context.Background(), 0)
+	srv := &EtcdServer{
+		node: &nodeRecorder{},
+		w:    &waitRecorder{},
+	}
+	_, err := srv.Do(ctx, pb.Request{Method: "PUT", ID: 1})
+	if err != ErrTimeout {
+		t.Fatalf("err = %v, want %v", err, ErrTimeout)
+	}
+}
+
 func TestDoProposalStopped(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -636,7 +680,7 @@ func TestDoProposalStopped(t *testing.T) {
 		// TODO: use fake node for better testability
 		node:    n,
 		store:   st,
-		send:    func(_ []raftpb.Message) {},
+		sender:  &nopSender{},
 		storage: &storageRecorder{},
 		Ticker:  tk,
 	}
@@ -666,15 +710,20 @@ func TestSync(t *testing.T) {
 	srv := &EtcdServer{
 		node: n,
 	}
-	start := time.Now()
-	srv.sync(defaultSyncTimeout)
+	done := make(chan struct{})
+	go func() {
+		srv.sync(10 * time.Second)
+		close(done)
+	}()
 
 	// check that sync is non-blocking
-	if d := time.Since(start); d > time.Millisecond {
-		t.Errorf("CallSyncTime = %v, want < %v", d, time.Millisecond)
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatalf("sync should be non-blocking but did not return after 1s!")
 	}
 
-	pkg.ForceGosched()
+	testutil.ForceGosched()
 	data := n.data()
 	if len(data) != 1 {
 		t.Fatalf("len(proposeData) = %d, want 1", len(data))
@@ -695,17 +744,22 @@ func TestSyncTimeout(t *testing.T) {
 	srv := &EtcdServer{
 		node: n,
 	}
-	start := time.Now()
-	srv.sync(0)
+	done := make(chan struct{})
+	go func() {
+		srv.sync(0)
+		close(done)
+	}()
 
 	// check that sync is non-blocking
-	if d := time.Since(start); d > time.Millisecond {
-		t.Errorf("CallSyncTime = %v, want < %v", d, time.Millisecond)
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatalf("sync should be non-blocking but did not return after 1s!")
 	}
 
 	// give time for goroutine in sync to cancel
 	// TODO: use fake clock
-	pkg.ForceGosched()
+	testutil.ForceGosched()
 	w := []action{action{name: "Propose blocked"}}
 	if g := n.Action(); !reflect.DeepEqual(g, w) {
 		t.Errorf("action = %v, want %v", g, w)
@@ -736,7 +790,7 @@ func TestSyncTrigger(t *testing.T) {
 	srv := &EtcdServer{
 		node:       n,
 		store:      &storeRecorder{},
-		send:       func(_ []raftpb.Message) {},
+		sender:     &nopSender{},
 		storage:    &storageRecorder{},
 		SyncTicker: st,
 	}
@@ -801,17 +855,20 @@ func TestTriggerSnap(t *testing.T) {
 	ctx := context.Background()
 	n := raft.StartNode(0xBAD0, mustMakePeerSlice(t, 0xBAD0), 10, 1)
 	<-n.Ready()
+	n.Advance()
 	n.ApplyConfChange(raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 0xBAD0})
 	n.Campaign(ctx)
 	st := &storeRecorder{}
 	p := &storageRecorder{}
+	cl := newCluster("abc")
+	cl.SetStore(store.New())
 	s := &EtcdServer{
 		store:     st,
-		send:      func(_ []raftpb.Message) {},
+		sender:    &nopSender{},
 		storage:   p,
 		node:      n,
 		snapCount: 10,
-		Cluster:   &Cluster{},
+		Cluster:   cl,
 	}
 
 	s.start()
@@ -841,7 +898,7 @@ func TestRecvSnapshot(t *testing.T) {
 	p := &storageRecorder{}
 	s := &EtcdServer{
 		store:   st,
-		send:    func(_ []raftpb.Message) {},
+		sender:  &nopSender{},
 		storage: p,
 		node:    n,
 	}
@@ -849,7 +906,7 @@ func TestRecvSnapshot(t *testing.T) {
 	s.start()
 	n.readyc <- raft.Ready{Snapshot: raftpb.Snapshot{Index: 1}}
 	// make goroutines move forward to receive snapshot
-	pkg.ForceGosched()
+	testutil.ForceGosched()
 	s.Stop()
 
 	wactions := []action{action{name: "Recovery"}}
@@ -869,7 +926,7 @@ func TestRecvSlowSnapshot(t *testing.T) {
 	st := &storeRecorder{}
 	s := &EtcdServer{
 		store:   st,
-		send:    func(_ []raftpb.Message) {},
+		sender:  &nopSender{},
 		storage: &storageRecorder{},
 		node:    n,
 	}
@@ -877,12 +934,12 @@ func TestRecvSlowSnapshot(t *testing.T) {
 	s.start()
 	n.readyc <- raft.Ready{Snapshot: raftpb.Snapshot{Index: 1}}
 	// make goroutines move forward to receive snapshot
-	pkg.ForceGosched()
+	testutil.ForceGosched()
 	action := st.Action()
 
 	n.readyc <- raft.Ready{Snapshot: raftpb.Snapshot{Index: 1}}
 	// make goroutines move forward to receive snapshot
-	pkg.ForceGosched()
+	testutil.ForceGosched()
 	s.Stop()
 
 	if g := st.Action(); !reflect.DeepEqual(g, action) {
@@ -900,11 +957,11 @@ func TestAddMember(t *testing.T) {
 		},
 	}
 	cl := newTestCluster(nil)
-	cl.SetStore(&storeRecorder{})
+	cl.SetStore(store.New())
 	s := &EtcdServer{
 		node:    n,
 		store:   &storeRecorder{},
-		send:    func(_ []raftpb.Message) {},
+		sender:  &nopSender{},
 		storage: &storageRecorder{},
 		Cluster: cl,
 	}
@@ -936,11 +993,10 @@ func TestRemoveMember(t *testing.T) {
 		},
 	}
 	cl := newTestCluster([]Member{{ID: 1234}})
-	cl.SetStore(&storeRecorder{})
 	s := &EtcdServer{
 		node:    n,
 		store:   &storeRecorder{},
-		send:    func(_ []raftpb.Message) {},
+		sender:  &nopSender{},
 		storage: &storageRecorder{},
 		Cluster: cl,
 	}
@@ -974,6 +1030,7 @@ func TestPublish(t *testing.T) {
 	srv := &EtcdServer{
 		id:         1,
 		attributes: Attributes{Name: "node1", ClientURLs: []string{"http://a", "http://b"}},
+		Cluster:    &Cluster{},
 		node:       n,
 		w:          w,
 	}
@@ -1007,6 +1064,8 @@ func TestPublish(t *testing.T) {
 func TestPublishStopped(t *testing.T) {
 	srv := &EtcdServer{
 		node:    &nodeRecorder{},
+		sender:  &nopSender{},
+		Cluster: &Cluster{},
 		w:       &waitRecorder{},
 		done:    make(chan struct{}),
 		stopped: make(chan struct{}),
@@ -1207,18 +1266,19 @@ type storageRecorder struct {
 	recorder
 }
 
-func (p *storageRecorder) Save(st raftpb.HardState, ents []raftpb.Entry) {
+func (p *storageRecorder) Save(st raftpb.HardState, ents []raftpb.Entry) error {
 	p.record(action{name: "Save"})
+	return nil
 }
 func (p *storageRecorder) Cut() error {
 	p.record(action{name: "Cut"})
 	return nil
 }
-func (p *storageRecorder) SaveSnap(st raftpb.Snapshot) {
-	if raft.IsEmptySnap(st) {
-		return
+func (p *storageRecorder) SaveSnap(st raftpb.Snapshot) error {
+	if !raft.IsEmptySnap(st) {
+		p.record(action{name: "SaveSnap"})
 	}
-	p.record(action{name: "SaveSnap"})
+	return nil
 }
 
 type readyNode struct {
@@ -1237,6 +1297,7 @@ func (n *readyNode) ProposeConfChange(ctx context.Context, conf raftpb.ConfChang
 }
 func (n *readyNode) Step(ctx context.Context, msg raftpb.Message) error { return nil }
 func (n *readyNode) Ready() <-chan raft.Ready                           { return n.readyc }
+func (n *readyNode) Advance()                                           {}
 func (n *readyNode) ApplyConfChange(conf raftpb.ConfChange)             {}
 func (n *readyNode) Stop()                                              {}
 func (n *readyNode) Compact(index uint64, nodes []uint64, d []byte)     {}
@@ -1245,9 +1306,8 @@ type nodeRecorder struct {
 	recorder
 }
 
-func (n *nodeRecorder) Tick() {
-	n.record(action{name: "Tick"})
-}
+func (n *nodeRecorder) Tick() { n.record(action{name: "Tick"}) }
+
 func (n *nodeRecorder) Campaign(ctx context.Context) error {
 	n.record(action{name: "Campaign"})
 	return nil
@@ -1265,6 +1325,7 @@ func (n *nodeRecorder) Step(ctx context.Context, msg raftpb.Message) error {
 	return nil
 }
 func (n *nodeRecorder) Ready() <-chan raft.Ready { return nil }
+func (n *nodeRecorder) Advance()                 {}
 func (n *nodeRecorder) ApplyConfChange(conf raftpb.ConfChange) {
 	n.record(action{name: "ApplyConfChange", params: []interface{}{conf}})
 }
@@ -1339,35 +1400,17 @@ func (w *waitWithResponse) Register(id uint64) <-chan interface{} {
 }
 func (w *waitWithResponse) Trigger(id uint64, x interface{}) {}
 
-type clusterStoreRecorder struct {
-	recorder
-}
+type nopSender struct{}
 
-func (cs *clusterStoreRecorder) Add(m Member) {
-	cs.record(action{name: "Add", params: []interface{}{m}})
-}
-func (cs *clusterStoreRecorder) Get() Cluster {
-	cs.record(action{name: "Get"})
-	return Cluster{}
-}
-func (cs *clusterStoreRecorder) Remove(id uint64) {
-	cs.record(action{name: "Remove", params: []interface{}{id}})
-}
-func (cs *clusterStoreRecorder) IsRemoved(id uint64) bool { return false }
-
-type removedClusterStore struct {
-	removed map[uint64]bool
-}
-
-func (cs *removedClusterStore) Add(m Member)             {}
-func (cs *removedClusterStore) Get() Cluster             { return Cluster{} }
-func (cs *removedClusterStore) Remove(id uint64)         {}
-func (cs *removedClusterStore) IsRemoved(id uint64) bool { return cs.removed[id] }
+func (s *nopSender) Send(m []raftpb.Message) {}
+func (s *nopSender) Add(m *Member)           {}
+func (s *nopSender) Remove(id types.ID)      {}
+func (s *nopSender) Stop()                   {}
 
 func mustMakePeerSlice(t *testing.T, ids ...uint64) []raft.Peer {
 	peers := make([]raft.Peer, len(ids))
 	for i, id := range ids {
-		m := Member{ID: id}
+		m := Member{ID: types.ID(id)}
 		b, err := json.Marshal(m)
 		if err != nil {
 			t.Fatal(err)

etcdserver/stats/leader.go

@@ -49,7 +49,7 @@ func (ls *LeaderStats) JSON() []byte {
 	b, err := json.Marshal(stats)
 	// TODO(jonboulle): appropriate error handling?
 	if err != nil {
-		log.Printf("error marshalling leader stats: %v", err)
+		log.Printf("stats: error marshalling leader stats: %v", err)
 	}
 	return b
 }

etcdserver/stats/server.go

@@ -64,7 +64,7 @@ func (ss *ServerStats) JSON() []byte {
 	b, err := json.Marshal(stats)
 	// TODO(jonboulle): appropriate error handling?
 	if err != nil {
-		log.Printf("error marshalling server stats: %v", err)
+		log.Printf("stats: error marshalling server stats: %v", err)
 	}
 	return b
 }

hack/insta-discovery/Procfile

@@ -1,4 +1,6 @@
 # Use goreman to run `go get github.com/mattn/goreman`
-etcd1: ../../bin/etcd -name node1 -listen-client-urls http://127.0.0.1:4001 -advertise-client-urls http://127.0.0.1:4001 -listen-peer-urls http://127.0.0.1:7001 -initial-advertise-peer-urls http://127.0.0.1:7001
-etcd2: ../../bin/etcd -name node2 -listen-client-urls http://127.0.0.1:4002 -advertise-client-urls http://127.0.0.1:4002 -listen-peer-urls http://127.0.0.1:7002 -initial-advertise-peer-urls http://127.0.0.1:7002
-etcd3: ../../bin/etcd -name node3 -listen-client-urls http://127.0.0.1:4003 -advertise-client-urls http://127.0.0.1:4003 -listen-peer-urls http://127.0.0.1:7003 -initial-advertise-peer-urls http://127.0.0.1:7003
+# One of the four etcd members falls back to a proxy
+etcd1: ../../bin/etcd -name infra1 -listen-client-urls http://127.0.0.1:4001 -advertise-client-urls http://127.0.0.1:4001 -listen-peer-urls http://127.0.0.1:7001 -initial-advertise-peer-urls http://127.0.0.1:7001
+etcd2: ../../bin/etcd -name infra2 -listen-client-urls http://127.0.0.1:4002 -advertise-client-urls http://127.0.0.1:4002 -listen-peer-urls http://127.0.0.1:7002 -initial-advertise-peer-urls http://127.0.0.1:7002
+etcd3: ../../bin/etcd -name infra3 -listen-client-urls http://127.0.0.1:4003 -advertise-client-urls http://127.0.0.1:4003 -listen-peer-urls http://127.0.0.1:7003 -initial-advertise-peer-urls http://127.0.0.1:7003
+etcd4: ../../bin/etcd -name infra4 -listen-client-urls http://127.0.0.1:4004 -advertise-client-urls http://127.0.0.1:4004 -listen-peer-urls http://127.0.0.1:7004 -initial-advertise-peer-urls http://127.0.0.1:7004

hack/insta-discovery/discovery

@@ -1,6 +1,7 @@
 #!/bin/sh
 
-disc=$(curl https://discovery.etcd.io/new?size=3)
+rm -rf infra*.etcd
+disc=$(curl -s https://discovery.etcd.io/new?size=3)
 echo ETCD_DISCOVERY=${disc} > .env
 echo "setup discovery start your cluster"
 cat .env

integration/cluster_test.go

@@ -1,3 +1,19 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
 package integration
 
 import (
@@ -7,21 +23,23 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
-	"net/url"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/coreos/etcd/client"
 	"github.com/coreos/etcd/etcdserver"
 	"github.com/coreos/etcd/etcdserver/etcdhttp"
-	"github.com/coreos/etcd/pkg/transport"
 	"github.com/coreos/etcd/pkg/types"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/code.google.com/p/go.net/context"
 )
 
 const (
-	tickDuration = 5 * time.Millisecond
-	clusterName  = "etcd"
+	tickDuration   = 10 * time.Millisecond
+	clusterName    = "etcd"
+	requestTimeout = 2 * time.Second
 )
 
 func init() {
@@ -36,41 +54,27 @@ func testCluster(t *testing.T, size int) {
 	defer afterTest(t)
 	c := &cluster{Size: size}
 	c.Launch(t)
+	defer c.Terminate(t)
 	for i := 0; i < size; i++ {
-		for _, u := range c.Members[i].ClientURLs {
+		for j, u := range c.Members[i].ClientURLs {
+			cc := mustNewHTTPClient(t, []string{u.String()})
+			kapi := client.NewKeysAPI(cc)
+			// TODO: we retry it here because MsgProp may be dropped due to
+			// sender reaches its max serving. make it reliable that we don't
+			// need to worry about it.
 			var err error
-			for j := 0; j < 3; j++ {
-				if err = setKey(u, "/foo", "bar"); err == nil {
+			for k := 0; k < 3; k++ {
+				ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
+				if _, err = kapi.Create(ctx, fmt.Sprintf("/%d%d%d", i, j, k), "bar", -1); err == nil {
 					break
 				}
+				cancel()
 			}
 			if err != nil {
-				t.Errorf("setKey on %v error: %v", u.String(), err)
+				t.Errorf("create on %s error: %v", u.String(), err)
 			}
 		}
 	}
-	c.Terminate(t)
-}
-
-// TODO: use etcd client
-func setKey(u url.URL, key string, value string) error {
-	u.Path = "/v2/keys" + key
-	v := url.Values{"value": []string{value}}
-	req, err := http.NewRequest("PUT", u.String(), strings.NewReader(v.Encode()))
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	ioutil.ReadAll(resp.Body)
-	resp.Body.Close()
-	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
-		return fmt.Errorf("statusCode = %d, want %d or %d", resp.StatusCode, http.StatusOK, http.StatusCreated)
-	}
-	return nil
 }
 
 type cluster struct {
@@ -113,20 +117,18 @@ func (c *cluster) Launch(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		m.ClusterState = etcdserver.ClusterStateValueNew
-		m.Transport, err = transport.NewTransport(transport.TLSInfo{})
-		if err != nil {
-			t.Fatal(err)
-		}
-		// TODO: need the support of graceful stop in Sender to remove this
-		m.Transport.DisableKeepAlives = true
-		m.Transport.Dial = (&net.Dialer{Timeout: 100 * time.Millisecond}).Dial
+		m.NewCluster = true
+		m.Transport = newTransport()
 
 		m.Launch(t)
 		c.Members = append(c.Members, m)
 	}
 }
 
+func (c *cluster) URL(i int) string {
+	return c.Members[i].ClientURLs[0].String()
+}
+
 func (c *cluster) Terminate(t *testing.T) {
 	for _, m := range c.Members {
 		m.Terminate(t)
@@ -156,9 +158,12 @@ type member struct {
 // Launch starts a member based on ServerConfig, PeerListeners
 // and ClientListeners.
 func (m *member) Launch(t *testing.T) {
-	m.s = etcdserver.NewServer(&m.ServerConfig)
+	var err error
+	if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
+		t.Fatalf("failed to initialize the etcd server: %v", err)
+	}
 	m.s.Ticker = time.Tick(tickDuration)
-	m.s.SyncTicker = nil
+	m.s.SyncTicker = time.Tick(10 * tickDuration)
 	m.s.Start()
 
 	for _, ln := range m.PeerListeners {
@@ -184,18 +189,35 @@ func (m *member) Stop(t *testing.T) {
 	panic("unimplemented")
 }
 
-// Start starts the member using preserved data dir.
+// Start starts the member using the preserved data dir.
 func (m *member) Start(t *testing.T) {
 	panic("unimplemented")
 }
 
-// Terminate stops the member and remove the data dir.
+// Terminate stops the member and removes the data dir.
 func (m *member) Terminate(t *testing.T) {
 	m.s.Stop()
 	for _, hs := range m.hss {
+		hs.CloseClientConnections()
 		hs.Close()
 	}
 	if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
 		t.Fatal(err)
 	}
 }
+
+func mustNewHTTPClient(t *testing.T, eps []string) client.HTTPClient {
+	cc, err := client.NewHTTPClient(newTransport(), eps)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cc
+}
+
+func newTransport() *http.Transport {
+	tr := &http.Transport{}
+	// TODO: need the support of graceful stop in Sender to remove this
+	tr.DisableKeepAlives = true
+	tr.Dial = (&net.Dialer{Timeout: 100 * time.Millisecond}).Dial
+	return tr
+}

integration/doc.go

@@ -1,25 +1,27 @@
-// Copyright 2014 CoreOS Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//  http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
 
 /*
-functional tests are built upon embeded etcd, and focus on etcd functional
-correctness.
+Package integration implements tests built upon embedded etcd, and focus on
+etcd correctness.
 
-Its goal:
-1. it tests the whole code base except the command line parse.
-2. it is able to check internal data, including raft, store and etc.
-3. it is based on goroutine, which is faster than process.
-4. it mainly tests user behavior and user-facing API.
+Features/goals of the integration tests:
+1. test the whole code base except command-line parsing.
+2. check internal data, including raft, store and etc.
+3. based on goroutines, which is faster than process.
+4. mainly tests user behavior and user-facing API.
 */
 package integration

main.go

@@ -14,297 +14,21 @@
    limitations under the License.
 */
 
+// Package main is a simple wrapper of the real etcd entrypoint package
+// (located at github.com/coreos/etcd/etcdmain) to ensure that etcd is still
+// "go getable"; e.g. `go get github.com/coreos/etcd` works as expected and
+// builds a binary in $GOBIN/etcd
+//
+// This package should NOT be extended or modified in any way; to modify the
+// etcd binary, work in the `github.com/coreos/etcd/etcdmain` package.
+//
+
 package main
 
 import (
-	"flag"
-	"fmt"
-	"log"
-	"net/http"
-	"os"
-	"strings"
-
-	"github.com/coreos/etcd/etcdserver"
-	"github.com/coreos/etcd/etcdserver/etcdhttp"
-	"github.com/coreos/etcd/pkg"
-	flagtypes "github.com/coreos/etcd/pkg/flags"
-	"github.com/coreos/etcd/pkg/transport"
-	"github.com/coreos/etcd/pkg/types"
-	"github.com/coreos/etcd/proxy"
-	"github.com/coreos/etcd/version"
+	"github.com/coreos/etcd/etcdmain"
 )
 
-const (
-	// the owner can make/remove files inside the directory
-	privateDirMode = 0700
-)
-
-var (
-	fs           = flag.NewFlagSet("etcd", flag.ContinueOnError)
-	name         = fs.String("name", "default", "Unique human-readable name for this node")
-	dir          = fs.String("data-dir", "", "Path to the data directory")
-	durl         = fs.String("discovery", "", "Discovery service used to bootstrap the cluster")
-	snapCount    = fs.Uint64("snapshot-count", etcdserver.DefaultSnapCount, "Number of committed transactions to trigger a snapshot")
-	printVersion = fs.Bool("version", false, "Print the version and exit")
-
-	initialCluster     = fs.String("initial-cluster", "default=http://localhost:2380,default=http://localhost:7001", "Initial cluster configuration for bootstrapping")
-	initialClusterName = fs.String("initial-cluster-name", "etcd", "Initial name for the etcd cluster during bootstrap")
-	clusterState       = new(etcdserver.ClusterState)
-
-	cors      = &pkg.CORSInfo{}
-	proxyFlag = new(flagtypes.Proxy)
-
-	clientTLSInfo = transport.TLSInfo{}
-	peerTLSInfo   = transport.TLSInfo{}
-
-	ignored = []string{
-		"cluster-active-size",
-		"cluster-remove-delay",
-		"cluster-sync-interval",
-		"config",
-		"force",
-		"max-result-buffer",
-		"max-retry-attempts",
-		"peer-heartbeat-interval",
-		"peer-election-timeout",
-		"retry-interval",
-		"snapshot",
-		"v",
-		"vv",
-	}
-)
-
-func init() {
-	fs.Var(clusterState, "initial-cluster-state", "Initial cluster configuration for bootstrapping")
-	clusterState.Set(etcdserver.ClusterStateValueNew)
-
-	fs.Var(flagtypes.NewURLsValue("http://localhost:2380,http://localhost:7001"), "initial-advertise-peer-urls", "List of this member's peer URLs to advertise to the rest of the cluster")
-	fs.Var(flagtypes.NewURLsValue("http://localhost:2379,http://localhost:4001"), "advertise-client-urls", "List of this member's client URLs to advertise to the rest of the cluster")
-	fs.Var(flagtypes.NewURLsValue("http://localhost:2380,http://localhost:7001"), "listen-peer-urls", "List of URLs to listen on for peer traffic")
-	fs.Var(flagtypes.NewURLsValue("http://localhost:2379,http://localhost:4001"), "listen-client-urls", "List of URLs to listen on for client traffic")
-
-	fs.Var(cors, "cors", "Comma-separated white list of origins for CORS (cross-origin resource sharing).")
-
-	fs.Var(proxyFlag, "proxy", fmt.Sprintf("Valid values include %s", strings.Join(flagtypes.ProxyValues, ", ")))
-	proxyFlag.Set(flagtypes.ProxyValueOff)
-
-	fs.StringVar(&clientTLSInfo.CAFile, "ca-file", "", "Path to the client server TLS CA file.")
-	fs.StringVar(&clientTLSInfo.CertFile, "cert-file", "", "Path to the client server TLS cert file.")
-	fs.StringVar(&clientTLSInfo.KeyFile, "key-file", "", "Path to the client server TLS key file.")
-
-	fs.StringVar(&peerTLSInfo.CAFile, "peer-ca-file", "", "Path to the peer server TLS CA file.")
-	fs.StringVar(&peerTLSInfo.CertFile, "peer-cert-file", "", "Path to the peer server TLS cert file.")
-	fs.StringVar(&peerTLSInfo.KeyFile, "peer-key-file", "", "Path to the peer server TLS key file.")
-
-	// backwards-compatibility with v0.4.6
-	fs.Var(&flagtypes.IPAddressPort{}, "addr", "DEPRECATED: Use -advertise-client-urls instead.")
-	fs.Var(&flagtypes.IPAddressPort{}, "bind-addr", "DEPRECATED: Use -listen-client-urls instead.")
-	fs.Var(&flagtypes.IPAddressPort{}, "peer-addr", "DEPRECATED: Use -initial-advertise-peer-urls instead.")
-	fs.Var(&flagtypes.IPAddressPort{}, "peer-bind-addr", "DEPRECATED: Use -listen-peer-urls instead.")
-
-	for _, f := range ignored {
-		fs.Var(&pkg.IgnoredFlag{Name: f}, f, "")
-	}
-
-	fs.Var(&pkg.DeprecatedFlag{Name: "peers"}, "peers", "DEPRECATED: Use -initial-cluster instead")
-	fs.Var(&pkg.DeprecatedFlag{Name: "peers-file"}, "peers-file", "DEPRECATED: Use -initial-cluster instead")
-}
-
 func main() {
-	fs.Usage = pkg.UsageWithIgnoredFlagsFunc(fs, ignored)
-	err := fs.Parse(os.Args[1:])
-	switch err {
-	case nil:
-	case flag.ErrHelp:
-		os.Exit(0)
-	default:
-		os.Exit(2)
-	}
-
-	if *printVersion {
-		fmt.Println("etcd version", version.Version)
-		os.Exit(0)
-	}
-
-	pkg.SetFlagsFromEnv(fs)
-
-	if string(*proxyFlag) == flagtypes.ProxyValueOff {
-		startEtcd()
-	} else {
-		startProxy()
-	}
-
-	// Block indefinitely
-	<-make(chan struct{})
-}
-
-// startEtcd launches the etcd server and HTTP handlers for client/server communication.
-func startEtcd() {
-	cls, err := setupCluster()
-	if err != nil {
-		log.Fatalf("etcd: error setting up initial cluster: %v", err)
-	}
-
-	if *dir == "" {
-		*dir = fmt.Sprintf("%v.etcd", *name)
-		log.Printf("etcd: no data-dir provided, using default data-dir ./%s", *dir)
-	}
-	if err := os.MkdirAll(*dir, privateDirMode); err != nil {
-		log.Fatalf("etcd: cannot create data directory: %v", err)
-	}
-
-	pt, err := transport.NewTransport(peerTLSInfo)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	acurls, err := pkg.URLsFromFlags(fs, "advertise-client-urls", "addr", clientTLSInfo)
-	if err != nil {
-		log.Fatal(err.Error())
-	}
-	cfg := &etcdserver.ServerConfig{
-		Name:         *name,
-		ClientURLs:   acurls,
-		DataDir:      *dir,
-		SnapCount:    *snapCount,
-		Cluster:      cls,
-		DiscoveryURL: *durl,
-		ClusterState: *clusterState,
-		Transport:    pt,
-	}
-	s := etcdserver.NewServer(cfg)
-	s.Start()
-
-	ch := &pkg.CORSHandler{
-		Handler: etcdhttp.NewClientHandler(s),
-		Info:    cors,
-	}
-	ph := etcdhttp.NewPeerHandler(s)
-
-	lpurls, err := pkg.URLsFromFlags(fs, "listen-peer-urls", "peer-bind-addr", peerTLSInfo)
-	if err != nil {
-		log.Fatal(err.Error())
-	}
-
-	for _, u := range lpurls {
-		l, err := transport.NewListener(u.Host, peerTLSInfo)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		// Start the peer server in a goroutine
-		urlStr := u.String()
-		go func() {
-			log.Print("Listening for peers on ", urlStr)
-			log.Fatal(http.Serve(l, ph))
-		}()
-	}
-
-	lcurls, err := pkg.URLsFromFlags(fs, "listen-client-urls", "bind-addr", clientTLSInfo)
-	if err != nil {
-		log.Fatal(err.Error())
-	}
-
-	// Start a client server goroutine for each listen address
-	for _, u := range lcurls {
-		l, err := transport.NewListener(u.Host, clientTLSInfo)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		urlStr := u.String()
-		go func() {
-			log.Print("Listening for client requests on ", urlStr)
-			log.Fatal(http.Serve(l, ch))
-		}()
-	}
-}
-
-// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
-func startProxy() {
-	cls, err := setupCluster()
-	if err != nil {
-		log.Fatalf("etcd: error setting up initial cluster: %v", err)
-	}
-
-	pt, err := transport.NewTransport(clientTLSInfo)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	// TODO(jonboulle): update peerURLs dynamically (i.e. when updating
-	// clientURLs) instead of just using the initial fixed list here
-	peerURLs := cls.PeerURLs()
-	uf := func() []string {
-		cls, err := etcdserver.GetClusterFromPeers(peerURLs)
-		if err != nil {
-			log.Printf("etcd: %v", err)
-			return []string{}
-		}
-		return cls.ClientURLs()
-	}
-	ph := proxy.NewHandler(pt, uf)
-	ph = &pkg.CORSHandler{
-		Handler: ph,
-		Info:    cors,
-	}
-
-	if string(*proxyFlag) == flagtypes.ProxyValueReadonly {
-		ph = proxy.NewReadonlyHandler(ph)
-	}
-
-	lcurls, err := pkg.URLsFromFlags(fs, "listen-client-urls", "bind-addr", clientTLSInfo)
-	if err != nil {
-		log.Fatal(err.Error())
-	}
-	// Start a proxy server goroutine for each listen address
-	for _, u := range lcurls {
-		l, err := transport.NewListener(u.Host, clientTLSInfo)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		host := u.Host
-		go func() {
-			log.Print("Listening for client requests on ", host)
-			log.Fatal(http.Serve(l, ph))
-		}()
-	}
-}
-
-// setupCluster sets up the cluster definition for bootstrap or discovery.
-func setupCluster() (*etcdserver.Cluster, error) {
-	set := make(map[string]bool)
-	fs.Visit(func(f *flag.Flag) {
-		set[f.Name] = true
-	})
-	if set["discovery"] && set["initial-cluster"] {
-		return nil, fmt.Errorf("both discovery and bootstrap-config are set")
-	}
-	apurls, err := pkg.URLsFromFlags(fs, "initial-advertise-peer-urls", "addr", peerTLSInfo)
-	if err != nil {
-		return nil, err
-	}
-
-	var cls *etcdserver.Cluster
-	switch {
-	case set["discovery"]:
-		clusterStr := genClusterString(*name, apurls)
-		cls, err = etcdserver.NewClusterFromString(*durl, clusterStr)
-	case set["initial-cluster"]:
-		fallthrough
-	default:
-		// We're statically configured, and cluster has appropriately been set.
-		// Try to configure by indexing the static cluster by name.
-		cls, err = etcdserver.NewClusterFromString(*initialClusterName, *initialCluster)
-	}
-	return cls, err
-}
-
-func genClusterString(name string, urls types.URLs) string {
-	addrs := make([]string, 0)
-	for _, u := range urls {
-		addrs = append(addrs, fmt.Sprintf("%v=%v", name, u.String()))
-	}
-	return strings.Join(addrs, ",")
+	etcdmain.Main()
 }

pkg/README.md

@@ -0,0 +1,2 @@
+pkg/ is a collection of utility packages used by etcd without being specific to etcd itself. A package belongs here
+only if it could possibly be moved out into its own repository in the future.

pkg/cors/cors.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package pkg
+package cors
 
 import (
 	"fmt"

pkg/fileutil/fileutil.go

@@ -14,30 +14,24 @@
    limitations under the License.
 */
 
-package etcdserver
+package fileutil
 
 import (
-	"testing"
+	"io/ioutil"
+	"os"
+	"path"
 )
 
-func TestClusterStateSet(t *testing.T) {
-	tests := []struct {
-		val  string
-		pass bool
-	}{
-		// known values
-		{"new", true},
+const (
+	privateFileMode = 0600
+)
 
-		// unrecognized values
-		{"foo", false},
-		{"", false},
-	}
-
-	for i, tt := range tests {
-		pf := new(ClusterState)
-		err := pf.Set(tt.val)
-		if tt.pass != (err == nil) {
-			t.Errorf("#%d: want pass=%t, but got err=%v", i, tt.pass, err)
-		}
+// IsDirWriteable checks if dir is writable by writing and removing a file
+// to dir. It returns nil if dir is writable.
+func IsDirWriteable(dir string) error {
+	f := path.Join(dir, ".touch")
+	if err := ioutil.WriteFile(f, []byte(""), privateFileMode); err != nil {
+		return err
 	}
+	return os.Remove(f)
 }

pkg/fileutil/fileutil_test.go

@@ -0,0 +1,39 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package fileutil
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestIsDirWriteable(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("unexpected ioutil.TempDir error: %v", err)
+	}
+	if err := IsDirWriteable(tmpdir); err != nil {
+		t.Fatalf("unexpected IsDirWriteable error: %v", err)
+	}
+	if err := os.Chmod(tmpdir, 0444); err != nil {
+		t.Fatalf("unexpected os.Chmod error: %v", err)
+	}
+	if err := IsDirWriteable(tmpdir); err == nil {
+		t.Fatalf("expected IsDirWriteable to error")
+	}
+}

pkg/flags/flag.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package pkg
+package flags
 
 import (
 	"flag"
@@ -24,7 +24,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/coreos/etcd/pkg/flags"
 	"github.com/coreos/etcd/pkg/transport"
 )
 
@@ -86,7 +85,8 @@ func UsageWithIgnoredFlagsFunc(fs *flag.FlagSet, ignore []string) func() {
 // environment variables. Environment variables take the name of the flag but
 // are UPPERCASE, have the prefix "ETCD_", and any dashes are replaced by
 // underscores - for example: some-flag => ETCD_SOME_FLAG
-func SetFlagsFromEnv(fs *flag.FlagSet) {
+func SetFlagsFromEnv(fs *flag.FlagSet) error {
+	var err error
 	alreadySet := make(map[string]bool)
 	fs.Visit(func(f *flag.Flag) {
 		alreadySet[f.Name] = true
@@ -96,10 +96,13 @@ func SetFlagsFromEnv(fs *flag.FlagSet) {
 			key := "ETCD_" + strings.ToUpper(strings.Replace(f.Name, "-", "_", -1))
 			val := os.Getenv(key)
 			if val != "" {
-				fs.Set(f.Name, val)
+				if serr := fs.Set(f.Name, val); serr != nil {
+					err = fmt.Errorf("invalid value %q for %s: %v", val, key, serr)
+				}
 			}
 		}
 	})
+	return err
 }
 
 // URLsFromFlags decides what URLs should be using two different flags
@@ -126,7 +129,7 @@ func URLsFromFlags(fs *flag.FlagSet, urlsFlagName string, addrFlagName string, t
 			return nil, fmt.Errorf("Set only one of flags -%s and -%s", urlsFlagName, addrFlagName)
 		}
 
-		addr := *fs.Lookup(addrFlagName).Value.(*flags.IPAddressPort)
+		addr := *fs.Lookup(addrFlagName).Value.(*IPAddressPort)
 		addrURL := url.URL{Scheme: "http", Host: addr.String()}
 		if !tlsInfo.Empty() {
 			addrURL.Scheme = "https"
@@ -134,5 +137,5 @@ func URLsFromFlags(fs *flag.FlagSet, urlsFlagName string, addrFlagName string, t
 		return []url.URL{addrURL}, nil
 	}
 
-	return []url.URL(*fs.Lookup(urlsFlagName).Value.(*flags.URLsValue)), nil
+	return []url.URL(*fs.Lookup(urlsFlagName).Value.(*URLsValue)), nil
 }

pkg/flags/flag_test.go

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-package pkg
+package flags
 
 import (
 	"flag"
@@ -23,7 +23,6 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/coreos/etcd/pkg/flags"
 	"github.com/coreos/etcd/pkg/transport"
 )
 
@@ -59,7 +58,10 @@ func TestSetFlagsFromEnv(t *testing.T) {
 	}
 
 	// now read the env and verify flags were updated as expected
-	SetFlagsFromEnv(fs)
+	err := SetFlagsFromEnv(fs)
+	if err != nil {
+		t.Errorf("err=%v, want nil", err)
+	}
 	for f, want := range map[string]string{
 		"a": "foo",
 		"b": "bar",
@@ -71,6 +73,16 @@ func TestSetFlagsFromEnv(t *testing.T) {
 	}
 }
 
+func TestSetFlagsFromEnvBad(t *testing.T) {
+	// now verify that an error is propagated
+	fs := flag.NewFlagSet("testing", flag.ExitOnError)
+	fs.Int("x", 0, "")
+	os.Setenv("ETCD_X", "not_a_number")
+	if err := SetFlagsFromEnv(fs); err == nil {
+		t.Errorf("err=nil, want != nil")
+	}
+}
+
 func TestURLsFromFlags(t *testing.T) {
 	tests := []struct {
 		args     []string
@@ -133,8 +145,8 @@ func TestURLsFromFlags(t *testing.T) {
 
 	for i, tt := range tests {
 		fs := flag.NewFlagSet("test", flag.PanicOnError)
-		fs.Var(flags.NewURLsValue("http://127.0.0.1:2379"), "urls", "")
-		fs.Var(&flags.IPAddressPort{}, "addr", "")
+		fs.Var(NewURLsValue("http://127.0.0.1:2379"), "urls", "")
+		fs.Var(&IPAddressPort{}, "addr", "")
 
 		if err := fs.Parse(tt.args); err != nil {
 			t.Errorf("#%d: failed to parse flags: %v", i, err)

pkg/flags/strings.go

@@ -16,40 +16,33 @@
 
 package flags
 
-import (
-	"errors"
-)
+import "errors"
 
-const (
-	ProxyValueOff      = "off"
-	ProxyValueReadonly = "readonly"
-	ProxyValueOn       = "on"
-)
+// NewStringsFlag creates a new string flag for which any one of the given
+// strings is a valid value, and any other value is an error.
+func NewStringsFlag(valids ...string) *StringsFlag {
+	return &StringsFlag{Values: valids}
+}
 
-var (
-	ProxyValues = []string{
-		ProxyValueOff,
-		ProxyValueReadonly,
-		ProxyValueOn,
-	}
-)
+// StringsFlag implements the flag.Value interface.
+type StringsFlag struct {
+	Values []string
+	val    string
+}
 
-// ProxyFlag implements the flag.Value interface.
-type Proxy string
-
-// Set verifies the argument to be a valid member of proxyFlagValues
+// Set verifies the argument to be a valid member of the allowed values
 // before setting the underlying flag value.
-func (pf *Proxy) Set(s string) error {
-	for _, v := range ProxyValues {
+func (ss *StringsFlag) Set(s string) error {
+	for _, v := range ss.Values {
 		if s == v {
-			*pf = Proxy(s)
+			ss.val = s
 			return nil
 		}
 	}
-
 	return errors.New("invalid value")
 }
 
-func (pf *Proxy) String() string {
-	return string(*pf)
+// String returns the set value (if any) of the StringsFlag
+func (ss *StringsFlag) String() string {
+	return ss.val
 }

pkg/flags/strings_test.go

@@ -20,23 +20,26 @@ import (
 	"testing"
 )
 
-func TestProxySet(t *testing.T) {
+func TestStringsSet(t *testing.T) {
 	tests := []struct {
+		vals []string
+
 		val  string
 		pass bool
 	}{
 		// known values
-		{"on", true},
-		{"off", true},
+		{[]string{"abc", "def"}, "abc", true},
+		{[]string{"on", "off", "false"}, "on", true},
 
 		// unrecognized values
-		{"foo", false},
-		{"", false},
+		{[]string{"abc", "def"}, "ghi", false},
+		{[]string{"on", "off"}, "", false},
+		{[]string{}, "asdf", false},
 	}
 
 	for i, tt := range tests {
-		pf := new(Proxy)
-		err := pf.Set(tt.val)
+		sf := NewStringsFlag(tt.vals...)
+		err := sf.Set(tt.val)
 		if tt.pass != (err == nil) {
 			t.Errorf("#%d: want pass=%t, but got err=%v", i, tt.pass, err)
 		}

pkg/flags/urls.go

@@ -17,6 +17,7 @@
 package flags
 
 import (
+	"log"
 	"strings"
 
 	"github.com/coreos/etcd/pkg/types"
@@ -47,6 +48,8 @@ func (us *URLsValue) String() string {
 
 func NewURLsValue(init string) *URLsValue {
 	v := &URLsValue{}
-	v.Set(init)
+	if err := v.Set(init); err != nil {
+		log.Panicf("new URLsValue should never fail: %v", err)
+	}
 	return v
 }