version: bump to 3.1.4

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>

.github/ISSUE_TEMPLATE.md

@@ -5,4 +5,4 @@ A good bug report has some very specific qualities, so please read over our shor
 
 To ask a question, go ahead and ignore this.
 
-[report_bugs]: ../Documentation/reporting_bugs.md
+[report_bugs]: https://github.com/coreos/etcd/blob/master/Documentation/reporting_bugs.md

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,4 +2,4 @@
 
 Please read our [contribution workflow][contributing] before submitting a pull request.
 
-[contributing]: ../CONTRIBUTING.md#contribution-flow
+[contributing]: https://github.com/coreos/etcd/blob/master/CONTRIBUTING.md#contribution-flow

.gitignore

@@ -1,5 +1,6 @@
 /coverage
 /gopath
+/gopath.proto
 /go-bindata
 /machine*
 /bin

.travis.yml

@@ -4,17 +4,19 @@ go_import_path: github.com/coreos/etcd
 sudo: false
 
 go:
-  - 1.7.1
-  - tip
+  - 1.7.5
+
+notifications:
+  on_success: never
+  on_failure: never
 
 env:
-  global:
-   - GO15VENDOREXPERIMENT=1
   matrix:
    - TARGET=amd64
    - TARGET=arm64
    - TARGET=arm
    - TARGET=386
+   - TARGET=ppc64le
 
 matrix:
   fast_finish: true
@@ -27,6 +29,8 @@ matrix:
     env: TARGET=arm64
   - go: tip
     env: TARGET=386
+  - go: tip
+    env: TARGET=ppc64le
 
 addons:
   apt:
@@ -55,6 +59,6 @@ script:
         ;;
       *)
         # test building out of gopath
-        GO_BUILD_FLAGS="-a -v" GOPATH=/bad-go-path GOARCH="${TARGET}" ./build
+        GO_BUILD_FLAGS="-a -v" GOPATH="" GOARCH="${TARGET}" ./build
         ;;
     esac

Documentation/README.md

@@ -0,0 +1 @@
+docs.md

Documentation/benchmarks/etcd-2-1-0-alpha-benchmarks.md

@@ -14,7 +14,7 @@ GCE n1-highcpu-2 machine type
 
 ## Testing
 
-Bootstrap another machine and use the [boom HTTP benchmark tool][boom] to send requests to each etcd member. Check the [benchmark hacking guide][hack-benchmark] for detailed instructions.
+Bootstrap another machine and use the [hey HTTP benchmark tool][hey] to send requests to each etcd member. Check the [benchmark hacking guide][hack-benchmark] for detailed instructions.
 
 ## Performance
 
@@ -48,5 +48,5 @@ Bootstrap another machine and use the [boom HTTP benchmark tool][boom] to send r
 | 256               | 64                | all servers        | 1033      | 121.5 |
 | 256               | 256               | all servers        | 3061      | 119.3 |
 
-[boom]: https://github.com/rakyll/boom
+[hey]: https://github.com/rakyll/hey
 [hack-benchmark]: /hack/benchmark/

Documentation/benchmarks/etcd-2-2-0-benchmarks.md

@@ -24,7 +24,7 @@ Go OS/Arch: linux/amd64
 
 ## Testing
 
-Bootstrap another machine, outside of the etcd cluster, and run the [`boom` HTTP benchmark tool](https://github.com/rakyll/boom) with a connection reuse patch to send requests to each etcd cluster member. See the [benchmark instructions](../../hack/benchmark/) for the patch and the steps to reproduce our procedures.
+Bootstrap another machine, outside of the etcd cluster, and run the [`hey` HTTP benchmark tool](https://github.com/rakyll/hey) with a connection reuse patch to send requests to each etcd cluster member. See the [benchmark instructions](../../hack/benchmark/) for the patch and the steps to reproduce our procedures.
 
 The performance is calulated through results of 100 benchmark rounds.
 
@@ -66,4 +66,4 @@ The performance is calulated through results of 100 benchmark rounds.
 
 - Write QPS to cluster leaders seems to be increased by a small margin. This is because the main loop and entry apply loops were decoupled in the etcd raft logic, eliminating several blocks between them.
 
-- Write QPS to all members seems to be increased by a significant margin, because followers now receive the latest commit index sooner, and commit proposals more quickly.
+- Write QPS to all members seems to be increased by a significant margin, because followers now receive the latest commit index sooner, and commit proposals more quickly.

Documentation/benchmarks/etcd-2-2-0-rc-benchmarks.md

@@ -24,7 +24,7 @@ Also, we use 3 etcd 2.1.0 alpha-stage members to form cluster to get base perfor
 
 ## Testing
 
-Bootstrap another machine and use the [boom HTTP benchmark tool][boom] to send requests to each etcd member. Check the [benchmark hacking guide][hack-benchmark] for detailed instructions.
+Bootstrap another machine and use the [hey HTTP benchmark tool][hey] to send requests to each etcd member. Check the [benchmark hacking guide][hack-benchmark] for detailed instructions.
 
 ## Performance
 
@@ -66,7 +66,7 @@ Bootstrap another machine and use the [boom HTTP benchmark tool][boom] to send r
 
 - write QPS to all servers is increased by 30~80% because follower could receive latest commit index earlier and commit proposals faster.
 
-[boom]: https://github.com/rakyll/boom
+[hey]: https://github.com/rakyll/hey
 [c7146bd5]: https://github.com/coreos/etcd/commits/c7146bd5f2c73716091262edc638401bb8229144
 [etcd-2.1-benchmark]: etcd-2-1-0-alpha-benchmarks.md
 [hack-benchmark]: /hack/benchmark/

Documentation/dev-guide/api_grpc_gateway.md

@@ -8,6 +8,8 @@ etcd v3 uses [gRPC][grpc] for its messaging protocol. The etcd project includes
 
 The gateway accepts a [JSON mapping][json-mapping] for etcd's [protocol buffer][api-ref] message definitions. Note that `key` and `value` fields are defined as byte arrays and therefore must be base64 encoded in JSON.
 
+Use `curl` to put and get a key:
+
 ```bash
 <<COMMENT
 https://www.base64encode.org/
@@ -17,11 +19,24 @@ COMMENT
 
 curl -L http://localhost:2379/v3alpha/kv/put \
 	-X POST -d '{"key": "Zm9v", "value": "YmFy"}'
+# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"}}
 
 curl -L http://localhost:2379/v3alpha/kv/range \
 	-X POST -d '{"key": "Zm9v"}'
+# {"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"3"},"kvs":[{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}],"count":"1"}
 ```
 
+Use `curl` to watch a key:
+
+```bash
+curl http://localhost:2379/v3alpha/watch \
+        -X POST -d '{"create_request": {"key":"Zm9v"} }' &
+# {"result":{"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"1","raft_term":"2"},"created":true}}
+
+curl -L http://localhost:2379/v3alpha/kv/put \
+	-X POST -d '{"key": "Zm9v", "value": "YmFy"}' >/dev/null 2>&1
+# {"result":{"header":{"cluster_id":"12585971608760269493","member_id":"13847567121247652255","revision":"2","raft_term":"2"},"events":[{"kv":{"key":"Zm9v","create_revision":"2","mod_revision":"2","version":"1","value":"YmFy"}}]}}
+```
 
 ## Swagger
 

Documentation/dev-guide/api_reference_v3.md

@@ -427,7 +427,7 @@ Empty field.
 | Field | Description | Type |
 | ----- | ----------- | ---- |
 | key | key is the first key to delete in the range. | bytes |
-| range_end | range_end is the key following the last key to delete for the range [key, range_end). If range_end is not given, the range is defined to contain only the key argument. If range_end is '\0', the range is all keys greater than or equal to the key argument. | bytes |
+| range_end | range_end is the key following the last key to delete for the range [key, range_end). If range_end is not given, the range is defined to contain only the key argument. If range_end is one bit larger than the given key, then the range is all the all keys with the prefix (the given key). If range_end is '\0', the range is all keys greater than or equal to the key argument. | bytes |
 | prev_kv | If prev_kv is set, etcd gets the previous key-value pairs before deleting it. The previous key-value pairs will be returned in the delte response. | bool |
 
 
@@ -762,7 +762,7 @@ From google paxosdb paper: Our implementation hinges around a powerful primitive
 | Field | Description | Type |
 | ----- | ----------- | ---- |
 | key | key is the key to register for watching. | bytes |
-| range_end | range_end is the end of the range [key, range_end) to watch. If range_end is not given, only the key argument is watched. If range_end is equal to '\0', all keys greater than or equal to the key argument are watched. | bytes |
+| range_end | range_end is the end of the range [key, range_end) to watch. If range_end is not given, only the key argument is watched. If range_end is equal to '\0', all keys greater than or equal to the key argument are watched. If the range_end is one bit larger than the given key, then all keys with the prefix (the given key) will be watched. | bytes |
 | start_revision | start_revision is an optional revision to watch from (inclusive). No start_revision is "now". | int64 |
 | progress_notify | progress_notify is set so that the etcd server will periodically send a WatchResponse with no events to the new watcher if there are no recent events. It is useful when clients wish to recover a disconnected watcher starting from a recent known revision. The etcd server may decide how often it will send notifications based on current load. | bool |
 | filters | filter out put event. filter out delete event. filters filter the events at server side before it sends back to the watcher. | (slice of) FilterType |

Documentation/dev-guide/apispec/swagger/rpc.swagger.json

@@ -978,7 +978,8 @@
       "enum": [
         "EQUAL",
         "GREATER",
-        "LESS"
+        "LESS",
+        "NOT_EQUAL"
       ],
       "default": "EQUAL"
     },
@@ -1518,7 +1519,7 @@
         "range_end": {
           "type": "string",
           "format": "byte",
-          "description": "range_end is the key following the last key to delete for the range [key, range_end).\nIf range_end is not given, the range is defined to contain only the key argument.\nIf range_end is '\\0', the range is all keys greater than or equal to the key argument."
+          "description": "range_end is the key following the last key to delete for the range [key, range_end).\nIf range_end is not given, the range is defined to contain only the key argument.\nIf range_end is one bit larger than the given key, then the range is all\nthe all keys with the prefix (the given key).\nIf range_end is '\\0', the range is all keys greater than or equal to the key argument."
         }
       }
     },

Documentation/dev-guide/experimental_apis.md

@@ -1,8 +1,11 @@
 # Experimental APIs and features
 
-For the most part, the etcd project is stable, but we are still moving fast! We believe in the release fast philosophy. We want to get early feedback on features still in development and stabilizing. Thus, there are, and will be more, experimental features and APIs. We plan to improve these features based on the early feedback from the community, or abandon them if there is little interest, in the next few releases. If you are running a production system, please do not rely on any experimental features or APIs.
+For the most part, the etcd project is stable, but we are still moving fast! We believe in the release fast philosophy. We want to get early feedback on features still in development and stabilizing. Thus, there are, and will be more, experimental features and APIs. We plan to improve these features based on the early feedback from the community, or abandon them if there is little interest, in the next few releases. Please do not rely on any experimental features or APIs in production environment.
 
 ## The current experimental API/features are:
 
-- v3 auth API: expect to be stable in 3.1 release
-- etcd gateway: expect to be stable in 3.1 release
+- [gateway][gateway]: beta, to be stable in 3.2 release
+- [gRPC proxy][grpc-proxy]: alpha, to be stable in 3.2 release
+
+[gateway]: ../op-guide/gateway.md
+[grpc-proxy]: ../op-guide/grpc_proxy.md

Documentation/dev-guide/grpc_naming.md

@@ -0,0 +1,65 @@
+# gRPC naming and discovery
+
+etcd provides a gRPC resolver to support an alternative name system that fetches endpoints from etcd for discovering gRPC services. The underlying mechanism is based on watching updates to keys prefixed with the service name.
+
+## Using etcd discovery with go-grpc
+
+The etcd client provides a gRPC resolver for resolving gRPC endpoints with an etcd backend. The resolver is initialized with an etcd client and given a target for resolution:
+
+```go
+import (
+	"github.com/coreos/etcd/clientv3"
+	etcdnaming "github.com/coreos/etcd/clientv3/naming"
+
+	"google.golang.org/grpc"
+)
+
+...
+
+cli, cerr := clientv3.NewFromURL("http://localhost:2379")
+r := &etcdnaming.GRPCResolver{Client: cli}
+b := grpc.RoundRobin(r)
+conn, gerr := grpc.Dial("my-service", grpc.WithBalancer(b))
+```
+
+## Managing service endpoints
+
+The etcd resolver treats all keys under the prefix of the resolution target following a "/" (e.g., "my-service/") with JSON-encoded go-grpc `naming.Update` values as potential service endpoints. Endpoints are added to the service by creating new keys and removed from the service by deleting keys.
+
+### Adding an endpoint
+
+New endpoints can be added to the service through `etcdctl`:
+
+```sh
+ETCDCTL_API=3 etcdctl put my-service/1.2.3.4 '{"Addr":"1.2.3.4","Metadata":"..."}'
+```
+
+The etcd client's `GRPCResolver.Update` method can also register new endpoints with a key matching the `Addr`:
+
+```go
+r.Update(context.TODO(), "my-service", naming.Update{Op: naming.Add, Addr: "1.2.3.4", Metadata: "..."})
+```
+
+### Deleting an endpoint
+
+Hosts can be deleted from the service through `etcdctl`:
+
+```sh
+ETCDCTL_API=3 etcdctl del my-service/1.2.3.4
+```
+
+The etcd client's `GRPCResolver.Update` method also supports deleting endpoints:
+
+```go
+r.Update(context.TODO(), "my-service", naming.Update{Op: naming.Delete, Addr: "1.2.3.4"})
+```
+
+### Registering an endpoint with a lease
+
+Registering an endpoint with a lease ensures that if the host can't maintain a keepalive heartbeat (e.g., its machine fails), it will be removed from the service:
+
+```sh
+lease=`ETCDCTL_API=3 etcdctl lease grant 5 | cut -f2 -d' '`
+ETCDCTL_API=3 etcdctl put --lease=$lease my-service/1.2.3.4 '{"Addr":"1.2.3.4","Metadata":"..."}'
+ETCDCTL_API=3 etcdctl lease keep-alive $lease
+```

Documentation/dev-guide/interacting_v3.md

@@ -4,30 +4,54 @@ Users mostly interact with etcd by putting or getting the value of a key. This s
 
 By default, etcdctl talks to the etcd server with the v2 API for backward compatibility. For etcdctl to speak to etcd using the v3 API, the API version must be set to version 3 via the `ETCDCTL_API` environment variable.
 
-``` bash
+```bash
 export ETCDCTL_API=3
 ```
 
+## Find versions
+
+etcdctl version and Server API version can be useful in finding the appropriate commands to be used for performing various opertions on etcd.
+
+Here is the command to find the versions:
+
+```bash
+$ etcdctl version
+etcdctl version: 3.1.0-alpha.0+git
+API version: 3.1
+```
+
 ## Write a key
 
 Applications store keys into the etcd cluster by writing to keys. Every stored key is replicated to all etcd cluster members through the Raft protocol to achieve consistency and reliability.
 
 Here is the command to set the value of key `foo` to `bar`:
 
-``` bash
+```bash
 $ etcdctl put foo bar
 OK
 ```
 
+Also a key can be set for a specified interval of time by attaching lease to it.
+
+Here is the command to set the value of key `foo1` to `bar1` for 10s.
+
+```bash
+$ etcdctl put foo1 bar1 --lease=1234abcd
+OK
+```
+
+Note: The lease id `1234abcd` in the above command refers to id returned on creating the lease of 10s. This id can then be attached to the key.
+
 ## Read keys
 
 Applications can read values of keys from an etcd cluster. Queries may read a single key, or a range of keys.
 
 Suppose the etcd cluster has stored the following keys:
 
-```
+```bash
 foo = bar
 foo1 = bar1
+foo2 = bar2
 foo3 = bar3
 ```
 
@@ -39,18 +63,59 @@ foo
 bar
 ```
 
-Here is the command to range over the keys from `foo` to `foo9`:
+Here is the command to read the value of key `foo` in hex format:
 
 ```bash
-$ etcdctl get foo foo9
+$ etcdctl get foo --hex
+\x66\x6f\x6f          # Key
+\x62\x61\x72          # Value
+```
+
+Here is the command to read only the value of key `foo`:
+
+```bash
+$ etcdctl get foo --print-value-only
+bar
+```
+
+Here is the command to range over the keys from `foo` to `foo3`:
+
+```bash
+$ etcdctl get foo foo3
 foo
 bar
 foo1
 bar1
+foo2
+bar2
+```
+
+Note that `foo3` is excluded since the range is over the half-open interval `[foo, foo3)`, excluding `foo3`.
+
+Here is the command to range over all keys prefixed with `foo`:
+
+```bash
+$ etcdctl get --prefix foo
+foo
+bar
+foo1
+bar1
+foo2
+bar2
 foo3
 bar3
 ```
 
+Here is the command to range over all keys prefixed with `foo`, limiting the number of results to 2:
+
+```bash
+$ etcdctl get --prefix --limit=2 foo
+foo
+bar
+foo1
+bar1
+```
+
 ## Read past version of keys
 
 Applications may want to read superseded versions of a key. For example, an application may wish to roll back to an old configuration by accessing an earlier version of a key. Alternatively, an application may want a consistent view over multiple keys through multiple requests by accessing key history.
@@ -58,45 +123,81 @@ Since every modification to the etcd cluster key-value store increments the glob
 
 Suppose an etcd cluster already has the following keys:
 
-``` bash
-$ etcdctl put foo bar         # revision = 2
-$ etcdctl put foo1 bar1       # revision = 3
-$ etcdctl put foo bar_new     # revision = 4
-$ etcdctl put foo1 bar1_new   # revision = 5
+```bash
+foo = bar         # revision = 2
+foo1 = bar1       # revision = 3
+foo = bar_new     # revision = 4
+foo1 = bar1_new   # revision = 5
 ```
 
 Here are an example to access the past versions of keys:
 
 ```bash
-$ etcdctl get foo foo9 # access the most recent versions of keys
+$ etcdctl get --prefix foo # access the most recent versions of keys
 foo
 bar_new
 foo1
 bar1_new
 
-$ etcdctl get --rev=4 foo foo9 # access the versions of keys at revision 4
+$ etcdctl get --prefix --rev=4 foo # access the versions of keys at revision 4
 foo
 bar_new
 foo1
 bar1
 
-$ etcdctl get --rev=3 foo foo9 # access the versions of keys at revision 3
+$ etcdctl get --prefix --rev=3 foo # access the versions of keys at revision 3
 foo
 bar
 foo1
 bar1
 
-$ etcdctl get --rev=2 foo foo9 # access the versions of keys at revision 2
+$ etcdctl get --prefix --rev=2 foo # access the versions of keys at revision 2
 foo
 bar
 
-$ etcdctl get --rev=1 foo foo9 # access the versions of keys at revision 1
+$ etcdctl get --prefix --rev=1 foo # access the versions of keys at revision 1
+```
+
+## Read keys which are greater than or equal to the byte value of the specified key
+
+Applications may want to read keys which are greater than or equal to the byte value of the specified key.
+
+Suppose an etcd cluster already has the following keys:
+
+```bash
+a = 123
+b = 456
+z = 789
+```
+
+Here is the command to read keys which are greater than or equal to the byte value of key `b` :
+
+```bash
+$ etcdctl get --from-key b
+b
+456
+z
+789
 ```
 
 ## Delete keys
 
 Applications can delete a key or a range of keys from an etcd cluster.
 
+Suppose an etcd cluster already has the following keys:
+
+```bash
+foo = bar
+foo1 = bar1
+foo3 = bar3
+zoo = val
+zoo1 = val1
+zoo2 = val2
+a = 123
+b = 456
+z = 789
+```
+
 Here is the command to delete key `foo`:
 
 ```bash
@@ -111,6 +212,29 @@ $ etcdctl del foo foo9
 2 # two keys are deleted
 ```
 
+Here is the command to delete key `zoo` with the deleted key value pair returned:
+
+```bash
+$ etcdctl del --prev-kv zoo 
+1   # one key is deleted
+zoo # deleted key
+val # the value of the deleted key
+```
+
+Here is the command to delete keys having prefix as `zoo`:
+
+```bash
+$ etcdctl del --prefix zoo 
+2 # two keys are deleted
+```
+
+Here is the command to delete keys which are greater than or equal to the byte value of key `b` :
+
+```bash
+$ etcdctl del --from-key b
+2 # two keys are deleted
+```
+
 ## Watch key changes
 
 Applications can watch on a key or a range of keys to monitor for any updates.
@@ -120,33 +244,80 @@ Here is the command to watch on key `foo`:
 ```bash
 $ etcdctl watch foo
 # in another terminal: etcdctl put foo bar
+PUT
 foo
 bar
 ```
 
+Here is the command to watch on key `foo` in hex format:
+
+```bash
+$ etcdctl watch foo --hex
+# in another terminal: etcdctl put foo bar
+PUT
+\x66\x6f\x6f          # Key
+\x62\x61\x72          # Value
+```
+
 Here is the command to watch on a range key from `foo` to `foo9`:
 
 ```bash
 $ etcdctl watch foo foo9
 # in another terminal: etcdctl put foo bar
+PUT
 foo
 bar
 # in another terminal: etcdctl put foo1 bar1
+PUT
 foo1
 bar1
 ```
 
+Here is the command to watch on keys having prefix `foo`:
+
+```bash
+$ etcdctl watch --prefix foo
+# in another terminal: etcdctl put foo bar
+PUT
+foo
+bar
+# in another terminal: etcdctl put fooz1 barz1
+PUT
+fooz1
+barz1
+```
+
+Here is the command to watch on multiple keys `foo` and `zoo`:
+
+```bash
+$ etcdctl watch -i 
+$ watch foo
+$ watch zoo
+# in another terminal: etcdctl put foo bar
+PUT
+foo
+bar
+# in another terminal: etcdctl put zoo val
+PUT
+zoo
+val
+```
+
 ## Watch historical changes of keys
 
 Applications may want to watch for historical changes of keys in etcd. For example, an application may wish to receive all the modifications of a key; if the application stays connected to etcd, then `watch` is good enough. However, if the application or etcd fails, a change may happen during the failure, and the application will not receive the update in real time. To guarantee the update is delivered, the application must be able to watch for historical changes to keys. To do this, an application can specify a historical revision on a watch, just like reading past version of keys.
 
 Suppose we finished the following sequence of operations:
 
-``` bash
-etcdctl put foo bar         # revision = 2
-etcdctl put foo1 bar1       # revision = 3
-etcdctl put foo bar_new     # revision = 4
-etcdctl put foo1 bar1_new   # revision = 5
+```bash
+$ etcdctl put foo bar         # revision = 2
+OK
+$ etcdctl put foo1 bar1       # revision = 3
+OK
+$ etcdctl put foo bar_new     # revision = 4
+OK
+$ etcdctl put foo1 bar1_new   # revision = 5
+OK
 ```
 
 Here is an example to watch the historical changes:
@@ -160,7 +331,9 @@ bar
 PUT
 foo
 bar_new
+```
 
+```bash
 # watch for changes on key `foo` since revision 3
 $ etcdctl watch --rev=3 foo
 PUT
@@ -168,6 +341,19 @@ foo
 bar_new
 ```
 
+Here is an example to watch only from the last historical change:
+
+```bash
+# watch for changes on key `foo` and return last revision value along with modified value
+$ etcdctl watch --prev-kv foo
+# in another terminal: etcdctl put foo bar_latest
+PUT
+foo         # key
+bar_new     # last value of foo key before modification
+foo         # key
+bar_latest  # value of foo key after modification
+```
+
 ## Compacted revisions
 
 As we mentioned, etcd keeps revisions so that applications can read past versions of keys. However, to avoid accumulating an unbounded amount of history, it is important to compact past revisions. After compacting, etcd removes historical revisions, releasing resources for future use. All superseded data with revisions before the compacted revision will be unavailable.
@@ -183,6 +369,13 @@ $ etcdctl get --rev=4 foo
 Error:  rpc error: code = 11 desc = etcdserver: mvcc: required revision has been compacted
 ```
 
+Note: The current revision of etcd server can be found using get command on any key (existent or non-existent) in json format. Example is shown below for mykey which does not exist in etcd server:
+
+```bash
+$ etcdctl get mykey -w=json
+{"header":{"cluster_id":14841639068965178418,"member_id":10276657743932975437,"revision":15,"raft_term":4}}
+```
+
 ## Grant leases
 
 Applications can grant leases for keys from an etcd cluster. When a key is attached to a lease, its lifetime is bound to the lease's lifetime which in turn is governed by a time-to-live (TTL). Each lease has a minimum time-to-live (TTL) value specified by the application at grant time. The lease's actual TTL value is at least the minimum TTL and is chosen by the etcd cluster. Once a lease's TTL elapses, the lease expires and all attached keys are deleted.
@@ -242,3 +435,41 @@ lease 32695410dcc0ca06 keepalived with TTL(100)
 lease 32695410dcc0ca06 keepalived with TTL(100)
 ...
 ```
+
+## Get lease information
+
+Applications may want to know about lease information, so that they can be renewed or to check if the lease still exists or it has expired. Applications may also want to know the keys to which a particular lease is attached.
+
+Suppose we finished the following sequence of operations:
+
+```bash
+# grant a lease with 500 second TTL
+$ etcdctl lease grant 500
+lease 694d5765fc71500b granted with TTL(500s)
+
+# attach key zoo1 to lease 694d5765fc71500b
+$ etcdctl put zoo1 val1 --lease=694d5765fc71500b
+OK
+
+# attach key zoo2 to lease 694d5765fc71500b
+$ etcdctl put zoo2 val2 --lease=694d5765fc71500b
+OK
+```
+
+Here is the command to get information about the lease:
+
+```bash
+$ etcdctl lease timetolive 694d5765fc71500b
+lease 694d5765fc71500b granted with TTL(500s), remaining(258s)
+```
+
+Here is the command to get information about the lease along with the keys attached with the lease:
+
+```bash
+$ etcdctl lease timetolive --keys 694d5765fc71500b
+lease 694d5765fc71500b granted with TTL(500s), remaining(132s), attached keys([zoo2 zoo1])
+
+# if the lease has expired or does not exist it will give the below response:
+Error:  etcdserver: requested lease not found
+```
+

Documentation/dev-guide/limit.md

@@ -0,0 +1,10 @@
+# System limits
+
+## Request size limit
+
+etcd is designed to handle small key value pairs typical for metadata. Larger requests will work, but may increase the latency of other requests. For the time being, etcd guarantees to support RPC requests with up to 1MB of data. In the future, the size limit may be loosened or made it configurable.
+
+## Storage size limit
+
+The default storage size limit is 2GB, configurable with `--quota-backend-bytes` flag; supports up to 8GB.
+

Documentation/dev-guide/local_cluster.md

@@ -45,7 +45,7 @@ To interact with the started cluster by using etcdctl:
 # use API version 3
 $ export ETCDCTL_API=3
 
-$ etcdctl --write-out=table --endpoints=localhost:12379 member list
+$ etcdctl --write-out=table --endpoints=localhost:2379 member list
 +------------------+---------+--------+------------------------+------------------------+
 |        ID        | STATUS  |  NAME  |       PEER ADDRS       |      CLIENT ADDRS      |
 +------------------+---------+--------+------------------------+------------------------+

Documentation/dl_build.md

@@ -10,29 +10,44 @@ The easiest way to get etcd is to use one of the pre-built release binaries whic
 
 ## Build the latest version
 
-For those wanting to try the very latest version, build etcd from the `master` branch.
-[Go](https://golang.org/) version 1.6+ (with HTTP2 support) is required to build the latest version of etcd.
-etcd vendors its dependency for official release binaries, while making vendoring optional to avoid import conflicts.
-[`build` script][build-script] would automatically include the vendored dependencies from [`cmd`][cmd-directory] directory.  
+For those wanting to try the very latest version, build etcd from the `master` branch. [Go](https://golang.org/) version 1.7+ is required to build the latest version of etcd. To ensure etcd is built against well-tested libraries, etcd vendors its dependencies for official release binaries. However, etcd's vendoring is also optional to avoid potential import conflicts when embedding the etcd server or using the etcd client.
 
-Here are the commands to build an etcd binary from the `master` branch:
+First, confirm go 1.7+ is installed:
 
-```
+```sh
 # go is required
 $ go version
-go version go1.6 darwin/amd64
+go version go1.7.3 darwin/amd64
 
-# GOPATH should be set correctly
-$ echo $GOPATH
-/Users/example/go
+```
 
-$ mkdir -p $GOPATH/src/github.com/coreos
-$ cd $GOPATH/src/github.com/coreos
-$ git clone github.com:coreos/etcd.git
+To build `etcd` from the `master` branch without a `GOPATH` using the official `build` script:
+
+```sh
+$ git clone https://github.com/coreos/etcd.git
 $ cd etcd
 $ ./build
 $ ./bin/etcd
-...
+```
+
+To build a vendored `etcd` from the `master` branch via `go get`:
+
+```sh
+# GOPATH should be set
+$ echo $GOPATH
+/Users/example/go
+$ go get github.com/coreos/etcd/cmd/etcd
+$ $GOPATH/bin/etcd
+```
+
+To build `etcd` from the `master` branch without vendoring (may not build due to upstream conflicts):
+
+```sh
+# GOPATH should be set
+$ echo $GOPATH
+/Users/example/go
+$ go get github.com/coreos/etcd
+$ $GOPATH/bin/etcd
 ```
 
 ## Test the installation

Documentation/docs.md

@@ -14,8 +14,10 @@ The easiest way to get started using etcd as a distributed key-value store is to
  - [Interacting with etcd][interacting]
  - [API references][api_ref]
  - [gRPC gateway][api_grpc_gateway]
+ - [gRPC naming and discovery][grpc_naming]
  - [Embedding etcd][embed_etcd]
  - [Experimental features and APIs][experimental]
+ - [System limits][system-limit]
 
 ## Operating etcd clusters
 
@@ -23,10 +25,12 @@ Administrators who need to create reliable and scalable key-value stores for the
 
  - [Setting up etcd clusters][clustering]
  - [Setting up etcd gateways][gateway]
+ - [Setting up etcd gRPC proxy (pre-alpha)][grpc_proxy]
  - [Run etcd clusters inside containers][container]
+ - [Hardware recommendations][hardware]
  - [Configuration][conf]
  - [Security][security]
- - Monitoring
+ - [Monitoring][monitoring]
  - [Maintenance][maintenance]
  - [Understand failures][failures]
  - [Disaster recovery][recovery]
@@ -38,7 +42,7 @@ Administrators who need to create reliable and scalable key-value stores for the
 
 To learn more about the concepts and internals behind etcd, read the following pages:
 
- - Why etcd (TODO)
+ - [Why etcd][why] (TODO)
  - [Understand data model][data_model]
  - [Understand APIs][understand_apis]
  - [Glossary][glossary]
@@ -48,26 +52,36 @@ To learn more about the concepts and internals behind etcd, read the following p
 
  - [Migrate applications from using API v2 to API v3][v2_migration]
  - [Updating v2.3 to v3.0][v3_upgrade]
+ - [Updating v3.0 to v3.1][v31_upgrade]
 
-## Troubleshooting
+## Frequently Asked Questions (FAQ)
+
+Answers to [common questions] about etcd.
 
 [api_ref]: dev-guide/api_reference_v3.md
 [api_grpc_gateway]: dev-guide/api_grpc_gateway.md
 [clustering]: op-guide/clustering.md
 [conf]: op-guide/configuration.md
+[system-limit]: dev-guide/limit.md
+[common questions]: faq.md
+[why]: learning/why.md
 [data_model]: learning/data_model.md
 [demo]: demo.md
 [download_build]: dl_build.md
 [embed_etcd]: https://godoc.org/github.com/coreos/etcd/embed
+[grpc_naming]: dev-guide/grpc_naming.md
 [failures]: op-guide/failures.md
 [gateway]: op-guide/gateway.md
 [glossary]: learning/glossary.md
+[grpc_proxy]: op-guide/grpc_proxy.md
+[hardware]: op-guide/hardware.md
 [interacting]: dev-guide/interacting_v3.md
 [local_cluster]: dev-guide/local_cluster.md
 [performance]: op-guide/performance.md
 [recovery]: op-guide/recovery.md
 [maintenance]: op-guide/maintenance.md
 [security]: op-guide/security.md
+[monitoring]: op-guide/monitoring.md
 [v2_migration]: op-guide/v2-migration.md
 [container]: op-guide/container.md
 [understand_apis]: learning/api.md
@@ -75,3 +89,4 @@ To learn more about the concepts and internals behind etcd, read the following p
 [supported_platform]: op-guide/supported-platform.md
 [experimental]: dev-guide/experimental_apis.md
 [v3_upgrade]: upgrades/upgrade_3_0.md
+[v31_upgrade]: upgrades/upgrade_3_1.md

Documentation/faq.md

@@ -0,0 +1,128 @@
+## Frequently Asked Questions (FAQ)
+
+### etcd, general
+
+#### Do clients have to send requests to the etcd leader?
+
+[Raft][raft] is leader-based; the leader handles all client requests which need cluster consensus. However, the client does not need to know which node is the leader. Any request that requires consensus sent to a follower is automatically forwarded to the leader. Requests that do not require consensus (e.g., serialized reads) can be processed by any cluster member.
+
+### Configuration
+
+#### What is the difference between advertise-urls and listen-urls?
+
+`listen-urls` specifies the local addresses etcd server binds to for accepting incoming connections. To listen on a port for all interfaces, specify `0.0.0.0` as the listen IP address.
+
+`advertise-urls` specifies the addresses etcd clients or other etcd members should use to contact the etcd server. The advertise addresses must be reachable from the remote machines. Do not advertise addresses like `localhost` or `0.0.0.0` for a production setup since these addresses are unreachable from remote machines.
+
+### Deployment
+
+#### System requirements
+
+Since etcd writes data to disk, SSD is highly recommended. To prevent performance degradation or unintentionally overloading the key-value store, etcd enforces a 2GB default storage size quota, configurable up to 8GB. To avoid swapping or running out of memory, the machine should have at least as much RAM to cover the quota. At CoreOS, an etcd cluster is usually deployed on dedicated CoreOS Container Linux machines with dual-core processors, 2GB of RAM, and 80GB of SSD *at the very least*. **Note that performance is intrinsically workload dependent; please test before production deployment**. See [hardware][hardware-setup] for more recommendations.
+
+Most stable production environment is Linux operating system with amd64 architecture; see [supported platform][supported-platform] for more.
+
+#### Why an odd number of cluster members?
+
+An etcd cluster needs a majority of nodes, a quorum, to agree on updates to the cluster state. For a cluster with n members, quorum is (n/2)+1. For any odd-sized cluster, adding one node will always increase the number of nodes necessary for quorum. Although adding a node to an odd-sized cluster appears better since there are more machines, the fault tolerance is worse since exactly the same number of nodes may fail without losing quorum but there are more nodes that can fail. If the cluster is in a state where it can't tolerate any more failures, adding a node before removing nodes is dangerous because if the new node fails to register with the cluster (e.g., the address is misconfigured), quorum will be permanently lost.
+
+#### What is maximum cluster size?
+
+Theoretically, there is no hard limit. However, an etcd cluster probably should have no more than seven nodes. [Google Chubby lock service][chubby], similar to etcd and widely deployed within Google for many years, suggests running five nodes. A 5-member etcd cluster can tolerate two member failures, which is enough in most cases. Although larger clusters provide better fault tolerance, the write performance suffers because data must be replicated across more machines.
+
+#### What is failure tolerance?
+
+An etcd cluster operates so long as a member quorum can be established. If quorum is lost through transient network failures (e.g., partitions), etcd automatically and safely resumes once the network recovers and restores quorum; Raft enforces cluster consistency. For power loss, etcd persists the Raft log to disk; etcd replays the log to the point of failure and resumes cluster participation. For permanent hardware failure, the node may be removed from the cluster through [runtime reconfiguration][runtime reconfiguration].
+
+It is recommended to have an odd number of members in a cluster. An odd-size cluster tolerates the same number of failures as an even-size cluster but with fewer nodes. The difference can be seen by comparing even and odd sized clusters:
+
+| Cluster Size | Majority | Failure Tolerance |
+|:-:|:-:|:-:|
+| 1 | 1 | 0 |
+| 2 | 2 | 0 |
+| 3 | 2 | 1 |
+| 4 | 3 | 1 |
+| 5 | 3 | 2 |
+| 6 | 4 | 2 |
+| 7 | 4 | 3 |
+| 8 | 5 | 3 |
+| 9 | 5 | 4 |
+
+Adding a member to bring the size of cluster up to an even number doesn't buy additional fault tolerance. Likewise, during a network partition, an odd number of members guarantees that there will always be a majority partition that can continue to operate and be the source of truth when the partition ends.
+
+#### Does etcd work in cross-region or cross data center deployments?
+
+Deploying etcd across regions improves etcd's fault tolerance since members are in separate failure domains. The cost is higher consensus request latency from crossing data center boundaries. Since etcd relies on a member quorum for consensus, the latency from crossing data centers will be somewhat pronounced because at least a majority of cluster members must respond to consensus requests. Additionally, cluster data must be replicated across all peers, so there will be bandwidth cost as well.
+
+With longer latencies, the default etcd configuration may cause frequent elections or heartbeat timeouts. See [tuning] for adjusting timeouts for high latency deployments.
+
+### Operation
+
+#### How to backup a etcd cluster?
+
+etcdctl provides a `snapshot` command to create backups. See [backup][backup] for more details.
+
+#### Should I add a member before removing an unhealthy member?
+
+When replacing an etcd node, it's important  to remove the member first and then add its replacement.
+
+etcd employs distributed consensus based on a quorum model; (n+1)/2 members, a majority, must agree on a proposal before it can be committed to the cluster. These proposals include key-value updates and membership changes. This model totally avoids any possibility of split brain inconsistency. The downside is permanent quorum loss is catastrophic.
+
+How this applies to membership: If a 3-member cluster has 1 downed member, it can still make forward progress because the quorum is 2 and 2 members are still live. However, adding a new member to a 3-member cluster will increase the quorum to 3 because 3 votes are required for a majority of 4 members. Since the quorum increased, this extra member buys nothing in terms of fault tolerance; the cluster is still one node failure away from being unrecoverable.
+
+Additionally, that new member is risky because it may turn out to be misconfigured or incapable of joining the cluster. In that case, there's no way to recover quorum because the cluster has two members down and two members up, but needs three votes to change membership to undo the botched membership addition. etcd will by default reject member add attempts that could take down the cluster in this manner.
+
+On the other hand, if the downed member is removed from cluster membership first, the number of members becomes 2 and the quorum remains at 2. Following that removal by adding a new member will also keep the quorum steady at 2. So, even if the new node can't be brought up, it's still possible to remove the new member through quorum on the remaining live members.
+
+#### Why won't etcd accept my membership changes?
+
+etcd sets `strict-reconfig-check` in order to reject reconfiguration requests that would cause quorum loss. Abandoning quorum is really risky (especially when the cluster is already unhealthy). Although it may be tempting to disable quorum checking if there's quorum loss to add a new member, this could lead to full fledged cluster inconsistency. For many applications, this will make the problem even worse ("disk geometry corruption" being a candidate for most terrifying).
+
+### Performance
+
+#### How should I benchmark etcd?
+
+Try the [benchmark] tool. Current [benchmark results][benchmark-result] are available for comparison.
+
+#### What does the etcd warning "apply entries took too long" mean?
+
+After a majority of etcd members agree to commit a request, each etcd server applies the request to its data store and persists the result to disk. Even with a slow mechanical disk or a virtualized network disk, such as Amazon’s EBS or Google’s PD, applying a request should normally take fewer than 50 milliseconds. If the average apply duration exceeds 100 milliseconds, etcd will warn that entries are taking too long to apply.
+ 
+Usually this issue is caused by a slow disk. The disk could be experiencing contention among etcd and other applications, or the disk is too simply slow (e.g., a shared virtualized disk). To rule out a slow disk from causing this warning, monitor  [backend_commit_duration_seconds][backend_commit_metrics] (p99 duration should be less than 25ms) to confirm the disk is reasonably fast. If the disk is too slow, assigning a dedicated disk to etcd or using faster disk will typically solve the problem.
+
+The second most common cause is CPU starvation. If monitoring of the machine’s CPU usage shows heavy utilization, there may not be enough compute capacity for etcd. Moving etcd to dedicated machine, increasing process resource isolation  cgroups, or renicing the etcd server process into a higher priority can usually solve the problem.
+
+Expensive user requests which access too many keys (e.g., fetching the entire keyspace) can also cause long apply latencies. Accessing fewer than a several hundred keys per request, however, should always be performant.
+
+If none of the above suggestions clear the warnings, please [open an issue][new_issue] with detailed logging, monitoring, metrics and optionally workload information.
+
+#### What does the etcd warning "failed to send out heartbeat on time" mean?
+
+etcd uses a leader-based consensus protocol for consistent data replication and log execution. Cluster members elect a single leader, all other members become followers. The elected leader must periodically send heartbeats to its followers to maintain its leadership. Followers infer leader failure if no heartbeats are received within an election interval and trigger an election. If a leader doesn’t send its heartbeats in time but is still running, the election is spurious and likely caused by insufficient resources. To catch these soft failures, if the leader skips two heartbeat intervals, etcd will warn it failed to send a heartbeat on time.
+
+Usually this issue is caused by a slow disk. Before the leader sends heartbeats attached with metadata, it may need to persist the metadata to disk. The disk could be experiencing contention among etcd and other applications, or the disk is too simply slow (e.g., a shared virtualized disk). To rule out a slow disk from causing this warning, monitor  [wal_fsync_duration_seconds][wal_fsync_duration_seconds] (p99 duration should be less than 10ms) to confirm the disk is reasonably fast. If the disk is too slow, assigning a dedicated disk to etcd or using faster disk will typically solve the problem.
+
+The second most common cause is CPU starvation. If monitoring of the machine’s CPU usage shows heavy utilization, there may not be enough compute capacity for etcd. Moving etcd to dedicated machine, increasing process resource isolation  with cgroups, or renicing the etcd server process into a higher priority can usually solve the problem.
+
+A slow network can also cause this issue. If network metrics among the etcd machines shows long latencies or high drop rate, there may not be enough network capacity for etcd. Moving etcd members to a less congested network will typically solve the problem. However, if the etcd cluster is deployed across data centers, long latency between members is expected. For such deployments, tune the `heartbeat-interval` configuration to roughly match the round trip time between the machines, and the `election-timeout` configuration to be at least 5 * `heartbeat-interval`. See [tuning documentation][tuning] for detailed information.
+
+If none of the above suggestions clear the warnings, please [open an issue][new_issue] with detailed logging, monitoring, metrics and optionally workload information.
+
+#### What does the etcd warning "request ignored (cluster ID mismatch)" mean?
+
+Every new etcd cluster generates a new cluster ID based on the initial cluster configuration and a user-provided unique `initial-cluster-token` value. By having unique cluster ID's, etcd is protected from cross-cluster interaction which could corrupt the cluster.
+
+Usually this warning happens after tearing down an old cluster, then reusing some of the peer addresses for the new cluster. If any etcd process from the old cluster is still running it will try to contact the new cluster. The new cluster will recognize a cluster ID mismatch, then ignore the request and emit this warning. This warning is often cleared by ensuring peer addresses among distinct clusters are disjoint.
+
+[hardware-setup]: ./op-guide/hardware.md
+[supported-platform]: ./op-guide/supported-platform.md
+[wal_fsync_duration_seconds]: ./metrics.md#disk
+[tuning]: ./tuning.md
+[new_issue]: https://github.com/coreos/etcd/issues/new
+[backend_commit_metrics]: ./metrics.md#disk
+[raft]: https://raft.github.io/raft.pdf
+[backup]: https://github.com/coreos/etcd/blob/master/Documentation/op-guide/recovery.md#snapshotting-the-keyspace
+[chubby]: http://static.googleusercontent.com/media/research.google.com/en//archive/chubby-osdi06.pdf
+[runtime reconfiguration]: https://github.com/coreos/etcd/blob/master/Documentation/op-guide/runtime-configuration.md
+[benchmark]: https://github.com/coreos/etcd/tree/master/tools/benchmark
+[benchmark-result]: https://github.com/coreos/etcd/blob/master/Documentation/op-guide/performance.md

Documentation/learning/glossary.md

@@ -2,15 +2,17 @@
 
 This document defines the various terms used in etcd documentation, command line and source code.
 
-## Node
+## Alarm
 
-Node is an instance of raft state machine.
+The etcd server raises an alarm whenever the cluster needs operator intervention to remain reliable.
 
-It has a unique identification, and records other nodes' progress internally when it is the leader.
+## Authentication
 
-## Member
+Authentication manages user access permissions for etcd resources.
 
-Member is an instance of etcd. It hosts a node, and provides service to clients.
+## Client
+
+A client connects to the etcd cluster to issue service requests such as fetching key-value pairs, writing data, or watching for updates.
 
 ## Cluster
 
@@ -18,6 +20,42 @@ Cluster consists of several members.
 
 The node in each member follows raft consensus protocol to replicate logs. Cluster receives proposals from members, commits them and apply to local store.
 
+## Compaction
+
+Compaction discards all etcd event history and superseded keys prior to a given revision. It is used to reclaim storage space in the etcd backend database.
+
+## Election
+
+The etcd cluster holds elections among its members to choose a leader as part of the raft consensus protocol.
+
+## Endpoint
+
+A URL pointing to an etcd service or resource.
+
+## Key
+
+A user-defined identifier for storing and retrieving user-defined values in etcd.
+
+## Key range
+
+A set of keys containing either an individual key, a lexical interval for all x such that a < x <= b, or all keys greater than a given key.
+
+## Keyspace
+
+The set of all keys in an etcd cluster.
+
+## Lease
+
+A short-lived renewable contract that deletes keys associated with it on its expiry.
+
+## Member
+
+A logical etcd server that participates in serving an etcd cluster.
+
+## Modification Revision
+
+The first revision to hold the last write to a given key.
+
 ## Peer
 
 Peer is another member of the same cluster.
@@ -26,10 +64,34 @@ Peer is another member of the same cluster.
 
 A proposal is a request (for example a write request, a configuration change request) that needs to go through raft protocol.
 
-## Client
+## Quorum
 
-Client is a caller of the cluster's HTTP API.
+The number of active members needed for consensus to modify the cluster state. etcd requires a member majority to reach quorum.
 
-## Machine (deprecated)
+## Revision
 
-The alternative of Member in etcd before 2.0
+A 64-bit cluster-wide counter that is incremented each time the keyspace is modified.
+
+## Role
+
+A unit of permissions over a set of key ranges which may be granted to a set of users for access control.
+
+## Snapshot
+
+A point-in-time backup of the etcd cluster state.
+
+## Store
+
+The physical storage backing the cluster keyspace.
+
+## Transaction
+
+An atomically executed set of operations. All modified keys in a transaction share the same modification revision.
+
+## Key Version
+
+The number of writes to a key since it was created, starting at 1. The version of a nonexistent or deleted key is 0.
+
+## Watcher
+
+A client opens a watcher to observe updates on a given key range.

Documentation/learning/why.md

@@ -0,0 +1,21 @@
+# Why etcd
+
+The name "etcd" originated from two ideas, the unix "/etc" folder and "d"istibuted systems. The "/etc" folder is a place to store configuration data for a single system whereas etcd stores configuration information for large scale distributed systems. Hence, a "d"istributed "/etc" is "etcd".
+
+etcd stores metadata in a consistent and fault-tolerant way. Distributed systems use etcd as a consistent key-value store for configuration management, service discovery, and coordinating distributed work. Common distributed patterns using etcd include leader election, [distributed locks][etcd-concurrency], and monitoring machine liveness.
+
+## Use cases
+
+- Container Linux by CoreOS: Application running on [Container Linux][container-linux] gets automatic, zero-downtime Linux kernel updates. Container Linux uses [locksmith] to coordinate updates. locksmith implements a distributed semaphore over etcd to ensure only a subset of a cluster is rebooting at any given time.
+- [Kubernetes][kubernetes] stores configuration data into etcd for service discovery and cluster management; etcd's consistency is crucial for correctly scheduling and operating services. The Kubernetes API server persists cluster state into etcd. It uses etcd's watch API to monitor the cluster and roll out critical configuration changes.
+
+
+## Features and system comparisons
+
+TODO
+
+[etcd-concurrency]: https://godoc.org/github.com/coreos/etcd/clientv3/concurrency
+[container-linux]: https://coreos.com/why
+[locksmith]: https://github.com/coreos/locksmith
+[kubernetes]: http://kubernetes.io/docs/whatisk8s
+

Documentation/libraries-and-tools.md

@@ -14,6 +14,7 @@
 - [etcdtool](https://github.com/mickep76/etcdtool) - Export/Import/Edit etcd directory as JSON/YAML/TOML and Validate directory using JSON schema
 - [etcd-rest](https://github.com/mickep76/etcd-rest) - Create generic REST API in Go using etcd as a backend with validation using JSON schema
 - [etcdsh](https://github.com/kamilhark/etcdsh) - A command line client with support of command history and tab completion. Supports v2
+- [etcdloadtest](https://github.com/sinsharat/etcdloadtest) - A command line load test client for etcd version 3.0 and above.
 
 **Go libraries**
 
@@ -34,9 +35,11 @@
 **Scala libraries**
 
 - [maciej/etcd-client](https://github.com/maciej/etcd-client) - Supports v2. Akka HTTP-based fully async client
+- [eiipii/etcdhttpclient](https://bitbucket.org/eiipii/etcdhttpclient) - Supports v2. Async HTTP client based on Netty and Scala Futures.
 
 **Python libraries**
 
+- [kragniz/python-etcd3](https://github.com/kragniz/python-etcd3) - Work in progress client for v3
 - [jplana/python-etcd](https://github.com/jplana/python-etcd) - Supports v2
 - [russellhaering/txetcd](https://github.com/russellhaering/txetcd) - a Twisted Python library
 - [cholcombe973/autodock](https://github.com/cholcombe973/autodock) - A docker deployment automation tool
@@ -93,6 +96,10 @@
 
 - [ropensci/etseed](https://github.com/ropensci/etseed)
 
+**Nim libraries**
+
+- [etcd_client](https://github.com/FedericoCeratto/nim-etcd-client)
+
 **Tcl libraries**
 
 - [efrecon/etcd-tcl](https://github.com/efrecon/etcd-tcl) - Supports v2, except wait.
@@ -117,7 +124,9 @@
 **Projects using etcd**
 
 - [binocarlos/yoda](https://github.com/binocarlos/yoda) - etcd + ZeroMQ
+- [blox/blox](https://github.com/blox/blox) - a collection of open source projects for container management and orchestration with AWS ECS
 - [calavera/active-proxy](https://github.com/calavera/active-proxy) - HTTP Proxy configured with etcd
+- [chain/chain](https://github.com/chain/chain) - software designed to operate and connect to highly scalable permissioned blockchain networks 
 - [derekchiang/etcdplus](https://github.com/derekchiang/etcdplus) - A set of distributed synchronization primitives built upon etcd
 - [go-discover](https://github.com/flynn/go-discover) - service discovery in Go
 - [gleicon/goreman](https://github.com/gleicon/goreman/tree/etcd) - Branch of the Go Foreman clone with etcd support

Documentation/metrics.md

@@ -82,30 +82,7 @@ All these metrics are prefixed with `etcd_network_`
 
 ### gRPC requests
 
-These metrics describe the requests served by a specific etcd member: total received requests, total failed requests, and processing latency. They are useful for tracking user-generated traffic hitting the etcd cluster.
-
-All these metrics are prefixed with `etcd_grpc_`
-
-| Name                           | Description                                                                         | Type                   |
-|--------------------------------|-------------------------------------------------------------------------------------|------------------------|
-| requests_total                 | Total number of received requests                                                   | Counter(method)        |
-| requests_failed_total                   | Total number of failed requests.                                                    | Counter(method,error)  |
-| unary_requests_duration_seconds     | Bucketed handling duration of the requests.                                         | Histogram(method)      |
-
-
-Example Prometheus queries that may be useful from these metrics (across all etcd members):
- 
- * `sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[1m]) by (grpc_method) / sum(rate(etcd_grpc_total{job="etcd"})[1m]) by (grpc_method)` 
-    
-    Shows the fraction of events that failed by gRPC method across all members, across a time window of `1m`.
- 
- * `sum(rate(etcd_grpc_requests_total{job="etcd",grpc_method="PUT"})[1m]) by (grpc_method)`
-    
-    Shows the rate of PUT requests across all members, across a time window of `1m`.
-    
- * `histogram_quantile(0.9, sum(rate(etcd_grpc_unary_requests_duration_seconds{job="etcd",grpc_method="PUT"}[5m]) ) by (le))`
-    
-    Show the 0.90-tile latency (in seconds) of PUT request handling across all members, with a window of `5m`. 
+These metrics are exposed via [go-grpc-prometheus][go-grpc-prometheus].
 
 ## etcd_debugging namespace metrics
 
@@ -136,3 +113,4 @@ Heavy file descriptor (`process_open_fds`) usage (i.e., near the process's file
 [prometheus-getting-started]: http://prometheus.io/docs/introduction/getting_started/
 [prometheus-naming]: http://prometheus.io/docs/practices/naming/
 [v2-http-metrics]: v2/metrics.md#http-requests
+[go-grpc-prometheus]: https://github.com/grpc-ecosystem/go-grpc-prometheus

Documentation/op-guide/clustering.md

@@ -83,7 +83,7 @@ A cluster using self-signed certificates both encrypts traffic and authenticates
 On each machine, etcd would be started with these flags:
 
 ```
-$ etcd --name infra0 --initial-advertise-peer-urls http://10.0.1.10:2380 \
+$ etcd --name infra0 --initial-advertise-peer-urls https://10.0.1.10:2380 \
   --listen-peer-urls https://10.0.1.10:2380 \
   --listen-client-urls https://10.0.1.10:2379,https://127.0.0.1:2379 \
   --advertise-client-urls https://10.0.1.10:2379 \
@@ -126,7 +126,7 @@ $ etcd --name infra2 --initial-advertise-peer-urls https://10.0.1.12:2380 \
 
 If the cluster needs encrypted communication but does not require authenticated connections, etcd can be configured to automatically generate its keys. On initialization, each member creates its own set of keys based on its advertised IP addresses and hosts.
 
-On each machine, etcd would be started with these flag:
+On each machine, etcd would be started with these flags:
 
 ```
 $ etcd --name infra0 --initial-advertise-peer-urls https://10.0.1.10:2380 \
@@ -205,7 +205,7 @@ exit 1
 
 ## Discovery
 
-In a number of cases, the IPs of the cluster peers may not be known ahead of time. This is common when utilizing cloud providers or when the network uses DHCP. In these cases, rather than specifying a static configuration, use an existing etcd cluster to bootstrap a new one. We call this process "discovery".
+In a number of cases, the IPs of the cluster peers may not be known ahead of time. This is common when utilizing cloud providers or when the network uses DHCP. In these cases, rather than specifying a static configuration, use an existing etcd cluster to bootstrap a new one. This process is called "discovery".
 
 There two methods that can be used for discovery:
 
@@ -214,17 +214,17 @@ There two methods that can be used for discovery:
 
 ### etcd discovery
 
-To better understand the design about discovery service protocol, we suggest reading the discovery service protocol [documentation][discovery-proto].
+To better understand the design of the discovery service protocol, we suggest reading the discovery service protocol [documentation][discovery-proto].
 
 #### Lifetime of a discovery URL
 
-A discovery URL identifies a unique etcd cluster. Instead of reusing a discovery URL, always create discovery URLs for new clusters.
+A discovery URL identifies a unique etcd cluster. Instead of reusing an existing discovery URL, each etcd instance shares a new discovery URL to bootstrap the new cluster.
 
 Moreover, discovery URLs should ONLY be used for the initial bootstrapping of a cluster. To change cluster membership after the cluster is already running, see the [runtime reconfiguration][runtime-conf] guide.
 
 #### Custom etcd discovery service
 
-Discovery uses an existing cluster to bootstrap itself. If using a private etcd cluster, can create a URL like so:
+Discovery uses an existing cluster to bootstrap itself. If using a private etcd cluster, create a URL like so:
 
 ```
 $ curl -X PUT https://myetcd.local/v2/keys/discovery/6c007a14875d53d9bf0ef5a6fc0257c817f0fb83/_config/size -d value=3
@@ -271,7 +271,7 @@ $ curl https://discovery.etcd.io/new?size=3
 https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
 ```
 
-This will create the cluster with an initial expected size of 3 members. If no size is specified, a default of 3 is used.
+This will create the cluster with an initial size of 3 members. If no size is specified, a default of 3 is used.
 
 ```
 ETCD_DISCOVERY=https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
@@ -281,7 +281,7 @@ ETCD_DISCOVERY=https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573d
 --discovery https://discovery.etcd.io/3e86b59982e49066c5d813af1c2e2579cbf573de
 ```
 
-**Each member must have a different name flag specified. `Hostname` or `machine-id` can be a good choice. Or discovery will fail due to duplicated name.**
+**Each member must have a different name flag specified or else discovery will fail due to duplicated names. `Hostname` or `machine-id` can be a good choice. **
 
 Now we start etcd with those relevant flags for each member:
 

Documentation/op-guide/configuration.md

@@ -247,7 +247,7 @@ The security flags help to [build a secure etcd cluster][security].
 + env variable: ETCD_DEBUG
 
 ### --log-package-levels
-+ Set individual etcd subpackages to specific log levels. An example being `etcdserver=WARNING,security=DEBUG` 
++ Set individual etcd subpackages to specific log levels. An example being `etcdserver=WARNING,security=DEBUG`
 + default: none (INFO for all packages)
 + env variable: ETCD_LOG_PACKAGE_LEVELS
 
@@ -279,6 +279,10 @@ Follow the instructions when using these flags.
 + Enable runtime profiling data via HTTP server. Address is at client URL + "/debug/pprof/"
 + default: false
 
+### --metrics
++ Set level of detail for exported metrics, specify 'extensive' to include histogram metrics.
++ default: basic
+
 [build-cluster]: clustering.md#static
 [reconfig]: runtime-configuration.md
 [discovery]: clustering.md#discovery

Documentation/op-guide/container.md

@@ -57,7 +57,7 @@ sudo rkt run --net=default:IP=${NODE3} coreos.com/etcd:v3.0.6 -- -name=node3 -ad
 Verify the cluster is healthy and can be reached.
 
 ```
-ETCDCTL_API=3 etcdctl --endpoints=http://172.16.28.21:2379,http://172.16.28.22:2379,http://172.16.28.23:2379 endpoint-health
+ETCDCTL_API=3 etcdctl --endpoints=http://172.16.28.21:2379,http://172.16.28.22:2379,http://172.16.28.23:2379 endpoint health
 ```
 
 ### DNS

Documentation/op-guide/grpc_proxy.md

@@ -0,0 +1,77 @@
+# gRPC proxy
+
+*This is an alpha feature, we are looking for early feedback.*
+
+The gRPC proxy is a stateless etcd reverse proxy operating at the gRPC layer (L7). The proxy is designed to reduce the total processing load on the core etcd cluster. For horizontal scalability, it coalesces watch and lease API requests. To protect the cluster against abusive clients, it caches key range requests.
+
+The gRPC proxy supports multiple etcd server endpoints. When the proxy starts, it randomly picks one etcd server endpoint to use. This endpoint serves all requests until the proxy detects an endpoint failure. If the gRPC proxy detects an endpoint failure, it switches to a different endpoint, if available, to hide failures from its clients. Other retry policies, such as weighted round-robin, may be supported in the future.
+
+## Scalable watch API
+
+The gRPC proxy coalesces multiple client watchers (`c-watchers`) on the same key or range into a single watcher (`s-watcher`) connected to an etcd server. The proxy broadcasts all events from the `s-watcher` to its `c-watchers`.
+
+Assuming N clients watch the same key, one gRPC proxy can reduce the watch load on the etcd server from N to 1. Users can deploy multiple gRPC proxies to further distribute server load.
+
+In the following example, three clients watch on key A. The gRPC proxy coalesces the three watchers, creating a single  watcher attached to the etcd server.
+
+```
+            +-------------+
+            | etcd server |
+            +------+------+
+                   ^ watch key A (s-watcher)
+                   |
+           +-------+-----+
+           | gRPC proxy  | <-------+
+           |             |         |
+           ++-----+------+         |watch key A (c-watcher)
+watch key A ^     ^ watch key A    |
+(c-watcher) |     | (c-watcher)    |
+    +-------+-+  ++--------+  +----+----+
+    |  client |  |  client |  |  client |
+    |         |  |         |  |         |
+    +---------+  +---------+  +---------+
+```
+
+### Limitations
+
+To effectively coalesce multiple client watchers into a single watcher, the gRPC proxy coalesces new `c-watchers` into an existing `s-watcher` when possible. This coalesced `s-watcher` may be out of sync with the etcd server due to network delays or buffered undelivered events. When the watch revision is unspecified, the gRPC proxy will not guarantee the `c-watcher` will start watching from the most recent store revision. For example, if a client watches from an etcd server with revision 1000, that watcher will begin at revision 1000. If a client watches from the gRPC proxy, may begin watching from revision 990.
+
+Similar limitations apply to cancellation. When the watcher is cancelled, the etcd server’s revision may be greater than the cancellation response revision. 
+
+These two limitations should not cause problems for most use cases. In the future, there may be additional options to force the watcher to bypass the gRPC proxy for more accurate revision responses. 
+
+## Scalable lease API
+
+TODO
+
+## Abusive clients protection
+
+The gRPC proxy caches responses for requests when it does not break consistency requirements. This can protect the etcd server from abusive clients in tight for loops.
+
+## Start etcd gRPC proxy
+
+Consider an etcd cluster with the following static endpoints:
+
+|Name|Address|Hostname|
+|------|---------|------------------|
+|infra0|10.0.1.10|infra0.example.com|
+|infra1|10.0.1.11|infra1.example.com|
+|infra2|10.0.1.12|infra2.example.com|
+
+Start the etcd gRPC proxy to use these static endpoints with the command:
+
+```bash
+$ etcd grpc-proxy start --endpoints=infra0.example.com,infra1.example.com,infra2.example.com --listen-addr=127.0.0.1:2379
+```
+
+The etcd gRPC proxy starts and listens on port 8080. It forwards client requests to one of the three endpoints provided above.
+
+Sending requests through the proxy:
+
+```bash
+$ ETCDCTL_API=3 ./etcdctl --endpoints=127.0.0.1:2379 put foo bar
+OK
+$ ETCDCTL_API=3 ./etcdctl --endpoints=127.0.0.1:2379 get foo
+foo
+bar
+```

Documentation/op-guide/hardware.md

@@ -0,0 +1,93 @@
+# Hardware recommendations
+
+etcd usually runs well with limited resources for development or testing purposes; it’s common to develop with etcd on a  laptop or a cheap cloud machine. However, when running etcd clusters in production, some hardware guidelines are useful for proper administration. These suggestions are not hard rules; they serve as a good starting point for a robust production deployment. As always, deployments should be tested with simulated workloads before running in production.
+
+## CPUs
+
+Few etcd deployments require a lot of CPU capacity. Typical clusters need two to four cores to run smoothly.
+Heavily loaded etcd deployments, serving thousands of clients or tens of thousands of requests per second, tend to be CPU bound since etcd can serve requests from memory. Such heavy deployments usually need eight to sixteen dedicated cores.
+
+
+## Memory
+
+etcd has a relatively small memory footprint but its performance still depends on having enough memory. An etcd server will aggressively cache key-value data and spends most of the rest of its memory tracking watchers. Typically 8GB is enough. For heavy deployments with thousands of watchers and millions of keys, allocate 16GB to 64GB memory accordingly.
+
+
+## Disks
+
+Fast disks are the most critical factor for etcd deployment performance and stability. 
+
+A slow disk will increase etcd request latency and potentially hurt cluster stability. Since etcd’s consensus protocol depends on persistently storing metadata to a log, a majority of etcd cluster members must write every request down to disk. Additionally, etcd will also incrementally checkpoint its state to disk so it can truncate this log. If these writes take too long, heartbeats may time out and trigger an election, undermining the stability of the cluster.
+
+etcd is very sensitive to disk write latency. Typically 50 sequential IOPS (e.g., a 7200 RPM disk) is required. For heavily loaded clusters, 500 sequential IOPS (e.g., a typical local SSD or a high performance virtualized block device) is recommended. Note that most cloud providers publish concurrent IOPS rather than sequential IOPS; the published concurrent IOPS can be 10x greater than the sequential IOPS. To measure actual sequential IOPS, we suggest using a disk benchmarking tool such as [diskbench][diskbench] or [fio][fio].
+
+etcd requires only modest disk bandwidth but more disk bandwidth buys faster recovery times when a failed member has to catch up with the cluster. Typically 10MB/s will recover 100MB data within 15 seconds. For large clusters, 100MB/s or higher is suggested for recovering 1GB data within 15 seconds.
+
+When possible, back etcd’s storage with a SSD. A SSD usually provides lower write latencies and with less variance than a spinning disk, thus improving the stability and reliability of etcd. If using spinning disk, get the fastest disks possible (15,000 RPM). Using RAID 0 is also an effective way to increase disk speed, for both spinning disks and SSD. With at least three cluster members, mirroring and/or parity variants of RAID are unnecessary; etcd's consistent replication already gets high availability.
+
+
+## Network
+
+Multi-member etcd deployments benefit from a fast and reliable network. In order for etcd to be both consistent and partition tolerant, an unreliable network with partitioning outages will lead to poor availability. Low latency ensures etcd members can communicate fast. High bandwidth can reduce the time to recover a failed etcd member. 1GbE is sufficient for common etcd deployments. For large etcd clusters, a 10GbE network will reduce mean time to recovery.
+
+Deploy etcd members within a single data center when possible to avoid latency overheads and lessen the possibility of partitioning events. If a failure domain in another data center is required, choose a data center closer to the existing one. Please also read the [tuning][tuning] documentation for more information on cross data center deployment.
+
+
+## Example hardware configurations
+
+Here are a few example hardware setups on AWS and GCE environments. As mentioned before, but must be stressed  regardless, administrators should test an etcd deployment with a simulated workload before putting it into production.
+
+Note that these configurations assume these machines are totally dedicated to etcd. Running other applications along with etcd on these machines may cause resource contentions and lead to cluster instability.
+
+### Small cluster
+
+A small cluster serves fewer than 100 clients, fewer than 200 of requests per second, and stores no more than 100MB of data.
+
+Example application workload: A 50-node Kubernetes cluster
+
+| Provider | Type | vCPUs | Memory (GB) | Max concurrent IOPS | Disk bandwidth (MB/s) |
+|----------|------|-------|--------|------|----------------|
+| AWS | m4.large | 2 | 8 | 3600 | 56.25 |
+| GCE | n1-standard-1 + 50GB PD SSD | 2 | 7.5 | 1500 | 25 |
+
+
+### Medium cluster
+
+A medium cluster serves fewer than 500 clients, fewer than 1,000 of requests per second, and stores no more than 500MB of data.
+
+Example application workload: A 250-node Kubernetes cluster
+
+| Provider | Type | vCPUs | Memory (GB) | Max concurrent IOPS | Disk bandwidth (MB/s) |
+|----------|------|-------|--------|------|----------------|
+| AWS | m4.xlarge | 4 | 16 | 6000 | 93.75 |
+| GCE | n1-standard-4 + 150GB PD SSD | 4 | 15 | 4500 | 75 |
+
+
+### Large cluster
+
+A large cluster serves fewer than 1,500 clients, fewer than 10,000 of requests per second, and stores no more  than 1GB of data.
+
+Example application workload: A 1,000-node Kubernetes cluster
+
+| Provider | Type | vCPUs | Memory (GB) | Max concurrent IOPS | Disk bandwidth (MB/s) |
+|----------|------|-------|--------|------|----------------|
+| AWS | m4.2xlarge | 8 | 32 | 8000 | 125 |
+| GCE | n1-standard-8 + 250GB PD SSD | 8 | 30 | 7500 | 125 |
+
+
+### xLarge cluster
+
+An xLarge cluster serves more than 1,500 clients, more than 10,000 of requests per second, and stores more than 1GB data.
+
+Example application workload: A 3,000 node Kubernetes cluster
+
+| Provider | Type | vCPUs | Memory (GB) | Max concurrent IOPS | Disk bandwidth (MB/s) |
+|----------|------|-------|--------|------|----------------|
+| AWS | m4.4xlarge | 16 | 64 | 16,000 | 250 |
+| GCE | n1-standard-16 + 500GB PD SSD | 16 | 60 | 15,000 | 250 |
+
+
+[diskbench]: https://github.com/ongardie/diskbenchmark
+[fio]: https://github.com/axboe/fio
+[tuning]: ../tuning.md
+

Documentation/op-guide/maintenance.md

@@ -49,51 +49,50 @@ Finished defragmenting etcd member[127.0.0.1:2379]
 
 ## Space quota
 
-The space quota in `etcd` ensures the cluster operates in a reliable fashion. Without a space quota, `etcd` may suffer from poor performance if the keyspace grows excessively large, or it may simply run out of storage space, leading to unpredictable cluster behavior. If the keyspace's backend database for any member exceeds the space quota, `etcd` raises a cluster-wide alarm that puts the cluster into a maintenance mode which only accepts key reads and deletes. After freeing enough space in the keyspace, the alarm can be disarmed and the cluster will resume normal operation.
+The space quota in `etcd` ensures the cluster operates in a reliable fashion. Without a space quota, `etcd` may suffer from poor performance if the keyspace grows excessively large, or it may simply run out of storage space, leading to unpredictable cluster behavior. If the keyspace's backend database for any member exceeds the space quota, `etcd` raises a cluster-wide alarm that puts the cluster into a maintenance mode which only accepts key reads and deletes. Only after freeing enough space in the keyspace and defragmenting the backend database, along with clearing the space quota alarm can the cluster resume normal operation.
 
 By default, `etcd` sets a conservative space quota suitable for most applications, but it may be configured on the command line, in bytes:
 
 ```sh
 # set a very small 16MB quota
-$ etcd --quota-backend-bytes=16777216
+$ etcd --quota-backend-bytes=$((16*1024*1024))
 ```
 
 The space quota can be triggered with a loop:
 
 ```sh
 # fill keyspace
-$ while [ 1 ]; do dd if=/dev/urandom bs=1024 count=1024  | etcdctl put key  || break; done
+$ while [ 1 ]; do dd if=/dev/urandom bs=1024 count=1024  | ETCDCTL_API=3 etcdctl put key  || break; done
 ...
 Error:  rpc error: code = 8 desc = etcdserver: mvcc: database space exceeded
 # confirm quota space is exceeded
-$ etcdctl --write-out=table endpoint status
+$ ETCDCTL_API=3 etcdctl --write-out=table endpoint status
 +----------------+------------------+-----------+---------+-----------+-----------+------------+
 |    ENDPOINT    |        ID        |  VERSION  | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX |
 +----------------+------------------+-----------+---------+-----------+-----------+------------+
 | 127.0.0.1:2379 | bf9071f4639c75cc | 2.3.0+git | 18 MB   | true      |         2 |       3332 |
 +----------------+------------------+-----------+---------+-----------+-----------+------------+
 # confirm alarm is raised
-$ etcdctl alarm list
+$ ETCDCTL_API=3 etcdctl alarm list
 memberID:13803658152347727308 alarm:NOSPACE 
 ```
 
-Removing excessive keyspace data will put the cluster back within the quota limits so the alarm can be disarmed:
+Removing excessive keyspace data and defragmenting the backend database will put the cluster back within the quota limits:
 
 ```sh
 # get current revision
-$ etcdctl --endpoints=:2379 endpoint status
-[{"Endpoint":"127.0.0.1:2379","Status":{"header":{"cluster_id":8925027824743593106,"member_id":13803658152347727308,"revision":1516,"raft_term":2},"version":"2.3.0+git","dbSize":17973248,"leader":13803658152347727308,"raftIndex":6359,"raftTerm":2}}]
+$ rev=$(ETCDCTL_API=3 etcdctl --endpoints=:2379 endpoint status --write-out="json" | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*')
 # compact away all old revisions
-$ etdctl compact 1516
+$ ETCDCTL_API=3 etcdctl compact $rev
 compacted revision 1516
 # defragment away excessive space
-$ etcdctl defrag
+$ ETCDCTL_API=3 etcdctl defrag
 Finished defragmenting etcd member[127.0.0.1:2379]
 # disarm alarm
-$ etcdctl alarm disarm
+$ ETCDCTL_API=3 etcdctl alarm disarm
 memberID:13803658152347727308 alarm:NOSPACE 
 # test puts are allowed again
-$ etdctl put newkey 123
+$ ETCDCTL_API=3 etcdctl put newkey 123
 OK
 ```
 

Documentation/op-guide/monitoring.md

@@ -0,0 +1,82 @@
+# Monitoring etcd
+
+Each etcd server exports metrics under the `/metrics` path on its client port.
+
+The metrics can be fetched with `curl`:
+
+```sh
+$ curl -L http://localhost:2379/metrics
+
+# HELP etcd_debugging_mvcc_keys_total Total number of keys.
+# TYPE etcd_debugging_mvcc_keys_total gauge
+etcd_debugging_mvcc_keys_total 0
+# HELP etcd_debugging_mvcc_pending_events_total Total number of pending events to be sent.
+# TYPE etcd_debugging_mvcc_pending_events_total gauge
+etcd_debugging_mvcc_pending_events_total 0
+...
+```
+
+
+## Prometheus
+
+Running a [Prometheus][prometheus] monitoring service is the easiest way to ingest and record etcd's metrics.
+
+First, install Prometheus:
+
+```sh
+PROMETHEUS_VERSION="1.3.1"
+wget https://github.com/prometheus/prometheus/releases/download/v$PROMETHEUS_VERSION/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz -O /tmp/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz
+tar -xvzf /tmp/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz --directory /tmp/ --strip-components=1
+/tmp/prometheus -version
+```
+
+Set Prometheus's scraper to target the etcd cluster endpoints:
+
+```sh
+cat > /tmp/test-etcd.yaml <<EOF
+global:
+  scrape_interval: 10s
+scrape_configs:
+  - job_name: test-etcd
+    static_configs:
+    - targets: ['10.240.0.32:2379','10.240.0.33:2379','10.240.0.34:2379']
+EOF
+cat /tmp/test-etcd.yaml
+```
+
+Set up the Prometheus handler:
+
+```sh
+nohup /tmp/prometheus \
+    -config.file /tmp/test-etcd.yaml \
+    -web.listen-address ":9090" \
+    -storage.local.path "test-etcd.data" >> /tmp/test-etcd.log  2>&1 &
+```
+
+Now Prometheus will scrape etcd metrics every 10 seconds.
+
+
+## Grafana
+
+[Grafana][grafana] has built-in Prometheus support; just add a Prometheus data source:
+
+```
+Name:   test-etcd
+Type:   Prometheus
+Url:    http://localhost:9090
+Access: proxy
+```
+
+Then import the default [etcd dashboard template][template] and customize. For instance, if Prometheus data source name is `my-etcd`, the `datasource` field values in JSON also need to be `my-etcd`.
+
+See the [demo][demo].
+
+Sample dashboard:
+
+![](./etcd-sample-grafana.png)
+
+
+[prometheus]: https://prometheus.io/
+[grafana]: http://grafana.org/
+[template]: ./grafana.json
+[demo]: http://dash.etcd.io/dashboard/db/test-etcd

Documentation/op-guide/recovery.md

@@ -11,7 +11,7 @@ To recover from disastrous failure, etcd v3 provides snapshot and restore facili
 Recovering a cluster first needs a snapshot of the keyspace from an etcd member. A snapshot may either be taken from a live member with the `etcdctl snapshot save` command or by copying the `member/snap/db` file from an etcd data directory. For example, the following command snapshots the keyspace served by `$ENDPOINT` to the file `snapshot.db`:
 
 ```sh
-$ etcdctl --endpoints $ENDPOINT snapshot save snapshot.db
+$ ETCDCTL_API=3 etcdctl --endpoints $ENDPOINT snapshot save snapshot.db
 ```
 
 ### Restoring a cluster
@@ -23,19 +23,19 @@ Snapshot integrity may be optionally verified at restore time. If the snapshot i
 A restore initializes a new member of a new cluster, with a fresh cluster configuration using `etcd`'s cluster configuration flags, but preserves the contents of the etcd keyspace. Continuing from the previous example, the following creates new etcd data directories (`m1.etcd`, `m2.etcd`, `m3.etcd`) for a three member cluster:
 
 ```sh
-$ etcdctl snapshot restore snapshot.db \
+$ ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \
   --name m1 \
-  --initial-cluster m1=http:/host1:2380,m2=http://host2:2380,m3=http://host3:2380 \
+  --initial-cluster m1=http://host1:2380,m2=http://host2:2380,m3=http://host3:2380 \
   --initial-cluster-token etcd-cluster-1 \
   --initial-advertise-peer-urls http://host1:2380
-$ etcdctl snapshot restore snapshot.db \
+$ ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \
   --name m2 \
-  --initial-cluster m1=http:/host1:2380,m2=http://host2:2380,m3=http://host3:2380 \
+  --initial-cluster m1=http://host1:2380,m2=http://host2:2380,m3=http://host3:2380 \
   --initial-cluster-token etcd-cluster-1 \
   --initial-advertise-peer-urls http://host2:2380
-$ etcdctl snapshot restore snapshot.db \
+$ ETCDCTL_API=3 etcdctl snapshot restore snapshot.db \
   --name m3 \
-  --initial-cluster m1=http:/host1:2380,m2=http://host2:2380,m3=http://host3:2380 \
+  --initial-cluster m1=http://host1:2380,m2=http://host2:2380,m3=http://host3:2380 \
   --initial-cluster-token etcd-cluster-1 \
   --initial-advertise-peer-urls http://host3:2380
 ```

Documentation/op-guide/runtime-configuration.md

@@ -169,7 +169,7 @@ As described in the above, the best practice of adding new members is to configu
 
 For avoiding this problem, etcd provides an option `-strict-reconfig-check`. If this option is passed to etcd, etcd rejects reconfiguration requests if the number of started members will be less than a quorum of the reconfigured cluster.
 
-It is recommended to enable this option. However, it is disabled by default because of keeping compatibility.
+It is enabled by default.
 
 [add member]: #add-a-new-member
 [cluster-reconf]: #cluster-reconfiguration-operations

Documentation/upgrades/upgrade_3_0.md

@@ -6,27 +6,27 @@ In the general case, upgrading from etcd 2.3 to 3.0 can be a zero-downtime, roll
 
 Before [starting an upgrade](#upgrade-procedure), read through the rest of this guide to prepare.
 
-### Upgrade Checklists
+### Upgrade checklists
 
-#### Upgrade Requirements
+#### Upgrade requirements
 
 To upgrade an existing etcd deployment to 3.0, the running cluster must be 2.3 or greater. If it's before 2.3, please upgrade to [2.3](https://github.com/coreos/etcd/releases/tag/v2.3.0) before upgrading to 3.0.
 
-Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. You can check the health of the cluster by using the `etcdctl cluster-health` command.
+Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl cluster-health` command before proceeding.
 
 #### Preparation
 
 Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment.
 
-Before beginning,  [backup the etcd data directory](../v2/admin_guide.md#backing-up-the-datastore). Should something go wrong with the upgrade, it is possible to use this backup to [downgrade](#downgrade) back to existing etcd version.
+Before beginning, [backup the etcd data directory](../v2/admin_guide.md#backing-up-the-datastore). Should something go wrong with the upgrade, it is possible to use this backup to [downgrade](#downgrade) back to existing etcd version.
 
-#### Mixed Versions
+#### Mixed versions
 
 While upgrading, an etcd cluster supports mixed versions of etcd members, and operates with the protocol of the lowest common version. The cluster is only considered upgraded once all of its members are upgraded to version 3.0. Internally, etcd members negotiate with each other to determine the overall cluster version, which controls the reported version and the supported features.
 
 #### Limitations
 
-It might take up to 2 minutes for the newly upgraded member to catch up with the existing cluster when the total data size is larger than 50MB. Check the size of a recent  snapshot to estimate  the total data size. In other words, it is safest to wait for 2 minutes between upgrading each member.
+It might take up to 2 minutes for the newly upgraded member to catch up with the existing cluster when the total data size is larger than 50MB. Check the size of a recent snapshot to estimate the total data size. In other words, it is safest to wait for 2 minutes between upgrading each member.
 
 For a much larger total data size, 100MB or more , this one-time process might take even more time. Administrators of very large etcd clusters of this magnitude can feel free to contact the [etcd team][etcd-contact] before upgrading, and we’ll be happy to provide advice on the procedure.
 
@@ -36,13 +36,13 @@ If all members have been upgraded to v3.0, the cluster will be upgraded to v3.0,
 
 Please [backup the data directory](../v2/admin_guide.md#backing-up-the-datastore) of all etcd members to make downgrading the cluster possible even after it has been completely upgraded.
 
-### Upgrade Procedure
+### Upgrade procedure
 
-This example details the  upgrade of a three-member v2.3 ectd cluster running on a local machine.
+This example details the upgrade of a three-member v2.3 ectd cluster running on a local machine.
 
 #### 1. Check upgrade requirements.
 
-Is the the cluster healthy and running v.2.3.x?
+Is the cluster healthy and running v.2.3.x?
 
 ```
 $ etcdctl cluster-health
@@ -64,7 +64,7 @@ When each etcd process is stopped, expected errors will be logged by other clust
 2016-06-27 15:21:48.624175 I | rafthttp: the connection with 8211f1d0f64f3269 became inactive
 ```
 
-It’s a good idea at this point to  [backup the etcd data directory](../v2/admin_guide.md#backing-up-the-datastore) to provide a downgrade path should any problems occur:
+It’s a good idea at this point to [backup the etcd data directory](../v2/admin_guide.md#backing-up-the-datastore) to provide a downgrade path should any problems occur:
 
 ```
 $ etcdctl backup \
@@ -102,7 +102,7 @@ Upgraded members will log warnings like the following until the entire cluster i
 
 #### 5. Finish
 
-When all members are upgraded, the cluster will report  upgrading to 3.0 successfully:
+When all members are upgraded, the cluster will report upgrading to 3.0 successfully:
 
 ```
 2016-06-27 15:22:19.873751 N | membership: updated the cluster version from 2.3 to 3.0

Documentation/upgrades/upgrade_3_1.md

@@ -0,0 +1,123 @@
+## Upgrade etcd from 3.0 to 3.1
+
+In the general case, upgrading from etcd 3.0 to 3.1 can be a zero-downtime, rolling upgrade:
+ - one by one, stop the etcd v3.0 processes and replace them with etcd v3.1 processes
+ - after running all v3.1 processes, new features in v3.1 are available to the cluster
+
+Before [starting an upgrade](#upgrade-procedure), read through the rest of this guide to prepare.
+
+### Upgrade checklists
+
+#### Upgrade requirements
+
+To upgrade an existing etcd deployment to 3.1, the running cluster must be 3.0 or greater. If it's before 3.0, please upgrade to [3.0](https://github.com/coreos/etcd/releases/tag/v3.0.16) before upgrading to 3.1.
+
+Also, to ensure a smooth rolling upgrade, the running cluster must be healthy. Check the health of the cluster by using the `etcdctl endpoint health` command before proceeding.
+
+#### Preparation
+
+Before upgrading etcd, always test the services relying on etcd in a staging environment before deploying the upgrade to the production environment.
+
+Before beginning, [backup the etcd data](../op-guide/maintenance.md#snapshot-backup). Should something go wrong with the upgrade, it is possible to use this backup to [downgrade](#downgrade) back to existing etcd version. Please note that the `snapshot` command only backs up the v3 data. For v2 data, see [backing up v2 datastore](../v2/admin_guide.md#backing-up-the-datastore).
+
+#### Mixed versions
+
+While upgrading, an etcd cluster supports mixed versions of etcd members, and operates with the protocol of the lowest common version. The cluster is only considered upgraded once all of its members are upgraded to version 3.1. Internally, etcd members negotiate with each other to determine the overall cluster version, which controls the reported version and the supported features.
+
+#### Limitations
+
+Note: If the cluster only has v3 data and no v2 data, it is not subject to this limitation.
+
+If the cluster is serving a v2 data set larger than 50MB, each newly upgraded member may take up to two minutes to catch up with the existing cluster. Check the size of a recent snapshot to estimate the total data size. In other words, it is safest to wait for 2 minutes between upgrading each member.
+
+For a much larger total data size, 100MB or more , this one-time process might take even more time. Administrators of very large etcd clusters of this magnitude can feel free to contact the [etcd team][etcd-contact] before upgrading, and we'll be happy to provide advice on the procedure.
+
+#### Downgrade
+
+If all members have been upgraded to v3.1, the cluster will be upgraded to v3.1, and downgrade from this completed state is **not possible**. If any single member is still v3.0, however, the cluster and its operations remains "v3.0", and it is possible from this mixed cluster state to return to using a v3.0 etcd binary on all members.
+
+Please [backup the data directory](../op-guide/maintenance.md#snapshot-backup) of all etcd members to make downgrading the cluster possible even after it has been completely upgraded.
+
+### Upgrade procedure
+
+This example shows how to upgrade a 3-member v3.0 ectd cluster running on a local machine.
+
+#### 1. Check upgrade requirements
+
+Is the cluster healthy and running v3.0.x?
+
+```
+$ ETCDCTL_API=3 etcdctl endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+localhost:2379 is healthy: successfully committed proposal: took = 6.600684ms
+localhost:22379 is healthy: successfully committed proposal: took = 8.540064ms
+localhost:32379 is healthy: successfully committed proposal: took = 8.763432ms
+
+$ curl http://localhost:2379/version
+{"etcdserver":"3.0.16","etcdcluster":"3.0.0"}
+```
+
+#### 2. Stop the existing etcd process
+
+When each etcd process is stopped, expected errors will be logged by other cluster members. This is normal since a cluster member connection has been (temporarily) broken:
+
+```
+2017-01-17 09:34:18.352662 I | raft: raft.node: 1640829d9eea5cfb elected leader 1640829d9eea5cfb at term 5
+2017-01-17 09:34:18.359630 W | etcdserver: failed to reach the peerURL(http://localhost:2380) of member fd32987dcd0511e0 (Get http://localhost:2380/version: dial tcp 127.0.0.1:2380: getsockopt: connection refused)
+2017-01-17 09:34:18.359679 W | etcdserver: cannot get the version of member fd32987dcd0511e0 (Get http://localhost:2380/version: dial tcp 127.0.0.1:2380: getsockopt: connection refused)
+2017-01-17 09:34:18.548116 W | rafthttp: lost the TCP streaming connection with peer fd32987dcd0511e0 (stream Message writer)
+2017-01-17 09:34:19.147816 W | rafthttp: lost the TCP streaming connection with peer fd32987dcd0511e0 (stream MsgApp v2 writer)
+2017-01-17 09:34:34.364907 W | etcdserver: failed to reach the peerURL(http://localhost:2380) of member fd32987dcd0511e0 (Get http://localhost:2380/version: dial tcp 127.0.0.1:2380: getsockopt: connection refused)
+```
+
+It's a good idea at this point to [backup the etcd data](../op-guide/maintenance.md#snapshot-backup) to provide a downgrade path should any problems occur:
+
+```
+$ etcdctl snapshot save backup.db
+```
+
+#### 3. Drop-in etcd v3.1 binary and start the new etcd process
+
+The new v3.1 etcd will publish its information to the cluster:
+
+```
+2017-01-17 09:36:00.996590 I | etcdserver: published {Name:my-etcd-1 ClientURLs:[http://localhost:2379]} to cluster 46bc3ce73049e678
+```
+
+Verify that each member, and then the entire cluster, becomes healthy with the new v3.1 etcd binary:
+
+```
+$ ETCDCTL_API=3 /etcdctl endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+localhost:22379 is healthy: successfully committed proposal: took = 5.540129ms
+localhost:32379 is healthy: successfully committed proposal: took = 7.321671ms
+localhost:2379 is healthy: successfully committed proposal: took = 10.629901ms
+```
+
+Upgraded members will log warnings like the following until the entire cluster is upgraded. This is expected and will cease after all etcd cluster members are upgraded to v3.1:
+
+```
+2017-01-17 09:36:38.406268 W | etcdserver: the local etcd version 3.0.16 is not up-to-date
+2017-01-17 09:36:38.406295 W | etcdserver: member fd32987dcd0511e0 has a higher version 3.1.0
+2017-01-17 09:36:42.407695 W | etcdserver: the local etcd version 3.0.16 is not up-to-date
+2017-01-17 09:36:42.407730 W | etcdserver: member fd32987dcd0511e0 has a higher version 3.1.0
+```
+
+#### 4. Repeat step 2 to step 3 for all other members
+
+#### 5. Finish
+
+When all members are upgraded, the cluster will report upgrading to 3.1 successfully:
+
+```
+2017-01-17 09:37:03.100015 I | etcdserver: updating the cluster version from 3.0 to 3.1
+2017-01-17 09:37:03.104263 N | etcdserver/membership: updated the cluster version from 3.0 to 3.1
+2017-01-17 09:37:03.104374 I | etcdserver/api: enabled capabilities for version 3.1
+```
+
+```
+$ ETCDCTL_API=3 /etcdctl endpoint health --endpoints=localhost:2379,localhost:22379,localhost:32379
+localhost:2379 is healthy: successfully committed proposal: took = 2.312897ms
+localhost:22379 is healthy: successfully committed proposal: took = 2.553476ms
+localhost:32379 is healthy: successfully committed proposal: took = 2.516902ms
+```
+
+[etcd-contact]: https://groups.google.com/forum/#!forum/etcd-dev

Documentation/v2/admin_guide.md

@@ -216,7 +216,7 @@ To recover from such scenarios, etcd provides functionality to backup and restor
 
 #### Backing up the datastore
 
-**NB:** Windows users must stop etcd before running the backup command.
+**Note:** Windows users must stop etcd before running the backup command.
 
 The first step of the recovery is to backup the data directory and wal directory, if stored separately, on a functioning etcd node. To do this, use the `etcdctl backup` command, passing in the original data (and wal) directory used by etcd. For example:
 
@@ -262,7 +262,9 @@ Once you have verified that etcd has started successfully, shut it down and move
 
 Now that the node is running successfully, [change its advertised peer URLs][update-a-member], as the `--force-new-cluster` option has set the peer URL to the default listening on localhost.
 
-You can then add more nodes to the cluster and restore resiliency. See the [add a new member][add-a-member] guide for more details. **NB:** If you are trying to restore your cluster using old failed etcd nodes, please make sure you have stopped old etcd instances and removed their old data directories specified by the data-dir configuration parameter.
+You can then add more nodes to the cluster and restore resiliency. See the [add a new member][add-a-member] guide for more details.
+
+**Note:** If you are trying to restore your cluster using old failed etcd nodes, please make sure you have stopped old etcd instances and removed their old data directories specified by the data-dir configuration parameter.
 
 ### Client Request Timeout
 

Documentation/v2/proxy.md

@@ -149,5 +149,5 @@ If an error occurs, check the [add member troubleshooting doc][runtime-configura
 
 [discovery-service]: clustering.md#discovery
 [goreman]: https://github.com/mattn/goreman
-[procfile]: /Procfile
+[procfile]: https://github.com/coreos/etcd/blob/master/Procfile
 [runtime-configuration]: runtime-configuration.md#error-cases-when-adding-members

NEWS

@@ -1,45 +1,81 @@
+etcd v3.1.0 (2017-01-20)
+- faster linearizable reads (implements Raft read-index)
+- automatic leadership transfer when leader steps down
+- etcd uses default route IP if advertise URL is not given
+- cluster rejects removing members if quorum will be lost
+- SRV records (e.g., infra1.example.com) must match the discovery domain
+  (i.e., example.com) if no custom certificate authority is given
+  - TLSConfig ServerName is ignored with user-provided certificates
+    for backwards compatibility; to be deprecated in 3.2
+- discovery now has upper limit for waiting on retries
+- etcd flags
+  - --strict-reconfig-check flag is set by default
+  - add --log-output flag
+  - add --metrics flag
+- v3 authentication API is now stable
+- v3 client
+  - add SetEndpoints method; update endpoints at runtime
+  - add Sync method; auto-update endpoints at runtime
+  - add Lease TimeToLive API; fetch lease information
+  - replace Config.Logger field with global logger
+  - Get API responses are sorted in ascending order by default
+- v3 etcdctl
+  - add lease timetolive command
+  - add --print-value-only flag to get command
+  - add --dest-prefix flag to make-mirror command
+  - command get responses are sorted in ascending order by default
+- recipes now conform to sessions defined in clientv3/concurrency
+- ACI has symlinks to /usr/local/bin/etcd*
+- warn on binding listeners through domain names; to be deprecated in 3.2
+- experimental gRPC proxy feature
+
+etcd v3.0.16 (2017-01-13)
+
+etcd v3.0.15 (2016-11-11)
+- fix cancel watch request with wrong range end
+
+etcd v3.0.14 (2016-11-04)
+- v3 etcdctl migrate command now supports --no-ttl flag to discard keys on transform
+
+etcd v3.0.13 (2016-10-24)
+
+etcd v3.0.12 (2016-10-07)
+
+etcd v3.0.11 (2016-10-07)
+- server returns previous key-value (optional)
+  - clientv3 WithPrevKV option
+  - v3 etcdctl put,watch,del --prev-kv flag
+
+etcd v3.0.10 (2016-09-23)
+
 etcd v3.0.9 (2016-09-15)
-
-  - warn on domain names on listen URLs (v3.2 will reject domain names)
-
+- warn on domain names on listen URLs (v3.2 will reject domain names)
 
 etcd v3.0.8 (2016-09-09)
-
-  - allow only IP addresses in listen URLs (domain names are rejected)
-
+- allow only IP addresses in listen URLs (domain names are rejected)
 
 etcd v3.0.7 (2016-08-31)
-
-  - SRV records only allow A records (RFC 2052)
-
+- SRV records only allow A records (RFC 2052)
 
 etcd v3.0.6 (2016-08-19)
 
-
 etcd v3.0.5 (2016-08-19)
-
-  - SRV records (e.g., infra1.example.com) must match the discovery domain
-    (i.e., example.com) when using the default certificate authority.
-
+- SRV records (e.g., infra1.example.com) must match the discovery domain
+  (i.e., example.com) if no custom certificate authority is given
 
 etcd v3.0.4 (2016-07-27)
-
-  - v2 auth can now use common name from TLS certificate when --client-cert-auth is enabled
-  - v2 etcdctl ls command now supports --output=json
-  - Add /var/lib/etcd directory to etcd official Docker image
-
+- v2 auth can now use common name from TLS certificate when --client-cert-auth is enabled
+- v2 etcdctl ls command now supports --output=json
+- Add /var/lib/etcd directory to etcd official Docker image
 
 etcd v3.0.3 (2016-07-15)
-
-  - Revert Dockerfile to use CMD, instead of ENTRYPOINT, to support etcdctl run
-  - Docker commands for v3.0.2 won't work without specifying executable binary paths
-  - v3 etcdctl default endpoints are now 127.0.0.1:2379
-
+- Revert Dockerfile to use CMD, instead of ENTRYPOINT, to support etcdctl run
+- Docker commands for v3.0.2 won't work without specifying executable binary paths
+- v3 etcdctl default endpoints are now 127.0.0.1:2379
 
 etcd v3.0.2 (2016-07-08)
-
-  - Dockerfile uses ENTRYPOINT, instead of CMD, to run etcd without binary path specified
-
+- Dockerfile uses ENTRYPOINT, instead of CMD, to run etcd without binary path specified
 
 etcd v3.0.1 (2016-07-01)
 
+etcd v3.0.0 (2016-06-30)

README.md

@@ -37,13 +37,14 @@ See [etcdctl][etcdctl] for a simple command line client.
 
 ### Getting etcd
 
-The easiest way to get etcd is to use one of the pre-built release binaries which are available for OSX, Linux, Windows, AppC (ACI), and Docker. Instructions for using these binaries are on the [GitHub releases page][github-release].
+The easiest way to get etcd is to use one of the pre-built release binaries which are available for OSX, Linux, Windows, [rkt][rkt], and Docker. Instructions for using these binaries are on the [GitHub releases page][github-release].
 
 For those wanting to try the very latest version, you can [build the latest version of etcd][dl-build] from the `master` branch.
-You will first need [*Go*](https://golang.org/) installed on your machine (version 1.6+ is required).
+You will first need [*Go*](https://golang.org/) installed on your machine (version 1.7+ is required).
 All development occurs on `master`, including new features and bug fixes.
 Bug fixes are first targeted at `master` and subsequently ported to release branches, as described in the [branch management][branch-management] guide.
 
+[rkt]: https://github.com/coreos/rkt/releases/
 [github-release]: https://github.com/coreos/etcd/releases/
 [branch-management]: ./Documentation/branch_management.md
 [dl-build]: ./Documentation/dl_build.md#build-the-latest-version
@@ -93,6 +94,10 @@ This will bring up 3 etcd members `infra1`, `infra2` and `infra3` and etcd proxy
 
 Every cluster member and proxy accepts key value reads and key value writes.
 
+### Running etcd on Kubernetes
+
+If you want to run etcd cluster on Kubernetes, try [etcd operator](https://github.com/coreos/etcd-operator).
+
 ### Next steps
 
 Now it's time to dig into the full etcd API and other guides.
@@ -132,3 +137,4 @@ See [reporting bugs](Documentation/reporting_bugs.md) for details about reportin
 
 etcd is under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.
 
+

ROADMAP.md

@@ -6,19 +6,18 @@ This document defines a high level roadmap for etcd development.
 
 The dates below should not be considered authoritative, but rather indicative of the projected timeline of the project. The [milestones defined in GitHub](https://github.com/coreos/etcd/milestones) represent the most up-to-date and issue-for-issue plans.
 
-etcd 3.0 is our current stable branch. The roadmap below outlines new features that will be added to etcd, and while subject to change, define what future stable will look like.
+etcd 3.1 is our current stable branch. The roadmap below outlines new features that will be added to etcd, and while subject to change, define what future stable will look like.
 
-### etcd 3.1 (2016-Oct)
-- Stable L4 gateway
-- Experimental support for scalable proxy
-- Automatic leadership transfer for the rolling upgrade
-- V3 API improvements
-  - Get previous key-value pair
-  - Get only keys (ignore values)
-  - Get only key count
-
-### etcd 3.2 (2017-Feb)
+### etcd 3.2 (2017-May)
 - Stable scalable proxy
-- JWT token based auth
+- Proxy-as-client interface passthrough
+- Lock service
+- Namespacing proxy
+- TLS Command Name and JWT token based authentication
+- Read-modify-write V3 Put
 - Improved watch performance
-- ...
+- Support non-blocking concurrent read
+
+### etcd 3.3 (?)
+- TBD
+

auth/authpb/auth.pb.go

@@ -21,9 +21,9 @@ import (
 	proto "github.com/golang/protobuf/proto"
 
 	math "math"
-)
 
-import io "io"
+	io "io"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -32,7 +32,9 @@ var _ = math.Inf
 
 // This is a compile-time assertion to ensure that this generated file
 // is compatible with the proto package it is being compiled against.
-const _ = proto.ProtoPackageIsVersion1
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
 type Permission_Type int32
 
@@ -99,113 +101,113 @@ func init() {
 	proto.RegisterType((*Role)(nil), "authpb.Role")
 	proto.RegisterEnum("authpb.Permission_Type", Permission_Type_name, Permission_Type_value)
 }
-func (m *User) Marshal() (data []byte, err error) {
+func (m *User) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
-	data = make([]byte, size)
-	n, err := m.MarshalTo(data)
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
 	if err != nil {
 		return nil, err
 	}
-	return data[:n], nil
+	return dAtA[:n], nil
 }
 
-func (m *User) MarshalTo(data []byte) (int, error) {
+func (m *User) MarshalTo(dAtA []byte) (int, error) {
 	var i int
 	_ = i
 	var l int
 	_ = l
 	if len(m.Name) > 0 {
-		data[i] = 0xa
+		dAtA[i] = 0xa
 		i++
-		i = encodeVarintAuth(data, i, uint64(len(m.Name)))
-		i += copy(data[i:], m.Name)
+		i = encodeVarintAuth(dAtA, i, uint64(len(m.Name)))
+		i += copy(dAtA[i:], m.Name)
 	}
 	if len(m.Password) > 0 {
-		data[i] = 0x12
+		dAtA[i] = 0x12
 		i++
-		i = encodeVarintAuth(data, i, uint64(len(m.Password)))
-		i += copy(data[i:], m.Password)
+		i = encodeVarintAuth(dAtA, i, uint64(len(m.Password)))
+		i += copy(dAtA[i:], m.Password)
 	}
 	if len(m.Roles) > 0 {
 		for _, s := range m.Roles {
-			data[i] = 0x1a
+			dAtA[i] = 0x1a
 			i++
 			l = len(s)
 			for l >= 1<<7 {
-				data[i] = uint8(uint64(l)&0x7f | 0x80)
+				dAtA[i] = uint8(uint64(l)&0x7f | 0x80)
 				l >>= 7
 				i++
 			}
-			data[i] = uint8(l)
+			dAtA[i] = uint8(l)
 			i++
-			i += copy(data[i:], s)
+			i += copy(dAtA[i:], s)
 		}
 	}
 	return i, nil
 }
 
-func (m *Permission) Marshal() (data []byte, err error) {
+func (m *Permission) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
-	data = make([]byte, size)
-	n, err := m.MarshalTo(data)
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
 	if err != nil {
 		return nil, err
 	}
-	return data[:n], nil
+	return dAtA[:n], nil
 }
 
-func (m *Permission) MarshalTo(data []byte) (int, error) {
+func (m *Permission) MarshalTo(dAtA []byte) (int, error) {
 	var i int
 	_ = i
 	var l int
 	_ = l
 	if m.PermType != 0 {
-		data[i] = 0x8
+		dAtA[i] = 0x8
 		i++
-		i = encodeVarintAuth(data, i, uint64(m.PermType))
+		i = encodeVarintAuth(dAtA, i, uint64(m.PermType))
 	}
 	if len(m.Key) > 0 {
-		data[i] = 0x12
+		dAtA[i] = 0x12
 		i++
-		i = encodeVarintAuth(data, i, uint64(len(m.Key)))
-		i += copy(data[i:], m.Key)
+		i = encodeVarintAuth(dAtA, i, uint64(len(m.Key)))
+		i += copy(dAtA[i:], m.Key)
 	}
 	if len(m.RangeEnd) > 0 {
-		data[i] = 0x1a
+		dAtA[i] = 0x1a
 		i++
-		i = encodeVarintAuth(data, i, uint64(len(m.RangeEnd)))
-		i += copy(data[i:], m.RangeEnd)
+		i = encodeVarintAuth(dAtA, i, uint64(len(m.RangeEnd)))
+		i += copy(dAtA[i:], m.RangeEnd)
 	}
 	return i, nil
 }
 
-func (m *Role) Marshal() (data []byte, err error) {
+func (m *Role) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
-	data = make([]byte, size)
-	n, err := m.MarshalTo(data)
+	dAtA = make([]byte, size)
+	n, err := m.MarshalTo(dAtA)
 	if err != nil {
 		return nil, err
 	}
-	return data[:n], nil
+	return dAtA[:n], nil
 }
 
-func (m *Role) MarshalTo(data []byte) (int, error) {
+func (m *Role) MarshalTo(dAtA []byte) (int, error) {
 	var i int
 	_ = i
 	var l int
 	_ = l
 	if len(m.Name) > 0 {
-		data[i] = 0xa
+		dAtA[i] = 0xa
 		i++
-		i = encodeVarintAuth(data, i, uint64(len(m.Name)))
-		i += copy(data[i:], m.Name)
+		i = encodeVarintAuth(dAtA, i, uint64(len(m.Name)))
+		i += copy(dAtA[i:], m.Name)
 	}
 	if len(m.KeyPermission) > 0 {
 		for _, msg := range m.KeyPermission {
-			data[i] = 0x12
+			dAtA[i] = 0x12
 			i++
-			i = encodeVarintAuth(data, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(data[i:])
+			i = encodeVarintAuth(dAtA, i, uint64(msg.Size()))
+			n, err := msg.MarshalTo(dAtA[i:])
 			if err != nil {
 				return 0, err
 			}
@@ -215,31 +217,31 @@ func (m *Role) MarshalTo(data []byte) (int, error) {
 	return i, nil
 }
 
-func encodeFixed64Auth(data []byte, offset int, v uint64) int {
-	data[offset] = uint8(v)
-	data[offset+1] = uint8(v >> 8)
-	data[offset+2] = uint8(v >> 16)
-	data[offset+3] = uint8(v >> 24)
-	data[offset+4] = uint8(v >> 32)
-	data[offset+5] = uint8(v >> 40)
-	data[offset+6] = uint8(v >> 48)
-	data[offset+7] = uint8(v >> 56)
+func encodeFixed64Auth(dAtA []byte, offset int, v uint64) int {
+	dAtA[offset] = uint8(v)
+	dAtA[offset+1] = uint8(v >> 8)
+	dAtA[offset+2] = uint8(v >> 16)
+	dAtA[offset+3] = uint8(v >> 24)
+	dAtA[offset+4] = uint8(v >> 32)
+	dAtA[offset+5] = uint8(v >> 40)
+	dAtA[offset+6] = uint8(v >> 48)
+	dAtA[offset+7] = uint8(v >> 56)
 	return offset + 8
 }
-func encodeFixed32Auth(data []byte, offset int, v uint32) int {
-	data[offset] = uint8(v)
-	data[offset+1] = uint8(v >> 8)
-	data[offset+2] = uint8(v >> 16)
-	data[offset+3] = uint8(v >> 24)
+func encodeFixed32Auth(dAtA []byte, offset int, v uint32) int {
+	dAtA[offset] = uint8(v)
+	dAtA[offset+1] = uint8(v >> 8)
+	dAtA[offset+2] = uint8(v >> 16)
+	dAtA[offset+3] = uint8(v >> 24)
 	return offset + 4
 }
-func encodeVarintAuth(data []byte, offset int, v uint64) int {
+func encodeVarintAuth(dAtA []byte, offset int, v uint64) int {
 	for v >= 1<<7 {
-		data[offset] = uint8(v&0x7f | 0x80)
+		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
-	data[offset] = uint8(v)
+	dAtA[offset] = uint8(v)
 	return offset + 1
 }
 func (m *User) Size() (n int) {
@@ -308,8 +310,8 @@ func sovAuth(x uint64) (n int) {
 func sozAuth(x uint64) (n int) {
 	return sovAuth(uint64((x << 1) ^ uint64((int64(x) >> 63))))
 }
-func (m *User) Unmarshal(data []byte) error {
-	l := len(data)
+func (m *User) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
 		preIndex := iNdEx
@@ -321,7 +323,7 @@ func (m *User) Unmarshal(data []byte) error {
 			if iNdEx >= l {
 				return io.ErrUnexpectedEOF
 			}
-			b := data[iNdEx]
+			b := dAtA[iNdEx]
 			iNdEx++
 			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
@@ -349,7 +351,7 @@ func (m *User) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				byteLen |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -363,7 +365,7 @@ func (m *User) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.Name = append(m.Name[:0], data[iNdEx:postIndex]...)
+			m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...)
 			if m.Name == nil {
 				m.Name = []byte{}
 			}
@@ -380,7 +382,7 @@ func (m *User) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				byteLen |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -394,7 +396,7 @@ func (m *User) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.Password = append(m.Password[:0], data[iNdEx:postIndex]...)
+			m.Password = append(m.Password[:0], dAtA[iNdEx:postIndex]...)
 			if m.Password == nil {
 				m.Password = []byte{}
 			}
@@ -411,7 +413,7 @@ func (m *User) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				stringLen |= (uint64(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -426,11 +428,11 @@ func (m *User) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.Roles = append(m.Roles, string(data[iNdEx:postIndex]))
+			m.Roles = append(m.Roles, string(dAtA[iNdEx:postIndex]))
 			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
-			skippy, err := skipAuth(data[iNdEx:])
+			skippy, err := skipAuth(dAtA[iNdEx:])
 			if err != nil {
 				return err
 			}
@@ -449,8 +451,8 @@ func (m *User) Unmarshal(data []byte) error {
 	}
 	return nil
 }
-func (m *Permission) Unmarshal(data []byte) error {
-	l := len(data)
+func (m *Permission) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
 		preIndex := iNdEx
@@ -462,7 +464,7 @@ func (m *Permission) Unmarshal(data []byte) error {
 			if iNdEx >= l {
 				return io.ErrUnexpectedEOF
 			}
-			b := data[iNdEx]
+			b := dAtA[iNdEx]
 			iNdEx++
 			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
@@ -490,7 +492,7 @@ func (m *Permission) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				m.PermType |= (Permission_Type(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -509,7 +511,7 @@ func (m *Permission) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				byteLen |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -523,7 +525,7 @@ func (m *Permission) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.Key = append(m.Key[:0], data[iNdEx:postIndex]...)
+			m.Key = append(m.Key[:0], dAtA[iNdEx:postIndex]...)
 			if m.Key == nil {
 				m.Key = []byte{}
 			}
@@ -540,7 +542,7 @@ func (m *Permission) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				byteLen |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -554,14 +556,14 @@ func (m *Permission) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.RangeEnd = append(m.RangeEnd[:0], data[iNdEx:postIndex]...)
+			m.RangeEnd = append(m.RangeEnd[:0], dAtA[iNdEx:postIndex]...)
 			if m.RangeEnd == nil {
 				m.RangeEnd = []byte{}
 			}
 			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
-			skippy, err := skipAuth(data[iNdEx:])
+			skippy, err := skipAuth(dAtA[iNdEx:])
 			if err != nil {
 				return err
 			}
@@ -580,8 +582,8 @@ func (m *Permission) Unmarshal(data []byte) error {
 	}
 	return nil
 }
-func (m *Role) Unmarshal(data []byte) error {
-	l := len(data)
+func (m *Role) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
 		preIndex := iNdEx
@@ -593,7 +595,7 @@ func (m *Role) Unmarshal(data []byte) error {
 			if iNdEx >= l {
 				return io.ErrUnexpectedEOF
 			}
-			b := data[iNdEx]
+			b := dAtA[iNdEx]
 			iNdEx++
 			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
@@ -621,7 +623,7 @@ func (m *Role) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				byteLen |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -635,7 +637,7 @@ func (m *Role) Unmarshal(data []byte) error {
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
-			m.Name = append(m.Name[:0], data[iNdEx:postIndex]...)
+			m.Name = append(m.Name[:0], dAtA[iNdEx:postIndex]...)
 			if m.Name == nil {
 				m.Name = []byte{}
 			}
@@ -652,7 +654,7 @@ func (m *Role) Unmarshal(data []byte) error {
 				if iNdEx >= l {
 					return io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				msglen |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -667,13 +669,13 @@ func (m *Role) Unmarshal(data []byte) error {
 				return io.ErrUnexpectedEOF
 			}
 			m.KeyPermission = append(m.KeyPermission, &Permission{})
-			if err := m.KeyPermission[len(m.KeyPermission)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
+			if err := m.KeyPermission[len(m.KeyPermission)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
 				return err
 			}
 			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
-			skippy, err := skipAuth(data[iNdEx:])
+			skippy, err := skipAuth(dAtA[iNdEx:])
 			if err != nil {
 				return err
 			}
@@ -692,8 +694,8 @@ func (m *Role) Unmarshal(data []byte) error {
 	}
 	return nil
 }
-func skipAuth(data []byte) (n int, err error) {
-	l := len(data)
+func skipAuth(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
 	iNdEx := 0
 	for iNdEx < l {
 		var wire uint64
@@ -704,7 +706,7 @@ func skipAuth(data []byte) (n int, err error) {
 			if iNdEx >= l {
 				return 0, io.ErrUnexpectedEOF
 			}
-			b := data[iNdEx]
+			b := dAtA[iNdEx]
 			iNdEx++
 			wire |= (uint64(b) & 0x7F) << shift
 			if b < 0x80 {
@@ -722,7 +724,7 @@ func skipAuth(data []byte) (n int, err error) {
 					return 0, io.ErrUnexpectedEOF
 				}
 				iNdEx++
-				if data[iNdEx-1] < 0x80 {
+				if dAtA[iNdEx-1] < 0x80 {
 					break
 				}
 			}
@@ -739,7 +741,7 @@ func skipAuth(data []byte) (n int, err error) {
 				if iNdEx >= l {
 					return 0, io.ErrUnexpectedEOF
 				}
-				b := data[iNdEx]
+				b := dAtA[iNdEx]
 				iNdEx++
 				length |= (int(b) & 0x7F) << shift
 				if b < 0x80 {
@@ -762,7 +764,7 @@ func skipAuth(data []byte) (n int, err error) {
 					if iNdEx >= l {
 						return 0, io.ErrUnexpectedEOF
 					}
-					b := data[iNdEx]
+					b := dAtA[iNdEx]
 					iNdEx++
 					innerWire |= (uint64(b) & 0x7F) << shift
 					if b < 0x80 {
@@ -773,7 +775,7 @@ func skipAuth(data []byte) (n int, err error) {
 				if innerWireType == 4 {
 					break
 				}
-				next, err := skipAuth(data[start:])
+				next, err := skipAuth(dAtA[start:])
 				if err != nil {
 					return 0, err
 				}
@@ -797,24 +799,26 @@ var (
 	ErrIntOverflowAuth   = fmt.Errorf("proto: integer overflow")
 )
 
+func init() { proto.RegisterFile("auth.proto", fileDescriptorAuth) }
+
 var fileDescriptorAuth = []byte{
-	// 280 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x4a, 0x2c, 0x2d, 0xc9,
-	0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x03, 0xb1, 0x0b, 0x92, 0xa4, 0x44, 0xd2, 0xf3,
-	0xd3, 0xf3, 0xc1, 0x42, 0xfa, 0x20, 0x16, 0x44, 0x56, 0xc9, 0x87, 0x8b, 0x25, 0xb4, 0x38, 0xb5,
-	0x48, 0x48, 0x88, 0x8b, 0x25, 0x2f, 0x31, 0x37, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x27, 0x08,
-	0xcc, 0x16, 0x92, 0xe2, 0xe2, 0x28, 0x48, 0x2c, 0x2e, 0x2e, 0xcf, 0x2f, 0x4a, 0x91, 0x60, 0x02,
-	0x8b, 0xc3, 0xf9, 0x42, 0x22, 0x5c, 0xac, 0x45, 0xf9, 0x39, 0xa9, 0xc5, 0x12, 0xcc, 0x0a, 0xcc,
-	0x1a, 0x9c, 0x41, 0x10, 0x8e, 0xd2, 0x1c, 0x46, 0x2e, 0xae, 0x80, 0xd4, 0xa2, 0xdc, 0xcc, 0xe2,
-	0xe2, 0xcc, 0xfc, 0x3c, 0x21, 0x63, 0xa0, 0x01, 0x40, 0x5e, 0x48, 0x65, 0x01, 0xc4, 0x60, 0x3e,
-	0x23, 0x71, 0x3d, 0x88, 0x6b, 0xf4, 0x10, 0xaa, 0xf4, 0x40, 0xd2, 0x41, 0x70, 0x85, 0x42, 0x02,
-	0x5c, 0xcc, 0xd9, 0xa9, 0x95, 0x50, 0x0b, 0x41, 0x4c, 0x21, 0x69, 0x2e, 0xce, 0xa2, 0xc4, 0xbc,
-	0xf4, 0xd4, 0xf8, 0xd4, 0xbc, 0x14, 0xa0, 0x7d, 0x60, 0x87, 0x80, 0x05, 0x5c, 0xf3, 0x52, 0x94,
-	0xb4, 0xb8, 0x58, 0xc0, 0xda, 0x38, 0xb8, 0x58, 0x82, 0x5c, 0x1d, 0x5d, 0x04, 0x18, 0x84, 0x38,
-	0xb9, 0x58, 0xc3, 0x83, 0x3c, 0x43, 0x5c, 0x05, 0x18, 0x85, 0x78, 0xb9, 0x38, 0x41, 0x82, 0x10,
-	0x2e, 0x93, 0x52, 0x08, 0x50, 0x0d, 0xd0, 0x9d, 0x58, 0x3d, 0x6b, 0xc1, 0xc5, 0x0b, 0xb4, 0x0b,
-	0xe1, 0x2c, 0xa0, 0x03, 0x98, 0x35, 0xb8, 0x8d, 0x84, 0x30, 0x1d, 0x1c, 0x84, 0xaa, 0xd0, 0x49,
-	0xe2, 0xc4, 0x43, 0x39, 0x86, 0x0b, 0x40, 0x7c, 0xe2, 0x91, 0x1c, 0xe3, 0x05, 0x20, 0x7e, 0x00,
-	0xc4, 0x33, 0x1e, 0xcb, 0x31, 0x24, 0xb1, 0x81, 0xc3, 0xd8, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff,
-	0xcc, 0x76, 0x8d, 0x4f, 0x8f, 0x01, 0x00, 0x00,
+	// 288 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x90, 0xc1, 0x4a, 0xc3, 0x30,
+	0x1c, 0xc6, 0x9b, 0xb6, 0x1b, 0xed, 0x5f, 0x27, 0x25, 0x0c, 0x0c, 0x13, 0x42, 0xe9, 0xa9, 0x78,
+	0xa8, 0xb0, 0x5d, 0xbc, 0x2a, 0xf6, 0x20, 0x78, 0x90, 0x50, 0xf1, 0x28, 0x1d, 0x0d, 0x75, 0x6c,
+	0x6d, 0x4a, 0x32, 0x91, 0xbe, 0x89, 0x07, 0x1f, 0x68, 0xc7, 0x3d, 0x82, 0xab, 0x2f, 0x22, 0x4d,
+	0x64, 0x43, 0xdc, 0xed, 0xfb, 0xbe, 0xff, 0x97, 0xe4, 0x97, 0x3f, 0x40, 0xfe, 0xb6, 0x7e, 0x4d,
+	0x1a, 0x29, 0xd6, 0x02, 0x0f, 0x7b, 0xdd, 0xcc, 0x27, 0xe3, 0x52, 0x94, 0x42, 0x47, 0x57, 0xbd,
+	0x32, 0xd3, 0xe8, 0x01, 0xdc, 0x27, 0xc5, 0x25, 0xc6, 0xe0, 0xd6, 0x79, 0xc5, 0x09, 0x0a, 0x51,
+	0x7c, 0xca, 0xb4, 0xc6, 0x13, 0xf0, 0x9a, 0x5c, 0xa9, 0x77, 0x21, 0x0b, 0x62, 0xeb, 0x7c, 0xef,
+	0xf1, 0x18, 0x06, 0x52, 0xac, 0xb8, 0x22, 0x4e, 0xe8, 0xc4, 0x3e, 0x33, 0x26, 0xfa, 0x44, 0x00,
+	0x8f, 0x5c, 0x56, 0x0b, 0xa5, 0x16, 0xa2, 0xc6, 0x33, 0xf0, 0x1a, 0x2e, 0xab, 0xac, 0x6d, 0xcc,
+	0xc5, 0x67, 0xd3, 0xf3, 0xc4, 0xd0, 0x24, 0x87, 0x56, 0xd2, 0x8f, 0xd9, 0xbe, 0x88, 0x03, 0x70,
+	0x96, 0xbc, 0xfd, 0x7d, 0xb0, 0x97, 0xf8, 0x02, 0x7c, 0x99, 0xd7, 0x25, 0x7f, 0xe1, 0x75, 0x41,
+	0x1c, 0x03, 0xa2, 0x83, 0xb4, 0x2e, 0xa2, 0x4b, 0x70, 0xf5, 0x31, 0x0f, 0x5c, 0x96, 0xde, 0xdc,
+	0x05, 0x16, 0xf6, 0x61, 0xf0, 0xcc, 0xee, 0xb3, 0x34, 0x40, 0x78, 0x04, 0x7e, 0x1f, 0x1a, 0x6b,
+	0x47, 0x19, 0xb8, 0x4c, 0xac, 0xf8, 0xd1, 0xcf, 0x5e, 0xc3, 0x68, 0xc9, 0xdb, 0x03, 0x16, 0xb1,
+	0x43, 0x27, 0x3e, 0x99, 0xe2, 0xff, 0xc0, 0xec, 0x6f, 0xf1, 0x96, 0x6c, 0x76, 0xd4, 0xda, 0xee,
+	0xa8, 0xb5, 0xe9, 0x28, 0xda, 0x76, 0x14, 0x7d, 0x75, 0x14, 0x7d, 0x7c, 0x53, 0x6b, 0x3e, 0xd4,
+	0x3b, 0x9e, 0xfd, 0x04, 0x00, 0x00, 0xff, 0xff, 0xcc, 0x76, 0x8d, 0x4f, 0x8f, 0x01, 0x00, 0x00,
 }

auth/range_perm_cache.go

@@ -49,38 +49,30 @@ func isRangeEqual(a, b *rangePerm) bool {
 
 // removeSubsetRangePerms removes any rangePerms that are subsets of other rangePerms.
 // If there are equal ranges, removeSubsetRangePerms only keeps one of them.
-func removeSubsetRangePerms(perms []*rangePerm) []*rangePerm {
-	// TODO(mitake): currently it is O(n^2), we need a better algorithm
-	var newp []*rangePerm
-
+// It returns a sorted rangePerm slice.
+func removeSubsetRangePerms(perms []*rangePerm) (newp []*rangePerm) {
+	sort.Sort(RangePermSliceByBegin(perms))
+	var prev *rangePerm
 	for i := range perms {
-		skip := false
-
-		for j := range perms {
-			if i == j {
-				continue
-			}
-
-			if isRangeEqual(perms[i], perms[j]) {
-				// if ranges are equal, we only keep the first range.
-				if i > j {
-					skip = true
-					break
-				}
-			} else if isSubset(perms[i], perms[j]) {
-				// if a range is a strict subset of the other one, we skip the subset.
-				skip = true
-				break
-			}
-		}
-
-		if skip {
+		if i == 0 {
+			prev = perms[i]
+			newp = append(newp, perms[i])
 			continue
 		}
-
+		if isRangeEqual(perms[i], prev) {
+			continue
+		}
+		if isSubset(perms[i], prev) {
+			continue
+		}
+		if isSubset(prev, perms[i]) {
+			prev = perms[i]
+			newp[len(newp)-1] = perms[i]
+			continue
+		}
+		prev = perms[i]
 		newp = append(newp, perms[i])
 	}
-
 	return newp
 }
 
@@ -88,7 +80,6 @@ func removeSubsetRangePerms(perms []*rangePerm) []*rangePerm {
 func mergeRangePerms(perms []*rangePerm) []*rangePerm {
 	var merged []*rangePerm
 	perms = removeSubsetRangePerms(perms)
-	sort.Sort(RangePermSliceByBegin(perms))
 
 	i := 0
 	for i < len(perms) {

auth/range_perm_cache_test.go

@@ -16,6 +16,8 @@ package auth
 
 import (
 	"bytes"
+	"fmt"
+	"reflect"
 	"testing"
 )
 
@@ -131,3 +133,47 @@ func TestGetMergedPerms(t *testing.T) {
 		}
 	}
 }
+
+func TestRemoveSubsetRangePerms(t *testing.T) {
+	tests := []struct {
+		perms  []*rangePerm
+		expect []*rangePerm
+	}{
+		{ // subsets converge
+			[]*rangePerm{{[]byte{2}, []byte{3}}, {[]byte{2}, []byte{5}}, {[]byte{1}, []byte{4}}},
+			[]*rangePerm{{[]byte{1}, []byte{4}}, {[]byte{2}, []byte{5}}},
+		},
+		{ // subsets converge
+			[]*rangePerm{{[]byte{0}, []byte{3}}, {[]byte{0}, []byte{1}}, {[]byte{2}, []byte{4}}, {[]byte{0}, []byte{2}}},
+			[]*rangePerm{{[]byte{0}, []byte{3}}, {[]byte{2}, []byte{4}}},
+		},
+		{ // biggest range at the end
+			[]*rangePerm{{[]byte{2}, []byte{3}}, {[]byte{0}, []byte{2}}, {[]byte{1}, []byte{4}}, {[]byte{0}, []byte{5}}},
+			[]*rangePerm{{[]byte{0}, []byte{5}}},
+		},
+		{ // biggest range at the beginning
+			[]*rangePerm{{[]byte{0}, []byte{5}}, {[]byte{2}, []byte{3}}, {[]byte{0}, []byte{2}}, {[]byte{1}, []byte{4}}},
+			[]*rangePerm{{[]byte{0}, []byte{5}}},
+		},
+		{ // no overlapping ranges
+			[]*rangePerm{{[]byte{2}, []byte{3}}, {[]byte{0}, []byte{1}}, {[]byte{4}, []byte{7}}, {[]byte{8}, []byte{15}}},
+			[]*rangePerm{{[]byte{0}, []byte{1}}, {[]byte{2}, []byte{3}}, {[]byte{4}, []byte{7}}, {[]byte{8}, []byte{15}}},
+		},
+	}
+	for i, tt := range tests {
+		rs := removeSubsetRangePerms(tt.perms)
+		if !reflect.DeepEqual(rs, tt.expect) {
+			t.Fatalf("#%d: unexpected rangePerms %q, got %q", i, printPerms(rs), printPerms(tt.expect))
+		}
+	}
+}
+
+func printPerms(rs []*rangePerm) (txt string) {
+	for i, p := range rs {
+		if i != 0 {
+			txt += ","
+		}
+		txt += fmt.Sprintf("%+v", *p)
+	}
+	return
+}

auth/simple_token.go

@@ -21,6 +21,8 @@ import (
 	"crypto/rand"
 	"math/big"
 	"strings"
+	"sync"
+	"time"
 )
 
 const (
@@ -28,6 +30,73 @@ const (
 	defaultSimpleTokenLength = 16
 )
 
+// var for testing purposes
+var (
+	simpleTokenTTL           = 5 * time.Minute
+	simpleTokenTTLResolution = 1 * time.Second
+)
+
+type simpleTokenTTLKeeper struct {
+	tokensMu        sync.Mutex
+	tokens          map[string]time.Time
+	stopCh          chan chan struct{}
+	deleteTokenFunc func(string)
+}
+
+func NewSimpleTokenTTLKeeper(deletefunc func(string)) *simpleTokenTTLKeeper {
+	stk := &simpleTokenTTLKeeper{
+		tokens:          make(map[string]time.Time),
+		stopCh:          make(chan chan struct{}),
+		deleteTokenFunc: deletefunc,
+	}
+	go stk.run()
+	return stk
+}
+
+func (tm *simpleTokenTTLKeeper) stop() {
+	waitCh := make(chan struct{})
+	tm.stopCh <- waitCh
+	<-waitCh
+	close(tm.stopCh)
+}
+
+func (tm *simpleTokenTTLKeeper) addSimpleToken(token string) {
+	tm.tokens[token] = time.Now().Add(simpleTokenTTL)
+}
+
+func (tm *simpleTokenTTLKeeper) resetSimpleToken(token string) {
+	if _, ok := tm.tokens[token]; ok {
+		tm.tokens[token] = time.Now().Add(simpleTokenTTL)
+	}
+}
+
+func (tm *simpleTokenTTLKeeper) deleteSimpleToken(token string) {
+	delete(tm.tokens, token)
+}
+
+func (tm *simpleTokenTTLKeeper) run() {
+	tokenTicker := time.NewTicker(simpleTokenTTLResolution)
+	defer tokenTicker.Stop()
+	for {
+		select {
+		case <-tokenTicker.C:
+			nowtime := time.Now()
+			tm.tokensMu.Lock()
+			for t, tokenendtime := range tm.tokens {
+				if nowtime.After(tokenendtime) {
+					tm.deleteTokenFunc(t)
+					delete(tm.tokens, t)
+				}
+			}
+			tm.tokensMu.Unlock()
+		case waitCh := <-tm.stopCh:
+			tm.tokens = make(map[string]time.Time)
+			waitCh <- struct{}{}
+			return
+		}
+	}
+}
+
 func (as *authStore) GenSimpleToken() (string, error) {
 	ret := make([]byte, defaultSimpleTokenLength)
 
@@ -44,6 +113,7 @@ func (as *authStore) GenSimpleToken() (string, error) {
 }
 
 func (as *authStore) assignSimpleTokenToUser(username, token string) {
+	as.simpleTokenKeeper.tokensMu.Lock()
 	as.simpleTokensMu.Lock()
 
 	_, ok := as.simpleTokens[token]
@@ -52,16 +122,23 @@ func (as *authStore) assignSimpleTokenToUser(username, token string) {
 	}
 
 	as.simpleTokens[token] = username
+	as.simpleTokenKeeper.addSimpleToken(token)
 	as.simpleTokensMu.Unlock()
+	as.simpleTokenKeeper.tokensMu.Unlock()
 }
 
 func (as *authStore) invalidateUser(username string) {
+	if as.simpleTokenKeeper == nil {
+		return
+	}
+	as.simpleTokenKeeper.tokensMu.Lock()
 	as.simpleTokensMu.Lock()
-	defer as.simpleTokensMu.Unlock()
-
 	for token, name := range as.simpleTokens {
 		if strings.Compare(name, username) == 0 {
 			delete(as.simpleTokens, token)
+			as.simpleTokenKeeper.deleteSimpleToken(token)
 		}
 	}
+	as.simpleTokensMu.Unlock()
+	as.simpleTokenKeeper.tokensMu.Unlock()
 }

auth/store.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -29,6 +30,7 @@ import (
 	"github.com/coreos/pkg/capnslog"
 	"golang.org/x/crypto/bcrypt"
 	"golang.org/x/net/context"
+	"google.golang.org/grpc/metadata"
 )
 
 var (
@@ -47,6 +49,7 @@ var (
 	ErrRootUserNotExist     = errors.New("auth: root user does not exist")
 	ErrRootRoleNotExist     = errors.New("auth: root user does not have root role")
 	ErrUserAlreadyExist     = errors.New("auth: user already exists")
+	ErrUserEmpty            = errors.New("auth: user name is empty")
 	ErrUserNotFound         = errors.New("auth: user not found")
 	ErrRoleAlreadyExist     = errors.New("auth: role already exists")
 	ErrRoleNotFound         = errors.New("auth: role not found")
@@ -56,6 +59,7 @@ var (
 	ErrPermissionNotGranted = errors.New("auth: permission is not granted to the role")
 	ErrAuthNotEnabled       = errors.New("auth: authentication is not enabled")
 	ErrAuthOldRevision      = errors.New("auth: revision in header is old")
+	ErrInvalidAuthToken     = errors.New("auth: invalid auth token")
 
 	// BcryptCost is the algorithm cost / strength for hashing auth passwords
 	BcryptCost = bcrypt.DefaultCost
@@ -146,6 +150,15 @@ type AuthStore interface {
 
 	// Revision gets current revision of authStore
 	Revision() uint64
+
+	// CheckPassword checks a given pair of username and password is correct
+	CheckPassword(username, password string) (uint64, error)
+
+	// Close does cleanup of AuthStore
+	Close() error
+
+	// AuthInfoFromCtx gets AuthInfo from gRPC's context
+	AuthInfoFromCtx(ctx context.Context) (*AuthInfo, error)
 }
 
 type authStore struct {
@@ -155,13 +168,33 @@ type authStore struct {
 
 	rangePermCache map[string]*unifiedRangePermissions // username -> unifiedRangePermissions
 
-	simpleTokensMu sync.RWMutex
-	simpleTokens   map[string]string // token -> username
-
 	revision uint64
+
+	// tokenSimple in v3.2+
+	indexWaiter       func(uint64) <-chan struct{}
+	simpleTokenKeeper *simpleTokenTTLKeeper
+	simpleTokensMu    sync.Mutex
+	simpleTokens      map[string]string // token -> username
+}
+
+func newDeleterFunc(as *authStore) func(string) {
+	return func(t string) {
+		as.simpleTokensMu.Lock()
+		defer as.simpleTokensMu.Unlock()
+		if username, ok := as.simpleTokens[t]; ok {
+			plog.Infof("deleting token %s for user %s", t, username)
+			delete(as.simpleTokens, t)
+		}
+	}
 }
 
 func (as *authStore) AuthEnable() error {
+	as.enabledMu.Lock()
+	defer as.enabledMu.Unlock()
+	if as.enabled {
+		plog.Noticef("Authentication already enabled")
+		return nil
+	}
 	b := as.be
 	tx := b.BatchTx()
 	tx.Lock()
@@ -181,9 +214,9 @@ func (as *authStore) AuthEnable() error {
 
 	tx.UnsafePut(authBucketName, enableFlagKey, authEnabled)
 
-	as.enabledMu.Lock()
 	as.enabled = true
-	as.enabledMu.Unlock()
+
+	as.simpleTokenKeeper = NewSimpleTokenTTLKeeper(newDeleterFunc(as))
 
 	as.rangePermCache = make(map[string]*unifiedRangePermissions)
 
@@ -195,6 +228,11 @@ func (as *authStore) AuthEnable() error {
 }
 
 func (as *authStore) AuthDisable() {
+	as.enabledMu.Lock()
+	defer as.enabledMu.Unlock()
+	if !as.enabled {
+		return
+	}
 	b := as.be
 	tx := b.BatchTx()
 	tx.Lock()
@@ -203,17 +241,32 @@ func (as *authStore) AuthDisable() {
 	tx.Unlock()
 	b.ForceCommit()
 
-	as.enabledMu.Lock()
 	as.enabled = false
-	as.enabledMu.Unlock()
 
 	as.simpleTokensMu.Lock()
 	as.simpleTokens = make(map[string]string) // invalidate all tokens
 	as.simpleTokensMu.Unlock()
+	if as.simpleTokenKeeper != nil {
+		as.simpleTokenKeeper.stop()
+		as.simpleTokenKeeper = nil
+	}
 
 	plog.Noticef("Authentication disabled")
 }
 
+func (as *authStore) Close() error {
+	as.enabledMu.Lock()
+	defer as.enabledMu.Unlock()
+	if !as.enabled {
+		return nil
+	}
+	if as.simpleTokenKeeper != nil {
+		as.simpleTokenKeeper.stop()
+		as.simpleTokenKeeper = nil
+	}
+	return nil
+}
+
 func (as *authStore) Authenticate(ctx context.Context, username, password string) (*pb.AuthenticateResponse, error) {
 	if !as.isAuthEnabled() {
 		return nil, ErrAuthNotEnabled
@@ -232,11 +285,6 @@ func (as *authStore) Authenticate(ctx context.Context, username, password string
 		return nil, ErrAuthFailed
 	}
 
-	if bcrypt.CompareHashAndPassword(user.Password, []byte(password)) != nil {
-		plog.Noticef("authentication failed, invalid password for user %s", username)
-		return &pb.AuthenticateResponse{}, ErrAuthFailed
-	}
-
 	token := fmt.Sprintf("%s.%d", simpleToken, index)
 	as.assignSimpleTokenToUser(username, token)
 
@@ -244,6 +292,24 @@ func (as *authStore) Authenticate(ctx context.Context, username, password string
 	return &pb.AuthenticateResponse{Token: token}, nil
 }
 
+func (as *authStore) CheckPassword(username, password string) (uint64, error) {
+	tx := as.be.BatchTx()
+	tx.Lock()
+	defer tx.Unlock()
+
+	user := getUser(tx, username)
+	if user == nil {
+		return 0, ErrAuthFailed
+	}
+
+	if bcrypt.CompareHashAndPassword(user.Password, []byte(password)) != nil {
+		plog.Noticef("authentication failed, invalid password for user %s", username)
+		return 0, ErrAuthFailed
+	}
+
+	return getRevision(tx), nil
+}
+
 func (as *authStore) Recover(be backend.Backend) {
 	enabled := false
 	as.be = be
@@ -266,6 +332,10 @@ func (as *authStore) Recover(be backend.Backend) {
 }
 
 func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, error) {
+	if len(r.Name) == 0 {
+		return nil, ErrUserEmpty
+	}
+
 	hashed, err := bcrypt.GenerateFromPassword([]byte(r.Password), BcryptCost)
 	if err != nil {
 		plog.Errorf("failed to hash password: %s", err)
@@ -400,11 +470,7 @@ func (as *authStore) UserGet(r *pb.AuthUserGetRequest) (*pb.AuthUserGetResponse,
 	if user == nil {
 		return nil, ErrUserNotFound
 	}
-
-	for _, role := range user.Roles {
-		resp.Roles = append(resp.Roles, role)
-	}
-
+	resp.Roles = append(resp.Roles, user.Roles...)
 	return &resp, nil
 }
 
@@ -470,11 +536,7 @@ func (as *authStore) RoleGet(r *pb.AuthRoleGetRequest) (*pb.AuthRoleGetResponse,
 	if role == nil {
 		return nil, ErrRoleNotFound
 	}
-
-	for _, perm := range role.KeyPermission {
-		resp.Perm = append(resp.Perm, perm)
-	}
-
+	resp.Perm = append(resp.Perm, role.KeyPermission...)
 	return &resp, nil
 }
 
@@ -584,10 +646,16 @@ func (as *authStore) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse,
 }
 
 func (as *authStore) AuthInfoFromToken(token string) (*AuthInfo, bool) {
-	as.simpleTokensMu.RLock()
-	defer as.simpleTokensMu.RUnlock()
-	t, ok := as.simpleTokens[token]
-	return &AuthInfo{Username: t, Revision: as.revision}, ok
+	// same as '(t *tokenSimple) info' in v3.2+
+	as.simpleTokenKeeper.tokensMu.Lock()
+	as.simpleTokensMu.Lock()
+	username, ok := as.simpleTokens[token]
+	if ok {
+		as.simpleTokenKeeper.resetSimpleToken(token)
+	}
+	as.simpleTokensMu.Unlock()
+	as.simpleTokenKeeper.tokensMu.Unlock()
+	return &AuthInfo{Username: username, Revision: as.revision}, ok
 }
 
 type permSlice []*authpb.Permission
@@ -652,6 +720,11 @@ func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeE
 		return nil
 	}
 
+	// only gets rev == 0 when passed AuthInfo{}; no user given
+	if revision == 0 {
+		return ErrUserEmpty
+	}
+
 	if revision < as.revision {
 		return ErrAuthOldRevision
 	}
@@ -694,6 +767,9 @@ func (as *authStore) IsAdminPermitted(authInfo *AuthInfo) error {
 	if !as.isAuthEnabled() {
 		return nil
 	}
+	if authInfo == nil {
+		return ErrUserEmpty
+	}
 
 	tx := as.be.BatchTx()
 	tx.Lock()
@@ -812,7 +888,7 @@ func (as *authStore) isAuthEnabled() bool {
 	return as.enabled
 }
 
-func NewAuthStore(be backend.Backend) *authStore {
+func NewAuthStore(be backend.Backend, indexWaiter func(uint64) <-chan struct{}) *authStore {
 	tx := be.BatchTx()
 	tx.Lock()
 
@@ -820,13 +896,30 @@ func NewAuthStore(be backend.Backend) *authStore {
 	tx.UnsafeCreateBucket(authUsersBucketName)
 	tx.UnsafeCreateBucket(authRolesBucketName)
 
-	as := &authStore{
-		be:           be,
-		simpleTokens: make(map[string]string),
-		revision:     0,
+	enabled := false
+	_, vs := tx.UnsafeRange(authBucketName, enableFlagKey, nil, 0)
+	if len(vs) == 1 {
+		if bytes.Equal(vs[0], authEnabled) {
+			enabled = true
+		}
 	}
 
-	as.commitRevision(tx)
+	as := &authStore{
+		be:             be,
+		simpleTokens:   make(map[string]string),
+		revision:       getRevision(tx),
+		indexWaiter:    indexWaiter,
+		enabled:        enabled,
+		rangePermCache: make(map[string]*unifiedRangePermissions),
+	}
+
+	if enabled {
+		as.simpleTokenKeeper = NewSimpleTokenTTLKeeper(newDeleterFunc(as))
+	}
+
+	if as.revision == 0 {
+		as.commitRevision(tx)
+	}
 
 	tx.Unlock()
 	be.ForceCommit()
@@ -853,7 +946,8 @@ func (as *authStore) commitRevision(tx backend.BatchTx) {
 func getRevision(tx backend.BatchTx) uint64 {
 	_, vs := tx.UnsafeRange(authBucketName, []byte(revisionKey), nil, 0)
 	if len(vs) != 1 {
-		plog.Panicf("failed to get the key of auth store revision")
+		// this can happen in the initialization phase
+		return 0
 	}
 
 	return binary.BigEndian.Uint64(vs[0])
@@ -862,3 +956,46 @@ func getRevision(tx backend.BatchTx) uint64 {
 func (as *authStore) Revision() uint64 {
 	return as.revision
 }
+
+func (as *authStore) isValidSimpleToken(token string, ctx context.Context) bool {
+	splitted := strings.Split(token, ".")
+	if len(splitted) != 2 {
+		return false
+	}
+	index, err := strconv.Atoi(splitted[1])
+	if err != nil {
+		return false
+	}
+
+	select {
+	case <-as.indexWaiter(uint64(index)):
+		return true
+	case <-ctx.Done():
+	}
+
+	return false
+}
+
+func (as *authStore) AuthInfoFromCtx(ctx context.Context) (*AuthInfo, error) {
+	md, ok := metadata.FromContext(ctx)
+	if !ok {
+		return nil, nil
+	}
+
+	ts, tok := md["token"]
+	if !tok {
+		return nil, nil
+	}
+
+	token := ts[0]
+	if !as.isValidSimpleToken(token, ctx) {
+		return nil, ErrInvalidAuthToken
+	}
+
+	authInfo, uok := as.AuthInfoFromToken(token)
+	if !uok {
+		plog.Warningf("invalid auth token: %s", token)
+		return nil, ErrInvalidAuthToken
+	}
+	return authInfo, nil
+}

auth/store_test.go

@@ -26,25 +26,38 @@ import (
 
 func init() { BcryptCost = bcrypt.MinCost }
 
-func TestUserAdd(t *testing.T) {
-	b, tPath := backend.NewDefaultTmpBackend()
-	defer func() {
-		b.Close()
-		os.Remove(tPath)
+func dummyIndexWaiter(index uint64) <-chan struct{} {
+	ch := make(chan struct{})
+	go func() {
+		ch <- struct{}{}
 	}()
+	return ch
+}
 
-	as := NewAuthStore(b)
-	ua := &pb.AuthUserAddRequest{Name: "foo"}
-	_, err := as.UserAdd(ua) // add a non-existing user
+// TestNewAuthStoreRevision ensures newly auth store
+// keeps the old revision when there are no changes.
+func TestNewAuthStoreRevision(t *testing.T) {
+	b, tPath := backend.NewDefaultTmpBackend()
+	defer os.Remove(tPath)
+
+	as := NewAuthStore(b, dummyIndexWaiter)
+	err := enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = as.UserAdd(ua) // add an existing user
-	if err == nil {
-		t.Fatalf("expected %v, got %v", ErrUserAlreadyExist, err)
-	}
-	if err != ErrUserAlreadyExist {
-		t.Fatalf("expected %v, got %v", ErrUserAlreadyExist, err)
+	old := as.Revision()
+	b.Close()
+	as.Close()
+
+	// no changes to commit
+	b2 := backend.NewDefaultBackend(tPath)
+	as = NewAuthStore(b2, dummyIndexWaiter)
+	new := as.Revision()
+	b2.Close()
+	as.Close()
+
+	if old != new {
+		t.Fatalf("expected revision %d, got %d", old, new)
 	}
 }
 
@@ -67,14 +80,15 @@ func enableAuthAndCreateRoot(as *authStore) error {
 	return as.AuthEnable()
 }
 
-func TestAuthenticate(t *testing.T) {
+func TestCheckPassword(t *testing.T) {
 	b, tPath := backend.NewDefaultTmpBackend()
 	defer func() {
 		b.Close()
 		os.Remove(tPath)
 	}()
 
-	as := NewAuthStore(b)
+	as := NewAuthStore(b, dummyIndexWaiter)
+	defer as.Close()
 	err := enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
@@ -87,8 +101,7 @@ func TestAuthenticate(t *testing.T) {
 	}
 
 	// auth a non-existing user
-	ctx1 := context.WithValue(context.WithValue(context.TODO(), "index", uint64(1)), "simpleToken", "dummy")
-	_, err = as.Authenticate(ctx1, "foo-test", "bar")
+	_, err = as.CheckPassword("foo-test", "bar")
 	if err == nil {
 		t.Fatalf("expected %v, got %v", ErrAuthFailed, err)
 	}
@@ -97,15 +110,13 @@ func TestAuthenticate(t *testing.T) {
 	}
 
 	// auth an existing user with correct password
-	ctx2 := context.WithValue(context.WithValue(context.TODO(), "index", uint64(2)), "simpleToken", "dummy")
-	_, err = as.Authenticate(ctx2, "foo", "bar")
+	_, err = as.CheckPassword("foo", "bar")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// auth an existing user but with wrong password
-	ctx3 := context.WithValue(context.WithValue(context.TODO(), "index", uint64(3)), "simpleToken", "dummy")
-	_, err = as.Authenticate(ctx3, "foo", "")
+	_, err = as.CheckPassword("foo", "")
 	if err == nil {
 		t.Fatalf("expected %v, got %v", ErrAuthFailed, err)
 	}
@@ -121,7 +132,8 @@ func TestUserDelete(t *testing.T) {
 		os.Remove(tPath)
 	}()
 
-	as := NewAuthStore(b)
+	as := NewAuthStore(b, dummyIndexWaiter)
+	defer as.Close()
 	err := enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
@@ -157,7 +169,8 @@ func TestUserChangePassword(t *testing.T) {
 		os.Remove(tPath)
 	}()
 
-	as := NewAuthStore(b)
+	as := NewAuthStore(b, dummyIndexWaiter)
+	defer as.Close()
 	err := enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
@@ -202,7 +215,8 @@ func TestRoleAdd(t *testing.T) {
 		os.Remove(tPath)
 	}()
 
-	as := NewAuthStore(b)
+	as := NewAuthStore(b, dummyIndexWaiter)
+	defer as.Close()
 	err := enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
@@ -222,7 +236,8 @@ func TestUserGrant(t *testing.T) {
 		os.Remove(tPath)
 	}()
 
-	as := NewAuthStore(b)
+	as := NewAuthStore(b, dummyIndexWaiter)
+	defer as.Close()
 	err := enableAuthAndCreateRoot(as)
 	if err != nil {
 		t.Fatal(err)
@@ -253,4 +268,93 @@ func TestUserGrant(t *testing.T) {
 	if err != ErrUserNotFound {
 		t.Fatalf("expected %v, got %v", ErrUserNotFound, err)
 	}
+
+	// non-admin user
+	err = as.IsAdminPermitted(&AuthInfo{Username: "foo", Revision: 1})
+	if err != ErrPermissionDenied {
+		t.Errorf("expected %v, got %v", ErrPermissionDenied, err)
+	}
+
+	// disabled auth should return nil
+	as.AuthDisable()
+	err = as.IsAdminPermitted(&AuthInfo{Username: "root", Revision: 1})
+	if err != nil {
+		t.Errorf("expected nil, got %v", err)
+	}
+}
+
+func TestRecoverFromSnapshot(t *testing.T) {
+	as, _ := setupAuthStore(t)
+
+	ua := &pb.AuthUserAddRequest{Name: "foo"}
+	_, err := as.UserAdd(ua) // add an existing user
+	if err == nil {
+		t.Fatalf("expected %v, got %v", ErrUserAlreadyExist, err)
+	}
+	if err != ErrUserAlreadyExist {
+		t.Fatalf("expected %v, got %v", ErrUserAlreadyExist, err)
+	}
+
+	ua = &pb.AuthUserAddRequest{Name: ""}
+	_, err = as.UserAdd(ua) // add a user with empty name
+	if err != ErrUserEmpty {
+		t.Fatal(err)
+	}
+
+	as.Close()
+
+	as2 := NewAuthStore(as.be, dummyIndexWaiter)
+	defer func(a *authStore) {
+		a.Close()
+	}(as2)
+
+	if !as2.isAuthEnabled() {
+		t.Fatal("recovering authStore from existing backend failed")
+	}
+
+	ul, err := as.UserList(&pb.AuthUserListRequest{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !contains(ul.Users, "root") {
+		t.Errorf("expected %v in %v", "root", ul.Users)
+	}
+}
+
+func contains(array []string, str string) bool {
+	for _, s := range array {
+		if s == str {
+			return true
+		}
+	}
+	return false
+}
+
+func setupAuthStore(t *testing.T) (store *authStore, teardownfunc func(t *testing.T)) {
+	b, tPath := backend.NewDefaultTmpBackend()
+
+	as := NewAuthStore(b, dummyIndexWaiter)
+	err := enableAuthAndCreateRoot(as)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// adds a new role
+	_, err = as.RoleAdd(&pb.AuthRoleAddRequest{Name: "role-test"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ua := &pb.AuthUserAddRequest{Name: "foo", Password: "bar"}
+	_, err = as.UserAdd(ua) // add a non-existing user
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tearDown := func(t *testing.T) {
+		b.Close()
+		os.Remove(tPath)
+		as.Close()
+	}
+	return as, tearDown
 }

build

@@ -4,12 +4,16 @@
 ORG_PATH="github.com/coreos"
 REPO_PATH="${ORG_PATH}/etcd"
 export GO15VENDOREXPERIMENT="1"
+
 eval $(go env)
 GIT_SHA=`git rev-parse --short HEAD || echo "GitNotFound"`
 if [ ! -z "$FAILPOINTS" ]; then
 	GIT_SHA="$GIT_SHA"-FAILPOINTS
 fi
 
+# Set GO_LDFLAGS="-s" for building without symbols for debugging.
+GO_LDFLAGS="$GO_LDFLAGS -X ${REPO_PATH}/cmd/vendor/${REPO_PATH}/version.GitSHA=${GIT_SHA}"
+
 # enable/disable failpoints
 toggle_failpoints() {
 	FAILPKGS="etcdserver/ mvcc/backend/"
@@ -31,8 +35,8 @@ etcd_build() {
 	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
 	toggle_failpoints
 	# Static compilation is useful when etcd is run in a container
-	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "-s -X ${REPO_PATH}/cmd/vendor/${REPO_PATH}/version.GitSHA=${GIT_SHA}" -o ${out}/etcd ${REPO_PATH}/cmd/etcd || return
-	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "-s" -o ${out}/etcdctl ${REPO_PATH}/cmd/etcdctl || return
+	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "$GO_LDFLAGS" -o ${out}/etcd ${REPO_PATH}/cmd/etcd || return
+	CGO_ENABLED=0 go build $GO_BUILD_FLAGS -installsuffix cgo -ldflags "$GO_LDFLAGS" -o ${out}/etcdctl ${REPO_PATH}/cmd/etcdctl || return
 }
 
 etcd_setup_gopath() {
@@ -40,8 +44,11 @@ etcd_setup_gopath() {
 	cd "$CDIR"
 	etcdGOPATH=${CDIR}/gopath
 	# preserve old gopath to support building with unvendored tooling deps (e.g., gofail)
-	export GOPATH=${etcdGOPATH}:$GOPATH
-	rm -f ${etcdGOPATH}/src
+	if [ -n "$GOPATH" ]; then
+		GOPATH=":$GOPATH"
+	fi
+	export GOPATH=${etcdGOPATH}$GOPATH
+	rm -rf ${etcdGOPATH}/src
 	mkdir -p ${etcdGOPATH}
 	ln -s ${CDIR}/cmd/vendor ${etcdGOPATH}/src
 }

build.ps1

@@ -1,9 +1,18 @@
 $ORG_PATH="github.com/coreos"
 $REPO_PATH="$ORG_PATH/etcd"
 $PWD = $((Get-Item -Path ".\" -Verbose).FullName)
+$FSROOT = $((Get-Location).Drive.Name+":")
+$FSYS = $((Get-WMIObject win32_logicaldisk -filter "DeviceID = '$FSROOT'").filesystem)
+
+if ($FSYS.StartsWith("FAT","CurrentCultureIgnoreCase")) {
+	echo "Error: Cannot build etcd using the $FSYS filesystem (use NTFS instead)"
+	exit 1
+}
+
+# Set $Env:GO_LDFLAGS="-s" for building without symbols.
+$GO_LDFLAGS="$Env:GO_LDFLAGS -X $REPO_PATH/cmd/vendor/$REPO_PATH/version.GitSHA=$GIT_SHA"
 
 # rebuild symlinks
-echo "Rebuilding symlinks"
 git ls-files -s cmd | select-string -pattern 120000 | ForEach {
 	$l = $_.ToString()
 	$lnkname = $l.Split('	')[1]
@@ -13,27 +22,54 @@ git ls-files -s cmd | select-string -pattern 120000 | ForEach {
 
 	$terms = $lnkname.Split("\")
 	$dirname = $terms[0..($terms.length-2)] -join "\"
-	
 	$lnkname = "$PWD\$lnkname"
 	$targetAbs = "$((Get-Item -Path "$dirname\$target").FullName)"
 	$targetAbs = $targetAbs.Replace("/", "\")
+
 	if (test-path -pathtype container "$targetAbs") {
-		# rd so deleting junction doesn't take files with it
-		cmd /c rd "$lnkname"
-		cmd /c del /A /F "$lnkname"
-		cmd /c mklink /J "$lnkname" "$targetAbs"
+		if (Test-Path "$lnkname") {
+			if ((Get-Item "$lnkname") -is [System.IO.DirectoryInfo]) {
+				# rd so deleting junction doesn't take files with it
+				cmd /c rd  "$lnkname"
+			}
+		}
+		if (Test-Path "$lnkname") {
+			if (!((Get-Item "$lnkname") -is [System.IO.DirectoryInfo])) {
+				cmd /c del /A /F  "$lnkname"
+			}
+		}
+		cmd /c mklink /J  "$lnkname"   "$targetAbs"  ">NUL"
 	} else {
-		cmd /c del /A /F "$lnkname"
-		cmd /c mklink /H "$lnkname" "$targetAbs"
+		# Remove file with symlink data (first run)
+		if (Test-Path "$lnkname") {
+			cmd /c del /A /F  "$lnkname"
+		}
+		cmd /c mklink /H  "$lnkname"   "$targetAbs"  ">NUL"
 	}
 }
 
 if (-not $env:GOPATH) {
 	$orgpath="$PWD\gopath\src\" + $ORG_PATH.Replace("/", "\")
-	cmd /c rd "$orgpath\etcd"
-	cmd /c del "$orgpath"
-	cmd /c mkdir "$orgpath"
-	cmd /c mklink /J "$orgpath\etcd" "$PWD"
+	if (Test-Path "$orgpath\etcd") {
+		if ((Get-Item "$orgpath\etcd") -is [System.IO.DirectoryInfo]) {
+			# rd so deleting junction doesn't take files with it
+			cmd /c rd  "$orgpath\etcd"
+		}
+	}
+	if (Test-Path "$orgpath") {
+		if ((Get-Item "$orgpath") -is [System.IO.DirectoryInfo]) {
+			# rd so deleting junction doesn't take files with it
+			cmd /c rd  "$orgpath"
+		}
+	}
+	if (Test-Path "$orgpath") {
+		if (!((Get-Item "$orgpath") -is [System.IO.DirectoryInfo])) {
+			# Remove file with symlink data (first run)
+			cmd /c del /A /F  "$orgpath"
+		}
+	}
+	cmd /c mkdir  "$orgpath"
+	cmd /c mklink /J  "$orgpath\etcd"   "$PWD"  ">NUL"
 	$env:GOPATH = "$PWD\gopath"
 }
 
@@ -41,5 +77,5 @@ if (-not $env:GOPATH) {
 $env:CGO_ENABLED = 0
 $env:GO15VENDOREXPERIMENT = 1
 $GIT_SHA="$(git rev-parse --short HEAD)"
-go build -a -installsuffix cgo -ldflags "-s -X $REPO_PATH/cmd/vendor/$REPO_PATH/version.GitSHA=$GIT_SHA" -o bin\etcd.exe "$REPO_PATH\cmd\etcd"
-go build -a -installsuffix cgo -ldflags "-s" -o bin\etcdctl.exe "$REPO_PATH\cmd\etcdctl"
+go build -a -installsuffix cgo -ldflags $GO_LDFLAGS -o bin\etcd.exe "$REPO_PATH\cmd\etcd"
+go build -a -installsuffix cgo -ldflags $GO_LDFLAGS -o bin\etcdctl.exe "$REPO_PATH\cmd\etcdctl"

client/README.md

@@ -114,4 +114,4 @@ if err != nil {
 
 3. Default etcd/client cannot handle the case that the remote server is SIGSTOPed now. TCP keepalive mechanism doesn't help in this scenario because operating system may still send TCP keep-alive packets. Over time we'd like to improve this functionality, but solving this issue isn't high priority because a real-life case in which a server is stopped, but the connection is kept alive, hasn't been brought to our attention.
 
-4. etcd/client cannot detect whether the member in use is healthy when doing read requests. If the member is isolated from the cluster, etcd/client may retrieve outdated data. As a workaround, users could monitor experimental /health endpoint for member healthy information. We are improving it at [#3265](https://github.com/coreos/etcd/issues/3265).
+4. etcd/client cannot detect whether a member is healthy with watches and non-quorum read requests. If the member is isolated from the cluster, etcd/client may retrieve outdated data. Instead, users can either issue quorum read requests or monitor the /health endpoint for member health information.

client/client.go

@@ -22,7 +22,6 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"reflect"
 	"sort"
 	"strconv"
 	"sync"
@@ -261,53 +260,67 @@ type httpClusterClient struct {
 	selectionMode EndpointSelectionMode
 }
 
-func (c *httpClusterClient) getLeaderEndpoint() (string, error) {
-	mAPI := NewMembersAPI(c)
-	leader, err := mAPI.Leader(context.Background())
+func (c *httpClusterClient) getLeaderEndpoint(ctx context.Context, eps []url.URL) (string, error) {
+	ceps := make([]url.URL, len(eps))
+	copy(ceps, eps)
+
+	// To perform a lookup on the new endpoint list without using the current
+	// client, we'll copy it
+	clientCopy := &httpClusterClient{
+		clientFactory: c.clientFactory,
+		credentials:   c.credentials,
+		rand:          c.rand,
+
+		pinned:    0,
+		endpoints: ceps,
+	}
+
+	mAPI := NewMembersAPI(clientCopy)
+	leader, err := mAPI.Leader(ctx)
 	if err != nil {
 		return "", err
 	}
+	if len(leader.ClientURLs) == 0 {
+		return "", ErrNoLeaderEndpoint
+	}
 
 	return leader.ClientURLs[0], nil // TODO: how to handle multiple client URLs?
 }
 
-func (c *httpClusterClient) SetEndpoints(eps []string) error {
+func (c *httpClusterClient) parseEndpoints(eps []string) ([]url.URL, error) {
 	if len(eps) == 0 {
-		return ErrNoEndpoints
+		return []url.URL{}, ErrNoEndpoints
 	}
 
 	neps := make([]url.URL, len(eps))
 	for i, ep := range eps {
 		u, err := url.Parse(ep)
 		if err != nil {
-			return err
+			return []url.URL{}, err
 		}
 		neps[i] = *u
 	}
+	return neps, nil
+}
 
-	switch c.selectionMode {
-	case EndpointSelectionRandom:
-		c.endpoints = shuffleEndpoints(c.rand, neps)
-		c.pinned = 0
-	case EndpointSelectionPrioritizeLeader:
-		c.endpoints = neps
-		lep, err := c.getLeaderEndpoint()
-		if err != nil {
-			return ErrNoLeaderEndpoint
-		}
-
-		for i := range c.endpoints {
-			if c.endpoints[i].String() == lep {
-				c.pinned = i
-				break
-			}
-		}
-		// If endpoints doesn't have the lu, just keep c.pinned = 0.
-		// Forwarding between follower and leader would be required but it works.
-	default:
-		return fmt.Errorf("invalid endpoint selection mode: %d", c.selectionMode)
+func (c *httpClusterClient) SetEndpoints(eps []string) error {
+	neps, err := c.parseEndpoints(eps)
+	if err != nil {
+		return err
 	}
 
+	c.Lock()
+	defer c.Unlock()
+
+	c.endpoints = shuffleEndpoints(c.rand, neps)
+	// We're not doing anything for PrioritizeLeader here. This is
+	// due to not having a context meaning we can't call getLeaderEndpoint
+	// However, if you're using PrioritizeLeader, you've already been told
+	// to regularly call sync, where we do have a ctx, and can figure the
+	// leader. PrioritizeLeader is also quite a loose guarantee, so deal
+	// with it
+	c.pinned = 0
+
 	return nil
 }
 
@@ -401,27 +414,51 @@ func (c *httpClusterClient) Sync(ctx context.Context) error {
 		return err
 	}
 
-	c.Lock()
-	defer c.Unlock()
-
 	var eps []string
 	for _, m := range ms {
 		eps = append(eps, m.ClientURLs...)
 	}
-	sort.Sort(sort.StringSlice(eps))
 
-	ceps := make([]string, len(c.endpoints))
-	for i, cep := range c.endpoints {
-		ceps[i] = cep.String()
-	}
-	sort.Sort(sort.StringSlice(ceps))
-	// fast path if no change happens
-	// this helps client to pin the endpoint when no cluster change
-	if reflect.DeepEqual(eps, ceps) {
-		return nil
+	neps, err := c.parseEndpoints(eps)
+	if err != nil {
+		return err
 	}
 
-	return c.SetEndpoints(eps)
+	npin := 0
+
+	switch c.selectionMode {
+	case EndpointSelectionRandom:
+		c.RLock()
+		eq := endpointsEqual(c.endpoints, neps)
+		c.RUnlock()
+
+		if eq {
+			return nil
+		}
+		// When items in the endpoint list changes, we choose a new pin
+		neps = shuffleEndpoints(c.rand, neps)
+	case EndpointSelectionPrioritizeLeader:
+		nle, err := c.getLeaderEndpoint(ctx, neps)
+		if err != nil {
+			return ErrNoLeaderEndpoint
+		}
+
+		for i, n := range neps {
+			if n.String() == nle {
+				npin = i
+				break
+			}
+		}
+	default:
+		return fmt.Errorf("invalid endpoint selection mode: %d", c.selectionMode)
+	}
+
+	c.Lock()
+	defer c.Unlock()
+	c.endpoints = neps
+	c.pinned = npin
+
+	return nil
 }
 
 func (c *httpClusterClient) AutoSync(ctx context.Context, interval time.Duration) error {
@@ -607,3 +644,27 @@ func shuffleEndpoints(r *rand.Rand, eps []url.URL) []url.URL {
 	}
 	return neps
 }
+
+func endpointsEqual(left, right []url.URL) bool {
+	if len(left) != len(right) {
+		return false
+	}
+
+	sLeft := make([]string, len(left))
+	sRight := make([]string, len(right))
+	for i, l := range left {
+		sLeft[i] = l.String()
+	}
+	for i, r := range right {
+		sRight[i] = r.String()
+	}
+
+	sort.Strings(sLeft)
+	sort.Strings(sRight)
+	for i := range sLeft {
+		if sLeft[i] != sRight[i] {
+			return false
+		}
+	}
+	return true
+}

client/client_test.go

@@ -900,6 +900,90 @@ func TestHTTPClusterClientSyncPinEndpoint(t *testing.T) {
 	}
 }
 
+// TestHTTPClusterClientSyncUnpinEndpoint tests that Sync() unpins the endpoint when
+// it gets a different member list than before.
+func TestHTTPClusterClientSyncUnpinEndpoint(t *testing.T) {
+	cf := newStaticHTTPClientFactory([]staticHTTPResponse{
+		{
+			resp: http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}},
+			body: []byte(`{"members":[{"id":"2745e2525fce8fe","peerURLs":["http://127.0.0.1:7003"],"name":"node3","clientURLs":["http://127.0.0.1:4003"]},{"id":"42134f434382925","peerURLs":["http://127.0.0.1:2380","http://127.0.0.1:7001"],"name":"node1","clientURLs":["http://127.0.0.1:2379","http://127.0.0.1:4001"]},{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}]}`),
+		},
+		{
+			resp: http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}},
+			body: []byte(`{"members":[{"id":"42134f434382925","peerURLs":["http://127.0.0.1:2380","http://127.0.0.1:7001"],"name":"node1","clientURLs":["http://127.0.0.1:2379","http://127.0.0.1:4001"]},{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}]}`),
+		},
+		{
+			resp: http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}},
+			body: []byte(`{"members":[{"id":"2745e2525fce8fe","peerURLs":["http://127.0.0.1:7003"],"name":"node3","clientURLs":["http://127.0.0.1:4003"]},{"id":"42134f434382925","peerURLs":["http://127.0.0.1:2380","http://127.0.0.1:7001"],"name":"node1","clientURLs":["http://127.0.0.1:2379","http://127.0.0.1:4001"]},{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}]}`),
+		},
+	})
+
+	hc := &httpClusterClient{
+		clientFactory: cf,
+		rand:          rand.New(rand.NewSource(0)),
+	}
+	err := hc.SetEndpoints([]string{"http://127.0.0.1:4003", "http://127.0.0.1:2379", "http://127.0.0.1:4001", "http://127.0.0.1:4002"})
+	if err != nil {
+		t.Fatalf("unexpected error during setup: %#v", err)
+	}
+	wants := []string{"http://127.0.0.1:2379", "http://127.0.0.1:4001", "http://127.0.0.1:4002"}
+
+	for i := 0; i < 3; i++ {
+		err = hc.Sync(context.Background())
+		if err != nil {
+			t.Fatalf("#%d: unexpected error during Sync: %#v", i, err)
+		}
+
+		if g := hc.endpoints[hc.pinned]; g.String() != wants[i] {
+			t.Errorf("#%d: pinned endpoint = %v, want %v", i, g, wants[i])
+		}
+	}
+}
+
+// TestHTTPClusterClientSyncPinLeaderEndpoint tests that Sync() pins the leader
+// when the selection mode is EndpointSelectionPrioritizeLeader
+func TestHTTPClusterClientSyncPinLeaderEndpoint(t *testing.T) {
+	cf := newStaticHTTPClientFactory([]staticHTTPResponse{
+		{
+			resp: http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}},
+			body: []byte(`{"members":[{"id":"2745e2525fce8fe","peerURLs":["http://127.0.0.1:7003"],"name":"node3","clientURLs":["http://127.0.0.1:4003"]},{"id":"42134f434382925","peerURLs":["http://127.0.0.1:2380","http://127.0.0.1:7001"],"name":"node1","clientURLs":["http://127.0.0.1:2379","http://127.0.0.1:4001"]},{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}]}`),
+		},
+		{
+			resp: http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}},
+			body: []byte(`{"id":"2745e2525fce8fe","peerURLs":["http://127.0.0.1:7003"],"name":"node3","clientURLs":["http://127.0.0.1:4003"]}`),
+		},
+		{
+			resp: http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}},
+			body: []byte(`{"members":[{"id":"2745e2525fce8fe","peerURLs":["http://127.0.0.1:7003"],"name":"node3","clientURLs":["http://127.0.0.1:4003"]},{"id":"42134f434382925","peerURLs":["http://127.0.0.1:2380","http://127.0.0.1:7001"],"name":"node1","clientURLs":["http://127.0.0.1:2379","http://127.0.0.1:4001"]},{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}]}`),
+		},
+		{
+			resp: http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}},
+			body: []byte(`{"id":"94088180e21eb87b","peerURLs":["http://127.0.0.1:7002"],"name":"node2","clientURLs":["http://127.0.0.1:4002"]}`),
+		},
+	})
+
+	hc := &httpClusterClient{
+		clientFactory: cf,
+		rand:          rand.New(rand.NewSource(0)),
+		selectionMode: EndpointSelectionPrioritizeLeader,
+		endpoints:     []url.URL{{}}, // Need somewhere to pretend to send to initially
+	}
+
+	wants := []string{"http://127.0.0.1:4003", "http://127.0.0.1:4002"}
+
+	for i, want := range wants {
+		err := hc.Sync(context.Background())
+		if err != nil {
+			t.Fatalf("#%d: unexpected error during Sync: %#v", i, err)
+		}
+
+		pinned := hc.endpoints[hc.pinned].String()
+		if pinned != want {
+			t.Errorf("#%d: pinned endpoint = %v, want %v", i, pinned, want)
+		}
+	}
+}
+
 func TestHTTPClusterClientResetFail(t *testing.T) {
 	tests := [][]string{
 		// need at least one endpoint

client/keys.go

@@ -272,6 +272,10 @@ type Response struct {
 	// Index holds the cluster-level index at the time the Response was generated.
 	// This index is not tied to the Node(s) contained in this Response.
 	Index uint64 `json:"-"`
+
+	// ClusterID holds the cluster-level ID reported by the server.  This
+	// should be different for different etcd clusters.
+	ClusterID string `json:"-"`
 }
 
 type Node struct {
@@ -665,6 +669,7 @@ func unmarshalSuccessfulKeysResponse(header http.Header, body []byte) (*Response
 			return nil, err
 		}
 	}
+	res.ClusterID = header.Get("X-Etcd-Cluster-ID")
 	return &res, nil
 }
 

client/keys_test.go

@@ -673,23 +673,24 @@ func TestUnmarshalSuccessfulResponse(t *testing.T) {
 	expiration.UnmarshalText([]byte("2015-04-07T04:40:23.044979686Z"))
 
 	tests := []struct {
-		hdr     string
-		body    string
-		wantRes *Response
-		wantErr bool
+		indexHdr     string
+		clusterIDHdr string
+		body         string
+		wantRes      *Response
+		wantErr      bool
 	}{
 		// Neither PrevNode or Node
 		{
-			hdr:     "1",
-			body:    `{"action":"delete"}`,
-			wantRes: &Response{Action: "delete", Index: 1},
-			wantErr: false,
+			indexHdr: "1",
+			body:     `{"action":"delete"}`,
+			wantRes:  &Response{Action: "delete", Index: 1},
+			wantErr:  false,
 		},
 
 		// PrevNode
 		{
-			hdr:  "15",
-			body: `{"action":"delete", "prevNode": {"key": "/foo", "value": "bar", "modifiedIndex": 12, "createdIndex": 10}}`,
+			indexHdr: "15",
+			body:     `{"action":"delete", "prevNode": {"key": "/foo", "value": "bar", "modifiedIndex": 12, "createdIndex": 10}}`,
 			wantRes: &Response{
 				Action: "delete",
 				Index:  15,
@@ -706,8 +707,8 @@ func TestUnmarshalSuccessfulResponse(t *testing.T) {
 
 		// Node
 		{
-			hdr:  "15",
-			body: `{"action":"get", "node": {"key": "/foo", "value": "bar", "modifiedIndex": 12, "createdIndex": 10, "ttl": 10, "expiration": "2015-04-07T04:40:23.044979686Z"}}`,
+			indexHdr: "15",
+			body:     `{"action":"get", "node": {"key": "/foo", "value": "bar", "modifiedIndex": 12, "createdIndex": 10, "ttl": 10, "expiration": "2015-04-07T04:40:23.044979686Z"}}`,
 			wantRes: &Response{
 				Action: "get",
 				Index:  15,
@@ -726,8 +727,9 @@ func TestUnmarshalSuccessfulResponse(t *testing.T) {
 
 		// Node Dir
 		{
-			hdr:  "15",
-			body: `{"action":"get", "node": {"key": "/foo", "dir": true, "modifiedIndex": 12, "createdIndex": 10}}`,
+			indexHdr:     "15",
+			clusterIDHdr: "abcdef",
+			body:         `{"action":"get", "node": {"key": "/foo", "dir": true, "modifiedIndex": 12, "createdIndex": 10}}`,
 			wantRes: &Response{
 				Action: "get",
 				Index:  15,
@@ -737,15 +739,16 @@ func TestUnmarshalSuccessfulResponse(t *testing.T) {
 					ModifiedIndex: 12,
 					CreatedIndex:  10,
 				},
-				PrevNode: nil,
+				PrevNode:  nil,
+				ClusterID: "abcdef",
 			},
 			wantErr: false,
 		},
 
 		// PrevNode and Node
 		{
-			hdr:  "15",
-			body: `{"action":"update", "prevNode": {"key": "/foo", "value": "baz", "modifiedIndex": 10, "createdIndex": 10}, "node": {"key": "/foo", "value": "bar", "modifiedIndex": 12, "createdIndex": 10}}`,
+			indexHdr: "15",
+			body:     `{"action":"update", "prevNode": {"key": "/foo", "value": "baz", "modifiedIndex": 10, "createdIndex": 10}, "node": {"key": "/foo", "value": "bar", "modifiedIndex": 12, "createdIndex": 10}}`,
 			wantRes: &Response{
 				Action: "update",
 				Index:  15,
@@ -767,24 +770,24 @@ func TestUnmarshalSuccessfulResponse(t *testing.T) {
 
 		// Garbage in body
 		{
-			hdr:     "",
-			body:    `garbage`,
-			wantRes: nil,
-			wantErr: true,
+			indexHdr: "",
+			body:     `garbage`,
+			wantRes:  nil,
+			wantErr:  true,
 		},
 
 		// non-integer index
 		{
-			hdr:     "poo",
-			body:    `{}`,
-			wantRes: nil,
-			wantErr: true,
+			indexHdr: "poo",
+			body:     `{}`,
+			wantRes:  nil,
+			wantErr:  true,
 		},
 	}
 
 	for i, tt := range tests {
 		h := make(http.Header)
-		h.Add("X-Etcd-Index", tt.hdr)
+		h.Add("X-Etcd-Index", tt.indexHdr)
 		res, err := unmarshalSuccessfulKeysResponse(h, []byte(tt.body))
 		if tt.wantErr != (err != nil) {
 			t.Errorf("#%d: wantErr=%t, err=%v", i, tt.wantErr, err)

clientv3/README.md

@@ -72,6 +72,10 @@ if err != nil {
 }
 ```
 
+## Metrics
+
+The etcd client optionally exposes RPC metrics through [go-grpc-prometheus](https://github.com/grpc-ecosystem/go-grpc-prometheus). See the [examples](https://github.com/coreos/etcd/blob/master/clientv3/example_metrics_test.go).
+
 ## Examples
 
 More code examples can be found at [GoDoc](https://godoc.org/github.com/coreos/etcd/clientv3).

clientv3/auth.go

@@ -116,12 +116,12 @@ func NewAuth(c *Client) Auth {
 }
 
 func (auth *auth) AuthEnable(ctx context.Context) (*AuthEnableResponse, error) {
-	resp, err := auth.remote.AuthEnable(ctx, &pb.AuthEnableRequest{})
+	resp, err := auth.remote.AuthEnable(ctx, &pb.AuthEnableRequest{}, grpc.FailFast(false))
 	return (*AuthEnableResponse)(resp), toErr(ctx, err)
 }
 
 func (auth *auth) AuthDisable(ctx context.Context) (*AuthDisableResponse, error) {
-	resp, err := auth.remote.AuthDisable(ctx, &pb.AuthDisableRequest{})
+	resp, err := auth.remote.AuthDisable(ctx, &pb.AuthDisableRequest{}, grpc.FailFast(false))
 	return (*AuthDisableResponse)(resp), toErr(ctx, err)
 }
 

clientv3/balancer.go

@@ -21,8 +21,14 @@ import (
 
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 )
 
+// ErrNoAddrAvilable is returned by Get() when the balancer does not have
+// any active connection to endpoints at the time.
+// This error is returned only when opts.BlockingWait is true.
+var ErrNoAddrAvilable = grpc.Errorf(codes.Unavailable, "there is no address available")
+
 // simpleBalancer does the bare minimum to expose multiple eps
 // to the grpc reconnection code path
 type simpleBalancer struct {
@@ -72,7 +78,7 @@ func newSimpleBalancer(eps []string) *simpleBalancer {
 	return sb
 }
 
-func (b *simpleBalancer) Start(target string) error { return nil }
+func (b *simpleBalancer) Start(target string, config grpc.BalancerConfig) error { return nil }
 
 func (b *simpleBalancer) ConnectNotify() <-chan struct{} {
 	b.mu.Lock()
@@ -162,6 +168,25 @@ func (b *simpleBalancer) Up(addr grpc.Address) func(error) {
 
 func (b *simpleBalancer) Get(ctx context.Context, opts grpc.BalancerGetOptions) (grpc.Address, func(), error) {
 	var addr string
+
+	// If opts.BlockingWait is false (for fail-fast RPCs), it should return
+	// an address it has notified via Notify immediately instead of blocking.
+	if !opts.BlockingWait {
+		b.mu.RLock()
+		closed := b.closed
+		addr = b.pinAddr
+		upEps := len(b.upEps)
+		b.mu.RUnlock()
+		if closed {
+			return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing
+		}
+
+		if upEps == 0 {
+			return grpc.Address{Addr: ""}, nil, ErrNoAddrAvilable
+		}
+		return grpc.Address{Addr: addr}, func() {}, nil
+	}
+
 	for {
 		b.mu.RLock()
 		ch := b.upc

clientv3/balancer_test.go

@@ -0,0 +1,106 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+)
+
+var (
+	endpoints = []string{"localhost:2379", "localhost:22379", "localhost:32379"}
+)
+
+func TestBalancerGetUnblocking(t *testing.T) {
+	sb := newSimpleBalancer(endpoints)
+	unblockingOpts := grpc.BalancerGetOptions{BlockingWait: false}
+
+	_, _, err := sb.Get(context.Background(), unblockingOpts)
+	if err != ErrNoAddrAvilable {
+		t.Errorf("Get() with no up endpoints should return ErrNoAddrAvailable, got: %v", err)
+	}
+
+	down1 := sb.Up(grpc.Address{Addr: endpoints[1]})
+	down2 := sb.Up(grpc.Address{Addr: endpoints[2]})
+	addrFirst, putFun, err := sb.Get(context.Background(), unblockingOpts)
+	if err != nil {
+		t.Errorf("Get() with up endpoints should success, got %v", err)
+	}
+	if addrFirst.Addr != endpoints[1] {
+		t.Errorf("Get() didn't return expected address, got %v", addrFirst)
+	}
+	if putFun == nil {
+		t.Errorf("Get() returned unexpected nil put function")
+	}
+	addrSecond, _, _ := sb.Get(context.Background(), unblockingOpts)
+	if addrFirst.Addr != addrSecond.Addr {
+		t.Errorf("Get() didn't return the same address as previous call, got %v and %v", addrFirst, addrSecond)
+	}
+
+	down1(errors.New("error"))
+	down2(errors.New("error"))
+	_, _, err = sb.Get(context.Background(), unblockingOpts)
+	if err != ErrNoAddrAvilable {
+		t.Errorf("Get() with no up endpoints should return ErrNoAddrAvailable, got: %v", err)
+	}
+}
+
+func TestBalancerGetBlocking(t *testing.T) {
+	sb := newSimpleBalancer(endpoints)
+	blockingOpts := grpc.BalancerGetOptions{BlockingWait: true}
+
+	ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*100)
+	_, _, err := sb.Get(ctx, blockingOpts)
+	if err != context.DeadlineExceeded {
+		t.Errorf("Get() with no up endpoints should timeout, got %v", err)
+	}
+
+	downC := make(chan func(error), 1)
+
+	go func() {
+		// ensure sb.Up() will be called after sb.Get() to see if Up() releases blocking Get()
+		time.Sleep(time.Millisecond * 100)
+		downC <- sb.Up(grpc.Address{Addr: endpoints[1]})
+	}()
+	addrFirst, putFun, err := sb.Get(context.Background(), blockingOpts)
+	if err != nil {
+		t.Errorf("Get() with up endpoints should success, got %v", err)
+	}
+	if addrFirst.Addr != endpoints[1] {
+		t.Errorf("Get() didn't return expected address, got %v", addrFirst)
+	}
+	if putFun == nil {
+		t.Errorf("Get() returned unexpected nil put function")
+	}
+	down1 := <-downC
+
+	down2 := sb.Up(grpc.Address{Addr: endpoints[2]})
+	addrSecond, _, _ := sb.Get(context.Background(), blockingOpts)
+	if addrFirst.Addr != addrSecond.Addr {
+		t.Errorf("Get() didn't return the same address as previous call, got %v and %v", addrFirst, addrSecond)
+	}
+
+	down1(errors.New("error"))
+	down2(errors.New("error"))
+	ctx, _ = context.WithTimeout(context.Background(), time.Millisecond*100)
+	_, _, err = sb.Get(ctx, blockingOpts)
+	if err != context.DeadlineExceeded {
+		t.Errorf("Get() with no up endpoints should timeout, got %v", err)
+	}
+}

clientv3/client.go

@@ -18,14 +18,14 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"io/ioutil"
-	"log"
 	"net"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -47,11 +47,12 @@ type Client struct {
 	Auth
 	Maintenance
 
-	conn         *grpc.ClientConn
-	cfg          Config
-	creds        *credentials.TransportCredentials
-	balancer     *simpleBalancer
-	retryWrapper retryRpcFunc
+	conn             *grpc.ClientConn
+	cfg              Config
+	creds            *credentials.TransportCredentials
+	balancer         *simpleBalancer
+	retryWrapper     retryRpcFunc
+	retryAuthWrapper retryRpcFunc
 
 	ctx    context.Context
 	cancel context.CancelFunc
@@ -60,6 +61,8 @@ type Client struct {
 	Username string
 	// Password is a password for authentication
 	Password string
+	// tokenCred is an instance of WithPerRPCCredentials()'s argument
+	tokenCred *authTokenCredential
 }
 
 // New creates a new etcdv3 client from a given configuration.
@@ -88,6 +91,8 @@ func NewFromConfigFile(path string) (*Client, error) {
 // Close shuts down the client's etcd connections.
 func (c *Client) Close() error {
 	c.cancel()
+	c.Watcher.Close()
+	c.Lease.Close()
 	return toErr(c.ctx, c.conn.Close())
 }
 
@@ -97,7 +102,12 @@ func (c *Client) Close() error {
 func (c *Client) Ctx() context.Context { return c.ctx }
 
 // Endpoints lists the registered endpoints for the client.
-func (c *Client) Endpoints() []string { return c.cfg.Endpoints }
+func (c *Client) Endpoints() (eps []string) {
+	// copy the slice; protect original endpoints from being changed
+	eps = make([]string, len(c.cfg.Endpoints))
+	copy(eps, c.cfg.Endpoints)
+	return
+}
 
 // SetEndpoints updates client's endpoints.
 func (c *Client) SetEndpoints(eps ...string) {
@@ -138,7 +148,8 @@ func (c *Client) autoSync() {
 }
 
 type authTokenCredential struct {
-	token string
+	token   string
+	tokenMu *sync.RWMutex
 }
 
 func (cred authTokenCredential) RequireTransportSecurity() bool {
@@ -146,19 +157,21 @@ func (cred authTokenCredential) RequireTransportSecurity() bool {
 }
 
 func (cred authTokenCredential) GetRequestMetadata(ctx context.Context, s ...string) (map[string]string, error) {
+	cred.tokenMu.RLock()
+	defer cred.tokenMu.RUnlock()
 	return map[string]string{
 		"token": cred.token,
 	}, nil
 }
 
-func parseEndpoint(endpoint string) (proto string, host string, scheme bool) {
+func parseEndpoint(endpoint string) (proto string, host string, scheme string) {
 	proto = "tcp"
 	host = endpoint
 	url, uerr := url.Parse(endpoint)
 	if uerr != nil || !strings.Contains(endpoint, "://") {
 		return
 	}
-	scheme = true
+	scheme = url.Scheme
 
 	// strip scheme:// prefix since grpc dials by host
 	host = url.Host
@@ -172,9 +185,9 @@ func parseEndpoint(endpoint string) (proto string, host string, scheme bool) {
 	return
 }
 
-func (c *Client) processCreds(protocol string) (creds *credentials.TransportCredentials) {
+func (c *Client) processCreds(scheme string) (creds *credentials.TransportCredentials) {
 	creds = c.creds
-	switch protocol {
+	switch scheme {
 	case "unix":
 	case "http":
 		creds = nil
@@ -208,13 +221,14 @@ func (c *Client) dialSetupOpts(endpoint string, dopts ...grpc.DialOption) (opts
 			return nil, c.ctx.Err()
 		default:
 		}
-		return net.DialTimeout(proto, host, t)
+		dialer := &net.Dialer{Timeout: t}
+		return dialer.DialContext(c.ctx, proto, host)
 	}
 	opts = append(opts, grpc.WithDialer(f))
 
 	creds := c.creds
-	if proto, _, scheme := parseEndpoint(endpoint); scheme {
-		creds = c.processCreds(proto)
+	if _, _, scheme := parseEndpoint(endpoint); len(scheme) != 0 {
+		creds = c.processCreds(scheme)
 	}
 	if creds != nil {
 		opts = append(opts, grpc.WithTransportCredentials(*creds))
@@ -230,24 +244,56 @@ func (c *Client) Dial(endpoint string) (*grpc.ClientConn, error) {
 	return c.dial(endpoint)
 }
 
+func (c *Client) getToken(ctx context.Context) error {
+	var err error // return last error in a case of fail
+	var auth *authenticator
+
+	for i := 0; i < len(c.cfg.Endpoints); i++ {
+		endpoint := c.cfg.Endpoints[i]
+		host := getHost(endpoint)
+		// use dial options without dopts to avoid reusing the client balancer
+		auth, err = newAuthenticator(host, c.dialSetupOpts(endpoint))
+		if err != nil {
+			continue
+		}
+		defer auth.close()
+
+		var resp *AuthenticateResponse
+		resp, err = auth.authenticate(ctx, c.Username, c.Password)
+		if err != nil {
+			continue
+		}
+
+		c.tokenCred.tokenMu.Lock()
+		c.tokenCred.token = resp.Token
+		c.tokenCred.tokenMu.Unlock()
+
+		return nil
+	}
+
+	return err
+}
+
 func (c *Client) dial(endpoint string, dopts ...grpc.DialOption) (*grpc.ClientConn, error) {
 	opts := c.dialSetupOpts(endpoint, dopts...)
 	host := getHost(endpoint)
 	if c.Username != "" && c.Password != "" {
-		// use dial options without dopts to avoid reusing the client balancer
-		auth, err := newAuthenticator(host, c.dialSetupOpts(endpoint))
-		if err != nil {
-			return nil, err
+		c.tokenCred = &authTokenCredential{
+			tokenMu: &sync.RWMutex{},
 		}
-		defer auth.close()
 
-		resp, err := auth.authenticate(c.ctx, c.Username, c.Password)
+		err := c.getToken(context.TODO())
 		if err != nil {
 			return nil, err
 		}
-		opts = append(opts, grpc.WithPerRPCCredentials(authTokenCredential{token: resp.Token}))
+
+		opts = append(opts, grpc.WithPerRPCCredentials(c.tokenCred))
 	}
 
+	// add metrics options
+	opts = append(opts, grpc.WithUnaryInterceptor(prometheus.UnaryClientInterceptor))
+	opts = append(opts, grpc.WithStreamInterceptor(prometheus.StreamClientInterceptor))
+
 	conn, err := grpc.Dial(host, opts...)
 	if err != nil {
 		return nil, err
@@ -293,6 +339,7 @@ func newClient(cfg *Config) (*Client, error) {
 	}
 	client.conn = conn
 	client.retryWrapper = client.newRetryWrapper()
+	client.retryAuthWrapper = client.newAuthRetryWrapper()
 
 	// wait for a connection
 	if cfg.DialTimeout > 0 {
@@ -317,12 +364,6 @@ func newClient(cfg *Config) (*Client, error) {
 	client.Watcher = NewWatcher(client)
 	client.Auth = NewAuth(client)
 	client.Maintenance = NewMaintenance(client)
-	if cfg.Logger != nil {
-		logger.Set(cfg.Logger)
-	} else {
-		// disable client side grpc by default
-		logger.Set(log.New(ioutil.Discard, "", 0))
-	}
 
 	go client.autoSync()
 	return client, nil

clientv3/client_test.go

@@ -16,6 +16,7 @@ package clientv3
 
 import (
 	"fmt"
+	"net"
 	"testing"
 	"time"
 
@@ -25,6 +26,47 @@ import (
 	"google.golang.org/grpc"
 )
 
+func TestDialCancel(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	// accept first connection so client is created with dial timeout
+	ln, err := net.Listen("unix", "dialcancel:12345")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+
+	ep := "unix://dialcancel:12345"
+	cfg := Config{
+		Endpoints:   []string{ep},
+		DialTimeout: 30 * time.Second}
+	c, err := New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// connect to ipv4 blackhole so dial blocks
+	c.SetEndpoints("http://254.0.0.1:12345")
+
+	// issue Get to force redial attempts
+	go c.Get(context.TODO(), "abc")
+
+	// wait a little bit so client close is after dial starts
+	time.Sleep(100 * time.Millisecond)
+
+	donec := make(chan struct{})
+	go func() {
+		defer close(donec)
+		c.Close()
+	}()
+
+	select {
+	case <-time.After(5 * time.Second):
+		t.Fatalf("failed to close")
+	case <-donec:
+	}
+}
+
 func TestDialTimeout(t *testing.T) {
 	defer testutil.AfterTest(t)
 

clientv3/cluster.go

@@ -78,7 +78,7 @@ func (c *cluster) MemberUpdate(ctx context.Context, id uint64, peerAddrs []strin
 	// it is safe to retry on update.
 	for {
 		r := &pb.MemberUpdateRequest{ID: id, PeerURLs: peerAddrs}
-		resp, err := c.remote.MemberUpdate(ctx, r)
+		resp, err := c.remote.MemberUpdate(ctx, r, grpc.FailFast(false))
 		if err == nil {
 			return (*MemberUpdateResponse)(resp), nil
 		}

clientv3/compare.go

@@ -36,6 +36,8 @@ func Compare(cmp Cmp, result string, v interface{}) Cmp {
 	switch result {
 	case "=":
 		r = pb.Compare_EQUAL
+	case "!=":
+		r = pb.Compare_NOT_EQUAL
 	case ">":
 		r = pb.Compare_GREATER
 	case "<":

clientv3/concurrency/session.go

@@ -15,6 +15,8 @@
 package concurrency
 
 import (
+	"time"
+
 	v3 "github.com/coreos/etcd/clientv3"
 	"golang.org/x/net/context"
 )
@@ -25,6 +27,7 @@ const defaultSessionTTL = 60
 // Fault-tolerant applications may use sessions to reason about liveness.
 type Session struct {
 	client *v3.Client
+	opts   *sessionOptions
 	id     v3.LeaseID
 
 	cancel context.CancelFunc
@@ -33,25 +36,25 @@ type Session struct {
 
 // NewSession gets the leased session for a client.
 func NewSession(client *v3.Client, opts ...SessionOption) (*Session, error) {
-	ops := &sessionOptions{ttl: defaultSessionTTL}
+	ops := &sessionOptions{ttl: defaultSessionTTL, ctx: client.Ctx()}
 	for _, opt := range opts {
 		opt(ops)
 	}
 
-	resp, err := client.Grant(client.Ctx(), int64(ops.ttl))
+	resp, err := client.Grant(ops.ctx, int64(ops.ttl))
 	if err != nil {
 		return nil, err
 	}
 	id := v3.LeaseID(resp.ID)
 
-	ctx, cancel := context.WithCancel(client.Ctx())
+	ctx, cancel := context.WithCancel(ops.ctx)
 	keepAlive, err := client.KeepAlive(ctx, id)
 	if err != nil || keepAlive == nil {
 		return nil, err
 	}
 
 	donec := make(chan struct{})
-	s := &Session{client: client, id: id, cancel: cancel, donec: donec}
+	s := &Session{client: client, opts: ops, id: id, cancel: cancel, donec: donec}
 
 	// keep the lease alive until client error or cancelled context
 	go func() {
@@ -87,12 +90,16 @@ func (s *Session) Orphan() {
 // Close orphans the session and revokes the session lease.
 func (s *Session) Close() error {
 	s.Orphan()
-	_, err := s.client.Revoke(s.client.Ctx(), s.id)
+	// if revoke takes longer than the ttl, lease is expired anyway
+	ctx, cancel := context.WithTimeout(s.opts.ctx, time.Duration(s.opts.ttl)*time.Second)
+	_, err := s.client.Revoke(ctx, s.id)
+	cancel()
 	return err
 }
 
 type sessionOptions struct {
 	ttl int
+	ctx context.Context
 }
 
 // SessionOption configures Session.
@@ -107,3 +114,14 @@ func WithTTL(ttl int) SessionOption {
 		}
 	}
 }
+
+// WithContext assigns a context to the session instead of defaulting to
+// using the client context. This is useful for canceling NewSession and
+// Close operations immediately without having to close the client. If the
+// context is canceled before Close() completes, the session's lease will be
+// abandoned and left to expire instead of being revoked.
+func WithContext(ctx context.Context) SessionOption {
+	return func(so *sessionOptions) {
+		so.ctx = ctx
+	}
+}

clientv3/concurrency/stm.go

@@ -249,11 +249,10 @@ func (s *stmReadCommitted) commit() *v3.TxnResponse {
 }
 
 func isKeyCurrent(k string, r *v3.GetResponse) v3.Cmp {
-	rev := r.Header.Revision + 1
 	if len(r.Kvs) != 0 {
-		rev = r.Kvs[0].ModRevision + 1
+		return v3.Compare(v3.ModRevision(k), "=", r.Kvs[0].ModRevision)
 	}
-	return v3.Compare(v3.ModRevision(k), "<", rev)
+	return v3.Compare(v3.ModRevision(k), "=", 0)
 }
 
 func respToValue(resp *v3.GetResponse) string {

clientv3/config.go

@@ -38,9 +38,6 @@ type Config struct {
 	// TLS holds the client secure credentials, if any.
 	TLS *tls.Config
 
-	// Logger is the logger used by client library.
-	Logger Logger
-
 	// Username is a username for authentication
 	Username string
 

clientv3/doc.go

@@ -44,7 +44,7 @@
 // etcd client returns 2 types of errors:
 //
 //	1. context error: canceled or deadline exceeded.
-//	2. gRPC error: see https://github.com/coreos/etcd/blob/master/etcdserver/api/v3rpc/error.go.
+//	2. gRPC error: see https://github.com/coreos/etcd/blob/master/etcdserver/api/v3rpc/rpctypes/error.go
 //
 // Here is the example code to handle client errors:
 //

clientv3/example_metrics_test.go

@@ -0,0 +1,46 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func ExampleMetrics_All() {
+	// listen for all prometheus metrics
+	go func() {
+		http.Handle("/metrics", prometheus.Handler())
+		log.Fatal(http.ListenAndServe(":47989", nil))
+	}()
+
+	url := "http://localhost:47989/metrics"
+
+	// make an http request to fetch all prometheus metrics
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Fatalf("fetch error: %v", err)
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		log.Fatalf("fetch error: reading %s: %v", url, err)
+	}
+	fmt.Printf("%s", b)
+}

clientv3/example_test.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/pkg/transport"
+	"github.com/coreos/pkg/capnslog"
 	"golang.org/x/net/context"
 )
 
@@ -30,6 +31,9 @@ var (
 )
 
 func Example() {
+	var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "clientv3")
+	clientv3.SetLogger(plog)
+
 	cli, err := clientv3.New(clientv3.Config{
 		Endpoints:   endpoints,
 		DialTimeout: dialTimeout,

clientv3/integration/kv_test.go

@@ -17,6 +17,7 @@ package integration
 import (
 	"bytes"
 	"math/rand"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -35,8 +36,8 @@ func TestKVPutError(t *testing.T) {
 	defer testutil.AfterTest(t)
 
 	var (
-		maxReqBytes = 1.5 * 1024 * 1024
-		quota       = int64(maxReqBytes * 1.2)
+		maxReqBytes = 1.5 * 1024 * 1024 // hard coded max in v3_server.go
+		quota       = int64(int(maxReqBytes) + 8*os.Getpagesize())
 	)
 	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1, QuotaBackendBytes: quota})
 	defer clus.Terminate(t)
@@ -49,7 +50,7 @@ func TestKVPutError(t *testing.T) {
 		t.Fatalf("expected %v, got %v", rpctypes.ErrEmptyKey, err)
 	}
 
-	_, err = kv.Put(ctx, "key", strings.Repeat("a", int(maxReqBytes+100))) // 1.5MB
+	_, err = kv.Put(ctx, "key", strings.Repeat("a", int(maxReqBytes+100)))
 	if err != rpctypes.ErrRequestTooLarge {
 		t.Fatalf("expected %v, got %v", rpctypes.ErrRequestTooLarge, err)
 	}
@@ -59,7 +60,7 @@ func TestKVPutError(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	time.Sleep(500 * time.Millisecond) // give enough time for commit
+	time.Sleep(1 * time.Second) // give enough time for commit
 
 	_, err = kv.Put(ctx, "foo2", strings.Repeat("a", int(maxReqBytes-50)))
 	if err != rpctypes.ErrNoSpace { // over quota
@@ -225,6 +226,21 @@ func TestKVRange(t *testing.T) {
 				{Key: []byte("fop"), Value: nil, CreateRevision: 9, ModRevision: 9, Version: 1},
 			},
 		},
+		// range all with SortByKey, missing sorting order (ASCEND by default)
+		{
+			"a", "x",
+			0,
+			[]clientv3.OpOption{clientv3.WithSort(clientv3.SortByKey, clientv3.SortNone)},
+
+			[]*mvccpb.KeyValue{
+				{Key: []byte("a"), Value: nil, CreateRevision: 2, ModRevision: 2, Version: 1},
+				{Key: []byte("b"), Value: nil, CreateRevision: 3, ModRevision: 3, Version: 1},
+				{Key: []byte("c"), Value: nil, CreateRevision: 4, ModRevision: 6, Version: 3},
+				{Key: []byte("foo"), Value: nil, CreateRevision: 7, ModRevision: 7, Version: 1},
+				{Key: []byte("foo/abc"), Value: nil, CreateRevision: 8, ModRevision: 8, Version: 1},
+				{Key: []byte("fop"), Value: nil, CreateRevision: 9, ModRevision: 9, Version: 1},
+			},
+		},
 		// range all with SortByCreateRevision, SortDescend
 		{
 			"a", "x",
@@ -240,6 +256,21 @@ func TestKVRange(t *testing.T) {
 				{Key: []byte("a"), Value: nil, CreateRevision: 2, ModRevision: 2, Version: 1},
 			},
 		},
+		// range all with SortByCreateRevision, missing sorting order (ASCEND by default)
+		{
+			"a", "x",
+			0,
+			[]clientv3.OpOption{clientv3.WithSort(clientv3.SortByCreateRevision, clientv3.SortNone)},
+
+			[]*mvccpb.KeyValue{
+				{Key: []byte("a"), Value: nil, CreateRevision: 2, ModRevision: 2, Version: 1},
+				{Key: []byte("b"), Value: nil, CreateRevision: 3, ModRevision: 3, Version: 1},
+				{Key: []byte("c"), Value: nil, CreateRevision: 4, ModRevision: 6, Version: 3},
+				{Key: []byte("foo"), Value: nil, CreateRevision: 7, ModRevision: 7, Version: 1},
+				{Key: []byte("foo/abc"), Value: nil, CreateRevision: 8, ModRevision: 8, Version: 1},
+				{Key: []byte("fop"), Value: nil, CreateRevision: 9, ModRevision: 9, Version: 1},
+			},
+		},
 		// range all with SortByModRevision, SortDescend
 		{
 			"a", "x",

clientv3/integration/lease_test.go

@@ -17,10 +17,12 @@ package integration
 import (
 	"reflect"
 	"sort"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/clientv3/concurrency"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
 	"github.com/coreos/etcd/integration"
 	"github.com/coreos/etcd/pkg/testutil"
@@ -154,6 +156,30 @@ func TestLeaseKeepAlive(t *testing.T) {
 	}
 }
 
+func TestLeaseKeepAliveOneSecond(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	cli := clus.Client(0)
+
+	resp, err := cli.Grant(context.Background(), 1)
+	if err != nil {
+		t.Errorf("failed to create lease %v", err)
+	}
+	rc, kerr := cli.KeepAlive(context.Background(), resp.ID)
+	if kerr != nil {
+		t.Errorf("failed to keepalive lease %v", kerr)
+	}
+
+	for i := 0; i < 3; i++ {
+		if _, ok := <-rc; !ok {
+			t.Errorf("chan is closed, want not closed")
+		}
+	}
+}
+
 // TODO: add a client that can connect to all the members of cluster via unix sock.
 // TODO: test handle more complicated failures.
 func TestLeaseKeepAliveHandleFailure(t *testing.T) {
@@ -510,3 +536,121 @@ func TestLeaseTimeToLive(t *testing.T) {
 		t.Fatalf("unexpected keys %+v", lresp.Keys)
 	}
 }
+
+// TestLeaseRenewLostQuorum ensures keepalives work after losing quorum
+// for a while.
+func TestLeaseRenewLostQuorum(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	cli := clus.Client(0)
+	r, err := cli.Grant(context.TODO(), 4)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	kctx, kcancel := context.WithCancel(context.Background())
+	defer kcancel()
+	ka, err := cli.KeepAlive(kctx, r.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// consume first keepalive so next message sends when cluster is down
+	<-ka
+
+	// force keepalive stream message to timeout
+	clus.Members[1].Stop(t)
+	clus.Members[2].Stop(t)
+	// Use TTL-1 since the client closes the keepalive channel if no
+	// keepalive arrives before the lease deadline.
+	// The cluster has 1 second to recover and reply to the keepalive.
+	time.Sleep(time.Duration(r.TTL-1) * time.Second)
+	clus.Members[1].Restart(t)
+	clus.Members[2].Restart(t)
+
+	select {
+	case _, ok := <-ka:
+		if !ok {
+			t.Fatalf("keepalive closed")
+		}
+	case <-time.After(time.Duration(r.TTL) * time.Second):
+		t.Fatalf("timed out waiting for keepalive")
+	}
+}
+
+func TestLeaseKeepAliveLoopExit(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	ctx := context.Background()
+	cli := clus.Client(0)
+
+	resp, err := cli.Grant(ctx, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cli.Lease.Close()
+
+	_, err = cli.KeepAlive(ctx, resp.ID)
+	if _, ok := err.(clientv3.ErrKeepAliveHalted); !ok {
+		t.Fatalf("expected %T, got %v(%T)", clientv3.ErrKeepAliveHalted{}, err, err)
+	}
+}
+
+// TestV3LeaseFailureOverlap issues Grant and Keepalive requests to a cluster
+// before, during, and after quorum loss to confirm Grant/Keepalive tolerates
+// transient cluster failure.
+func TestV3LeaseFailureOverlap(t *testing.T) {
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2})
+	defer clus.Terminate(t)
+
+	numReqs := 5
+	cli := clus.Client(0)
+
+	// bring up a session, tear it down
+	updown := func(i int) error {
+		sess, err := concurrency.NewSession(cli)
+		if err != nil {
+			return err
+		}
+		ch := make(chan struct{})
+		go func() {
+			defer close(ch)
+			sess.Close()
+		}()
+		select {
+		case <-ch:
+		case <-time.After(time.Minute / 4):
+			t.Fatalf("timeout %d", i)
+		}
+		return nil
+	}
+
+	var wg sync.WaitGroup
+	mkReqs := func(n int) {
+		wg.Add(numReqs)
+		for i := 0; i < numReqs; i++ {
+			go func() {
+				defer wg.Done()
+				err := updown(n)
+				if err == nil || err == rpctypes.ErrTimeoutDueToConnectionLost {
+					return
+				}
+				t.Fatal(err)
+			}()
+		}
+	}
+
+	mkReqs(1)
+	clus.Members[1].Stop(t)
+	mkReqs(2)
+	time.Sleep(time.Second)
+	mkReqs(3)
+	clus.Members[1].Restart(t)
+	mkReqs(4)
+	wg.Wait()
+}

clientv3/integration/metrics_test.go

@@ -0,0 +1,169 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"bufio"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/integration"
+	"github.com/coreos/etcd/pkg/testutil"
+	"github.com/coreos/etcd/pkg/transport"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/net/context"
+)
+
+func TestV3ClientMetrics(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	var (
+		addr string = "localhost:27989"
+		ln   net.Listener
+		err  error
+	)
+
+	// listen for all prometheus metrics
+	donec := make(chan struct{})
+	go func() {
+		defer close(donec)
+
+		srv := &http.Server{Handler: prometheus.Handler()}
+		srv.SetKeepAlivesEnabled(false)
+
+		ln, err = transport.NewUnixListener(addr)
+		if err != nil {
+			t.Fatalf("Error: %v occurred while listening on addr: %v", err, addr)
+		}
+
+		err = srv.Serve(ln)
+		if err != nil && !strings.Contains(err.Error(), "use of closed network connection") {
+			t.Fatalf("Err serving http requests: %v", err)
+		}
+	}()
+
+	url := "unix://" + addr + "/metrics"
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+
+	client := clus.Client(0)
+
+	w := clientv3.NewWatcher(client)
+	defer w.Close()
+
+	kv := clientv3.NewKV(client)
+
+	wc := w.Watch(context.Background(), "foo")
+
+	wBefore := sumCountersForMetricAndLabels(t, url, "grpc_client_msg_received_total", "Watch", "bidi_stream")
+
+	pBefore := sumCountersForMetricAndLabels(t, url, "grpc_client_started_total", "Put", "unary")
+
+	_, err = kv.Put(context.Background(), "foo", "bar")
+	if err != nil {
+		t.Errorf("Error putting value in key store")
+	}
+
+	pAfter := sumCountersForMetricAndLabels(t, url, "grpc_client_started_total", "Put", "unary")
+	if pBefore+1 != pAfter {
+		t.Errorf("grpc_client_started_total expected %d, got %d", 1, pAfter-pBefore)
+	}
+
+	// consume watch response
+	select {
+	case <-wc:
+	case <-time.After(10 * time.Second):
+		t.Error("Timeout occurred for getting watch response")
+	}
+
+	wAfter := sumCountersForMetricAndLabels(t, url, "grpc_client_msg_received_total", "Watch", "bidi_stream")
+	if wBefore+1 != wAfter {
+		t.Errorf("grpc_client_msg_received_total expected %d, got %d", 1, wAfter-wBefore)
+	}
+
+	ln.Close()
+	<-donec
+}
+
+func sumCountersForMetricAndLabels(t *testing.T, url string, metricName string, matchingLabelValues ...string) int {
+	count := 0
+	for _, line := range getHTTPBodyAsLines(t, url) {
+		ok := true
+		if !strings.HasPrefix(line, metricName) {
+			continue
+		}
+
+		for _, labelValue := range matchingLabelValues {
+			if !strings.Contains(line, `"`+labelValue+`"`) {
+				ok = false
+				break
+			}
+		}
+
+		if !ok {
+			continue
+		}
+
+		valueString := line[strings.LastIndex(line, " ")+1 : len(line)-1]
+		valueFloat, err := strconv.ParseFloat(valueString, 32)
+		if err != nil {
+			t.Fatalf("failed parsing value for line: %v and matchingLabelValues: %v", line, matchingLabelValues)
+		}
+		count += int(valueFloat)
+	}
+	return count
+}
+
+func getHTTPBodyAsLines(t *testing.T, url string) []string {
+	cfgtls := transport.TLSInfo{}
+	tr, err := transport.NewTransport(cfgtls, time.Second)
+	if err != nil {
+		t.Fatalf("Error getting transport: %v", err)
+	}
+
+	tr.MaxIdleConns = -1
+	tr.DisableKeepAlives = true
+
+	cli := &http.Client{Transport: tr}
+
+	resp, err := cli.Get(url)
+	if err != nil {
+		t.Fatalf("Error fetching: %v", err)
+	}
+
+	reader := bufio.NewReader(resp.Body)
+	lines := []string{}
+	for {
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				t.Fatalf("error reading: %v", err)
+			}
+		}
+		lines = append(lines, line)
+	}
+	resp.Body.Close()
+	return lines
+}

clientv3/integration/watch_test.go

@@ -349,6 +349,9 @@ func putAndWatch(t *testing.T, wctx *watchctx, key, val string) {
 
 // TestWatchResumeComapcted checks that the watcher gracefully closes in case
 // that it tries to resume to a revision that's been compacted out of the store.
+// Since the watcher's server restarts with stale data, the watcher will receive
+// either a compaction error or all keys by staying in sync before the compaction
+// is finally applied.
 func TestWatchResumeCompacted(t *testing.T) {
 	defer testutil.AfterTest(t)
 
@@ -377,8 +380,9 @@ func TestWatchResumeCompacted(t *testing.T) {
 	}
 
 	// put some data and compact away
+	numPuts := 5
 	kv := clientv3.NewKV(clus.Client(1))
-	for i := 0; i < 5; i++ {
+	for i := 0; i < numPuts; i++ {
 		if _, err := kv.Put(context.TODO(), "foo", "bar"); err != nil {
 			t.Fatal(err)
 		}
@@ -389,17 +393,48 @@ func TestWatchResumeCompacted(t *testing.T) {
 
 	clus.Members[0].Restart(t)
 
-	// get compacted error message
-	wresp, ok := <-wch
-	if !ok {
-		t.Fatalf("expected wresp, but got closed channel")
+	// since watch's server isn't guaranteed to be synced with the cluster when
+	// the watch resumes, there is a window where the watch can stay synced and
+	// read off all events; if the watcher misses the window, it will go out of
+	// sync and get a compaction error.
+	wRev := int64(2)
+	for int(wRev) <= numPuts+1 {
+		var wresp clientv3.WatchResponse
+		var ok bool
+		select {
+		case wresp, ok = <-wch:
+			if !ok {
+				t.Fatalf("expected wresp, but got closed channel")
+			}
+		case <-time.After(5 * time.Second):
+			t.Fatalf("compacted watch timed out")
+		}
+		for _, ev := range wresp.Events {
+			if ev.Kv.ModRevision != wRev {
+				t.Fatalf("expected modRev %v, got %+v", wRev, ev)
+			}
+			wRev++
+		}
+		if wresp.Err() == nil {
+			continue
+		}
+		if wresp.Err() != rpctypes.ErrCompacted {
+			t.Fatalf("wresp.Err() expected %v, but got %v %+v", rpctypes.ErrCompacted, wresp.Err())
+		}
+		break
 	}
-	if wresp.Err() != rpctypes.ErrCompacted {
-		t.Fatalf("wresp.Err() expected %v, but got %v", rpctypes.ErrCompacted, wresp.Err())
+	if int(wRev) > numPuts+1 {
+		// got data faster than the compaction
+		return
 	}
-	// ensure the channel is closed
-	if wresp, ok = <-wch; ok {
-		t.Fatalf("expected closed channel, but got %v", wresp)
+	// received compaction error; ensure the channel closes
+	select {
+	case wresp, ok := <-wch:
+		if ok {
+			t.Fatalf("expected closed channel, but got %v", wresp)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timed out waiting for channel close")
 	}
 }
 
@@ -579,18 +614,19 @@ func TestWatchErrConnClosed(t *testing.T) {
 	defer clus.Terminate(t)
 
 	cli := clus.Client(0)
+	defer cli.Close()
 	wc := clientv3.NewWatcher(cli)
 
 	donec := make(chan struct{})
 	go func() {
 		defer close(donec)
-		wc.Watch(context.TODO(), "foo")
-		if err := wc.Close(); err != nil && err != grpc.ErrClientConnClosing {
-			t.Fatalf("expected %v, got %v", grpc.ErrClientConnClosing, err)
+		ch := wc.Watch(context.TODO(), "foo")
+		if wr := <-ch; grpc.ErrorDesc(wr.Err()) != grpc.ErrClientConnClosing.Error() {
+			t.Fatalf("expected %v, got %v", grpc.ErrClientConnClosing, grpc.ErrorDesc(wr.Err()))
 		}
 	}()
 
-	if err := cli.Close(); err != nil {
+	if err := cli.ActiveConnection().Close(); err != nil {
 		t.Fatal(err)
 	}
 	clus.TakeClient(0)
@@ -637,8 +673,12 @@ func TestWatchWithRequireLeader(t *testing.T) {
 	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
 	defer clus.Terminate(t)
 
-	// something for the non-require leader watch to read as an event
-	if _, err := clus.Client(1).Put(context.TODO(), "foo", "bar"); err != nil {
+	// Put a key for the non-require leader watch to read as an event.
+	// The watchers will be on member[0]; put key through member[0] to
+	// ensure that it receives the update so watching after killing quorum
+	// is guaranteed to have the key.
+	liveClient := clus.Client(0)
+	if _, err := liveClient.Put(context.TODO(), "foo", "bar"); err != nil {
 		t.Fatal(err)
 	}
 
@@ -653,8 +693,8 @@ func TestWatchWithRequireLeader(t *testing.T) {
 	tickDuration := 10 * time.Millisecond
 	time.Sleep(time.Duration(3*clus.Members[0].ElectionTicks) * tickDuration)
 
-	chLeader := clus.Client(0).Watch(clientv3.WithRequireLeader(context.TODO()), "foo", clientv3.WithRev(1))
-	chNoLeader := clus.Client(0).Watch(context.TODO(), "foo", clientv3.WithRev(1))
+	chLeader := liveClient.Watch(clientv3.WithRequireLeader(context.TODO()), "foo", clientv3.WithRev(1))
+	chNoLeader := liveClient.Watch(context.TODO(), "foo", clientv3.WithRev(1))
 
 	select {
 	case resp, ok := <-chLeader:
@@ -736,6 +776,36 @@ func TestWatchWithCreatedNotification(t *testing.T) {
 	}
 }
 
+// TestWatchWithCreatedNotificationDropConn ensures that
+// a watcher with created notify does not post duplicate
+// created events from disconnect.
+func TestWatchWithCreatedNotificationDropConn(t *testing.T) {
+	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer cluster.Terminate(t)
+
+	client := cluster.RandClient()
+
+	wch := client.Watch(context.Background(), "a", clientv3.WithCreatedNotify())
+
+	resp := <-wch
+
+	if !resp.Created {
+		t.Fatalf("expected created event, got %v", resp)
+	}
+
+	cluster.Members[0].DropConnections()
+
+	// try to receive from watch channel again
+	// ensure it doesn't post another createNotify
+	select {
+	case wresp := <-wch:
+		t.Fatalf("got unexpected watch response: %+v\n", wresp)
+	case <-time.After(time.Second):
+		// watcher may not reconnect by the time it hits the select,
+		// so it wouldn't have a chance to filter out the second create event
+	}
+}
+
 // TestWatchCancelOnServer ensures client watcher cancels propagate back to the server.
 func TestWatchCancelOnServer(t *testing.T) {
 	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
@@ -770,15 +840,22 @@ func TestWatchCancelOnServer(t *testing.T) {
 //     4. watcher client finishes tearing down stream on "ctx"
 //     5. w2 comes back canceled
 func TestWatchOverlapContextCancel(t *testing.T) {
+	f := func(clus *integration.ClusterV3) {}
+	testWatchOverlapContextCancel(t, f)
+}
+
+func TestWatchOverlapDropConnContextCancel(t *testing.T) {
+	f := func(clus *integration.ClusterV3) {
+		clus.Members[0].DropConnections()
+	}
+	testWatchOverlapContextCancel(t, f)
+}
+
+func testWatchOverlapContextCancel(t *testing.T, f func(*integration.ClusterV3)) {
 	defer testutil.AfterTest(t)
 	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
 	defer clus.Terminate(t)
 
-	cli := clus.RandClient()
-	if _, err := cli.Put(context.TODO(), "abc", "def"); err != nil {
-		t.Fatal(err)
-	}
-
 	// each unique context "%v" has a unique grpc stream
 	n := 100
 	ctxs, ctxc := make([]context.Context, 5), make([]chan struct{}, 5)
@@ -788,17 +865,30 @@ func TestWatchOverlapContextCancel(t *testing.T) {
 		// limits the maximum number of outstanding watchers per stream
 		ctxc[i] = make(chan struct{}, 2)
 	}
+
+	// issue concurrent watches on "abc" with cancel
+	cli := clus.RandClient()
+	if _, err := cli.Put(context.TODO(), "abc", "def"); err != nil {
+		t.Fatal(err)
+	}
 	ch := make(chan struct{}, n)
-	// issue concurrent watches with cancel
 	for i := 0; i < n; i++ {
 		go func() {
 			defer func() { ch <- struct{}{} }()
 			idx := rand.Intn(len(ctxs))
 			ctx, cancel := context.WithCancel(ctxs[idx])
 			ctxc[idx] <- struct{}{}
-			ch := cli.Watch(ctx, "abc", clientv3.WithRev(1))
-			if _, ok := <-ch; !ok {
-				t.Fatalf("unexpected closed channel")
+			wch := cli.Watch(ctx, "abc", clientv3.WithRev(1))
+			f(clus)
+			select {
+			case _, ok := <-wch:
+				if !ok {
+					t.Fatalf("unexpected closed channel %p", wch)
+				}
+			// may take a second or two to reestablish a watcher because of
+			// grpc backoff policies for disconnects
+			case <-time.After(5 * time.Second):
+				t.Errorf("timed out waiting for watch on %p", wch)
 			}
 			// randomize how cancel overlaps with watch creation
 			if rand.Intn(2) == 0 {
@@ -814,8 +904,79 @@ func TestWatchOverlapContextCancel(t *testing.T) {
 	for i := 0; i < n; i++ {
 		select {
 		case <-ch:
-		case <-time.After(time.Second):
+		case <-time.After(5 * time.Second):
 			t.Fatalf("timed out waiting for completed watch")
 		}
 	}
 }
+
+// TestWatchCanelAndCloseClient ensures that canceling a watcher then immediately
+// closing the client does not return a client closing error.
+func TestWatchCancelAndCloseClient(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+	cli := clus.Client(0)
+	ctx, cancel := context.WithCancel(context.Background())
+	wch := cli.Watch(ctx, "abc")
+	donec := make(chan struct{})
+	go func() {
+		defer close(donec)
+		select {
+		case wr, ok := <-wch:
+			if ok {
+				t.Fatalf("expected closed watch after cancel(), got resp=%+v err=%v", wr, wr.Err())
+			}
+		case <-time.After(5 * time.Second):
+			t.Fatal("timed out waiting for closed channel")
+		}
+	}()
+	cancel()
+	if err := cli.Close(); err != nil {
+		t.Fatal(err)
+	}
+	<-donec
+	clus.TakeClient(0)
+}
+
+// TestWatchStressResumeClose establishes a bunch of watchers, disconnects
+// to put them in resuming mode, cancels them so some resumes by cancel fail,
+// then closes the watcher interface to ensure correct clean up.
+func TestWatchStressResumeClose(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+	cli := clus.Client(0)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	// add more watches than can be resumed before the cancel
+	wchs := make([]clientv3.WatchChan, 2000)
+	for i := range wchs {
+		wchs[i] = cli.Watch(ctx, "abc")
+	}
+	clus.Members[0].DropConnections()
+	cancel()
+	if err := cli.Close(); err != nil {
+		t.Fatal(err)
+	}
+	clus.TakeClient(0)
+}
+
+// TestWatchCancelDisconnected ensures canceling a watcher works when
+// its grpc stream is disconnected / reconnecting.
+func TestWatchCancelDisconnected(t *testing.T) {
+	defer testutil.AfterTest(t)
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+	cli := clus.Client(0)
+	ctx, cancel := context.WithCancel(context.Background())
+	// add more watches than can be resumed before the cancel
+	wch := cli.Watch(ctx, "abc")
+	clus.Members[0].Stop(t)
+	cancel()
+	select {
+	case <-wch:
+	case <-time.After(time.Second):
+		t.Fatal("took too long to cancel disconnected watcher")
+	}
+}

clientv3/kv.go

@@ -105,7 +105,7 @@ func (kv *kv) Delete(ctx context.Context, key string, opts ...OpOption) (*Delete
 }
 
 func (kv *kv) Compact(ctx context.Context, rev int64, opts ...CompactOption) (*CompactResponse, error) {
-	resp, err := kv.remote.Compact(ctx, OpCompact(rev, opts...).toRequest(), grpc.FailFast(false))
+	resp, err := kv.remote.Compact(ctx, OpCompact(rev, opts...).toRequest())
 	if err != nil {
 		return nil, toErr(ctx, err)
 	}
@@ -125,6 +125,7 @@ func (kv *kv) Do(ctx context.Context, op Op) (OpResponse, error) {
 		if err == nil {
 			return resp, nil
 		}
+
 		if isHaltErr(ctx, err) {
 			return resp, toErr(ctx, err)
 		}

clientv3/lease.go

@@ -69,6 +69,21 @@ const (
 	NoLease LeaseID = 0
 )
 
+// ErrKeepAliveHalted is returned if client keep alive loop halts with an unexpected error.
+//
+// This usually means that automatic lease renewal via KeepAlive is broken, but KeepAliveOnce will still work as expected.
+type ErrKeepAliveHalted struct {
+	Reason error
+}
+
+func (e ErrKeepAliveHalted) Error() string {
+	s := "etcdclient: leases keep alive halted"
+	if e.Reason != nil {
+		s += ": " + e.Reason.Error()
+	}
+	return s
+}
+
 type Lease interface {
 	// Grant creates a new lease.
 	Grant(ctx context.Context, ttl int64) (*LeaseGrantResponse, error)
@@ -94,8 +109,9 @@ type Lease interface {
 type lessor struct {
 	mu sync.Mutex // guards all fields
 
-	// donec is closed when recvKeepAliveLoop stops
-	donec chan struct{}
+	// donec is closed and loopErr is set when recvKeepAliveLoop stops
+	donec   chan struct{}
+	loopErr error
 
 	remote pb.LeaseClient
 
@@ -161,9 +177,6 @@ func (l *lessor) Grant(ctx context.Context, ttl int64) (*LeaseGrantResponse, err
 		if isHaltErr(cctx, err) {
 			return nil, toErr(cctx, err)
 		}
-		if nerr := l.newStream(); nerr != nil {
-			return nil, nerr
-		}
 	}
 }
 
@@ -182,9 +195,6 @@ func (l *lessor) Revoke(ctx context.Context, id LeaseID) (*LeaseRevokeResponse,
 		if isHaltErr(ctx, err) {
 			return nil, toErr(ctx, err)
 		}
-		if nerr := l.newStream(); nerr != nil {
-			return nil, nerr
-		}
 	}
 }
 
@@ -195,7 +205,7 @@ func (l *lessor) TimeToLive(ctx context.Context, id LeaseID, opts ...LeaseOption
 
 	for {
 		r := toLeaseTimeToLiveRequest(id, opts...)
-		resp, err := l.remote.LeaseTimeToLive(cctx, r)
+		resp, err := l.remote.LeaseTimeToLive(cctx, r, grpc.FailFast(false))
 		if err == nil {
 			gresp := &LeaseTimeToLiveResponse{
 				ResponseHeader: resp.GetHeader(),
@@ -216,6 +226,15 @@ func (l *lessor) KeepAlive(ctx context.Context, id LeaseID) (<-chan *LeaseKeepAl
 	ch := make(chan *LeaseKeepAliveResponse, leaseResponseChSize)
 
 	l.mu.Lock()
+	// ensure that recvKeepAliveLoop is still running
+	select {
+	case <-l.donec:
+		err := l.loopErr
+		l.mu.Unlock()
+		close(ch)
+		return ch, ErrKeepAliveHalted{Reason: err}
+	default:
+	}
 	ka, ok := l.keepAlives[id]
 	if !ok {
 		// create fresh keep alive
@@ -255,10 +274,6 @@ func (l *lessor) KeepAliveOnce(ctx context.Context, id LeaseID) (*LeaseKeepAlive
 		if isHaltErr(ctx, err) {
 			return nil, toErr(ctx, err)
 		}
-
-		if nerr := l.newStream(); nerr != nil {
-			return nil, nerr
-		}
 	}
 }
 
@@ -327,10 +342,11 @@ func (l *lessor) keepAliveOnce(ctx context.Context, id LeaseID) (*LeaseKeepAlive
 	return karesp, nil
 }
 
-func (l *lessor) recvKeepAliveLoop() {
+func (l *lessor) recvKeepAliveLoop() (gerr error) {
 	defer func() {
 		l.mu.Lock()
 		close(l.donec)
+		l.loopErr = gerr
 		for _, ka := range l.keepAlives {
 			ka.Close()
 		}
@@ -343,21 +359,35 @@ func (l *lessor) recvKeepAliveLoop() {
 		resp, err := stream.Recv()
 		if err != nil {
 			if isHaltErr(l.stopCtx, err) {
-				return
+				return err
 			}
 			stream, serr = l.resetRecv()
 			continue
 		}
 		l.recvKeepAlive(resp)
 	}
+	return serr
 }
 
 // resetRecv opens a new lease stream and starts sending LeaseKeepAliveRequests
 func (l *lessor) resetRecv() (pb.Lease_LeaseKeepAliveClient, error) {
-	if err := l.newStream(); err != nil {
+	sctx, cancel := context.WithCancel(l.stopCtx)
+	stream, err := l.remote.LeaseKeepAlive(sctx, grpc.FailFast(false))
+	if err = toErr(sctx, err); err != nil {
+		cancel()
 		return nil, err
 	}
-	stream := l.getKeepAliveStream()
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.stream != nil && l.streamCancel != nil {
+		l.stream.CloseSend()
+		l.streamCancel()
+	}
+
+	l.streamCancel = cancel
+	l.stream = stream
+
 	go l.sendKeepAliveLoop(stream)
 	return stream, nil
 }
@@ -386,7 +416,7 @@ func (l *lessor) recvKeepAlive(resp *pb.LeaseKeepAliveResponse) {
 	}
 
 	// send update to all channels
-	nextKeepAlive := time.Now().Add(1 + time.Duration(karesp.TTL/3)*time.Second)
+	nextKeepAlive := time.Now().Add((time.Duration(karesp.TTL) * time.Second) / 3.0)
 	ka.deadline = time.Now().Add(time.Duration(karesp.TTL) * time.Second)
 	for _, ch := range ka.chs {
 		select {
@@ -453,32 +483,6 @@ func (l *lessor) sendKeepAliveLoop(stream pb.Lease_LeaseKeepAliveClient) {
 	}
 }
 
-func (l *lessor) getKeepAliveStream() pb.Lease_LeaseKeepAliveClient {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	return l.stream
-}
-
-func (l *lessor) newStream() error {
-	sctx, cancel := context.WithCancel(l.stopCtx)
-	stream, err := l.remote.LeaseKeepAlive(sctx, grpc.FailFast(false))
-	if err != nil {
-		cancel()
-		return toErr(sctx, err)
-	}
-
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	if l.stream != nil && l.streamCancel != nil {
-		l.stream.CloseSend()
-		l.streamCancel()
-	}
-
-	l.streamCancel = cancel
-	l.stream = stream
-	return nil
-}
-
 func (ka *keepAlive) Close() {
 	close(ka.donec)
 	for _, ch := range ka.chs {

clientv3/logger.go

@@ -15,13 +15,15 @@
 package clientv3
 
 import (
+	"io/ioutil"
 	"log"
-	"os"
 	"sync"
 
 	"google.golang.org/grpc/grpclog"
 )
 
+// Logger is the logger used by client library.
+// It implements grpclog.Logger interface.
 type Logger grpclog.Logger
 
 var (
@@ -34,20 +36,36 @@ type settableLogger struct {
 }
 
 func init() {
-	// use go's standard logger by default like grpc
+	// disable client side logs by default
 	logger.mu.Lock()
-	logger.l = log.New(os.Stderr, "", log.LstdFlags)
+	logger.l = log.New(ioutil.Discard, "", 0)
+
+	// logger has to override the grpclog at initialization so that
+	// any changes to the grpclog go through logger with locking
+	// instead of through SetLogger
+	//
+	// now updates only happen through settableLogger.set
 	grpclog.SetLogger(&logger)
 	logger.mu.Unlock()
 }
 
-func (s *settableLogger) Set(l Logger) {
+// SetLogger sets client-side Logger. By default, logs are disabled.
+func SetLogger(l Logger) {
+	logger.set(l)
+}
+
+// GetLogger returns the current logger.
+func GetLogger() Logger {
+	return logger.get()
+}
+
+func (s *settableLogger) set(l Logger) {
 	s.mu.Lock()
 	logger.l = l
 	s.mu.Unlock()
 }
 
-func (s *settableLogger) Get() Logger {
+func (s *settableLogger) get() Logger {
 	s.mu.RLock()
 	l := logger.l
 	s.mu.RUnlock()
@@ -56,9 +74,9 @@ func (s *settableLogger) Get() Logger {
 
 // implement the grpclog.Logger interface
 
-func (s *settableLogger) Fatal(args ...interface{})                 { s.Get().Fatal(args...) }
-func (s *settableLogger) Fatalf(format string, args ...interface{}) { s.Get().Fatalf(format, args...) }
-func (s *settableLogger) Fatalln(args ...interface{})               { s.Get().Fatalln(args...) }
-func (s *settableLogger) Print(args ...interface{})                 { s.Get().Print(args...) }
-func (s *settableLogger) Printf(format string, args ...interface{}) { s.Get().Printf(format, args...) }
-func (s *settableLogger) Println(args ...interface{})               { s.Get().Println(args...) }
+func (s *settableLogger) Fatal(args ...interface{})                 { s.get().Fatal(args...) }
+func (s *settableLogger) Fatalf(format string, args ...interface{}) { s.get().Fatalf(format, args...) }
+func (s *settableLogger) Fatalln(args ...interface{})               { s.get().Fatalln(args...) }
+func (s *settableLogger) Print(args ...interface{})                 { s.get().Print(args...) }
+func (s *settableLogger) Printf(format string, args ...interface{}) { s.get().Printf(format, args...) }
+func (s *settableLogger) Println(args ...interface{})               { s.get().Println(args...) }

clientv3/naming/grpc.go

@@ -16,99 +16,112 @@ package naming
 
 import (
 	"encoding/json"
-	"time"
 
-	"github.com/coreos/etcd/clientv3"
-	"github.com/coreos/etcd/mvcc/mvccpb"
+	etcd "github.com/coreos/etcd/clientv3"
 	"golang.org/x/net/context"
-	"google.golang.org/grpc/naming"
-)
 
-const (
-	gRPCNamingPrefix = "/github.com/grpc/"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/naming"
 )
 
 // GRPCResolver creates a grpc.Watcher for a target to track its resolution changes.
 type GRPCResolver struct {
-	// Client is an initialized etcd client
-	Client *clientv3.Client
-	// Timeout for update/delete request.
-	Timeout time.Duration
+	// Client is an initialized etcd client.
+	Client *etcd.Client
 }
 
-func (gr *GRPCResolver) Add(target string, addr string, metadata interface{}) error {
-	update := naming.Update{
-		Addr:     addr,
-		Metadata: metadata,
+func (gr *GRPCResolver) Update(ctx context.Context, target string, nm naming.Update, opts ...etcd.OpOption) (err error) {
+	switch nm.Op {
+	case naming.Add:
+		var v []byte
+		if v, err = json.Marshal(nm); err != nil {
+			return grpc.Errorf(codes.InvalidArgument, err.Error())
+		}
+		_, err = gr.Client.KV.Put(ctx, target+"/"+nm.Addr, string(v), opts...)
+	case naming.Delete:
+		_, err = gr.Client.Delete(ctx, target+"/"+nm.Addr, opts...)
+	default:
+		return grpc.Errorf(codes.InvalidArgument, "naming: bad naming op")
 	}
-	val, err := json.Marshal(update)
-	if err != nil {
-		return err
-	}
-
-	ctx := context.Background()
-	if gr.Timeout != 0 {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(context.Background(), gr.Timeout)
-		defer cancel()
-	}
-
-	_, err = gr.Client.KV.Put(ctx, gRPCNamingPrefix+target, string(val))
-	return err
-}
-
-func (gr *GRPCResolver) Delete(target string) error {
-	ctx := context.Background()
-	if gr.Timeout != 0 {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(context.Background(), gr.Timeout)
-		defer cancel()
-	}
-
-	_, err := gr.Client.Delete(ctx, gRPCNamingPrefix+target)
 	return err
 }
 
 func (gr *GRPCResolver) Resolve(target string) (naming.Watcher, error) {
-	cctx, cancel := context.WithCancel(context.Background())
-
-	wch := gr.Client.Watch(cctx, gRPCNamingPrefix+target)
-
-	w := &gRPCWatcher{
-		cancel: cancel,
-		wch:    wch,
-	}
-
+	ctx, cancel := context.WithCancel(context.Background())
+	w := &gRPCWatcher{c: gr.Client, target: target + "/", ctx: ctx, cancel: cancel}
 	return w, nil
 }
 
 type gRPCWatcher struct {
+	c      *etcd.Client
+	target string
+	ctx    context.Context
 	cancel context.CancelFunc
-	wch    clientv3.WatchChan
+	wch    etcd.WatchChan
+	err    error
 }
 
+// Next gets the next set of updates from the etcd resolver.
+// Calls to Next should be serialized; concurrent calls are not safe since
+// there is no way to reconcile the update ordering.
 func (gw *gRPCWatcher) Next() ([]*naming.Update, error) {
+	if gw.wch == nil {
+		// first Next() returns all addresses
+		return gw.firstNext()
+	}
+	if gw.err != nil {
+		return nil, gw.err
+	}
+
+	// process new events on target/*
 	wr, ok := <-gw.wch
 	if !ok {
-		return nil, wr.Err()
+		gw.err = grpc.Errorf(codes.Unavailable, "naming: watch closed")
+		return nil, gw.err
+	}
+	if gw.err = wr.Err(); gw.err != nil {
+		return nil, gw.err
 	}
 
 	updates := make([]*naming.Update, 0, len(wr.Events))
-
 	for _, e := range wr.Events {
+		var jupdate naming.Update
+		var err error
 		switch e.Type {
-		case mvccpb.PUT:
-			var jupdate naming.Update
-			err := json.Unmarshal(e.Kv.Value, &jupdate)
-			if err != nil {
-				continue
-			}
+		case etcd.EventTypePut:
+			err = json.Unmarshal(e.Kv.Value, &jupdate)
+			jupdate.Op = naming.Add
+		case etcd.EventTypeDelete:
+			err = json.Unmarshal(e.PrevKv.Value, &jupdate)
+			jupdate.Op = naming.Delete
+		}
+		if err == nil {
 			updates = append(updates, &jupdate)
-		case mvccpb.DELETE:
-			updates = append(updates, &naming.Update{Op: naming.Delete})
 		}
 	}
+	return updates, nil
+}
 
+func (gw *gRPCWatcher) firstNext() ([]*naming.Update, error) {
+	// Use serialized request so resolution still works if the target etcd
+	// server is partitioned away from the quorum.
+	resp, err := gw.c.Get(gw.ctx, gw.target, etcd.WithPrefix(), etcd.WithSerializable())
+	if gw.err = err; err != nil {
+		return nil, err
+	}
+
+	updates := make([]*naming.Update, 0, len(resp.Kvs))
+	for _, kv := range resp.Kvs {
+		var jupdate naming.Update
+		if err := json.Unmarshal(kv.Value, &jupdate); err != nil {
+			continue
+		}
+		updates = append(updates, &jupdate)
+	}
+
+	opts := []etcd.OpOption{etcd.WithRev(resp.Header.Revision + 1), etcd.WithPrefix(), etcd.WithPrevKV()}
+	gw.wch = gw.c.Watch(gw.ctx, gw.target, opts...)
 	return updates, nil
 }
 

clientv3/naming/grpc_test.go

@@ -15,11 +15,14 @@
 package naming
 
 import (
+	"encoding/json"
 	"reflect"
 	"testing"
 
+	"golang.org/x/net/context"
 	"google.golang.org/grpc/naming"
 
+	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/integration"
 	"github.com/coreos/etcd/pkg/testutil"
 )
@@ -40,7 +43,8 @@ func TestGRPCResolver(t *testing.T) {
 	}
 	defer w.Close()
 
-	err = r.Add("foo", "127.0.0.1", "metadata")
+	addOp := naming.Update{Op: naming.Add, Addr: "127.0.0.1", Metadata: "metadata"}
+	err = r.Update(context.TODO(), "foo", addOp)
 	if err != nil {
 		t.Fatal("failed to add foo", err)
 	}
@@ -60,7 +64,8 @@ func TestGRPCResolver(t *testing.T) {
 		t.Fatalf("up = %#v, want %#v", us[0], wu)
 	}
 
-	err = r.Delete("foo")
+	delOp := naming.Update{Op: naming.Delete, Addr: "127.0.0.1"}
+	err = r.Update(context.TODO(), "foo", delOp)
 
 	us, err = w.Next()
 	if err != nil {
@@ -68,10 +73,63 @@ func TestGRPCResolver(t *testing.T) {
 	}
 
 	wu = &naming.Update{
-		Op: naming.Delete,
+		Op:       naming.Delete,
+		Addr:     "127.0.0.1",
+		Metadata: "metadata",
 	}
 
 	if !reflect.DeepEqual(us[0], wu) {
 		t.Fatalf("up = %#v, want %#v", us[0], wu)
 	}
 }
+
+// TestGRPCResolverMultiInit ensures the resolver will initialize
+// correctly with multiple hosts and correctly receive multiple
+// updates in a single revision.
+func TestGRPCResolverMulti(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
+	defer clus.Terminate(t)
+	c := clus.RandClient()
+
+	v, verr := json.Marshal(naming.Update{Addr: "127.0.0.1", Metadata: "md"})
+	if verr != nil {
+		t.Fatal(verr)
+	}
+	if _, err := c.Put(context.TODO(), "foo/host", string(v)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := c.Put(context.TODO(), "foo/host2", string(v)); err != nil {
+		t.Fatal(err)
+	}
+
+	r := GRPCResolver{c}
+
+	w, err := r.Resolve("foo")
+	if err != nil {
+		t.Fatal("failed to resolve foo", err)
+	}
+	defer w.Close()
+
+	updates, nerr := w.Next()
+	if nerr != nil {
+		t.Fatal(nerr)
+	}
+	if len(updates) != 2 {
+		t.Fatalf("expected two updates, got %+v", updates)
+	}
+
+	_, err = c.Txn(context.TODO()).Then(etcd.OpDelete("foo/host"), etcd.OpDelete("foo/host2")).Commit()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	updates, nerr = w.Next()
+	if nerr != nil {
+		t.Fatal(nerr)
+	}
+	if len(updates) != 2 || (updates[0].Op != naming.Delete && updates[1].Op != naming.Delete) {
+		t.Fatalf("expected two updates, got %+v", updates)
+	}
+}

clientv3/op.go

@@ -223,7 +223,7 @@ func WithSort(target SortTarget, order SortOrder) OpOption {
 			// If order != SortNone, server fetches the entire key-space,
 			// and then applies the sort and limit, if provided.
 			// Since current mvcc.Range implementation returns results
-			// sorted by keys in lexiographically ascending order,
+			// sorted by keys in lexicographically ascending order,
 			// client should ignore SortOrder if the target is SortByKey.
 			order = SortNone
 		}
@@ -261,14 +261,15 @@ func WithPrefix() OpOption {
 	}
 }
 
-// WithRange specifies the range of 'Get' or 'Delete' requests.
+// WithRange specifies the range of 'Get', 'Delete', 'Watch' requests.
 // For example, 'Get' requests with 'WithRange(end)' returns
 // the keys in the range [key, end).
+// endKey must be lexicographically greater than start key.
 func WithRange(endKey string) OpOption {
 	return func(op *Op) { op.end = []byte(endKey) }
 }
 
-// WithFromKey specifies the range of 'Get' or 'Delete' requests
+// WithFromKey specifies the range of 'Get', 'Delete', 'Watch' requests
 // to be equal or greater than the key in the argument.
 func WithFromKey() OpOption { return WithRange("\x00") }
 

clientv3/retry.go

@@ -23,70 +23,109 @@ import (
 )
 
 type rpcFunc func(ctx context.Context) error
-type retryRpcFunc func(context.Context, rpcFunc)
+type retryRpcFunc func(context.Context, rpcFunc) error
 
 func (c *Client) newRetryWrapper() retryRpcFunc {
-	return func(rpcCtx context.Context, f rpcFunc) {
+	return func(rpcCtx context.Context, f rpcFunc) error {
 		for {
 			err := f(rpcCtx)
 			if err == nil {
-				return
+				return nil
 			}
+
+			eErr := rpctypes.Error(err)
+			// always stop retry on etcd errors
+			if _, ok := eErr.(rpctypes.EtcdError); ok {
+				return err
+			}
+
 			// only retry if unavailable
 			if grpc.Code(err) != codes.Unavailable {
-				return
-			}
-			// always stop retry on etcd errors
-			eErr := rpctypes.Error(err)
-			if _, ok := eErr.(rpctypes.EtcdError); ok {
-				return
+				return err
 			}
+
 			select {
 			case <-c.balancer.ConnectNotify():
 			case <-rpcCtx.Done():
+				return rpcCtx.Err()
 			case <-c.ctx.Done():
-				return
+				return c.ctx.Err()
 			}
 		}
 	}
 }
 
-type retryKVClient struct {
-	pb.KVClient
-	retryf retryRpcFunc
+func (c *Client) newAuthRetryWrapper() retryRpcFunc {
+	return func(rpcCtx context.Context, f rpcFunc) error {
+		for {
+			err := f(rpcCtx)
+			if err == nil {
+				return nil
+			}
+
+			// always stop retry on etcd errors other than invalid auth token
+			if rpctypes.Error(err) == rpctypes.ErrInvalidAuthToken {
+				gterr := c.getToken(rpcCtx)
+				if gterr != nil {
+					return err // return the original error for simplicity
+				}
+				continue
+			}
+
+			return err
+		}
+	}
 }
 
 // RetryKVClient implements a KVClient that uses the client's FailFast retry policy.
 func RetryKVClient(c *Client) pb.KVClient {
-	return &retryKVClient{pb.NewKVClient(c.conn), c.retryWrapper}
+	retryWrite := &retryWriteKVClient{pb.NewKVClient(c.conn), c.retryWrapper}
+	return &retryKVClient{&retryWriteKVClient{retryWrite, c.retryAuthWrapper}}
 }
 
-func (rkv *retryKVClient) Put(ctx context.Context, in *pb.PutRequest, opts ...grpc.CallOption) (resp *pb.PutResponse, err error) {
-	rkv.retryf(ctx, func(rctx context.Context) error {
+type retryKVClient struct {
+	*retryWriteKVClient
+}
+
+func (rkv *retryKVClient) Range(ctx context.Context, in *pb.RangeRequest, opts ...grpc.CallOption) (resp *pb.RangeResponse, err error) {
+	err = rkv.retryf(ctx, func(rctx context.Context) error {
+		resp, err = rkv.retryWriteKVClient.Range(rctx, in, opts...)
+		return err
+	})
+	return resp, err
+}
+
+type retryWriteKVClient struct {
+	pb.KVClient
+	retryf retryRpcFunc
+}
+
+func (rkv *retryWriteKVClient) Put(ctx context.Context, in *pb.PutRequest, opts ...grpc.CallOption) (resp *pb.PutResponse, err error) {
+	err = rkv.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rkv.KVClient.Put(rctx, in, opts...)
 		return err
 	})
 	return resp, err
 }
 
-func (rkv *retryKVClient) DeleteRange(ctx context.Context, in *pb.DeleteRangeRequest, opts ...grpc.CallOption) (resp *pb.DeleteRangeResponse, err error) {
-	rkv.retryf(ctx, func(rctx context.Context) error {
+func (rkv *retryWriteKVClient) DeleteRange(ctx context.Context, in *pb.DeleteRangeRequest, opts ...grpc.CallOption) (resp *pb.DeleteRangeResponse, err error) {
+	err = rkv.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rkv.KVClient.DeleteRange(rctx, in, opts...)
 		return err
 	})
 	return resp, err
 }
 
-func (rkv *retryKVClient) Txn(ctx context.Context, in *pb.TxnRequest, opts ...grpc.CallOption) (resp *pb.TxnResponse, err error) {
-	rkv.retryf(ctx, func(rctx context.Context) error {
+func (rkv *retryWriteKVClient) Txn(ctx context.Context, in *pb.TxnRequest, opts ...grpc.CallOption) (resp *pb.TxnResponse, err error) {
+	err = rkv.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rkv.KVClient.Txn(rctx, in, opts...)
 		return err
 	})
 	return resp, err
 }
 
-func (rkv *retryKVClient) Compact(ctx context.Context, in *pb.CompactionRequest, opts ...grpc.CallOption) (resp *pb.CompactionResponse, err error) {
-	rkv.retryf(ctx, func(rctx context.Context) error {
+func (rkv *retryWriteKVClient) Compact(ctx context.Context, in *pb.CompactionRequest, opts ...grpc.CallOption) (resp *pb.CompactionResponse, err error) {
+	err = rkv.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rkv.KVClient.Compact(rctx, in, opts...)
 		return err
 	})
@@ -100,11 +139,12 @@ type retryLeaseClient struct {
 
 // RetryLeaseClient implements a LeaseClient that uses the client's FailFast retry policy.
 func RetryLeaseClient(c *Client) pb.LeaseClient {
-	return &retryLeaseClient{pb.NewLeaseClient(c.conn), c.retryWrapper}
+	retry := &retryLeaseClient{pb.NewLeaseClient(c.conn), c.retryWrapper}
+	return &retryLeaseClient{retry, c.retryAuthWrapper}
 }
 
 func (rlc *retryLeaseClient) LeaseGrant(ctx context.Context, in *pb.LeaseGrantRequest, opts ...grpc.CallOption) (resp *pb.LeaseGrantResponse, err error) {
-	rlc.retryf(ctx, func(rctx context.Context) error {
+	err = rlc.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rlc.LeaseClient.LeaseGrant(rctx, in, opts...)
 		return err
 	})
@@ -113,7 +153,7 @@ func (rlc *retryLeaseClient) LeaseGrant(ctx context.Context, in *pb.LeaseGrantRe
 }
 
 func (rlc *retryLeaseClient) LeaseRevoke(ctx context.Context, in *pb.LeaseRevokeRequest, opts ...grpc.CallOption) (resp *pb.LeaseRevokeResponse, err error) {
-	rlc.retryf(ctx, func(rctx context.Context) error {
+	err = rlc.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rlc.LeaseClient.LeaseRevoke(rctx, in, opts...)
 		return err
 	})
@@ -131,7 +171,7 @@ func RetryClusterClient(c *Client) pb.ClusterClient {
 }
 
 func (rcc *retryClusterClient) MemberAdd(ctx context.Context, in *pb.MemberAddRequest, opts ...grpc.CallOption) (resp *pb.MemberAddResponse, err error) {
-	rcc.retryf(ctx, func(rctx context.Context) error {
+	err = rcc.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rcc.ClusterClient.MemberAdd(rctx, in, opts...)
 		return err
 	})
@@ -139,7 +179,7 @@ func (rcc *retryClusterClient) MemberAdd(ctx context.Context, in *pb.MemberAddRe
 }
 
 func (rcc *retryClusterClient) MemberRemove(ctx context.Context, in *pb.MemberRemoveRequest, opts ...grpc.CallOption) (resp *pb.MemberRemoveResponse, err error) {
-	rcc.retryf(ctx, func(rctx context.Context) error {
+	err = rcc.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rcc.ClusterClient.MemberRemove(rctx, in, opts...)
 		return err
 	})
@@ -147,7 +187,7 @@ func (rcc *retryClusterClient) MemberRemove(ctx context.Context, in *pb.MemberRe
 }
 
 func (rcc *retryClusterClient) MemberUpdate(ctx context.Context, in *pb.MemberUpdateRequest, opts ...grpc.CallOption) (resp *pb.MemberUpdateResponse, err error) {
-	rcc.retryf(ctx, func(rctx context.Context) error {
+	err = rcc.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rcc.ClusterClient.MemberUpdate(rctx, in, opts...)
 		return err
 	})
@@ -165,7 +205,7 @@ func RetryAuthClient(c *Client) pb.AuthClient {
 }
 
 func (rac *retryAuthClient) AuthEnable(ctx context.Context, in *pb.AuthEnableRequest, opts ...grpc.CallOption) (resp *pb.AuthEnableResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.AuthEnable(rctx, in, opts...)
 		return err
 	})
@@ -173,7 +213,7 @@ func (rac *retryAuthClient) AuthEnable(ctx context.Context, in *pb.AuthEnableReq
 }
 
 func (rac *retryAuthClient) AuthDisable(ctx context.Context, in *pb.AuthDisableRequest, opts ...grpc.CallOption) (resp *pb.AuthDisableResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.AuthDisable(rctx, in, opts...)
 		return err
 	})
@@ -181,7 +221,7 @@ func (rac *retryAuthClient) AuthDisable(ctx context.Context, in *pb.AuthDisableR
 }
 
 func (rac *retryAuthClient) UserAdd(ctx context.Context, in *pb.AuthUserAddRequest, opts ...grpc.CallOption) (resp *pb.AuthUserAddResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.UserAdd(rctx, in, opts...)
 		return err
 	})
@@ -189,7 +229,7 @@ func (rac *retryAuthClient) UserAdd(ctx context.Context, in *pb.AuthUserAddReque
 }
 
 func (rac *retryAuthClient) UserDelete(ctx context.Context, in *pb.AuthUserDeleteRequest, opts ...grpc.CallOption) (resp *pb.AuthUserDeleteResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.UserDelete(rctx, in, opts...)
 		return err
 	})
@@ -197,7 +237,7 @@ func (rac *retryAuthClient) UserDelete(ctx context.Context, in *pb.AuthUserDelet
 }
 
 func (rac *retryAuthClient) UserChangePassword(ctx context.Context, in *pb.AuthUserChangePasswordRequest, opts ...grpc.CallOption) (resp *pb.AuthUserChangePasswordResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.UserChangePassword(rctx, in, opts...)
 		return err
 	})
@@ -205,7 +245,7 @@ func (rac *retryAuthClient) UserChangePassword(ctx context.Context, in *pb.AuthU
 }
 
 func (rac *retryAuthClient) UserGrantRole(ctx context.Context, in *pb.AuthUserGrantRoleRequest, opts ...grpc.CallOption) (resp *pb.AuthUserGrantRoleResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.UserGrantRole(rctx, in, opts...)
 		return err
 	})
@@ -213,7 +253,7 @@ func (rac *retryAuthClient) UserGrantRole(ctx context.Context, in *pb.AuthUserGr
 }
 
 func (rac *retryAuthClient) UserRevokeRole(ctx context.Context, in *pb.AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (resp *pb.AuthUserRevokeRoleResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.UserRevokeRole(rctx, in, opts...)
 		return err
 	})
@@ -221,7 +261,7 @@ func (rac *retryAuthClient) UserRevokeRole(ctx context.Context, in *pb.AuthUserR
 }
 
 func (rac *retryAuthClient) RoleAdd(ctx context.Context, in *pb.AuthRoleAddRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleAddResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.RoleAdd(rctx, in, opts...)
 		return err
 	})
@@ -229,7 +269,7 @@ func (rac *retryAuthClient) RoleAdd(ctx context.Context, in *pb.AuthRoleAddReque
 }
 
 func (rac *retryAuthClient) RoleDelete(ctx context.Context, in *pb.AuthRoleDeleteRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleDeleteResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.RoleDelete(rctx, in, opts...)
 		return err
 	})
@@ -237,7 +277,7 @@ func (rac *retryAuthClient) RoleDelete(ctx context.Context, in *pb.AuthRoleDelet
 }
 
 func (rac *retryAuthClient) RoleGrantPermission(ctx context.Context, in *pb.AuthRoleGrantPermissionRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleGrantPermissionResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.RoleGrantPermission(rctx, in, opts...)
 		return err
 	})
@@ -245,7 +285,7 @@ func (rac *retryAuthClient) RoleGrantPermission(ctx context.Context, in *pb.Auth
 }
 
 func (rac *retryAuthClient) RoleRevokePermission(ctx context.Context, in *pb.AuthRoleRevokePermissionRequest, opts ...grpc.CallOption) (resp *pb.AuthRoleRevokePermissionResponse, err error) {
-	rac.retryf(ctx, func(rctx context.Context) error {
+	err = rac.retryf(ctx, func(rctx context.Context) error {
 		resp, err = rac.AuthClient.RoleRevokePermission(rctx, in, opts...)
 		return err
 	})

clientv3/watch.go

@@ -92,7 +92,7 @@ func (wr *WatchResponse) Err() error {
 
 // IsProgressNotify returns true if the WatchResponse is progress notification.
 func (wr *WatchResponse) IsProgressNotify() bool {
-	return len(wr.Events) == 0 && !wr.Canceled && !wr.Created
+	return len(wr.Events) == 0 && !wr.Canceled && !wr.Created && wr.CompactRevision == 0 && wr.Header.Revision != 0
 }
 
 // watcher implements the Watcher interface
@@ -106,6 +106,7 @@ type watcher struct {
 	streams map[string]*watchGrpcStream
 }
 
+// watchGrpcStream tracks all watch resources attached to a single grpc stream.
 type watchGrpcStream struct {
 	owner  *watcher
 	remote pb.WatchClient
@@ -116,17 +117,15 @@ type watchGrpcStream struct {
 	ctxKey string
 	cancel context.CancelFunc
 
-	// mu protects the streams map
-	mu sync.RWMutex
-	// streams holds all active watchers
-	streams map[int64]*watcherStream
+	// substreams holds all active watchers on this grpc stream
+	substreams map[int64]*watcherStream
+	// resuming holds all resuming watchers on this grpc stream
+	resuming []*watcherStream
 
 	// reqc sends a watch request from Watch() to the main goroutine
 	reqc chan *watchRequest
 	// respc receives data from the watch client
 	respc chan *pb.WatchResponse
-	// stopc is sent to the main goroutine to stop all processing
-	stopc chan struct{}
 	// donec closes to broadcast shutdown
 	donec chan struct{}
 	// errc transmits errors from grpc Recv to the watch stream reconn logic
@@ -134,7 +133,9 @@ type watchGrpcStream struct {
 	// closingc gets the watcherStream of closing watchers
 	closingc chan *watcherStream
 
-	// the error that closed the watch stream
+	// resumec closes to signal that all substreams should begin resuming
+	resumec chan struct{}
+	// closeErr is the error that closed the watch stream
 	closeErr error
 }
 
@@ -162,15 +163,18 @@ type watcherStream struct {
 	initReq watchRequest
 
 	// outc publishes watch responses to subscriber
-	outc chan<- WatchResponse
+	outc chan WatchResponse
 	// recvc buffers watch responses before publishing
 	recvc chan *WatchResponse
-	id    int64
+	// donec closes when the watcherStream goroutine stops.
+	donec chan struct{}
+	// closing is set to true when stream should be scheduled to shutdown.
+	closing bool
+	// id is the registered watch id on the grpc stream
+	id int64
 
-	// lastRev is revision last successfully sent over outc
-	lastRev int64
-	// resumec indicates the stream must recover at a given revision
-	resumec chan int64
+	// buf holds all events received from etcd but not yet consumed by the client
+	buf []*WatchResponse
 }
 
 func NewWatcher(c *Client) Watcher {
@@ -198,19 +202,19 @@ func (vc *valCtx) Err() error                  { return nil }
 func (w *watcher) newWatcherGrpcStream(inctx context.Context) *watchGrpcStream {
 	ctx, cancel := context.WithCancel(&valCtx{inctx})
 	wgs := &watchGrpcStream{
-		owner:   w,
-		remote:  w.remote,
-		ctx:     ctx,
-		ctxKey:  fmt.Sprintf("%v", inctx),
-		cancel:  cancel,
-		streams: make(map[int64]*watcherStream),
+		owner:      w,
+		remote:     w.remote,
+		ctx:        ctx,
+		ctxKey:     fmt.Sprintf("%v", inctx),
+		cancel:     cancel,
+		substreams: make(map[int64]*watcherStream),
 
 		respc:    make(chan *pb.WatchResponse),
 		reqc:     make(chan *watchRequest),
-		stopc:    make(chan struct{}),
 		donec:    make(chan struct{}),
 		errc:     make(chan error, 1),
 		closingc: make(chan *watcherStream),
+		resumec:  make(chan struct{}),
 	}
 	go wgs.run()
 	return wgs
@@ -220,8 +224,6 @@ func (w *watcher) newWatcherGrpcStream(inctx context.Context) *watchGrpcStream {
 func (w *watcher) Watch(ctx context.Context, key string, opts ...OpOption) WatchChan {
 	ow := opWatch(key, opts...)
 
-	retc := make(chan chan WatchResponse, 1)
-
 	var filters []pb.WatchCreateRequest_FilterType
 	if ow.filterPut {
 		filters = append(filters, pb.WatchCreateRequest_NOPUT)
@@ -239,7 +241,7 @@ func (w *watcher) Watch(ctx context.Context, key string, opts ...OpOption) Watch
 		progressNotify: ow.progressNotify,
 		filters:        filters,
 		prevKV:         ow.prevKV,
-		retc:           retc,
+		retc:           make(chan chan WatchResponse, 1),
 	}
 
 	ok := false
@@ -283,7 +285,7 @@ func (w *watcher) Watch(ctx context.Context, key string, opts ...OpOption) Watch
 	// receive channel
 	if ok {
 		select {
-		case ret := <-retc:
+		case ret := <-wr.retc:
 			return ret
 		case <-ctx.Done():
 		case <-donec:
@@ -314,12 +316,7 @@ func (w *watcher) Close() (err error) {
 }
 
 func (w *watchGrpcStream) Close() (err error) {
-	w.mu.Lock()
-	if w.stopc != nil {
-		close(w.stopc)
-		w.stopc = nil
-	}
-	w.mu.Unlock()
+	w.cancel()
 	<-w.donec
 	select {
 	case err = <-w.errc:
@@ -328,71 +325,57 @@ func (w *watchGrpcStream) Close() (err error) {
 	return toErr(w.ctx, err)
 }
 
-func (w *watchGrpcStream) addStream(resp *pb.WatchResponse, pendingReq *watchRequest) {
-	if pendingReq == nil {
-		// no pending request; ignore
-		return
-	}
-	if resp.Canceled || resp.CompactRevision != 0 {
-		// a cancel at id creation time means the start revision has
-		// been compacted out of the store
-		ret := make(chan WatchResponse, 1)
-		ret <- WatchResponse{
-			Header:          *resp.Header,
-			CompactRevision: resp.CompactRevision,
-			Canceled:        true}
-		close(ret)
-		pendingReq.retc <- ret
-		return
-	}
-
-	ret := make(chan WatchResponse)
-	if resp.WatchId == -1 {
-		// failed; no channel
-		close(ret)
-		pendingReq.retc <- ret
-		return
-	}
-
-	ws := &watcherStream{
-		initReq: *pendingReq,
-		id:      resp.WatchId,
-		outc:    ret,
-		// buffered so unlikely to block on sending while holding mu
-		recvc:   make(chan *WatchResponse, 4),
-		resumec: make(chan int64),
-	}
-
-	if pendingReq.rev == 0 {
-		// note the header revision so that a put following a current watcher
-		// disconnect will arrive on the watcher channel after reconnect
-		ws.initReq.rev = resp.Header.Revision
-	}
-
+func (w *watcher) closeStream(wgs *watchGrpcStream) {
 	w.mu.Lock()
-	w.streams[ws.id] = ws
+	close(wgs.donec)
+	wgs.cancel()
+	if w.streams != nil {
+		delete(w.streams, wgs.ctxKey)
+	}
 	w.mu.Unlock()
-
-	// pass back the subscriber channel for the watcher
-	pendingReq.retc <- ret
-
-	// send messages to subscriber
-	go w.serveStream(ws)
 }
 
-func (w *watchGrpcStream) closeStream(ws *watcherStream) bool {
-	w.mu.Lock()
-	// cancels request stream; subscriber receives nil channel
-	close(ws.initReq.retc)
-	// close subscriber's channel
-	close(ws.outc)
-	delete(w.streams, ws.id)
-	empty := len(w.streams) == 0
-	if empty && w.stopc != nil {
-		w.stopc = nil
+func (w *watchGrpcStream) addSubstream(resp *pb.WatchResponse, ws *watcherStream) {
+	if resp.WatchId == -1 {
+		// failed; no channel
+		close(ws.recvc)
+		return
+	}
+	ws.id = resp.WatchId
+	w.substreams[ws.id] = ws
+}
+
+func (w *watchGrpcStream) sendCloseSubstream(ws *watcherStream, resp *WatchResponse) {
+	select {
+	case ws.outc <- *resp:
+	case <-ws.initReq.ctx.Done():
+	case <-time.After(closeSendErrTimeout):
+	}
+	close(ws.outc)
+}
+
+func (w *watchGrpcStream) closeSubstream(ws *watcherStream) {
+	// send channel response in case stream was never established
+	select {
+	case ws.initReq.retc <- ws.outc:
+	default:
+	}
+	// close subscriber's channel
+	if closeErr := w.closeErr; closeErr != nil && ws.initReq.ctx.Err() == nil {
+		go w.sendCloseSubstream(ws, &WatchResponse{closeErr: w.closeErr})
+	} else if ws.outc != nil {
+		close(ws.outc)
+	}
+	if ws.id != -1 {
+		delete(w.substreams, ws.id)
+		return
+	}
+	for i := range w.resuming {
+		if w.resuming[i] == ws {
+			w.resuming[i] = nil
+			return
+		}
 	}
-	w.mu.Unlock()
-	return empty
 }
 
 // run is the root of the goroutines for managing a watcher client
@@ -400,67 +383,81 @@ func (w *watchGrpcStream) run() {
 	var wc pb.Watch_WatchClient
 	var closeErr error
 
-	defer func() {
-		w.owner.mu.Lock()
-		w.closeErr = closeErr
-		if w.owner.streams != nil {
-			delete(w.owner.streams, w.ctxKey)
-		}
-		close(w.donec)
-		w.owner.mu.Unlock()
-		w.cancel()
-	}()
+	// substreams marked to close but goroutine still running; needed for
+	// avoiding double-closing recvc on grpc stream teardown
+	closing := make(map[*watcherStream]struct{})
 
-	// already stopped?
-	w.mu.RLock()
-	stopc := w.stopc
-	w.mu.RUnlock()
-	if stopc == nil {
-		return
-	}
+	defer func() {
+		w.closeErr = closeErr
+		// shutdown substreams and resuming substreams
+		for _, ws := range w.substreams {
+			if _, ok := closing[ws]; !ok {
+				close(ws.recvc)
+				closing[ws] = struct{}{}
+			}
+		}
+		for _, ws := range w.resuming {
+			if _, ok := closing[ws]; ws != nil && !ok {
+				close(ws.recvc)
+				closing[ws] = struct{}{}
+			}
+		}
+		w.joinSubstreams()
+		for range closing {
+			w.closeSubstream(<-w.closingc)
+		}
+
+		w.owner.closeStream(w)
+	}()
 
 	// start a stream with the etcd grpc server
 	if wc, closeErr = w.newWatchClient(); closeErr != nil {
 		return
 	}
 
-	var pendingReq, failedReq *watchRequest
-	curReqC := w.reqc
 	cancelSet := make(map[int64]struct{})
 
 	for {
 		select {
 		// Watch() requested
-		case pendingReq = <-curReqC:
-			// no more watch requests until there's a response
-			curReqC = nil
-			if err := wc.Send(pendingReq.toPB()); err == nil {
-				// pendingReq now waits on w.respc
-				break
+		case wreq := <-w.reqc:
+			outc := make(chan WatchResponse, 1)
+			ws := &watcherStream{
+				initReq: *wreq,
+				id:      -1,
+				outc:    outc,
+				// unbufffered so resumes won't cause repeat events
+				recvc: make(chan *WatchResponse),
+			}
+
+			ws.donec = make(chan struct{})
+			go w.serveSubstream(ws, w.resumec)
+
+			// queue up for watcher creation/resume
+			w.resuming = append(w.resuming, ws)
+			if len(w.resuming) == 1 {
+				// head of resume queue, can register a new watcher
+				wc.Send(ws.initReq.toPB())
 			}
-			failedReq = pendingReq
 		// New events from the watch client
 		case pbresp := <-w.respc:
 			switch {
 			case pbresp.Created:
-				// response to pending req, try to add
-				w.addStream(pbresp, pendingReq)
-				pendingReq = nil
-				curReqC = w.reqc
-				w.dispatchEvent(pbresp)
+				// response to head of queue creation
+				if ws := w.resuming[0]; ws != nil {
+					w.addSubstream(pbresp, ws)
+					w.dispatchEvent(pbresp)
+					w.resuming[0] = nil
+				}
+				if ws := w.nextResume(); ws != nil {
+					wc.Send(ws.initReq.toPB())
+				}
 			case pbresp.Canceled:
 				delete(cancelSet, pbresp.WatchId)
-				// shutdown serveStream, if any
-				w.mu.Lock()
-				if ws, ok := w.streams[pbresp.WatchId]; ok {
+				if ws, ok := w.substreams[pbresp.WatchId]; ok {
+					// signal to stream goroutine to update closingc
 					close(ws.recvc)
-					delete(w.streams, ws.id)
-				}
-				numStreams := len(w.streams)
-				w.mu.Unlock()
-				if numStreams == 0 {
-					// don't leak watcher streams
-					return
+					closing[ws] = struct{}{}
 				}
 			default:
 				// dispatch to appropriate watch stream
@@ -481,7 +478,6 @@ func (w *watchGrpcStream) run() {
 				wc.Send(req)
 			}
 		// watch client failed to recv; spawn another if possible
-		// TODO report watch client errors from errc?
 		case err := <-w.errc:
 			if isHaltErr(w.ctx, err) || toErr(w.ctx, err) == v3rpc.ErrNoLeader {
 				closeErr = err
@@ -490,43 +486,41 @@ func (w *watchGrpcStream) run() {
 			if wc, closeErr = w.newWatchClient(); closeErr != nil {
 				return
 			}
-			curReqC = w.reqc
-			if pendingReq != nil {
-				failedReq = pendingReq
+			if ws := w.nextResume(); ws != nil {
+				wc.Send(ws.initReq.toPB())
 			}
 			cancelSet = make(map[int64]struct{})
-		case <-stopc:
+		case <-w.ctx.Done():
 			return
 		case ws := <-w.closingc:
-			if w.closeStream(ws) {
+			w.closeSubstream(ws)
+			delete(closing, ws)
+			if len(w.substreams)+len(w.resuming) == 0 {
+				// no more watchers on this stream, shutdown
 				return
 			}
 		}
-
-		// send failed; queue for retry
-		if failedReq != nil {
-			go func(wr *watchRequest) {
-				select {
-				case w.reqc <- wr:
-				case <-wr.ctx.Done():
-				case <-w.donec:
-				}
-			}(pendingReq)
-			failedReq = nil
-			pendingReq = nil
-		}
 	}
 }
 
+// nextResume chooses the next resuming to register with the grpc stream. Abandoned
+// streams are marked as nil in the queue since the head must wait for its inflight registration.
+func (w *watchGrpcStream) nextResume() *watcherStream {
+	for len(w.resuming) != 0 {
+		if w.resuming[0] != nil {
+			return w.resuming[0]
+		}
+		w.resuming = w.resuming[1:len(w.resuming)]
+	}
+	return nil
+}
+
 // dispatchEvent sends a WatchResponse to the appropriate watcher stream
 func (w *watchGrpcStream) dispatchEvent(pbresp *pb.WatchResponse) bool {
-	w.mu.RLock()
-	defer w.mu.RUnlock()
-	ws, ok := w.streams[pbresp.WatchId]
+	ws, ok := w.substreams[pbresp.WatchId]
 	if !ok {
 		return false
 	}
-
 	events := make([]*Event, len(pbresp.Events))
 	for i, ev := range pbresp.Events {
 		events[i] = (*Event)(ev)
@@ -538,7 +532,11 @@ func (w *watchGrpcStream) dispatchEvent(pbresp *pb.WatchResponse) bool {
 		Created:         pbresp.Created,
 		Canceled:        pbresp.Canceled,
 	}
-	ws.recvc <- wr
+	select {
+	case ws.recvc <- wr:
+	case <-ws.donec:
+		return false
+	}
 	return true
 }
 
@@ -561,140 +559,187 @@ func (w *watchGrpcStream) serveWatchClient(wc pb.Watch_WatchClient) {
 	}
 }
 
-// serveStream forwards watch responses from run() to the subscriber
-func (w *watchGrpcStream) serveStream(ws *watcherStream) {
+// serveSubstream forwards watch responses from run() to the subscriber
+func (w *watchGrpcStream) serveSubstream(ws *watcherStream, resumec chan struct{}) {
+	if ws.closing {
+		panic("created substream goroutine but substream is closing")
+	}
+
+	// nextRev is the minimum expected next revision
+	nextRev := ws.initReq.rev
+	resuming := false
 	defer func() {
-		// signal that this watcherStream is finished
-		select {
-		case w.closingc <- ws:
-		case <-w.donec:
-			w.closeStream(ws)
+		if !resuming {
+			ws.closing = true
+		}
+		close(ws.donec)
+		if !resuming {
+			w.closingc <- ws
 		}
 	}()
 
-	var closeErr error
 	emptyWr := &WatchResponse{}
-	wrs := []*WatchResponse{}
-	resuming := false
-	closing := false
-	for !closing {
+	for {
 		curWr := emptyWr
 		outc := ws.outc
 
-		// ignore created event if create notify is not requested or
-		// we already sent the initial created event (when we are on the resume path).
-		if len(wrs) > 0 && wrs[0].Created &&
-			(!ws.initReq.createdNotify || ws.lastRev != 0) {
-			wrs = wrs[1:]
-		}
-
-		if len(wrs) > 0 {
-			curWr = wrs[0]
+		if len(ws.buf) > 0 {
+			curWr = ws.buf[0]
 		} else {
 			outc = nil
 		}
 		select {
 		case outc <- *curWr:
-			if wrs[0].Err() != nil {
-				closing = true
-				break
-			}
-			var newRev int64
-			if len(wrs[0].Events) > 0 {
-				newRev = wrs[0].Events[len(wrs[0].Events)-1].Kv.ModRevision
-			} else {
-				newRev = wrs[0].Header.Revision
-			}
-			if newRev != ws.lastRev {
-				ws.lastRev = newRev
-			}
-			wrs[0] = nil
-			wrs = wrs[1:]
-		case wr, ok := <-ws.recvc:
-			if !ok {
-				// shutdown from closeStream
+			if ws.buf[0].Err() != nil {
 				return
 			}
-			// resume up to last seen event if disconnected
-			if resuming && wr.Err() == nil {
-				resuming = false
-				// trim events already seen
-				for i := 0; i < len(wr.Events); i++ {
-					if wr.Events[i].Kv.ModRevision > ws.lastRev {
-						wr.Events = wr.Events[i:]
-						break
+			ws.buf[0] = nil
+			ws.buf = ws.buf[1:]
+		case wr, ok := <-ws.recvc:
+			if !ok {
+				// shutdown from closeSubstream
+				return
+			}
+
+			if wr.Created {
+				if ws.initReq.retc != nil {
+					ws.initReq.retc <- ws.outc
+					// to prevent next write from taking the slot in buffered channel
+					// and posting duplicate create events
+					ws.initReq.retc = nil
+
+					// send first creation event only if requested
+					if ws.initReq.createdNotify {
+						ws.outc <- *wr
 					}
 				}
-				// only forward new events
-				if wr.Events[0].Kv.ModRevision == ws.lastRev {
-					break
-				}
 			}
-			resuming = false
-			// TODO don't keep buffering if subscriber stops reading
-			wrs = append(wrs, wr)
-		case resumeRev := <-ws.resumec:
-			wrs = nil
-			resuming = true
-			if resumeRev == -1 {
-				// pause serving stream while resume gets set up
-				break
+
+			nextRev = wr.Header.Revision
+			if len(wr.Events) > 0 {
+				nextRev = wr.Events[len(wr.Events)-1].Kv.ModRevision + 1
 			}
-			if resumeRev != ws.lastRev {
-				panic("unexpected resume revision")
+			ws.initReq.rev = nextRev
+
+			// created event is already sent above,
+			// watcher should not post duplicate events
+			if wr.Created {
+				continue
 			}
-		case <-w.donec:
-			closing = true
-			closeErr = w.closeErr
+
+			// TODO pause channel if buffer gets too large
+			ws.buf = append(ws.buf, wr)
+		case <-w.ctx.Done():
+			return
 		case <-ws.initReq.ctx.Done():
-			closing = true
+			return
+		case <-resumec:
+			resuming = true
+			return
 		}
 	}
-
-	// try to send off close error
-	if closeErr != nil {
-		select {
-		case ws.outc <- WatchResponse{closeErr: w.closeErr}:
-		case <-w.donec:
-		case <-time.After(closeSendErrTimeout):
-		}
-	}
-
 	// lazily send cancel message if events on missing id
 }
 
 func (w *watchGrpcStream) newWatchClient() (pb.Watch_WatchClient, error) {
-	ws, rerr := w.resume()
-	if rerr != nil {
-		return nil, rerr
+	// mark all substreams as resuming
+	close(w.resumec)
+	w.resumec = make(chan struct{})
+	w.joinSubstreams()
+	for _, ws := range w.substreams {
+		ws.id = -1
+		w.resuming = append(w.resuming, ws)
 	}
-	go w.serveWatchClient(ws)
-	return ws, nil
-}
-
-// resume creates a new WatchClient with all current watchers reestablished
-func (w *watchGrpcStream) resume() (ws pb.Watch_WatchClient, err error) {
-	for {
-		if ws, err = w.openWatchClient(); err != nil {
-			break
-		} else if err = w.resumeWatchers(ws); err == nil {
-			break
+	// strip out nils, if any
+	var resuming []*watcherStream
+	for _, ws := range w.resuming {
+		if ws != nil {
+			resuming = append(resuming, ws)
+		}
+	}
+	w.resuming = resuming
+	w.substreams = make(map[int64]*watcherStream)
+
+	// connect to grpc stream while accepting watcher cancelation
+	stopc := make(chan struct{})
+	donec := w.waitCancelSubstreams(stopc)
+	wc, err := w.openWatchClient()
+	close(stopc)
+	<-donec
+
+	// serve all non-closing streams, even if there's a client error
+	// so that the teardown path can shutdown the streams as expected.
+	for _, ws := range w.resuming {
+		if ws.closing {
+			continue
+		}
+		ws.donec = make(chan struct{})
+		go w.serveSubstream(ws, w.resumec)
+	}
+
+	if err != nil {
+		return nil, v3rpc.Error(err)
+	}
+
+	// receive data from new grpc stream
+	go w.serveWatchClient(wc)
+	return wc, nil
+}
+
+func (w *watchGrpcStream) waitCancelSubstreams(stopc <-chan struct{}) <-chan struct{} {
+	var wg sync.WaitGroup
+	wg.Add(len(w.resuming))
+	donec := make(chan struct{})
+	for i := range w.resuming {
+		go func(ws *watcherStream) {
+			defer wg.Done()
+			if ws.closing {
+				if ws.initReq.ctx.Err() != nil && ws.outc != nil {
+					close(ws.outc)
+					ws.outc = nil
+				}
+				return
+			}
+			select {
+			case <-ws.initReq.ctx.Done():
+				// closed ws will be removed from resuming
+				ws.closing = true
+				close(ws.outc)
+				ws.outc = nil
+				go func() { w.closingc <- ws }()
+			case <-stopc:
+			}
+		}(w.resuming[i])
+	}
+	go func() {
+		defer close(donec)
+		wg.Wait()
+	}()
+	return donec
+}
+
+// joinSubstream waits for all substream goroutines to complete
+func (w *watchGrpcStream) joinSubstreams() {
+	for _, ws := range w.substreams {
+		<-ws.donec
+	}
+	for _, ws := range w.resuming {
+		if ws != nil {
+			<-ws.donec
 		}
 	}
-	return ws, v3rpc.Error(err)
 }
 
 // openWatchClient retries opening a watchclient until retryConnection fails
 func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error) {
 	for {
-		w.mu.Lock()
-		stopc := w.stopc
-		w.mu.Unlock()
-		if stopc == nil {
+		select {
+		case <-w.ctx.Done():
 			if err == nil {
-				err = context.Canceled
+				return nil, w.ctx.Err()
 			}
 			return nil, err
+		default:
 		}
 		if ws, err = w.remote.Watch(w.ctx, grpc.FailFast(false)); ws != nil && err == nil {
 			break
@@ -706,63 +751,6 @@ func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error)
 	return ws, nil
 }
 
-// resumeWatchers rebuilds every registered watcher on a new client
-func (w *watchGrpcStream) resumeWatchers(wc pb.Watch_WatchClient) error {
-	w.mu.RLock()
-	streams := make([]*watcherStream, 0, len(w.streams))
-	for _, ws := range w.streams {
-		streams = append(streams, ws)
-	}
-	w.mu.RUnlock()
-
-	for _, ws := range streams {
-		// drain recvc so no old WatchResponses (e.g., Created messages)
-		// are processed while resuming
-		ws.drain()
-
-		// pause serveStream
-		ws.resumec <- -1
-
-		// reconstruct watcher from initial request
-		if ws.lastRev != 0 {
-			ws.initReq.rev = ws.lastRev
-		}
-		if err := wc.Send(ws.initReq.toPB()); err != nil {
-			return err
-		}
-
-		// wait for request ack
-		resp, err := wc.Recv()
-		if err != nil {
-			return err
-		} else if len(resp.Events) != 0 || !resp.Created {
-			return fmt.Errorf("watcher: unexpected response (%+v)", resp)
-		}
-
-		// id may be different since new remote watcher; update map
-		w.mu.Lock()
-		delete(w.streams, ws.id)
-		ws.id = resp.WatchId
-		w.streams[ws.id] = ws
-		w.mu.Unlock()
-
-		// unpause serveStream
-		ws.resumec <- ws.lastRev
-	}
-	return nil
-}
-
-// drain removes all buffered WatchResponses from the stream's receive channel.
-func (ws *watcherStream) drain() {
-	for {
-		select {
-		case <-ws.recvc:
-		default:
-			return
-		}
-	}
-}
-
 // toPB converts an internal watch request structure to its protobuf messagefunc (wr *watchRequest)
 func (wr *watchRequest) toPB() *pb.WatchRequest {
 	req := &pb.WatchCreateRequest{

cmd/vendor/github.com/akrennmair/gopcap/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Andreas Krennmair nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

cmd/vendor/github.com/akrennmair/gopcap/decode.go

@@ -1,527 +0,0 @@
-package pcap
-
-import (
-	"encoding/binary"
-	"fmt"
-	"net"
-	"reflect"
-	"strings"
-)
-
-const (
-	TYPE_IP   = 0x0800
-	TYPE_ARP  = 0x0806
-	TYPE_IP6  = 0x86DD
-	TYPE_VLAN = 0x8100
-
-	IP_ICMP = 1
-	IP_INIP = 4
-	IP_TCP  = 6
-	IP_UDP  = 17
-)
-
-const (
-	ERRBUF_SIZE = 256
-
-	// According to pcap-linktype(7).
-	LINKTYPE_NULL             = 0
-	LINKTYPE_ETHERNET         = 1
-	LINKTYPE_TOKEN_RING       = 6
-	LINKTYPE_ARCNET           = 7
-	LINKTYPE_SLIP             = 8
-	LINKTYPE_PPP              = 9
-	LINKTYPE_FDDI             = 10
-	LINKTYPE_ATM_RFC1483      = 100
-	LINKTYPE_RAW              = 101
-	LINKTYPE_PPP_HDLC         = 50
-	LINKTYPE_PPP_ETHER        = 51
-	LINKTYPE_C_HDLC           = 104
-	LINKTYPE_IEEE802_11       = 105
-	LINKTYPE_FRELAY           = 107
-	LINKTYPE_LOOP             = 108
-	LINKTYPE_LINUX_SLL        = 113
-	LINKTYPE_LTALK            = 104
-	LINKTYPE_PFLOG            = 117
-	LINKTYPE_PRISM_HEADER     = 119
-	LINKTYPE_IP_OVER_FC       = 122
-	LINKTYPE_SUNATM           = 123
-	LINKTYPE_IEEE802_11_RADIO = 127
-	LINKTYPE_ARCNET_LINUX     = 129
-	LINKTYPE_LINUX_IRDA       = 144
-	LINKTYPE_LINUX_LAPD       = 177
-)
-
-type addrHdr interface {
-	SrcAddr() string
-	DestAddr() string
-	Len() int
-}
-
-type addrStringer interface {
-	String(addr addrHdr) string
-}
-
-func decodemac(pkt []byte) uint64 {
-	mac := uint64(0)
-	for i := uint(0); i < 6; i++ {
-		mac = (mac << 8) + uint64(pkt[i])
-	}
-	return mac
-}
-
-// Decode decodes the headers of a Packet.
-func (p *Packet) Decode() {
-	if len(p.Data) <= 14 {
-		return
-	}
-
-	p.Type = int(binary.BigEndian.Uint16(p.Data[12:14]))
-	p.DestMac = decodemac(p.Data[0:6])
-	p.SrcMac = decodemac(p.Data[6:12])
-
-	if len(p.Data) >= 15 {
-		p.Payload = p.Data[14:]
-	}
-
-	switch p.Type {
-	case TYPE_IP:
-		p.decodeIp()
-	case TYPE_IP6:
-		p.decodeIp6()
-	case TYPE_ARP:
-		p.decodeArp()
-	case TYPE_VLAN:
-		p.decodeVlan()
-	}
-}
-
-func (p *Packet) headerString(headers []interface{}) string {
-	// If there's just one header, return that.
-	if len(headers) == 1 {
-		if hdr, ok := headers[0].(fmt.Stringer); ok {
-			return hdr.String()
-		}
-	}
-	// If there are two headers (IPv4/IPv6 -> TCP/UDP/IP..)
-	if len(headers) == 2 {
-		// Commonly the first header is an address.
-		if addr, ok := p.Headers[0].(addrHdr); ok {
-			if hdr, ok := p.Headers[1].(addrStringer); ok {
-				return fmt.Sprintf("%s %s", p.Time, hdr.String(addr))
-			}
-		}
-	}
-	// For IP in IP, we do a recursive call.
-	if len(headers) >= 2 {
-		if addr, ok := headers[0].(addrHdr); ok {
-			if _, ok := headers[1].(addrHdr); ok {
-				return fmt.Sprintf("%s > %s IP in IP: ",
-					addr.SrcAddr(), addr.DestAddr(), p.headerString(headers[1:]))
-			}
-		}
-	}
-
-	var typeNames []string
-	for _, hdr := range headers {
-		typeNames = append(typeNames, reflect.TypeOf(hdr).String())
-	}
-
-	return fmt.Sprintf("unknown [%s]", strings.Join(typeNames, ","))
-}
-
-// String prints a one-line representation of the packet header.
-// The output is suitable for use in a tcpdump program.
-func (p *Packet) String() string {
-	// If there are no headers, print "unsupported protocol".
-	if len(p.Headers) == 0 {
-		return fmt.Sprintf("%s unsupported protocol %d", p.Time, int(p.Type))
-	}
-	return fmt.Sprintf("%s %s", p.Time, p.headerString(p.Headers))
-}
-
-// Arphdr is a ARP packet header.
-type Arphdr struct {
-	Addrtype          uint16
-	Protocol          uint16
-	HwAddressSize     uint8
-	ProtAddressSize   uint8
-	Operation         uint16
-	SourceHwAddress   []byte
-	SourceProtAddress []byte
-	DestHwAddress     []byte
-	DestProtAddress   []byte
-}
-
-func (arp *Arphdr) String() (s string) {
-	switch arp.Operation {
-	case 1:
-		s = "ARP request"
-	case 2:
-		s = "ARP Reply"
-	}
-	if arp.Addrtype == LINKTYPE_ETHERNET && arp.Protocol == TYPE_IP {
-		s = fmt.Sprintf("%012x (%s) > %012x (%s)",
-			decodemac(arp.SourceHwAddress), arp.SourceProtAddress,
-			decodemac(arp.DestHwAddress), arp.DestProtAddress)
-	} else {
-		s = fmt.Sprintf("addrtype = %d protocol = %d", arp.Addrtype, arp.Protocol)
-	}
-	return
-}
-
-func (p *Packet) decodeArp() {
-	if len(p.Payload) < 8 {
-		return
-	}
-
-	pkt := p.Payload
-	arp := new(Arphdr)
-	arp.Addrtype = binary.BigEndian.Uint16(pkt[0:2])
-	arp.Protocol = binary.BigEndian.Uint16(pkt[2:4])
-	arp.HwAddressSize = pkt[4]
-	arp.ProtAddressSize = pkt[5]
-	arp.Operation = binary.BigEndian.Uint16(pkt[6:8])
-
-	if len(pkt) < int(8+2*arp.HwAddressSize+2*arp.ProtAddressSize) {
-		return
-	}
-	arp.SourceHwAddress = pkt[8 : 8+arp.HwAddressSize]
-	arp.SourceProtAddress = pkt[8+arp.HwAddressSize : 8+arp.HwAddressSize+arp.ProtAddressSize]
-	arp.DestHwAddress = pkt[8+arp.HwAddressSize+arp.ProtAddressSize : 8+2*arp.HwAddressSize+arp.ProtAddressSize]
-	arp.DestProtAddress = pkt[8+2*arp.HwAddressSize+arp.ProtAddressSize : 8+2*arp.HwAddressSize+2*arp.ProtAddressSize]
-
-	p.Headers = append(p.Headers, arp)
-
-	if len(pkt) >= int(8+2*arp.HwAddressSize+2*arp.ProtAddressSize) {
-		p.Payload = p.Payload[8+2*arp.HwAddressSize+2*arp.ProtAddressSize:]
-	}
-}
-
-// IPadr is the header of an IP packet.
-type Iphdr struct {
-	Version    uint8
-	Ihl        uint8
-	Tos        uint8
-	Length     uint16
-	Id         uint16
-	Flags      uint8
-	FragOffset uint16
-	Ttl        uint8
-	Protocol   uint8
-	Checksum   uint16
-	SrcIp      []byte
-	DestIp     []byte
-}
-
-func (p *Packet) decodeIp() {
-	if len(p.Payload) < 20 {
-		return
-	}
-
-	pkt := p.Payload
-	ip := new(Iphdr)
-
-	ip.Version = uint8(pkt[0]) >> 4
-	ip.Ihl = uint8(pkt[0]) & 0x0F
-	ip.Tos = pkt[1]
-	ip.Length = binary.BigEndian.Uint16(pkt[2:4])
-	ip.Id = binary.BigEndian.Uint16(pkt[4:6])
-	flagsfrags := binary.BigEndian.Uint16(pkt[6:8])
-	ip.Flags = uint8(flagsfrags >> 13)
-	ip.FragOffset = flagsfrags & 0x1FFF
-	ip.Ttl = pkt[8]
-	ip.Protocol = pkt[9]
-	ip.Checksum = binary.BigEndian.Uint16(pkt[10:12])
-	ip.SrcIp = pkt[12:16]
-	ip.DestIp = pkt[16:20]
-
-	pEnd := int(ip.Length)
-	if pEnd > len(pkt) {
-		pEnd = len(pkt)
-	}
-
-	if len(pkt) >= pEnd && int(ip.Ihl*4) < pEnd {
-		p.Payload = pkt[ip.Ihl*4 : pEnd]
-	} else {
-		p.Payload = []byte{}
-	}
-
-	p.Headers = append(p.Headers, ip)
-	p.IP = ip
-
-	switch ip.Protocol {
-	case IP_TCP:
-		p.decodeTcp()
-	case IP_UDP:
-		p.decodeUdp()
-	case IP_ICMP:
-		p.decodeIcmp()
-	case IP_INIP:
-		p.decodeIp()
-	}
-}
-
-func (ip *Iphdr) SrcAddr() string  { return net.IP(ip.SrcIp).String() }
-func (ip *Iphdr) DestAddr() string { return net.IP(ip.DestIp).String() }
-func (ip *Iphdr) Len() int         { return int(ip.Length) }
-
-type Vlanhdr struct {
-	Priority       byte
-	DropEligible   bool
-	VlanIdentifier int
-	Type           int // Not actually part of the vlan header, but the type of the actual packet
-}
-
-func (v *Vlanhdr) String() {
-	fmt.Sprintf("VLAN Priority:%d Drop:%v Tag:%d", v.Priority, v.DropEligible, v.VlanIdentifier)
-}
-
-func (p *Packet) decodeVlan() {
-	pkt := p.Payload
-	vlan := new(Vlanhdr)
-	if len(pkt) < 4 {
-		return
-	}
-
-	vlan.Priority = (pkt[2] & 0xE0) >> 13
-	vlan.DropEligible = pkt[2]&0x10 != 0
-	vlan.VlanIdentifier = int(binary.BigEndian.Uint16(pkt[:2])) & 0x0FFF
-	vlan.Type = int(binary.BigEndian.Uint16(p.Payload[2:4]))
-	p.Headers = append(p.Headers, vlan)
-
-	if len(pkt) >= 5 {
-		p.Payload = p.Payload[4:]
-	}
-
-	switch vlan.Type {
-	case TYPE_IP:
-		p.decodeIp()
-	case TYPE_IP6:
-		p.decodeIp6()
-	case TYPE_ARP:
-		p.decodeArp()
-	}
-}
-
-type Tcphdr struct {
-	SrcPort    uint16
-	DestPort   uint16
-	Seq        uint32
-	Ack        uint32
-	DataOffset uint8
-	Flags      uint16
-	Window     uint16
-	Checksum   uint16
-	Urgent     uint16
-	Data       []byte
-}
-
-const (
-	TCP_FIN = 1 << iota
-	TCP_SYN
-	TCP_RST
-	TCP_PSH
-	TCP_ACK
-	TCP_URG
-	TCP_ECE
-	TCP_CWR
-	TCP_NS
-)
-
-func (p *Packet) decodeTcp() {
-	if len(p.Payload) < 20 {
-		return
-	}
-
-	pkt := p.Payload
-	tcp := new(Tcphdr)
-	tcp.SrcPort = binary.BigEndian.Uint16(pkt[0:2])
-	tcp.DestPort = binary.BigEndian.Uint16(pkt[2:4])
-	tcp.Seq = binary.BigEndian.Uint32(pkt[4:8])
-	tcp.Ack = binary.BigEndian.Uint32(pkt[8:12])
-	tcp.DataOffset = (pkt[12] & 0xF0) >> 4
-	tcp.Flags = binary.BigEndian.Uint16(pkt[12:14]) & 0x1FF
-	tcp.Window = binary.BigEndian.Uint16(pkt[14:16])
-	tcp.Checksum = binary.BigEndian.Uint16(pkt[16:18])
-	tcp.Urgent = binary.BigEndian.Uint16(pkt[18:20])
-	if len(pkt) >= int(tcp.DataOffset*4) {
-		p.Payload = pkt[tcp.DataOffset*4:]
-	}
-	p.Headers = append(p.Headers, tcp)
-	p.TCP = tcp
-}
-
-func (tcp *Tcphdr) String(hdr addrHdr) string {
-	return fmt.Sprintf("TCP %s:%d > %s:%d %s SEQ=%d ACK=%d LEN=%d",
-		hdr.SrcAddr(), int(tcp.SrcPort), hdr.DestAddr(), int(tcp.DestPort),
-		tcp.FlagsString(), int64(tcp.Seq), int64(tcp.Ack), hdr.Len())
-}
-
-func (tcp *Tcphdr) FlagsString() string {
-	var sflags []string
-	if 0 != (tcp.Flags & TCP_SYN) {
-		sflags = append(sflags, "syn")
-	}
-	if 0 != (tcp.Flags & TCP_FIN) {
-		sflags = append(sflags, "fin")
-	}
-	if 0 != (tcp.Flags & TCP_ACK) {
-		sflags = append(sflags, "ack")
-	}
-	if 0 != (tcp.Flags & TCP_PSH) {
-		sflags = append(sflags, "psh")
-	}
-	if 0 != (tcp.Flags & TCP_RST) {
-		sflags = append(sflags, "rst")
-	}
-	if 0 != (tcp.Flags & TCP_URG) {
-		sflags = append(sflags, "urg")
-	}
-	if 0 != (tcp.Flags & TCP_NS) {
-		sflags = append(sflags, "ns")
-	}
-	if 0 != (tcp.Flags & TCP_CWR) {
-		sflags = append(sflags, "cwr")
-	}
-	if 0 != (tcp.Flags & TCP_ECE) {
-		sflags = append(sflags, "ece")
-	}
-	return fmt.Sprintf("[%s]", strings.Join(sflags, " "))
-}
-
-type Udphdr struct {
-	SrcPort  uint16
-	DestPort uint16
-	Length   uint16
-	Checksum uint16
-}
-
-func (p *Packet) decodeUdp() {
-	if len(p.Payload) < 8 {
-		return
-	}
-
-	pkt := p.Payload
-	udp := new(Udphdr)
-	udp.SrcPort = binary.BigEndian.Uint16(pkt[0:2])
-	udp.DestPort = binary.BigEndian.Uint16(pkt[2:4])
-	udp.Length = binary.BigEndian.Uint16(pkt[4:6])
-	udp.Checksum = binary.BigEndian.Uint16(pkt[6:8])
-	p.Headers = append(p.Headers, udp)
-	p.UDP = udp
-	if len(p.Payload) >= 8 {
-		p.Payload = pkt[8:]
-	}
-}
-
-func (udp *Udphdr) String(hdr addrHdr) string {
-	return fmt.Sprintf("UDP %s:%d > %s:%d LEN=%d CHKSUM=%d",
-		hdr.SrcAddr(), int(udp.SrcPort), hdr.DestAddr(), int(udp.DestPort),
-		int(udp.Length), int(udp.Checksum))
-}
-
-type Icmphdr struct {
-	Type     uint8
-	Code     uint8
-	Checksum uint16
-	Id       uint16
-	Seq      uint16
-	Data     []byte
-}
-
-func (p *Packet) decodeIcmp() *Icmphdr {
-	if len(p.Payload) < 8 {
-		return nil
-	}
-
-	pkt := p.Payload
-	icmp := new(Icmphdr)
-	icmp.Type = pkt[0]
-	icmp.Code = pkt[1]
-	icmp.Checksum = binary.BigEndian.Uint16(pkt[2:4])
-	icmp.Id = binary.BigEndian.Uint16(pkt[4:6])
-	icmp.Seq = binary.BigEndian.Uint16(pkt[6:8])
-	p.Payload = pkt[8:]
-	p.Headers = append(p.Headers, icmp)
-	return icmp
-}
-
-func (icmp *Icmphdr) String(hdr addrHdr) string {
-	return fmt.Sprintf("ICMP %s > %s Type = %d Code = %d ",
-		hdr.SrcAddr(), hdr.DestAddr(), icmp.Type, icmp.Code)
-}
-
-func (icmp *Icmphdr) TypeString() (result string) {
-	switch icmp.Type {
-	case 0:
-		result = fmt.Sprintf("Echo reply seq=%d", icmp.Seq)
-	case 3:
-		switch icmp.Code {
-		case 0:
-			result = "Network unreachable"
-		case 1:
-			result = "Host unreachable"
-		case 2:
-			result = "Protocol unreachable"
-		case 3:
-			result = "Port unreachable"
-		default:
-			result = "Destination unreachable"
-		}
-	case 8:
-		result = fmt.Sprintf("Echo request seq=%d", icmp.Seq)
-	case 30:
-		result = "Traceroute"
-	}
-	return
-}
-
-type Ip6hdr struct {
-	// http://www.networksorcery.com/enp/protocol/ipv6.htm
-	Version      uint8  // 4 bits
-	TrafficClass uint8  // 8 bits
-	FlowLabel    uint32 // 20 bits
-	Length       uint16 // 16 bits
-	NextHeader   uint8  // 8 bits, same as Protocol in Iphdr
-	HopLimit     uint8  // 8 bits
-	SrcIp        []byte // 16 bytes
-	DestIp       []byte // 16 bytes
-}
-
-func (p *Packet) decodeIp6() {
-	if len(p.Payload) < 40 {
-		return
-	}
-
-	pkt := p.Payload
-	ip6 := new(Ip6hdr)
-	ip6.Version = uint8(pkt[0]) >> 4
-	ip6.TrafficClass = uint8((binary.BigEndian.Uint16(pkt[0:2]) >> 4) & 0x00FF)
-	ip6.FlowLabel = binary.BigEndian.Uint32(pkt[0:4]) & 0x000FFFFF
-	ip6.Length = binary.BigEndian.Uint16(pkt[4:6])
-	ip6.NextHeader = pkt[6]
-	ip6.HopLimit = pkt[7]
-	ip6.SrcIp = pkt[8:24]
-	ip6.DestIp = pkt[24:40]
-
-	if len(p.Payload) >= 40 {
-		p.Payload = pkt[40:]
-	}
-
-	p.Headers = append(p.Headers, ip6)
-
-	switch ip6.NextHeader {
-	case IP_TCP:
-		p.decodeTcp()
-	case IP_UDP:
-		p.decodeUdp()
-	case IP_ICMP:
-		p.decodeIcmp()
-	case IP_INIP:
-		p.decodeIp()
-	}
-}
-
-func (ip6 *Ip6hdr) SrcAddr() string  { return net.IP(ip6.SrcIp).String() }
-func (ip6 *Ip6hdr) DestAddr() string { return net.IP(ip6.DestIp).String() }
-func (ip6 *Ip6hdr) Len() int         { return int(ip6.Length) }

cmd/vendor/github.com/akrennmair/gopcap/io.go

@@ -1,206 +0,0 @@
-package pcap
-
-import (
-	"encoding/binary"
-	"fmt"
-	"io"
-	"time"
-)
-
-// FileHeader is the parsed header of a pcap file.
-// http://wiki.wireshark.org/Development/LibpcapFileFormat
-type FileHeader struct {
-	MagicNumber  uint32
-	VersionMajor uint16
-	VersionMinor uint16
-	TimeZone     int32
-	SigFigs      uint32
-	SnapLen      uint32
-	Network      uint32
-}
-
-type PacketTime struct {
-	Sec  int32
-	Usec int32
-}
-
-// Convert the PacketTime to a go Time struct.
-func (p *PacketTime) Time() time.Time {
-	return time.Unix(int64(p.Sec), int64(p.Usec)*1000)
-}
-
-// Packet is a single packet parsed from a pcap file.
-//
-// Convenient access to IP, TCP, and UDP headers is provided after Decode()
-// is called if the packet is of the appropriate type.
-type Packet struct {
-	Time   time.Time // packet send/receive time
-	Caplen uint32    // bytes stored in the file (caplen <= len)
-	Len    uint32    // bytes sent/received
-	Data   []byte    // packet data
-
-	Type    int // protocol type, see LINKTYPE_*
-	DestMac uint64
-	SrcMac  uint64
-
-	Headers []interface{} // decoded headers, in order
-	Payload []byte        // remaining non-header bytes
-
-	IP  *Iphdr  // IP header (for IP packets, after decoding)
-	TCP *Tcphdr // TCP header (for TCP packets, after decoding)
-	UDP *Udphdr // UDP header (for UDP packets after decoding)
-}
-
-// Reader parses pcap files.
-type Reader struct {
-	flip         bool
-	buf          io.Reader
-	err          error
-	fourBytes    []byte
-	twoBytes     []byte
-	sixteenBytes []byte
-	Header       FileHeader
-}
-
-// NewReader reads pcap data from an io.Reader.
-func NewReader(reader io.Reader) (*Reader, error) {
-	r := &Reader{
-		buf:          reader,
-		fourBytes:    make([]byte, 4),
-		twoBytes:     make([]byte, 2),
-		sixteenBytes: make([]byte, 16),
-	}
-	switch magic := r.readUint32(); magic {
-	case 0xa1b2c3d4:
-		r.flip = false
-	case 0xd4c3b2a1:
-		r.flip = true
-	default:
-		return nil, fmt.Errorf("pcap: bad magic number: %0x", magic)
-	}
-	r.Header = FileHeader{
-		MagicNumber:  0xa1b2c3d4,
-		VersionMajor: r.readUint16(),
-		VersionMinor: r.readUint16(),
-		TimeZone:     r.readInt32(),
-		SigFigs:      r.readUint32(),
-		SnapLen:      r.readUint32(),
-		Network:      r.readUint32(),
-	}
-	return r, nil
-}
-
-// Next returns the next packet or nil if no more packets can be read.
-func (r *Reader) Next() *Packet {
-	d := r.sixteenBytes
-	r.err = r.read(d)
-	if r.err != nil {
-		return nil
-	}
-	timeSec := asUint32(d[0:4], r.flip)
-	timeUsec := asUint32(d[4:8], r.flip)
-	capLen := asUint32(d[8:12], r.flip)
-	origLen := asUint32(d[12:16], r.flip)
-
-	data := make([]byte, capLen)
-	if r.err = r.read(data); r.err != nil {
-		return nil
-	}
-	return &Packet{
-		Time:   time.Unix(int64(timeSec), int64(timeUsec)),
-		Caplen: capLen,
-		Len:    origLen,
-		Data:   data,
-	}
-}
-
-func (r *Reader) read(data []byte) error {
-	var err error
-	n, err := r.buf.Read(data)
-	for err == nil && n != len(data) {
-		var chunk int
-		chunk, err = r.buf.Read(data[n:])
-		n += chunk
-	}
-	if len(data) == n {
-		return nil
-	}
-	return err
-}
-
-func (r *Reader) readUint32() uint32 {
-	data := r.fourBytes
-	if r.err = r.read(data); r.err != nil {
-		return 0
-	}
-	return asUint32(data, r.flip)
-}
-
-func (r *Reader) readInt32() int32 {
-	data := r.fourBytes
-	if r.err = r.read(data); r.err != nil {
-		return 0
-	}
-	return int32(asUint32(data, r.flip))
-}
-
-func (r *Reader) readUint16() uint16 {
-	data := r.twoBytes
-	if r.err = r.read(data); r.err != nil {
-		return 0
-	}
-	return asUint16(data, r.flip)
-}
-
-// Writer writes a pcap file.
-type Writer struct {
-	writer io.Writer
-	buf    []byte
-}
-
-// NewWriter creates a Writer that stores output in an io.Writer.
-// The FileHeader is written immediately.
-func NewWriter(writer io.Writer, header *FileHeader) (*Writer, error) {
-	w := &Writer{
-		writer: writer,
-		buf:    make([]byte, 24),
-	}
-	binary.LittleEndian.PutUint32(w.buf, header.MagicNumber)
-	binary.LittleEndian.PutUint16(w.buf[4:], header.VersionMajor)
-	binary.LittleEndian.PutUint16(w.buf[6:], header.VersionMinor)
-	binary.LittleEndian.PutUint32(w.buf[8:], uint32(header.TimeZone))
-	binary.LittleEndian.PutUint32(w.buf[12:], header.SigFigs)
-	binary.LittleEndian.PutUint32(w.buf[16:], header.SnapLen)
-	binary.LittleEndian.PutUint32(w.buf[20:], header.Network)
-	if _, err := writer.Write(w.buf); err != nil {
-		return nil, err
-	}
-	return w, nil
-}
-
-// Writer writes a packet to the underlying writer.
-func (w *Writer) Write(pkt *Packet) error {
-	binary.LittleEndian.PutUint32(w.buf, uint32(pkt.Time.Unix()))
-	binary.LittleEndian.PutUint32(w.buf[4:], uint32(pkt.Time.Nanosecond()))
-	binary.LittleEndian.PutUint32(w.buf[8:], uint32(pkt.Time.Unix()))
-	binary.LittleEndian.PutUint32(w.buf[12:], pkt.Len)
-	if _, err := w.writer.Write(w.buf[:16]); err != nil {
-		return err
-	}
-	_, err := w.writer.Write(pkt.Data)
-	return err
-}
-
-func asUint32(data []byte, flip bool) uint32 {
-	if flip {
-		return binary.BigEndian.Uint32(data)
-	}
-	return binary.LittleEndian.Uint32(data)
-}
-
-func asUint16(data []byte, flip bool) uint16 {
-	if flip {
-		return binary.BigEndian.Uint16(data)
-	}
-	return binary.LittleEndian.Uint16(data)
-}

cmd/vendor/github.com/akrennmair/gopcap/pcap.go

@@ -1,266 +0,0 @@
-// Interface to both live and offline pcap parsing.
-package pcap
-
-/*
-#cgo linux LDFLAGS: -lpcap
-#cgo freebsd LDFLAGS: -lpcap
-#cgo darwin LDFLAGS: -lpcap
-#cgo windows CFLAGS: -I C:/WpdPack/Include
-#cgo windows,386 LDFLAGS: -L C:/WpdPack/Lib -lwpcap
-#cgo windows,amd64 LDFLAGS: -L C:/WpdPack/Lib/x64 -lwpcap
-#include <stdlib.h>
-#include <pcap.h>
-
-// Workaround for not knowing how to cast to const u_char**
-int hack_pcap_next_ex(pcap_t *p, struct pcap_pkthdr **pkt_header,
-                      u_char **pkt_data) {
-    return pcap_next_ex(p, pkt_header, (const u_char **)pkt_data);
-}
-*/
-import "C"
-import (
-	"errors"
-	"net"
-	"syscall"
-	"time"
-	"unsafe"
-)
-
-type Pcap struct {
-	cptr *C.pcap_t
-}
-
-type Stat struct {
-	PacketsReceived  uint32
-	PacketsDropped   uint32
-	PacketsIfDropped uint32
-}
-
-type Interface struct {
-	Name        string
-	Description string
-	Addresses   []IFAddress
-	// TODO: add more elements
-}
-
-type IFAddress struct {
-	IP      net.IP
-	Netmask net.IPMask
-	// TODO: add broadcast + PtP dst ?
-}
-
-func (p *Pcap) Next() (pkt *Packet) {
-	rv, _ := p.NextEx()
-	return rv
-}
-
-// Openlive opens a device and returns a *Pcap handler
-func Openlive(device string, snaplen int32, promisc bool, timeout_ms int32) (handle *Pcap, err error) {
-	var buf *C.char
-	buf = (*C.char)(C.calloc(ERRBUF_SIZE, 1))
-	h := new(Pcap)
-	var pro int32
-	if promisc {
-		pro = 1
-	}
-
-	dev := C.CString(device)
-	defer C.free(unsafe.Pointer(dev))
-
-	h.cptr = C.pcap_open_live(dev, C.int(snaplen), C.int(pro), C.int(timeout_ms), buf)
-	if nil == h.cptr {
-		handle = nil
-		err = errors.New(C.GoString(buf))
-	} else {
-		handle = h
-	}
-	C.free(unsafe.Pointer(buf))
-	return
-}
-
-func Openoffline(file string) (handle *Pcap, err error) {
-	var buf *C.char
-	buf = (*C.char)(C.calloc(ERRBUF_SIZE, 1))
-	h := new(Pcap)
-
-	cf := C.CString(file)
-	defer C.free(unsafe.Pointer(cf))
-
-	h.cptr = C.pcap_open_offline(cf, buf)
-	if nil == h.cptr {
-		handle = nil
-		err = errors.New(C.GoString(buf))
-	} else {
-		handle = h
-	}
-	C.free(unsafe.Pointer(buf))
-	return
-}
-
-func (p *Pcap) NextEx() (pkt *Packet, result int32) {
-	var pkthdr *C.struct_pcap_pkthdr
-
-	var buf_ptr *C.u_char
-	var buf unsafe.Pointer
-	result = int32(C.hack_pcap_next_ex(p.cptr, &pkthdr, &buf_ptr))
-
-	buf = unsafe.Pointer(buf_ptr)
-	if nil == buf {
-		return
-	}
-
-	pkt = new(Packet)
-	pkt.Time = time.Unix(int64(pkthdr.ts.tv_sec), int64(pkthdr.ts.tv_usec)*1000)
-	pkt.Caplen = uint32(pkthdr.caplen)
-	pkt.Len = uint32(pkthdr.len)
-	pkt.Data = C.GoBytes(buf, C.int(pkthdr.caplen))
-	return
-}
-
-func (p *Pcap) Close() {
-	C.pcap_close(p.cptr)
-}
-
-func (p *Pcap) Geterror() error {
-	return errors.New(C.GoString(C.pcap_geterr(p.cptr)))
-}
-
-func (p *Pcap) Getstats() (stat *Stat, err error) {
-	var cstats _Ctype_struct_pcap_stat
-	if -1 == C.pcap_stats(p.cptr, &cstats) {
-		return nil, p.Geterror()
-	}
-	stats := new(Stat)
-	stats.PacketsReceived = uint32(cstats.ps_recv)
-	stats.PacketsDropped = uint32(cstats.ps_drop)
-	stats.PacketsIfDropped = uint32(cstats.ps_ifdrop)
-
-	return stats, nil
-}
-
-func (p *Pcap) Setfilter(expr string) (err error) {
-	var bpf _Ctype_struct_bpf_program
-	cexpr := C.CString(expr)
-	defer C.free(unsafe.Pointer(cexpr))
-
-	if -1 == C.pcap_compile(p.cptr, &bpf, cexpr, 1, 0) {
-		return p.Geterror()
-	}
-
-	if -1 == C.pcap_setfilter(p.cptr, &bpf) {
-		C.pcap_freecode(&bpf)
-		return p.Geterror()
-	}
-	C.pcap_freecode(&bpf)
-	return nil
-}
-
-func Version() string {
-	return C.GoString(C.pcap_lib_version())
-}
-
-func (p *Pcap) Datalink() int {
-	return int(C.pcap_datalink(p.cptr))
-}
-
-func (p *Pcap) Setdatalink(dlt int) error {
-	if -1 == C.pcap_set_datalink(p.cptr, C.int(dlt)) {
-		return p.Geterror()
-	}
-	return nil
-}
-
-func DatalinkValueToName(dlt int) string {
-	if name := C.pcap_datalink_val_to_name(C.int(dlt)); name != nil {
-		return C.GoString(name)
-	}
-	return ""
-}
-
-func DatalinkValueToDescription(dlt int) string {
-	if desc := C.pcap_datalink_val_to_description(C.int(dlt)); desc != nil {
-		return C.GoString(desc)
-	}
-	return ""
-}
-
-func Findalldevs() (ifs []Interface, err error) {
-	var buf *C.char
-	buf = (*C.char)(C.calloc(ERRBUF_SIZE, 1))
-	defer C.free(unsafe.Pointer(buf))
-	var alldevsp *C.pcap_if_t
-
-	if -1 == C.pcap_findalldevs((**C.pcap_if_t)(&alldevsp), buf) {
-		return nil, errors.New(C.GoString(buf))
-	}
-	defer C.pcap_freealldevs((*C.pcap_if_t)(alldevsp))
-	dev := alldevsp
-	var i uint32
-	for i = 0; dev != nil; dev = (*C.pcap_if_t)(dev.next) {
-		i++
-	}
-	ifs = make([]Interface, i)
-	dev = alldevsp
-	for j := uint32(0); dev != nil; dev = (*C.pcap_if_t)(dev.next) {
-		var iface Interface
-		iface.Name = C.GoString(dev.name)
-		iface.Description = C.GoString(dev.description)
-		iface.Addresses = findalladdresses(dev.addresses)
-		// TODO: add more elements
-		ifs[j] = iface
-		j++
-	}
-	return
-}
-
-func findalladdresses(addresses *_Ctype_struct_pcap_addr) (retval []IFAddress) {
-	// TODO - make it support more than IPv4 and IPv6?
-	retval = make([]IFAddress, 0, 1)
-	for curaddr := addresses; curaddr != nil; curaddr = (*_Ctype_struct_pcap_addr)(curaddr.next) {
-		var a IFAddress
-		var err error
-		if a.IP, err = sockaddr_to_IP((*syscall.RawSockaddr)(unsafe.Pointer(curaddr.addr))); err != nil {
-			continue
-		}
-		if a.Netmask, err = sockaddr_to_IP((*syscall.RawSockaddr)(unsafe.Pointer(curaddr.addr))); err != nil {
-			continue
-		}
-		retval = append(retval, a)
-	}
-	return
-}
-
-func sockaddr_to_IP(rsa *syscall.RawSockaddr) (IP []byte, err error) {
-	switch rsa.Family {
-	case syscall.AF_INET:
-		pp := (*syscall.RawSockaddrInet4)(unsafe.Pointer(rsa))
-		IP = make([]byte, 4)
-		for i := 0; i < len(IP); i++ {
-			IP[i] = pp.Addr[i]
-		}
-		return
-	case syscall.AF_INET6:
-		pp := (*syscall.RawSockaddrInet6)(unsafe.Pointer(rsa))
-		IP = make([]byte, 16)
-		for i := 0; i < len(IP); i++ {
-			IP[i] = pp.Addr[i]
-		}
-		return
-	}
-	err = errors.New("Unsupported address type")
-	return
-}
-
-func (p *Pcap) Inject(data []byte) (err error) {
-	buf := (*C.char)(C.malloc((C.size_t)(len(data))))
-
-	for i := 0; i < len(data); i++ {
-		*(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(buf)) + uintptr(i))) = data[i]
-	}
-
-	if -1 == C.pcap_sendpacket(p.cptr, (*C.u_char)(unsafe.Pointer(buf)), (C.int)(len(data))) {
-		err = p.Geterror()
-	}
-	C.free(unsafe.Pointer(buf))
-	return
-}

cmd/vendor/github.com/beorn7/perks/LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2013 Blake Mizerany
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

cmd/vendor/github.com/beorn7/perks/quantile/stream.go

@@ -133,7 +133,7 @@ func (s *Stream) Query(q float64) float64 {
 		if l == 0 {
 			return 0
 		}
-		i := int(float64(l) * q)
+		i := int(math.Ceil(float64(l) * q))
 		if i > 0 {
 			i -= 1
 		}

cmd/vendor/github.com/coreos/go-semver/example.go

@@ -1,20 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"github.com/coreos/go-semver/semver"
-	"os"
-)
-
-func main() {
-	vA, err := semver.NewVersion(os.Args[1])
-	if err != nil {
-		fmt.Println(err.Error())
-	}
-	vB, err := semver.NewVersion(os.Args[2])
-	if err != nil {
-		fmt.Println(err.Error())
-	}
-
-	fmt.Printf("%s < %s == %t\n", vA, vB, vA.LessThan(*vB))
-}

cmd/vendor/github.com/coreos/go-systemd/daemon/sdnotify.go

@@ -1,3 +1,18 @@
+// Copyright 2014 Docker, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
 // Code forked from Docker project
 package daemon
 
@@ -7,11 +22,14 @@ import (
 )
 
 // SdNotify sends a message to the init daemon. It is common to ignore the error.
+// If `unsetEnvironment` is true, the environment variable `NOTIFY_SOCKET`
+// will be unconditionally unset.
+//
 // It returns one of the following:
 // (false, nil) - notification not supported (i.e. NOTIFY_SOCKET is unset)
 // (false, err) - notification supported, but failure happened (e.g. error connecting to NOTIFY_SOCKET or while sending data)
 // (true, nil) - notification supported, data has been sent
-func SdNotify(state string) (sent bool, err error) {
+func SdNotify(unsetEnvironment bool, state string) (sent bool, err error) {
 	socketAddr := &net.UnixAddr{
 		Name: os.Getenv("NOTIFY_SOCKET"),
 		Net:  "unixgram",
@@ -22,6 +40,13 @@ func SdNotify(state string) (sent bool, err error) {
 		return false, nil
 	}
 
+	if unsetEnvironment {
+		err = os.Unsetenv("NOTIFY_SOCKET")
+	}
+	if err != nil {
+		return false, err
+	}
+
 	conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr)
 	// Error connecting to NOTIFY_SOCKET
 	if err != nil {

cmd/vendor/github.com/coreos/go-systemd/daemon/watchdog.go

@@ -0,0 +1,72 @@
+// Copyright 2016 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package daemon
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+)
+
+// SdWatchdogEnabled return watchdog information for a service.
+// Process should send daemon.SdNotify("WATCHDOG=1") every time / 2.
+// If `unsetEnvironment` is true, the environment variables `WATCHDOG_USEC`
+// and `WATCHDOG_PID` will be unconditionally unset.
+//
+// It returns one of the following:
+// (0, nil) - watchdog isn't enabled or we aren't the watched PID.
+// (0, err) - an error happened (e.g. error converting time).
+// (time, nil) - watchdog is enabled and we can send ping.
+//   time is delay before inactive service will be killed.
+func SdWatchdogEnabled(unsetEnvironment bool) (time.Duration, error) {
+	wusec := os.Getenv("WATCHDOG_USEC")
+	wpid := os.Getenv("WATCHDOG_PID")
+	if unsetEnvironment {
+		wusecErr := os.Unsetenv("WATCHDOG_USEC")
+		wpidErr := os.Unsetenv("WATCHDOG_PID")
+		if wusecErr != nil {
+			return 0, wusecErr
+		}
+		if wpidErr != nil {
+			return 0, wpidErr
+		}
+	}
+
+	if wusec == "" {
+		return 0, nil
+	}
+	s, err := strconv.Atoi(wusec)
+	if err != nil {
+		return 0, fmt.Errorf("error converting WATCHDOG_USEC: %s", err)
+	}
+	if s <= 0 {
+		return 0, fmt.Errorf("error WATCHDOG_USEC must be a positive number")
+	}
+	interval := time.Duration(s) * time.Microsecond
+
+	if wpid == "" {
+		return interval, nil
+	}
+	p, err := strconv.Atoi(wpid)
+	if err != nil {
+		return 0, fmt.Errorf("error converting WATCHDOG_PID: %s", err)
+	}
+	if os.Getpid() != p {
+		return 0, nil
+	}
+
+	return interval, nil
+}

cmd/vendor/github.com/cpuguy83/go-md2man/md2man.go

@@ -1,44 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"os"
-
-	"github.com/cpuguy83/go-md2man/md2man"
-)
-
-var inFilePath = flag.String("in", "", "Path to file to be processed")
-var outFilePath = flag.String("out", "", "Path to output processed file")
-
-func main() {
-	flag.Parse()
-
-	inFile, err := os.Open(*inFilePath)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	defer inFile.Close()
-
-	doc, err := ioutil.ReadAll(inFile)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
-	out := md2man.Render(doc)
-
-	outFile, err := os.Create(*outFilePath)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	defer outFile.Close()
-	_, err = outFile.Write(out)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}

cmd/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go

@@ -11,6 +11,8 @@ import (
 
 type roffRenderer struct{}
 
+var listCounter int
+
 func RoffRenderer(flags int) blackfriday.Renderer {
 	return &roffRenderer{}
 }
@@ -33,8 +35,12 @@ func (r *roffRenderer) TitleBlock(out *bytes.Buffer, text []byte) {
 		line = append(line, []byte("\" ")...)
 		out.Write(line)
 	}
+	out.WriteString("\n")
 
-	out.WriteString(" \"\"\n")
+	// disable hyphenation
+	out.WriteString(".nh\n")
+	// disable justification (adjust text to left margin only)
+	out.WriteString(".ad l\n")
 }
 
 func (r *roffRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
@@ -80,23 +86,24 @@ func (r *roffRenderer) HRule(out *bytes.Buffer) {
 
 func (r *roffRenderer) List(out *bytes.Buffer, text func() bool, flags int) {
 	marker := out.Len()
-	out.WriteString(".IP ")
 	if flags&blackfriday.LIST_TYPE_ORDERED != 0 {
-		out.WriteString("\\(bu 2")
-	} else {
-		out.WriteString("\\n+[step" + string(flags) + "]")
+		listCounter = 1
 	}
-	out.WriteString("\n")
 	if !text() {
 		out.Truncate(marker)
 		return
 	}
-
 }
 
 func (r *roffRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
-	out.WriteString("\n\\item ")
+	if flags&blackfriday.LIST_TYPE_ORDERED != 0 {
+		out.WriteString(fmt.Sprintf(".IP \"%3d.\" 5\n", listCounter))
+		listCounter += 1
+	} else {
+		out.WriteString(".IP \\(bu 2\n")
+	}
 	out.Write(text)
+	out.WriteString("\n")
 }
 
 func (r *roffRenderer) Paragraph(out *bytes.Buffer, text func() bool) {
@@ -185,6 +192,7 @@ func (r *roffRenderer) LineBreak(out *bytes.Buffer) {
 }
 
 func (r *roffRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
+	out.Write(content)
 	r.AutoLink(out, link, 0)
 }
 
@@ -249,6 +257,11 @@ func needsBackslash(c byte) bool {
 
 func escapeSpecialChars(out *bytes.Buffer, text []byte) {
 	for i := 0; i < len(text); i++ {
+		// escape initial apostrophe or period
+		if len(text) >= 1 && (text[0] == '\'' || text[0] == '.') {
+			out.WriteString("\\&")
+		}
+
 		// directly copy normal characters
 		org := i
 

cmd/vendor/github.com/davecgh/go-spew/LICENSE

@@ -0,0 +1,15 @@
+ISC License
+
+Copyright (c) 2012-2013 Dave Collins <dave@davec.name>
+
+Permission to use, copy, modify, and distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

cmd/vendor/github.com/davecgh/go-spew/spew/bypass.go

@@ -0,0 +1,152 @@
+// Copyright (c) 2015 Dave Collins <dave@davec.name>
+//
+// Permission to use, copy, modify, and distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+// NOTE: Due to the following build constraints, this file will only be compiled
+// when the code is not running on Google App Engine, compiled by GopherJS, and
+// "-tags safe" is not added to the go build command line.  The "disableunsafe"
+// tag is deprecated and thus should not be used.
+// +build !js,!appengine,!safe,!disableunsafe
+
+package spew
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+const (
+	// UnsafeDisabled is a build-time constant which specifies whether or
+	// not access to the unsafe package is available.
+	UnsafeDisabled = false
+
+	// ptrSize is the size of a pointer on the current arch.
+	ptrSize = unsafe.Sizeof((*byte)(nil))
+)
+
+var (
+	// offsetPtr, offsetScalar, and offsetFlag are the offsets for the
+	// internal reflect.Value fields.  These values are valid before golang
+	// commit ecccf07e7f9d which changed the format.  The are also valid
+	// after commit 82f48826c6c7 which changed the format again to mirror
+	// the original format.  Code in the init function updates these offsets
+	// as necessary.
+	offsetPtr    = uintptr(ptrSize)
+	offsetScalar = uintptr(0)
+	offsetFlag   = uintptr(ptrSize * 2)
+
+	// flagKindWidth and flagKindShift indicate various bits that the
+	// reflect package uses internally to track kind information.
+	//
+	// flagRO indicates whether or not the value field of a reflect.Value is
+	// read-only.
+	//
+	// flagIndir indicates whether the value field of a reflect.Value is
+	// the actual data or a pointer to the data.
+	//
+	// These values are valid before golang commit 90a7c3c86944 which
+	// changed their positions.  Code in the init function updates these
+	// flags as necessary.
+	flagKindWidth = uintptr(5)
+	flagKindShift = uintptr(flagKindWidth - 1)
+	flagRO        = uintptr(1 << 0)
+	flagIndir     = uintptr(1 << 1)
+)
+
+func init() {
+	// Older versions of reflect.Value stored small integers directly in the
+	// ptr field (which is named val in the older versions).  Versions
+	// between commits ecccf07e7f9d and 82f48826c6c7 added a new field named
+	// scalar for this purpose which unfortunately came before the flag
+	// field, so the offset of the flag field is different for those
+	// versions.
+	//
+	// This code constructs a new reflect.Value from a known small integer
+	// and checks if the size of the reflect.Value struct indicates it has
+	// the scalar field. When it does, the offsets are updated accordingly.
+	vv := reflect.ValueOf(0xf00)
+	if unsafe.Sizeof(vv) == (ptrSize * 4) {
+		offsetScalar = ptrSize * 2
+		offsetFlag = ptrSize * 3
+	}
+
+	// Commit 90a7c3c86944 changed the flag positions such that the low
+	// order bits are the kind.  This code extracts the kind from the flags
+	// field and ensures it's the correct type.  When it's not, the flag
+	// order has been changed to the newer format, so the flags are updated
+	// accordingly.
+	upf := unsafe.Pointer(uintptr(unsafe.Pointer(&vv)) + offsetFlag)
+	upfv := *(*uintptr)(upf)
+	flagKindMask := uintptr((1<<flagKindWidth - 1) << flagKindShift)
+	if (upfv&flagKindMask)>>flagKindShift != uintptr(reflect.Int) {
+		flagKindShift = 0
+		flagRO = 1 << 5
+		flagIndir = 1 << 6
+
+		// Commit adf9b30e5594 modified the flags to separate the
+		// flagRO flag into two bits which specifies whether or not the
+		// field is embedded.  This causes flagIndir to move over a bit
+		// and means that flagRO is the combination of either of the
+		// original flagRO bit and the new bit.
+		//
+		// This code detects the change by extracting what used to be
+		// the indirect bit to ensure it's set.  When it's not, the flag
+		// order has been changed to the newer format, so the flags are
+		// updated accordingly.
+		if upfv&flagIndir == 0 {
+			flagRO = 3 << 5
+			flagIndir = 1 << 7
+		}
+	}
+}
+
+// unsafeReflectValue converts the passed reflect.Value into a one that bypasses
+// the typical safety restrictions preventing access to unaddressable and
+// unexported data.  It works by digging the raw pointer to the underlying
+// value out of the protected value and generating a new unprotected (unsafe)
+// reflect.Value to it.
+//
+// This allows us to check for implementations of the Stringer and error
+// interfaces to be used for pretty printing ordinarily unaddressable and
+// inaccessible values such as unexported struct fields.
+func unsafeReflectValue(v reflect.Value) (rv reflect.Value) {
+	indirects := 1
+	vt := v.Type()
+	upv := unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetPtr)
+	rvf := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetFlag))
+	if rvf&flagIndir != 0 {
+		vt = reflect.PtrTo(v.Type())
+		indirects++
+	} else if offsetScalar != 0 {
+		// The value is in the scalar field when it's not one of the
+		// reference types.
+		switch vt.Kind() {
+		case reflect.Uintptr:
+		case reflect.Chan:
+		case reflect.Func:
+		case reflect.Map:
+		case reflect.Ptr:
+		case reflect.UnsafePointer:
+		default:
+			upv = unsafe.Pointer(uintptr(unsafe.Pointer(&v)) +
+				offsetScalar)
+		}
+	}
+
+	pv := reflect.NewAt(vt, upv)
+	rv = pv
+	for i := 0; i < indirects; i++ {
+		rv = rv.Elem()
+	}
+	return rv
+}

cmd/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go

@@ -0,0 +1,38 @@
+// Copyright (c) 2015 Dave Collins <dave@davec.name>
+//
+// Permission to use, copy, modify, and distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+// NOTE: Due to the following build constraints, this file will only be compiled
+// when the code is running on Google App Engine, compiled by GopherJS, or
+// "-tags safe" is added to the go build command line.  The "disableunsafe"
+// tag is deprecated and thus should not be used.
+// +build js appengine safe disableunsafe
+
+package spew
+
+import "reflect"
+
+const (
+	// UnsafeDisabled is a build-time constant which specifies whether or
+	// not access to the unsafe package is available.
+	UnsafeDisabled = true
+)
+
+// unsafeReflectValue typically converts the passed reflect.Value into a one
+// that bypasses the typical safety restrictions preventing access to
+// unaddressable and unexported data.  However, doing this relies on access to
+// the unsafe package.  This is a stub version which simply returns the passed
+// reflect.Value when the unsafe package is not available.
+func unsafeReflectValue(v reflect.Value) reflect.Value {
+	return v
+}

cmd/vendor/github.com/davecgh/go-spew/spew/common.go

@@ -0,0 +1,341 @@
+/*
+ * Copyright (c) 2013 Dave Collins <dave@davec.name>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package spew
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"reflect"
+	"sort"
+	"strconv"
+)
+
+// Some constants in the form of bytes to avoid string overhead.  This mirrors
+// the technique used in the fmt package.
+var (
+	panicBytes            = []byte("(PANIC=")
+	plusBytes             = []byte("+")
+	iBytes                = []byte("i")
+	trueBytes             = []byte("true")
+	falseBytes            = []byte("false")
+	interfaceBytes        = []byte("(interface {})")
+	commaNewlineBytes     = []byte(",\n")
+	newlineBytes          = []byte("\n")
+	openBraceBytes        = []byte("{")
+	openBraceNewlineBytes = []byte("{\n")
+	closeBraceBytes       = []byte("}")
+	asteriskBytes         = []byte("*")
+	colonBytes            = []byte(":")
+	colonSpaceBytes       = []byte(": ")
+	openParenBytes        = []byte("(")
+	closeParenBytes       = []byte(")")
+	spaceBytes            = []byte(" ")
+	pointerChainBytes     = []byte("->")
+	nilAngleBytes         = []byte("<nil>")
+	maxNewlineBytes       = []byte("<max depth reached>\n")
+	maxShortBytes         = []byte("<max>")
+	circularBytes         = []byte("<already shown>")
+	circularShortBytes    = []byte("<shown>")
+	invalidAngleBytes     = []byte("<invalid>")
+	openBracketBytes      = []byte("[")
+	closeBracketBytes     = []byte("]")
+	percentBytes          = []byte("%")
+	precisionBytes        = []byte(".")
+	openAngleBytes        = []byte("<")
+	closeAngleBytes       = []byte(">")
+	openMapBytes          = []byte("map[")
+	closeMapBytes         = []byte("]")
+	lenEqualsBytes        = []byte("len=")
+	capEqualsBytes        = []byte("cap=")
+)
+
+// hexDigits is used to map a decimal value to a hex digit.
+var hexDigits = "0123456789abcdef"
+
+// catchPanic handles any panics that might occur during the handleMethods
+// calls.
+func catchPanic(w io.Writer, v reflect.Value) {
+	if err := recover(); err != nil {
+		w.Write(panicBytes)
+		fmt.Fprintf(w, "%v", err)
+		w.Write(closeParenBytes)
+	}
+}
+
+// handleMethods attempts to call the Error and String methods on the underlying
+// type the passed reflect.Value represents and outputes the result to Writer w.
+//
+// It handles panics in any called methods by catching and displaying the error
+// as the formatted value.
+func handleMethods(cs *ConfigState, w io.Writer, v reflect.Value) (handled bool) {
+	// We need an interface to check if the type implements the error or
+	// Stringer interface.  However, the reflect package won't give us an
+	// interface on certain things like unexported struct fields in order
+	// to enforce visibility rules.  We use unsafe, when it's available,
+	// to bypass these restrictions since this package does not mutate the
+	// values.
+	if !v.CanInterface() {
+		if UnsafeDisabled {
+			return false
+		}
+
+		v = unsafeReflectValue(v)
+	}
+
+	// Choose whether or not to do error and Stringer interface lookups against
+	// the base type or a pointer to the base type depending on settings.
+	// Technically calling one of these methods with a pointer receiver can
+	// mutate the value, however, types which choose to satisify an error or
+	// Stringer interface with a pointer receiver should not be mutating their
+	// state inside these interface methods.
+	if !cs.DisablePointerMethods && !UnsafeDisabled && !v.CanAddr() {
+		v = unsafeReflectValue(v)
+	}
+	if v.CanAddr() {
+		v = v.Addr()
+	}
+
+	// Is it an error or Stringer?
+	switch iface := v.Interface().(type) {
+	case error:
+		defer catchPanic(w, v)
+		if cs.ContinueOnMethod {
+			w.Write(openParenBytes)
+			w.Write([]byte(iface.Error()))
+			w.Write(closeParenBytes)
+			w.Write(spaceBytes)
+			return false
+		}
+
+		w.Write([]byte(iface.Error()))
+		return true
+
+	case fmt.Stringer:
+		defer catchPanic(w, v)
+		if cs.ContinueOnMethod {
+			w.Write(openParenBytes)
+			w.Write([]byte(iface.String()))
+			w.Write(closeParenBytes)
+			w.Write(spaceBytes)
+			return false
+		}
+		w.Write([]byte(iface.String()))
+		return true
+	}
+	return false
+}
+
+// printBool outputs a boolean value as true or false to Writer w.
+func printBool(w io.Writer, val bool) {
+	if val {
+		w.Write(trueBytes)
+	} else {
+		w.Write(falseBytes)
+	}
+}
+
+// printInt outputs a signed integer value to Writer w.
+func printInt(w io.Writer, val int64, base int) {
+	w.Write([]byte(strconv.FormatInt(val, base)))
+}
+
+// printUint outputs an unsigned integer value to Writer w.
+func printUint(w io.Writer, val uint64, base int) {
+	w.Write([]byte(strconv.FormatUint(val, base)))
+}
+
+// printFloat outputs a floating point value using the specified precision,
+// which is expected to be 32 or 64bit, to Writer w.
+func printFloat(w io.Writer, val float64, precision int) {
+	w.Write([]byte(strconv.FormatFloat(val, 'g', -1, precision)))
+}
+
+// printComplex outputs a complex value using the specified float precision
+// for the real and imaginary parts to Writer w.
+func printComplex(w io.Writer, c complex128, floatPrecision int) {
+	r := real(c)
+	w.Write(openParenBytes)
+	w.Write([]byte(strconv.FormatFloat(r, 'g', -1, floatPrecision)))
+	i := imag(c)
+	if i >= 0 {
+		w.Write(plusBytes)
+	}
+	w.Write([]byte(strconv.FormatFloat(i, 'g', -1, floatPrecision)))
+	w.Write(iBytes)
+	w.Write(closeParenBytes)
+}
+
+// printHexPtr outputs a uintptr formatted as hexidecimal with a leading '0x'
+// prefix to Writer w.
+func printHexPtr(w io.Writer, p uintptr) {
+	// Null pointer.
+	num := uint64(p)
+	if num == 0 {
+		w.Write(nilAngleBytes)
+		return
+	}
+
+	// Max uint64 is 16 bytes in hex + 2 bytes for '0x' prefix
+	buf := make([]byte, 18)
+
+	// It's simpler to construct the hex string right to left.
+	base := uint64(16)
+	i := len(buf) - 1
+	for num >= base {
+		buf[i] = hexDigits[num%base]
+		num /= base
+		i--
+	}
+	buf[i] = hexDigits[num]
+
+	// Add '0x' prefix.
+	i--
+	buf[i] = 'x'
+	i--
+	buf[i] = '0'
+
+	// Strip unused leading bytes.
+	buf = buf[i:]
+	w.Write(buf)
+}
+
+// valuesSorter implements sort.Interface to allow a slice of reflect.Value
+// elements to be sorted.
+type valuesSorter struct {
+	values  []reflect.Value
+	strings []string // either nil or same len and values
+	cs      *ConfigState
+}
+
+// newValuesSorter initializes a valuesSorter instance, which holds a set of
+// surrogate keys on which the data should be sorted.  It uses flags in
+// ConfigState to decide if and how to populate those surrogate keys.
+func newValuesSorter(values []reflect.Value, cs *ConfigState) sort.Interface {
+	vs := &valuesSorter{values: values, cs: cs}
+	if canSortSimply(vs.values[0].Kind()) {
+		return vs
+	}
+	if !cs.DisableMethods {
+		vs.strings = make([]string, len(values))
+		for i := range vs.values {
+			b := bytes.Buffer{}
+			if !handleMethods(cs, &b, vs.values[i]) {
+				vs.strings = nil
+				break
+			}
+			vs.strings[i] = b.String()
+		}
+	}
+	if vs.strings == nil && cs.SpewKeys {
+		vs.strings = make([]string, len(values))
+		for i := range vs.values {
+			vs.strings[i] = Sprintf("%#v", vs.values[i].Interface())
+		}
+	}
+	return vs
+}
+
+// canSortSimply tests whether a reflect.Kind is a primitive that can be sorted
+// directly, or whether it should be considered for sorting by surrogate keys
+// (if the ConfigState allows it).
+func canSortSimply(kind reflect.Kind) bool {
+	// This switch parallels valueSortLess, except for the default case.
+	switch kind {
+	case reflect.Bool:
+		return true
+	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
+		return true
+	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
+		return true
+	case reflect.Float32, reflect.Float64:
+		return true
+	case reflect.String:
+		return true
+	case reflect.Uintptr:
+		return true
+	case reflect.Array:
+		return true
+	}
+	return false
+}
+
+// Len returns the number of values in the slice.  It is part of the
+// sort.Interface implementation.
+func (s *valuesSorter) Len() int {
+	return len(s.values)
+}
+
+// Swap swaps the values at the passed indices.  It is part of the
+// sort.Interface implementation.
+func (s *valuesSorter) Swap(i, j int) {
+	s.values[i], s.values[j] = s.values[j], s.values[i]
+	if s.strings != nil {
+		s.strings[i], s.strings[j] = s.strings[j], s.strings[i]
+	}
+}
+
+// valueSortLess returns whether the first value should sort before the second
+// value.  It is used by valueSorter.Less as part of the sort.Interface
+// implementation.
+func valueSortLess(a, b reflect.Value) bool {
+	switch a.Kind() {
+	case reflect.Bool:
+		return !a.Bool() && b.Bool()
+	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
+		return a.Int() < b.Int()
+	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
+		return a.Uint() < b.Uint()
+	case reflect.Float32, reflect.Float64:
+		return a.Float() < b.Float()
+	case reflect.String:
+		return a.String() < b.String()
+	case reflect.Uintptr:
+		return a.Uint() < b.Uint()
+	case reflect.Array:
+		// Compare the contents of both arrays.
+		l := a.Len()
+		for i := 0; i < l; i++ {
+			av := a.Index(i)
+			bv := b.Index(i)
+			if av.Interface() == bv.Interface() {
+				continue
+			}
+			return valueSortLess(av, bv)
+		}
+	}
+	return a.String() < b.String()
+}
+
+// Less returns whether the value at index i should sort before the
+// value at index j.  It is part of the sort.Interface implementation.
+func (s *valuesSorter) Less(i, j int) bool {
+	if s.strings == nil {
+		return valueSortLess(s.values[i], s.values[j])
+	}
+	return s.strings[i] < s.strings[j]
+}
+
+// sortValues is a sort function that handles both native types and any type that
+// can be converted to error or Stringer.  Other inputs are sorted according to
+// their Value.String() value to ensure display stability.
+func sortValues(values []reflect.Value, cs *ConfigState) {
+	if len(values) == 0 {
+		return
+	}
+	sort.Sort(newValuesSorter(values, cs))
+}

cmd/vendor/github.com/davecgh/go-spew/spew/config.go

@@ -0,0 +1,297 @@
+/*
+ * Copyright (c) 2013 Dave Collins <dave@davec.name>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package spew
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+)
+
+// ConfigState houses the configuration options used by spew to format and
+// display values.  There is a global instance, Config, that is used to control
+// all top-level Formatter and Dump functionality.  Each ConfigState instance
+// provides methods equivalent to the top-level functions.
+//
+// The zero value for ConfigState provides no indentation.  You would typically
+// want to set it to a space or a tab.
+//
+// Alternatively, you can use NewDefaultConfig to get a ConfigState instance
+// with default settings.  See the documentation of NewDefaultConfig for default
+// values.
+type ConfigState struct {
+	// Indent specifies the string to use for each indentation level.  The
+	// global config instance that all top-level functions use set this to a
+	// single space by default.  If you would like more indentation, you might
+	// set this to a tab with "\t" or perhaps two spaces with "  ".
+	Indent string
+
+	// MaxDepth controls the maximum number of levels to descend into nested
+	// data structures.  The default, 0, means there is no limit.
+	//
+	// NOTE: Circular data structures are properly detected, so it is not
+	// necessary to set this value unless you specifically want to limit deeply
+	// nested data structures.
+	MaxDepth int
+
+	// DisableMethods specifies whether or not error and Stringer interfaces are
+	// invoked for types that implement them.
+	DisableMethods bool
+
+	// DisablePointerMethods specifies whether or not to check for and invoke
+	// error and Stringer interfaces on types which only accept a pointer
+	// receiver when the current type is not a pointer.
+	//
+	// NOTE: This might be an unsafe action since calling one of these methods
+	// with a pointer receiver could technically mutate the value, however,
+	// in practice, types which choose to satisify an error or Stringer
+	// interface with a pointer receiver should not be mutating their state
+	// inside these interface methods.  As a result, this option relies on
+	// access to the unsafe package, so it will not have any effect when
+	// running in environments without access to the unsafe package such as
+	// Google App Engine or with the "safe" build tag specified.
+	DisablePointerMethods bool
+
+	// ContinueOnMethod specifies whether or not recursion should continue once
+	// a custom error or Stringer interface is invoked.  The default, false,
+	// means it will print the results of invoking the custom error or Stringer
+	// interface and return immediately instead of continuing to recurse into
+	// the internals of the data type.
+	//
+	// NOTE: This flag does not have any effect if method invocation is disabled
+	// via the DisableMethods or DisablePointerMethods options.
+	ContinueOnMethod bool
+
+	// SortKeys specifies map keys should be sorted before being printed. Use
+	// this to have a more deterministic, diffable output.  Note that only
+	// native types (bool, int, uint, floats, uintptr and string) and types
+	// that support the error or Stringer interfaces (if methods are
+	// enabled) are supported, with other types sorted according to the
+	// reflect.Value.String() output which guarantees display stability.
+	SortKeys bool
+
+	// SpewKeys specifies that, as a last resort attempt, map keys should
+	// be spewed to strings and sorted by those strings.  This is only
+	// considered if SortKeys is true.
+	SpewKeys bool
+}
+
+// Config is the active configuration of the top-level functions.
+// The configuration can be changed by modifying the contents of spew.Config.
+var Config = ConfigState{Indent: " "}
+
+// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were
+// passed with a Formatter interface returned by c.NewFormatter.  It returns
+// the formatted string as a value that satisfies error.  See NewFormatter
+// for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Errorf(format, c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Errorf(format string, a ...interface{}) (err error) {
+	return fmt.Errorf(format, c.convertArgs(a)...)
+}
+
+// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were
+// passed with a Formatter interface returned by c.NewFormatter.  It returns
+// the number of bytes written and any write error encountered.  See
+// NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Fprint(w, c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Fprint(w io.Writer, a ...interface{}) (n int, err error) {
+	return fmt.Fprint(w, c.convertArgs(a)...)
+}
+
+// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were
+// passed with a Formatter interface returned by c.NewFormatter.  It returns
+// the number of bytes written and any write error encountered.  See
+// NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Fprintf(w, format, c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
+	return fmt.Fprintf(w, format, c.convertArgs(a)...)
+}
+
+// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it
+// passed with a Formatter interface returned by c.NewFormatter.  See
+// NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Fprintln(w, c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Fprintln(w io.Writer, a ...interface{}) (n int, err error) {
+	return fmt.Fprintln(w, c.convertArgs(a)...)
+}
+
+// Print is a wrapper for fmt.Print that treats each argument as if it were
+// passed with a Formatter interface returned by c.NewFormatter.  It returns
+// the number of bytes written and any write error encountered.  See
+// NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Print(c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Print(a ...interface{}) (n int, err error) {
+	return fmt.Print(c.convertArgs(a)...)
+}
+
+// Printf is a wrapper for fmt.Printf that treats each argument as if it were
+// passed with a Formatter interface returned by c.NewFormatter.  It returns
+// the number of bytes written and any write error encountered.  See
+// NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Printf(format, c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Printf(format string, a ...interface{}) (n int, err error) {
+	return fmt.Printf(format, c.convertArgs(a)...)
+}
+
+// Println is a wrapper for fmt.Println that treats each argument as if it were
+// passed with a Formatter interface returned by c.NewFormatter.  It returns
+// the number of bytes written and any write error encountered.  See
+// NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Println(c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Println(a ...interface{}) (n int, err error) {
+	return fmt.Println(c.convertArgs(a)...)
+}
+
+// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were
+// passed with a Formatter interface returned by c.NewFormatter.  It returns
+// the resulting string.  See NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Sprint(c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Sprint(a ...interface{}) string {
+	return fmt.Sprint(c.convertArgs(a)...)
+}
+
+// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were
+// passed with a Formatter interface returned by c.NewFormatter.  It returns
+// the resulting string.  See NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Sprintf(format, c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Sprintf(format string, a ...interface{}) string {
+	return fmt.Sprintf(format, c.convertArgs(a)...)
+}
+
+// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it
+// were passed with a Formatter interface returned by c.NewFormatter.  It
+// returns the resulting string.  See NewFormatter for formatting details.
+//
+// This function is shorthand for the following syntax:
+//
+//	fmt.Sprintln(c.NewFormatter(a), c.NewFormatter(b))
+func (c *ConfigState) Sprintln(a ...interface{}) string {
+	return fmt.Sprintln(c.convertArgs(a)...)
+}
+
+/*
+NewFormatter returns a custom formatter that satisfies the fmt.Formatter
+interface.  As a result, it integrates cleanly with standard fmt package
+printing functions.  The formatter is useful for inline printing of smaller data
+types similar to the standard %v format specifier.
+
+The custom formatter only responds to the %v (most compact), %+v (adds pointer
+addresses), %#v (adds types), and %#+v (adds types and pointer addresses) verb
+combinations.  Any other verbs such as %x and %q will be sent to the the
+standard fmt package for formatting.  In addition, the custom formatter ignores
+the width and precision arguments (however they will still work on the format
+specifiers not handled by the custom formatter).
+
+Typically this function shouldn't be called directly.  It is much easier to make
+use of the custom formatter by calling one of the convenience functions such as
+c.Printf, c.Println, or c.Printf.
+*/
+func (c *ConfigState) NewFormatter(v interface{}) fmt.Formatter {
+	return newFormatter(c, v)
+}
+
+// Fdump formats and displays the passed arguments to io.Writer w.  It formats
+// exactly the same as Dump.
+func (c *ConfigState) Fdump(w io.Writer, a ...interface{}) {
+	fdump(c, w, a...)
+}
+
+/*
+Dump displays the passed parameters to standard out with newlines, customizable
+indentation, and additional debug information such as complete types and all
+pointer addresses used to indirect to the final value.  It provides the
+following features over the built-in printing facilities provided by the fmt
+package:
+
+	* Pointers are dereferenced and followed
+	* Circular data structures are detected and handled properly
+	* Custom Stringer/error interfaces are optionally invoked, including
+	  on unexported types
+	* Custom types which only implement the Stringer/error interfaces via
+	  a pointer receiver are optionally invoked when passing non-pointer
+	  variables
+	* Byte arrays and slices are dumped like the hexdump -C command which
+	  includes offsets, byte values in hex, and ASCII output
+
+The configuration options are controlled by modifying the public members
+of c.  See ConfigState for options documentation.
+
+See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to
+get the formatted result as a string.
+*/
+func (c *ConfigState) Dump(a ...interface{}) {
+	fdump(c, os.Stdout, a...)
+}
+
+// Sdump returns a string with the passed arguments formatted exactly the same
+// as Dump.
+func (c *ConfigState) Sdump(a ...interface{}) string {
+	var buf bytes.Buffer
+	fdump(c, &buf, a...)
+	return buf.String()
+}
+
+// convertArgs accepts a slice of arguments and returns a slice of the same
+// length with each argument converted to a spew Formatter interface using
+// the ConfigState associated with s.
+func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{}) {
+	formatters = make([]interface{}, len(args))
+	for index, arg := range args {
+		formatters[index] = newFormatter(c, arg)
+	}
+	return formatters
+}
+
+// NewDefaultConfig returns a ConfigState with the following default settings.
+//
+// 	Indent: " "
+// 	MaxDepth: 0
+// 	DisableMethods: false
+// 	DisablePointerMethods: false
+// 	ContinueOnMethod: false
+// 	SortKeys: false
+func NewDefaultConfig() *ConfigState {
+	return &ConfigState{Indent: " "}
+}

cmd/vendor/github.com/davecgh/go-spew/spew/doc.go

@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2013 Dave Collins <dave@davec.name>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+Package spew implements a deep pretty printer for Go data structures to aid in
+debugging.
+
+A quick overview of the additional features spew provides over the built-in
+printing facilities for Go data types are as follows:
+
+	* Pointers are dereferenced and followed
+	* Circular data structures are detected and handled properly
+	* Custom Stringer/error interfaces are optionally invoked, including
+	  on unexported types
+	* Custom types which only implement the Stringer/error interfaces via
+	  a pointer receiver are optionally invoked when passing non-pointer
+	  variables
+	* Byte arrays and slices are dumped like the hexdump -C command which
+	  includes offsets, byte values in hex, and ASCII output (only when using
+	  Dump style)
+
+There are two different approaches spew allows for dumping Go data structures:
+
+	* Dump style which prints with newlines, customizable indentation,
+	  and additional debug information such as types and all pointer addresses
+	  used to indirect to the final value
+	* A custom Formatter interface that integrates cleanly with the standard fmt
+	  package and replaces %v, %+v, %#v, and %#+v to provide inline printing
+	  similar to the default %v while providing the additional functionality
+	  outlined above and passing unsupported format verbs such as %x and %q
+	  along to fmt
+
+Quick Start
+
+This section demonstrates how to quickly get started with spew.  See the
+sections below for further details on formatting and configuration options.
+
+To dump a variable with full newlines, indentation, type, and pointer
+information use Dump, Fdump, or Sdump:
+	spew.Dump(myVar1, myVar2, ...)
+	spew.Fdump(someWriter, myVar1, myVar2, ...)
+	str := spew.Sdump(myVar1, myVar2, ...)
+
+Alternatively, if you would prefer to use format strings with a compacted inline
+printing style, use the convenience wrappers Printf, Fprintf, etc with
+%v (most compact), %+v (adds pointer addresses), %#v (adds types), or
+%#+v (adds types and pointer addresses):
+	spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2)
+	spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
+	spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2)
+	spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
+
+Configuration Options
+
+Configuration of spew is handled by fields in the ConfigState type.  For
+convenience, all of the top-level functions use a global state available
+via the spew.Config global.
+
+It is also possible to create a ConfigState instance that provides methods
+equivalent to the top-level functions.  This allows concurrent configuration
+options.  See the ConfigState documentation for more details.
+
+The following configuration options are available:
+	* Indent
+		String to use for each indentation level for Dump functions.
+		It is a single space by default.  A popular alternative is "\t".
+
+	* MaxDepth
+		Maximum number of levels to descend into nested data structures.
+		There is no limit by default.
+
+	* DisableMethods
+		Disables invocation of error and Stringer interface methods.
+		Method invocation is enabled by default.
+
+	* DisablePointerMethods
+		Disables invocation of error and Stringer interface methods on types
+		which only accept pointer receivers from non-pointer variables.
+		Pointer method invocation is enabled by default.
+
+	* ContinueOnMethod
+		Enables recursion into types after invoking error and Stringer interface
+		methods. Recursion after method invocation is disabled by default.
+
+	* SortKeys
+		Specifies map keys should be sorted before being printed. Use
+		this to have a more deterministic, diffable output.  Note that
+		only native types (bool, int, uint, floats, uintptr and string)
+		and types which implement error or Stringer interfaces are
+		supported with other types sorted according to the
+		reflect.Value.String() output which guarantees display
+		stability.  Natural map order is used by default.
+
+	* SpewKeys
+		Specifies that, as a last resort attempt, map keys should be
+		spewed to strings and sorted by those strings.  This is only
+		considered if SortKeys is true.
+
+Dump Usage
+
+Simply call spew.Dump with a list of variables you want to dump:
+
+	spew.Dump(myVar1, myVar2, ...)
+
+You may also call spew.Fdump if you would prefer to output to an arbitrary
+io.Writer.  For example, to dump to standard error:
+
+	spew.Fdump(os.Stderr, myVar1, myVar2, ...)
+
+A third option is to call spew.Sdump to get the formatted output as a string:
+
+	str := spew.Sdump(myVar1, myVar2, ...)
+
+Sample Dump Output
+
+See the Dump example for details on the setup of the types and variables being
+shown here.
+
+	(main.Foo) {
+	 unexportedField: (*main.Bar)(0xf84002e210)({
+	  flag: (main.Flag) flagTwo,
+	  data: (uintptr) <nil>
+	 }),
+	 ExportedField: (map[interface {}]interface {}) (len=1) {
+	  (string) (len=3) "one": (bool) true
+	 }
+	}
+
+Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C
+command as shown.
+	([]uint8) (len=32 cap=32) {
+	 00000000  11 12 13 14 15 16 17 18  19 1a 1b 1c 1d 1e 1f 20  |............... |
+	 00000010  21 22 23 24 25 26 27 28  29 2a 2b 2c 2d 2e 2f 30  |!"#$%&'()*+,-./0|
+	 00000020  31 32                                             |12|
+	}
+
+Custom Formatter
+
+Spew provides a custom formatter that implements the fmt.Formatter interface
+so that it integrates cleanly with standard fmt package printing functions. The
+formatter is useful for inline printing of smaller data types similar to the
+standard %v format specifier.
+
+The custom formatter only responds to the %v (most compact), %+v (adds pointer
+addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb
+combinations.  Any other verbs such as %x and %q will be sent to the the
+standard fmt package for formatting.  In addition, the custom formatter ignores
+the width and precision arguments (however they will still work on the format
+specifiers not handled by the custom formatter).
+
+Custom Formatter Usage
+
+The simplest way to make use of the spew custom formatter is to call one of the
+convenience functions such as spew.Printf, spew.Println, or spew.Printf.  The
+functions have syntax you are most likely already familiar with:
+
+	spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2)
+	spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
+	spew.Println(myVar, myVar2)
+	spew.Fprintf(os.Stderr, "myVar1: %v -- myVar2: %+v", myVar1, myVar2)
+	spew.Fprintf(os.Stderr, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
+
+See the Index for the full list convenience functions.
+
+Sample Formatter Output
+
+Double pointer to a uint8:
+	  %v: <**>5
+	 %+v: <**>(0xf8400420d0->0xf8400420c8)5
+	 %#v: (**uint8)5
+	%#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5
+
+Pointer to circular struct with a uint8 field and a pointer to itself:
+	  %v: <*>{1 <*><shown>}
+	 %+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)<shown>}
+	 %#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)<shown>}
+	%#+v: (*main.circular)(0xf84003e260){ui8:(uint8)1 c:(*main.circular)(0xf84003e260)<shown>}
+
+See the Printf example for details on the setup of variables being shown
+here.
+
+Errors
+
+Since it is possible for custom Stringer/error interfaces to panic, spew
+detects them and handles them internally by printing the panic information
+inline with the output.  Since spew is intended to provide deep pretty printing
+capabilities on structures, it intentionally does not return any errors.
+*/
+package spew

cmd/vendor/github.com/davecgh/go-spew/spew/dump.go

@@ -0,0 +1,509 @@
+/*
+ * Copyright (c) 2013 Dave Collins <dave@davec.name>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package spew
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	// uint8Type is a reflect.Type representing a uint8.  It is used to
+	// convert cgo types to uint8 slices for hexdumping.
+	uint8Type = reflect.TypeOf(uint8(0))
+
+	// cCharRE is a regular expression that matches a cgo char.
+	// It is used to detect character arrays to hexdump them.
+	cCharRE = regexp.MustCompile("^.*\\._Ctype_char$")
+
+	// cUnsignedCharRE is a regular expression that matches a cgo unsigned
+	// char.  It is used to detect unsigned character arrays to hexdump
+	// them.
+	cUnsignedCharRE = regexp.MustCompile("^.*\\._Ctype_unsignedchar$")
+
+	// cUint8tCharRE is a regular expression that matches a cgo uint8_t.
+	// It is used to detect uint8_t arrays to hexdump them.
+	cUint8tCharRE = regexp.MustCompile("^.*\\._Ctype_uint8_t$")
+)
+
+// dumpState contains information about the state of a dump operation.
+type dumpState struct {
+	w                io.Writer
+	depth            int
+	pointers         map[uintptr]int
+	ignoreNextType   bool
+	ignoreNextIndent bool
+	cs               *ConfigState
+}
+
+// indent performs indentation according to the depth level and cs.Indent
+// option.
+func (d *dumpState) indent() {
+	if d.ignoreNextIndent {
+		d.ignoreNextIndent = false
+		return
+	}
+	d.w.Write(bytes.Repeat([]byte(d.cs.Indent), d.depth))
+}
+
+// unpackValue returns values inside of non-nil interfaces when possible.
+// This is useful for data types like structs, arrays, slices, and maps which
+// can contain varying types packed inside an interface.
+func (d *dumpState) unpackValue(v reflect.Value) reflect.Value {
+	if v.Kind() == reflect.Interface && !v.IsNil() {
+		v = v.Elem()
+	}
+	return v
+}
+
+// dumpPtr handles formatting of pointers by indirecting them as necessary.
+func (d *dumpState) dumpPtr(v reflect.Value) {
+	// Remove pointers at or below the current depth from map used to detect
+	// circular refs.
+	for k, depth := range d.pointers {
+		if depth >= d.depth {
+			delete(d.pointers, k)
+		}
+	}
+
+	// Keep list of all dereferenced pointers to show later.
+	pointerChain := make([]uintptr, 0)
+
+	// Figure out how many levels of indirection there are by dereferencing
+	// pointers and unpacking interfaces down the chain while detecting circular
+	// references.
+	nilFound := false
+	cycleFound := false
+	indirects := 0
+	ve := v
+	for ve.Kind() == reflect.Ptr {
+		if ve.IsNil() {
+			nilFound = true
+			break
+		}
+		indirects++
+		addr := ve.Pointer()
+		pointerChain = append(pointerChain, addr)
+		if pd, ok := d.pointers[addr]; ok && pd < d.depth {
+			cycleFound = true
+			indirects--
+			break
+		}
+		d.pointers[addr] = d.depth
+
+		ve = ve.Elem()
+		if ve.Kind() == reflect.Interface {
+			if ve.IsNil() {
+				nilFound = true
+				break
+			}
+			ve = ve.Elem()
+		}
+	}
+
+	// Display type information.
+	d.w.Write(openParenBytes)
+	d.w.Write(bytes.Repeat(asteriskBytes, indirects))
+	d.w.Write([]byte(ve.Type().String()))
+	d.w.Write(closeParenBytes)
+
+	// Display pointer information.
+	if len(pointerChain) > 0 {
+		d.w.Write(openParenBytes)
+		for i, addr := range pointerChain {
+			if i > 0 {
+				d.w.Write(pointerChainBytes)
+			}
+			printHexPtr(d.w, addr)
+		}
+		d.w.Write(closeParenBytes)
+	}
+
+	// Display dereferenced value.
+	d.w.Write(openParenBytes)
+	switch {
+	case nilFound == true:
+		d.w.Write(nilAngleBytes)
+
+	case cycleFound == true:
+		d.w.Write(circularBytes)
+
+	default:
+		d.ignoreNextType = true
+		d.dump(ve)
+	}
+	d.w.Write(closeParenBytes)
+}
+
+// dumpSlice handles formatting of arrays and slices.  Byte (uint8 under
+// reflection) arrays and slices are dumped in hexdump -C fashion.
+func (d *dumpState) dumpSlice(v reflect.Value) {
+	// Determine whether this type should be hex dumped or not.  Also,
+	// for types which should be hexdumped, try to use the underlying data
+	// first, then fall back to trying to convert them to a uint8 slice.
+	var buf []uint8
+	doConvert := false
+	doHexDump := false
+	numEntries := v.Len()
+	if numEntries > 0 {
+		vt := v.Index(0).Type()
+		vts := vt.String()
+		switch {
+		// C types that need to be converted.
+		case cCharRE.MatchString(vts):
+			fallthrough
+		case cUnsignedCharRE.MatchString(vts):
+			fallthrough
+		case cUint8tCharRE.MatchString(vts):
+			doConvert = true
+
+		// Try to use existing uint8 slices and fall back to converting
+		// and copying if that fails.
+		case vt.Kind() == reflect.Uint8:
+			// We need an addressable interface to convert the type
+			// to a byte slice.  However, the reflect package won't
+			// give us an interface on certain things like
+			// unexported struct fields in order to enforce
+			// visibility rules.  We use unsafe, when available, to
+			// bypass these restrictions since this package does not
+			// mutate the values.
+			vs := v
+			if !vs.CanInterface() || !vs.CanAddr() {
+				vs = unsafeReflectValue(vs)
+			}
+			if !UnsafeDisabled {
+				vs = vs.Slice(0, numEntries)
+
+				// Use the existing uint8 slice if it can be
+				// type asserted.
+				iface := vs.Interface()
+				if slice, ok := iface.([]uint8); ok {
+					buf = slice
+					doHexDump = true
+					break
+				}
+			}
+
+			// The underlying data needs to be converted if it can't
+			// be type asserted to a uint8 slice.
+			doConvert = true
+		}
+
+		// Copy and convert the underlying type if needed.
+		if doConvert && vt.ConvertibleTo(uint8Type) {
+			// Convert and copy each element into a uint8 byte
+			// slice.
+			buf = make([]uint8, numEntries)
+			for i := 0; i < numEntries; i++ {
+				vv := v.Index(i)
+				buf[i] = uint8(vv.Convert(uint8Type).Uint())
+			}
+			doHexDump = true
+		}
+	}
+
+	// Hexdump the entire slice as needed.
+	if doHexDump {
+		indent := strings.Repeat(d.cs.Indent, d.depth)
+		str := indent + hex.Dump(buf)
+		str = strings.Replace(str, "\n", "\n"+indent, -1)
+		str = strings.TrimRight(str, d.cs.Indent)
+		d.w.Write([]byte(str))
+		return
+	}
+
+	// Recursively call dump for each item.
+	for i := 0; i < numEntries; i++ {
+		d.dump(d.unpackValue(v.Index(i)))
+		if i < (numEntries - 1) {
+			d.w.Write(commaNewlineBytes)
+		} else {
+			d.w.Write(newlineBytes)
+		}
+	}
+}
+
+// dump is the main workhorse for dumping a value.  It uses the passed reflect
+// value to figure out what kind of object we are dealing with and formats it
+// appropriately.  It is a recursive function, however circular data structures
+// are detected and handled properly.
+func (d *dumpState) dump(v reflect.Value) {
+	// Handle invalid reflect values immediately.
+	kind := v.Kind()
+	if kind == reflect.Invalid {
+		d.w.Write(invalidAngleBytes)
+		return
+	}
+
+	// Handle pointers specially.
+	if kind == reflect.Ptr {
+		d.indent()
+		d.dumpPtr(v)
+		return
+	}
+
+	// Print type information unless already handled elsewhere.
+	if !d.ignoreNextType {
+		d.indent()
+		d.w.Write(openParenBytes)
+		d.w.Write([]byte(v.Type().String()))
+		d.w.Write(closeParenBytes)
+		d.w.Write(spaceBytes)
+	}
+	d.ignoreNextType = false
+
+	// Display length and capacity if the built-in len and cap functions
+	// work with the value's kind and the len/cap itself is non-zero.
+	valueLen, valueCap := 0, 0
+	switch v.Kind() {
+	case reflect.Array, reflect.Slice, reflect.Chan:
+		valueLen, valueCap = v.Len(), v.Cap()
+	case reflect.Map, reflect.String:
+		valueLen = v.Len()
+	}
+	if valueLen != 0 || valueCap != 0 {
+		d.w.Write(openParenBytes)
+		if valueLen != 0 {
+			d.w.Write(lenEqualsBytes)
+			printInt(d.w, int64(valueLen), 10)
+		}
+		if valueCap != 0 {
+			if valueLen != 0 {
+				d.w.Write(spaceBytes)
+			}
+			d.w.Write(capEqualsBytes)
+			printInt(d.w, int64(valueCap), 10)
+		}
+		d.w.Write(closeParenBytes)
+		d.w.Write(spaceBytes)
+	}
+
+	// Call Stringer/error interfaces if they exist and the handle methods flag
+	// is enabled
+	if !d.cs.DisableMethods {
+		if (kind != reflect.Invalid) && (kind != reflect.Interface) {
+			if handled := handleMethods(d.cs, d.w, v); handled {
+				return
+			}
+		}
+	}
+
+	switch kind {
+	case reflect.Invalid:
+		// Do nothing.  We should never get here since invalid has already
+		// been handled above.
+
+	case reflect.Bool:
+		printBool(d.w, v.Bool())
+
+	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
+		printInt(d.w, v.Int(), 10)
+
+	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
+		printUint(d.w, v.Uint(), 10)
+
+	case reflect.Float32:
+		printFloat(d.w, v.Float(), 32)
+
+	case reflect.Float64:
+		printFloat(d.w, v.Float(), 64)
+
+	case reflect.Complex64:
+		printComplex(d.w, v.Complex(), 32)
+
+	case reflect.Complex128:
+		printComplex(d.w, v.Complex(), 64)
+
+	case reflect.Slice:
+		if v.IsNil() {
+			d.w.Write(nilAngleBytes)
+			break
+		}
+		fallthrough
+
+	case reflect.Array:
+		d.w.Write(openBraceNewlineBytes)
+		d.depth++
+		if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
+			d.indent()
+			d.w.Write(maxNewlineBytes)
+		} else {
+			d.dumpSlice(v)
+		}
+		d.depth--
+		d.indent()
+		d.w.Write(closeBraceBytes)
+
+	case reflect.String:
+		d.w.Write([]byte(strconv.Quote(v.String())))
+
+	case reflect.Interface:
+		// The only time we should get here is for nil interfaces due to
+		// unpackValue calls.
+		if v.IsNil() {
+			d.w.Write(nilAngleBytes)
+		}
+
+	case reflect.Ptr:
+		// Do nothing.  We should never get here since pointers have already
+		// been handled above.
+
+	case reflect.Map:
+		// nil maps should be indicated as different than empty maps
+		if v.IsNil() {
+			d.w.Write(nilAngleBytes)
+			break
+		}
+
+		d.w.Write(openBraceNewlineBytes)
+		d.depth++
+		if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
+			d.indent()
+			d.w.Write(maxNewlineBytes)
+		} else {
+			numEntries := v.Len()
+			keys := v.MapKeys()
+			if d.cs.SortKeys {
+				sortValues(keys, d.cs)
+			}
+			for i, key := range keys {
+				d.dump(d.unpackValue(key))
+				d.w.Write(colonSpaceBytes)
+				d.ignoreNextIndent = true
+				d.dump(d.unpackValue(v.MapIndex(key)))
+				if i < (numEntries - 1) {
+					d.w.Write(commaNewlineBytes)
+				} else {
+					d.w.Write(newlineBytes)
+				}
+			}
+		}
+		d.depth--
+		d.indent()
+		d.w.Write(closeBraceBytes)
+
+	case reflect.Struct:
+		d.w.Write(openBraceNewlineBytes)
+		d.depth++
+		if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) {
+			d.indent()
+			d.w.Write(maxNewlineBytes)
+		} else {
+			vt := v.Type()
+			numFields := v.NumField()
+			for i := 0; i < numFields; i++ {
+				d.indent()
+				vtf := vt.Field(i)
+				d.w.Write([]byte(vtf.Name))
+				d.w.Write(colonSpaceBytes)
+				d.ignoreNextIndent = true
+				d.dump(d.unpackValue(v.Field(i)))
+				if i < (numFields - 1) {
+					d.w.Write(commaNewlineBytes)
+				} else {
+					d.w.Write(newlineBytes)
+				}
+			}
+		}
+		d.depth--
+		d.indent()
+		d.w.Write(closeBraceBytes)
+
+	case reflect.Uintptr:
+		printHexPtr(d.w, uintptr(v.Uint()))
+
+	case reflect.UnsafePointer, reflect.Chan, reflect.Func:
+		printHexPtr(d.w, v.Pointer())
+
+	// There were not any other types at the time this code was written, but
+	// fall back to letting the default fmt package handle it in case any new
+	// types are added.
+	default:
+		if v.CanInterface() {
+			fmt.Fprintf(d.w, "%v", v.Interface())
+		} else {
+			fmt.Fprintf(d.w, "%v", v.String())
+		}
+	}
+}
+
+// fdump is a helper function to consolidate the logic from the various public
+// methods which take varying writers and config states.
+func fdump(cs *ConfigState, w io.Writer, a ...interface{}) {
+	for _, arg := range a {
+		if arg == nil {
+			w.Write(interfaceBytes)
+			w.Write(spaceBytes)
+			w.Write(nilAngleBytes)
+			w.Write(newlineBytes)
+			continue
+		}
+
+		d := dumpState{w: w, cs: cs}
+		d.pointers = make(map[uintptr]int)
+		d.dump(reflect.ValueOf(arg))
+		d.w.Write(newlineBytes)
+	}
+}
+
+// Fdump formats and displays the passed arguments to io.Writer w.  It formats
+// exactly the same as Dump.
+func Fdump(w io.Writer, a ...interface{}) {
+	fdump(&Config, w, a...)
+}
+
+// Sdump returns a string with the passed arguments formatted exactly the same
+// as Dump.
+func Sdump(a ...interface{}) string {
+	var buf bytes.Buffer
+	fdump(&Config, &buf, a...)
+	return buf.String()
+}
+
+/*
+Dump displays the passed parameters to standard out with newlines, customizable
+indentation, and additional debug information such as complete types and all
+pointer addresses used to indirect to the final value.  It provides the
+following features over the built-in printing facilities provided by the fmt
+package:
+
+	* Pointers are dereferenced and followed
+	* Circular data structures are detected and handled properly
+	* Custom Stringer/error interfaces are optionally invoked, including
+	  on unexported types
+	* Custom types which only implement the Stringer/error interfaces via
+	  a pointer receiver are optionally invoked when passing non-pointer
+	  variables
+	* Byte arrays and slices are dumped like the hexdump -C command which
+	  includes offsets, byte values in hex, and ASCII output
+
+The configuration options are controlled by an exported package global,
+spew.Config.  See ConfigState for options documentation.
+
+See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to
+get the formatted result as a string.
+*/
+func Dump(a ...interface{}) {
+	fdump(&Config, os.Stdout, a...)
+}

cmd/vendor/github.com/davecgh/go-spew/spew/format.go

@@ -0,0 +1,419 @@
+/*
+ * Copyright (c) 2013 Dave Collins <dave@davec.name>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+package spew
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+// supportedFlags is a list of all the character flags supported by fmt package.
+const supportedFlags = "0-+# "
+
+// formatState implements the fmt.Formatter interface and contains information
+// about the state of a formatting operation.  The NewFormatter function can
+// be used to get a new Formatter which can be used directly as arguments
+// in standard fmt package printing calls.
+type formatState struct {
+	value          interface{}
+	fs             fmt.State
+	depth          int
+	pointers       map[uintptr]int
+	ignoreNextType bool
+	cs             *ConfigState
+}
+
+// buildDefaultFormat recreates the original format string without precision
+// and width information to pass in to fmt.Sprintf in the case of an
+// unrecognized type.  Unless new types are added to the language, this
+// function won't ever be called.
+func (f *formatState) buildDefaultFormat() (format string) {
+	buf := bytes.NewBuffer(percentBytes)
+
+	for _, flag := range supportedFlags {
+		if f.fs.Flag(int(flag)) {
+			buf.WriteRune(flag)
+		}
+	}
+
+	buf.WriteRune('v')
+
+	format = buf.String()
+	return format
+}
+
+// constructOrigFormat recreates the original format string including precision
+// and width information to pass along to the standard fmt package.  This allows
+// automatic deferral of all format strings this package doesn't support.
+func (f *formatState) constructOrigFormat(verb rune) (format string) {
+	buf := bytes.NewBuffer(percentBytes)
+
+	for _, flag := range supportedFlags {
+		if f.fs.Flag(int(flag)) {
+			buf.WriteRune(flag)
+		}
+	}
+
+	if width, ok := f.fs.Width(); ok {
+		buf.WriteString(strconv.Itoa(width))
+	}
+
+	if precision, ok := f.fs.Precision(); ok {
+		buf.Write(precisionBytes)
+		buf.WriteString(strconv.Itoa(precision))
+	}
+
+	buf.WriteRune(verb)
+
+	format = buf.String()
+	return format
+}
+
+// unpackValue returns values inside of non-nil interfaces when possible and
+// ensures that types for values which have been unpacked from an interface
+// are displayed when the show types flag is also set.
+// This is useful for data types like structs, arrays, slices, and maps which
+// can contain varying types packed inside an interface.
+func (f *formatState) unpackValue(v reflect.Value) reflect.Value {
+	if v.Kind() == reflect.Interface {
+		f.ignoreNextType = false
+		if !v.IsNil() {
+			v = v.Elem()
+		}
+	}
+	return v
+}
+
+// formatPtr handles formatting of pointers by indirecting them as necessary.
+func (f *formatState) formatPtr(v reflect.Value) {
+	// Display nil if top level pointer is nil.
+	showTypes := f.fs.Flag('#')
+	if v.IsNil() && (!showTypes || f.ignoreNextType) {
+		f.fs.Write(nilAngleBytes)
+		return
+	}
+
+	// Remove pointers at or below the current depth from map used to detect
+	// circular refs.
+	for k, depth := range f.pointers {
+		if depth >= f.depth {
+			delete(f.pointers, k)
+		}
+	}
+
+	// Keep list of all dereferenced pointers to possibly show later.
+	pointerChain := make([]uintptr, 0)
+
+	// Figure out how many levels of indirection there are by derferencing
+	// pointers and unpacking interfaces down the chain while detecting circular
+	// references.
+	nilFound := false
+	cycleFound := false
+	indirects := 0
+	ve := v
+	for ve.Kind() == reflect.Ptr {
+		if ve.IsNil() {
+			nilFound = true
+			break
+		}
+		indirects++
+		addr := ve.Pointer()
+		pointerChain = append(pointerChain, addr)
+		if pd, ok := f.pointers[addr]; ok && pd < f.depth {
+			cycleFound = true
+			indirects--
+			break
+		}
+		f.pointers[addr] = f.depth
+
+		ve = ve.Elem()
+		if ve.Kind() == reflect.Interface {
+			if ve.IsNil() {
+				nilFound = true
+				break
+			}
+			ve = ve.Elem()
+		}
+	}
+
+	// Display type or indirection level depending on flags.
+	if showTypes && !f.ignoreNextType {
+		f.fs.Write(openParenBytes)
+		f.fs.Write(bytes.Repeat(asteriskBytes, indirects))
+		f.fs.Write([]byte(ve.Type().String()))
+		f.fs.Write(closeParenBytes)
+	} else {
+		if nilFound || cycleFound {
+			indirects += strings.Count(ve.Type().String(), "*")
+		}
+		f.fs.Write(openAngleBytes)
+		f.fs.Write([]byte(strings.Repeat("*", indirects)))
+		f.fs.Write(closeAngleBytes)
+	}
+
+	// Display pointer information depending on flags.
+	if f.fs.Flag('+') && (len(pointerChain) > 0) {
+		f.fs.Write(openParenBytes)
+		for i, addr := range pointerChain {
+			if i > 0 {
+				f.fs.Write(pointerChainBytes)
+			}
+			printHexPtr(f.fs, addr)
+		}
+		f.fs.Write(closeParenBytes)
+	}
+
+	// Display dereferenced value.
+	switch {
+	case nilFound == true:
+		f.fs.Write(nilAngleBytes)
+
+	case cycleFound == true:
+		f.fs.Write(circularShortBytes)
+
+	default:
+		f.ignoreNextType = true
+		f.format(ve)
+	}
+}
+
+// format is the main workhorse for providing the Formatter interface.  It
+// uses the passed reflect value to figure out what kind of object we are
+// dealing with and formats it appropriately.  It is a recursive function,
+// however circular data structures are detected and handled properly.
+func (f *formatState) format(v reflect.Value) {
+	// Handle invalid reflect values immediately.
+	kind := v.Kind()
+	if kind == reflect.Invalid {
+		f.fs.Write(invalidAngleBytes)
+		return
+	}
+
+	// Handle pointers specially.
+	if kind == reflect.Ptr {
+		f.formatPtr(v)
+		return
+	}
+
+	// Print type information unless already handled elsewhere.
+	if !f.ignoreNextType && f.fs.Flag('#') {
+		f.fs.Write(openParenBytes)
+		f.fs.Write([]byte(v.Type().String()))
+		f.fs.Write(closeParenBytes)
+	}
+	f.ignoreNextType = false
+
+	// Call Stringer/error interfaces if they exist and the handle methods
+	// flag is enabled.
+	if !f.cs.DisableMethods {
+		if (kind != reflect.Invalid) && (kind != reflect.Interface) {
+			if handled := handleMethods(f.cs, f.fs, v); handled {
+				return
+			}
+		}
+	}
+
+	switch kind {
+	case reflect.Invalid:
+		// Do nothing.  We should never get here since invalid has already
+		// been handled above.
+
+	case reflect.Bool:
+		printBool(f.fs, v.Bool())
+
+	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
+		printInt(f.fs, v.Int(), 10)
+
+	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
+		printUint(f.fs, v.Uint(), 10)
+
+	case reflect.Float32:
+		printFloat(f.fs, v.Float(), 32)
+
+	case reflect.Float64:
+		printFloat(f.fs, v.Float(), 64)
+
+	case reflect.Complex64:
+		printComplex(f.fs, v.Complex(), 32)
+
+	case reflect.Complex128:
+		printComplex(f.fs, v.Complex(), 64)
+
+	case reflect.Slice:
+		if v.IsNil() {
+			f.fs.Write(nilAngleBytes)
+			break
+		}
+		fallthrough
+
+	case reflect.Array:
+		f.fs.Write(openBracketBytes)
+		f.depth++
+		if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
+			f.fs.Write(maxShortBytes)
+		} else {
+			numEntries := v.Len()
+			for i := 0; i < numEntries; i++ {
+				if i > 0 {
+					f.fs.Write(spaceBytes)
+				}
+				f.ignoreNextType = true
+				f.format(f.unpackValue(v.Index(i)))
+			}
+		}
+		f.depth--
+		f.fs.Write(closeBracketBytes)
+
+	case reflect.String:
+		f.fs.Write([]byte(v.String()))
+
+	case reflect.Interface:
+		// The only time we should get here is for nil interfaces due to
+		// unpackValue calls.
+		if v.IsNil() {
+			f.fs.Write(nilAngleBytes)
+		}
+
+	case reflect.Ptr:
+		// Do nothing.  We should never get here since pointers have already
+		// been handled above.
+
+	case reflect.Map:
+		// nil maps should be indicated as different than empty maps
+		if v.IsNil() {
+			f.fs.Write(nilAngleBytes)
+			break
+		}
+
+		f.fs.Write(openMapBytes)
+		f.depth++
+		if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
+			f.fs.Write(maxShortBytes)
+		} else {
+			keys := v.MapKeys()
+			if f.cs.SortKeys {
+				sortValues(keys, f.cs)
+			}
+			for i, key := range keys {
+				if i > 0 {
+					f.fs.Write(spaceBytes)
+				}
+				f.ignoreNextType = true
+				f.format(f.unpackValue(key))
+				f.fs.Write(colonBytes)
+				f.ignoreNextType = true
+				f.format(f.unpackValue(v.MapIndex(key)))
+			}
+		}
+		f.depth--
+		f.fs.Write(closeMapBytes)
+
+	case reflect.Struct:
+		numFields := v.NumField()
+		f.fs.Write(openBraceBytes)
+		f.depth++
+		if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) {
+			f.fs.Write(maxShortBytes)
+		} else {
+			vt := v.Type()
+			for i := 0; i < numFields; i++ {
+				if i > 0 {
+					f.fs.Write(spaceBytes)
+				}
+				vtf := vt.Field(i)
+				if f.fs.Flag('+') || f.fs.Flag('#') {
+					f.fs.Write([]byte(vtf.Name))
+					f.fs.Write(colonBytes)
+				}
+				f.format(f.unpackValue(v.Field(i)))
+			}
+		}
+		f.depth--
+		f.fs.Write(closeBraceBytes)
+
+	case reflect.Uintptr:
+		printHexPtr(f.fs, uintptr(v.Uint()))
+
+	case reflect.UnsafePointer, reflect.Chan, reflect.Func:
+		printHexPtr(f.fs, v.Pointer())
+
+	// There were not any other types at the time this code was written, but
+	// fall back to letting the default fmt package handle it if any get added.
+	default:
+		format := f.buildDefaultFormat()
+		if v.CanInterface() {
+			fmt.Fprintf(f.fs, format, v.Interface())
+		} else {
+			fmt.Fprintf(f.fs, format, v.String())
+		}
+	}
+}
+
+// Format satisfies the fmt.Formatter interface. See NewFormatter for usage
+// details.
+func (f *formatState) Format(fs fmt.State, verb rune) {
+	f.fs = fs
+
+	// Use standard formatting for verbs that are not v.
+	if verb != 'v' {
+		format := f.constructOrigFormat(verb)
+		fmt.Fprintf(fs, format, f.value)
+		return
+	}
+
+	if f.value == nil {
+		if fs.Flag('#') {
+			fs.Write(interfaceBytes)
+		}
+		fs.Write(nilAngleBytes)
+		return
+	}
+
+	f.format(reflect.ValueOf(f.value))
+}
+
+// newFormatter is a helper function to consolidate the logic from the various
+// public methods which take varying config states.
+func newFormatter(cs *ConfigState, v interface{}) fmt.Formatter {
+	fs := &formatState{value: v, cs: cs}
+	fs.pointers = make(map[uintptr]int)
+	return fs
+}
+
+/*
+NewFormatter returns a custom formatter that satisfies the fmt.Formatter
+interface.  As a result, it integrates cleanly with standard fmt package
+printing functions.  The formatter is useful for inline printing of smaller data
+types similar to the standard %v format specifier.
+
+The custom formatter only responds to the %v (most compact), %+v (adds pointer
+addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb
+combinations.  Any other verbs such as %x and %q will be sent to the the
+standard fmt package for formatting.  In addition, the custom formatter ignores
+the width and precision arguments (however they will still work on the format
+specifiers not handled by the custom formatter).
+
+Typically this function shouldn't be called directly.  It is much easier to make
+use of the custom formatter by calling one of the convenience functions such as
+Printf, Println, or Fprintf.
+*/
+func NewFormatter(v interface{}) fmt.Formatter {
+	return newFormatter(&Config, v)
+}