From 9d0de611a7a0d546deb8708c10f66ec3f8855fbf Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Wed, 30 Oct 2013 17:36:15 -0700 Subject: [PATCH] feat add snapCount parameter --- Documentation/external-documentation.md | 104 +++ config.go | 143 ---- etcd.go | 237 ++---- server/config.go | 406 ++++++++++ server/config_test.go | 479 +++++++++++ server/info.go | 19 + server/join_command.go | 10 +- server/peer_server.go | 84 +- server/peer_server_handlers.go | 43 +- server/registry.go | 5 +- server/server.go | 11 +- server/tls_config.go | 1 + server/tls_info.go | 69 ++ server/transporter.go | 14 +- server/transporter_test.go | 2 + server/util.go | 13 + server/v1/delete_key_handler.go | 3 +- server/v1/set_key_handler.go | 23 +- server/v2/delete_handler.go | 8 +- server/v2/post_handler.go | 8 +- server/v2/put_handler.go | 26 +- server/v2/tests/delete_handler_test.go | 29 + server/v2/tests/get_handler_test.go | 14 +- server/version.go | 5 - store/command_factory.go | 58 ++ store/node.go | 2 +- store/store.go | 31 +- store/v2/command_factory.go | 73 ++ store/{ => v2}/compare_and_swap_command.go | 5 +- store/{ => v2}/create_command.go | 10 +- store/{ => v2}/delete_command.go | 5 +- store/{ => v2}/set_command.go | 10 +- store/{ => v2}/update_command.go | 5 +- test.sh | 3 + tests/functional/version_check_test.go | 46 ++ tests/http_utils.go | 8 + tests/server_utils.go | 17 +- third_party/deps | 1 + .../github.com/BurntSushi/toml/.gitignore | 4 + .../github.com/BurntSushi/toml/COMPATIBLE | 3 + .../github.com/BurntSushi/toml/COPYING | 14 + .../github.com/BurntSushi/toml/Makefile | 14 + .../github.com/BurntSushi/toml/README.md | 163 ++++ .../BurntSushi/toml/_examples/example.go | 59 ++ .../BurntSushi/toml/_examples/example.toml | 35 + .../BurntSushi/toml/_examples/hard.toml | 22 + .../BurntSushi/toml/_examples/implicit.toml | 4 + .../toml/_examples/invalid-apples.toml | 6 + .../BurntSushi/toml/_examples/invalid.toml | 35 + .../BurntSushi/toml/_examples/readme1.toml | 5 + .../BurntSushi/toml/_examples/readme2.toml | 1 + .../github.com/BurntSushi/toml/decode.go | 429 ++++++++++ .../github.com/BurntSushi/toml/decode_test.go | 343 ++++++++ third_party/github.com/BurntSushi/toml/doc.go | 10 + .../github.com/BurntSushi/toml/encode.go | 99 +++ .../github.com/BurntSushi/toml/encode_test.go | 25 + third_party/github.com/BurntSushi/toml/lex.go | 741 ++++++++++++++++++ .../github.com/BurntSushi/toml/lex_test.go | 59 ++ .../github.com/BurntSushi/toml/out_test.go | 19 + .../github.com/BurntSushi/toml/parse.go | 388 +++++++++ .../github.com/BurntSushi/toml/parse_test.go | 61 ++ .../github.com/BurntSushi/toml/session.vim | 1 + .../BurntSushi/toml/toml-test-go/COPYING | 14 + .../BurntSushi/toml/toml-test-go/README.md | 14 + .../BurntSushi/toml/toml-test-go/main.go | 89 +++ .../github.com/BurntSushi/toml/tomlv/COPYING | 14 + .../BurntSushi/toml/tomlv/README.md | 22 + .../github.com/BurntSushi/toml/tomlv/main.go | 60 ++ .../github.com/BurntSushi/toml/type-check.go | 78 ++ util.go | 88 --- 70 files changed, 4413 insertions(+), 536 deletions(-) create mode 100644 Documentation/external-documentation.md delete mode 100644 config.go create mode 100644 server/config.go create mode 100644 server/config_test.go create mode 100644 server/info.go create mode 100644 server/v2/tests/delete_handler_test.go create mode 100644 store/command_factory.go create mode 100644 store/v2/command_factory.go rename store/{ => v2}/compare_and_swap_command.go (91%) rename store/{ => v2}/create_command.go (88%) rename store/{ => v2}/delete_command.go (87%) rename store/{ => v2}/set_command.go (88%) rename store/{ => v2}/update_command.go (88%) create mode 100644 tests/functional/version_check_test.go create mode 100644 third_party/github.com/BurntSushi/toml/.gitignore create mode 100644 third_party/github.com/BurntSushi/toml/COMPATIBLE create mode 100644 third_party/github.com/BurntSushi/toml/COPYING create mode 100644 third_party/github.com/BurntSushi/toml/Makefile create mode 100644 third_party/github.com/BurntSushi/toml/README.md create mode 100644 third_party/github.com/BurntSushi/toml/_examples/example.go create mode 100644 third_party/github.com/BurntSushi/toml/_examples/example.toml create mode 100644 third_party/github.com/BurntSushi/toml/_examples/hard.toml create mode 100644 third_party/github.com/BurntSushi/toml/_examples/implicit.toml create mode 100644 third_party/github.com/BurntSushi/toml/_examples/invalid-apples.toml create mode 100644 third_party/github.com/BurntSushi/toml/_examples/invalid.toml create mode 100644 third_party/github.com/BurntSushi/toml/_examples/readme1.toml create mode 100644 third_party/github.com/BurntSushi/toml/_examples/readme2.toml create mode 100644 third_party/github.com/BurntSushi/toml/decode.go create mode 100644 third_party/github.com/BurntSushi/toml/decode_test.go create mode 100644 third_party/github.com/BurntSushi/toml/doc.go create mode 100644 third_party/github.com/BurntSushi/toml/encode.go create mode 100644 third_party/github.com/BurntSushi/toml/encode_test.go create mode 100644 third_party/github.com/BurntSushi/toml/lex.go create mode 100644 third_party/github.com/BurntSushi/toml/lex_test.go create mode 100644 third_party/github.com/BurntSushi/toml/out_test.go create mode 100644 third_party/github.com/BurntSushi/toml/parse.go create mode 100644 third_party/github.com/BurntSushi/toml/parse_test.go create mode 100644 third_party/github.com/BurntSushi/toml/session.vim create mode 100644 third_party/github.com/BurntSushi/toml/toml-test-go/COPYING create mode 100644 third_party/github.com/BurntSushi/toml/toml-test-go/README.md create mode 100644 third_party/github.com/BurntSushi/toml/toml-test-go/main.go create mode 100644 third_party/github.com/BurntSushi/toml/tomlv/COPYING create mode 100644 third_party/github.com/BurntSushi/toml/tomlv/README.md create mode 100644 third_party/github.com/BurntSushi/toml/tomlv/main.go create mode 100644 third_party/github.com/BurntSushi/toml/type-check.go delete mode 100644 util.go diff --git a/Documentation/external-documentation.md b/Documentation/external-documentation.md new file mode 100644 index 000000000..a7d957aef --- /dev/null +++ b/Documentation/external-documentation.md @@ -0,0 +1,104 @@ +# Etcd Configuration + +Configuration options can be set in three places: + + 1. Command line flags + 2. Environment variables + 3. Configuration file + +Options set on the command line take precedence over all other sources. +Options set in environment variables take precedence over options set in +configuration files. + +## Command Line Flags + +### Required + +* `-n` - The node name. Defaults to `default-name`. + +### Optional + +* `-c` - The advertised public hostname:port for client communication. Defaults to `127.0.0.1:4001`. +* `-cl` - The listening hostname for client communication. Defaults to advertised ip. +* `-C` - A comma separated list of machines in the cluster (i.e `"203.0.113.101:7001,203.0.113.102:7001"`). +* `-CF` - The file path containing a comma separated list of machines in the cluster. +* `-clientCAFile` - The path of the client CAFile. Enables client cert authentication when present. +* `-clientCert` - The cert file of the client. +* `-clientKey` - The key file of the client. +* `-configfile` - The path of the etcd config file. Defaults to `/etc/etcd/etcd.conf`. +* `-cors` - A comma separated white list of origins for cross-origin resource sharing. +* `-cpuprofile` - The path to a file to output cpu profile data. Enables cpu profiling when present. +* `-d` - The directory to store log and snapshot. Defaults to the current working directory. +* `-m` - The max size of result buffer. Defaults to `1024`. +* `-maxsize` - The max size of the cluster. Defaults to `9`. +* `-r` - The max retry attempts when trying to join a cluster. Defaults to `3`. +* `-s` - The advertised public hostname:port for server communication. Defaults to `127.0.0.1:7001`. +* `-sl` - The listening hostname for server communication. Defaults to advertised ip. +* `-serverCAFile` - The path of the CAFile. Enables client/peer cert authentication when present. +* `-serverCert` - The cert file of the server. +* `-serverKey` - The key file of the server. +* `-snapshot` - Open or close snapshot. Defaults to `false`. +* `-v` - Enable verbose logging. Defaults to `false`. +* `-vv` - Enable very verbose logging. Defaults to `false`. +* `-version` - Print the version and exit. +* `-w` - The hostname:port of web interface. + +## Configuration File + +The etcd configuration file is written in [TOML](https://github.com/mojombo/toml) +and read from `/etc/etcd/etcd.conf` by default. + +```TOML +advertised_url = "127.0.0.1:4001" +ca_file = "" +cert_file = "" +cors = [] +cpu_profile_file = "" +datadir = "." +key_file = "" +listen_host = "127.0.0.1:4001" +machines = [] +machines_file = "" +max_cluster_size = 9 +max_result_buffer = 1024 +max_retry_attempts = 3 +name = "default-name" +snapshot = false +verbose = false +very_verbose = false +web_url = "" + +[peer] +advertised_url = "127.0.0.1:7001" +ca_file = "" +cert_file = "" +key_file = "" +listen_host = "127.0.0.1:7001" +``` + +## Environment Variables + + * `ETCD_ADVERTISED_URL` + * `ETCD_CA_FILE` + * `ETCD_CERT_FILE` + * `ETCD_CORS` + * `ETCD_CONFIG_FILE` + * `ETCD_CPU_PROFILE_FILE` + * `ETCD_DATADIR` + * `ETCD_KEY_FILE` + * `ETCD_LISTEN_HOST` + * `ETCD_MACHINES` + * `ETCD_MACHINES_FILE` + * `ETCD_MAX_RETRY_ATTEMPTS` + * `ETCD_MAX_CLUSTER_SIZE` + * `ETCD_MAX_RESULT_BUFFER` + * `ETCD_NAME` + * `ETCD_SNAPSHOT` + * `ETCD_VERBOSE` + * `ETCD_VERY_VERBOSE` + * `ETCD_WEB_URL` + * `ETCD_PEER_ADVERTISED_URL` + * `ETCD_PEER_CA_FILE` + * `ETCD_PEER_CERT_FILE` + * `ETCD_PEER_KEY_FILE` + * `ETCD_PEER_LISTEN_HOST` diff --git a/config.go b/config.go deleted file mode 100644 index d1549c7be..000000000 --- a/config.go +++ /dev/null @@ -1,143 +0,0 @@ -package main - -import ( - "crypto/tls" - "crypto/x509" - "encoding/json" - "encoding/pem" - "io/ioutil" - "os" - "path/filepath" - - "github.com/coreos/etcd/log" - "github.com/coreos/etcd/server" -) - -//-------------------------------------- -// Config -//-------------------------------------- - -// Get the server info from previous conf file -// or from the user -func getInfo(path string) *Info { - - infoPath := filepath.Join(path, "info") - - if force { - // Delete the old configuration if exist - logPath := filepath.Join(path, "log") - confPath := filepath.Join(path, "conf") - snapshotPath := filepath.Join(path, "snapshot") - os.Remove(infoPath) - os.Remove(logPath) - os.Remove(confPath) - os.RemoveAll(snapshotPath) - } else if info := readInfo(infoPath); info != nil { - log.Infof("Found node configuration in '%s'. Ignoring flags", infoPath) - return info - } - - // Read info from command line - info := &argInfo - - // Write to file. - content, _ := json.MarshalIndent(info, "", " ") - content = []byte(string(content) + "\n") - if err := ioutil.WriteFile(infoPath, content, 0644); err != nil { - log.Fatalf("Unable to write info to file: %v", err) - } - - log.Infof("Wrote node configuration to '%s'", infoPath) - - return info -} - -// readInfo reads from info file and decode to Info struct -func readInfo(path string) *Info { - file, err := os.Open(path) - - if err != nil { - if os.IsNotExist(err) { - return nil - } - log.Fatal(err) - } - defer file.Close() - - info := &Info{} - - content, err := ioutil.ReadAll(file) - if err != nil { - log.Fatalf("Unable to read info: %v", err) - return nil - } - - if err = json.Unmarshal(content, &info); err != nil { - log.Fatalf("Unable to parse info: %v", err) - return nil - } - - return info -} - -func tlsConfigFromInfo(info server.TLSInfo) (t server.TLSConfig, ok bool) { - var keyFile, certFile, CAFile string - var tlsCert tls.Certificate - var err error - - t.Scheme = "http" - - keyFile = info.KeyFile - certFile = info.CertFile - CAFile = info.CAFile - - // If the user do not specify key file, cert file and - // CA file, the type will be HTTP - if keyFile == "" && certFile == "" && CAFile == "" { - return t, true - } - - // both the key and cert must be present - if keyFile == "" || certFile == "" { - return t, false - } - - tlsCert, err = tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - log.Fatal(err) - } - - t.Scheme = "https" - t.Server.ClientAuth, t.Server.ClientCAs = newCertPool(CAFile) - - // The client should trust the RootCA that the Server uses since - // everyone is a peer in the network. - t.Client.Certificates = []tls.Certificate{tlsCert} - t.Client.RootCAs = t.Server.ClientCAs - - return t, true -} - -// newCertPool creates x509 certPool and corresponding Auth Type. -// If the given CAfile is valid, add the cert into the pool and verify the clients' -// certs against the cert in the pool. -// If the given CAfile is empty, do not verify the clients' cert. -// If the given CAfile is not valid, fatal. -func newCertPool(CAFile string) (tls.ClientAuthType, *x509.CertPool) { - if CAFile == "" { - return tls.NoClientCert, nil - } - pemByte, err := ioutil.ReadFile(CAFile) - check(err) - - block, pemByte := pem.Decode(pemByte) - - cert, err := x509.ParseCertificate(block.Bytes) - check(err) - - certPool := x509.NewCertPool() - - certPool.AddCert(cert) - - return tls.RequireAndVerifyClientCert, certPool -} diff --git a/etcd.go b/etcd.go index f9df25c6d..ec79e519d 100644 --- a/etcd.go +++ b/etcd.go @@ -5,7 +5,8 @@ import ( "fmt" "io/ioutil" "os" - "strings" + "os/signal" + "runtime/pprof" "github.com/coreos/etcd/log" "github.com/coreos/etcd/server" @@ -13,189 +14,103 @@ import ( "github.com/coreos/go-raft" ) -//------------------------------------------------------------------------------ -// -// Initialization -// -//------------------------------------------------------------------------------ - -var ( - veryVerbose bool - - machines string - machinesFile string - - cluster []string - - argInfo Info - dirPath string - - force bool - - printVersion bool - - maxSize int - - snapshot bool - - retryTimes int - - maxClusterSize int - - cpuprofile string - - cors string -) - -func init() { - flag.BoolVar(&printVersion, "version", false, "print the version and exit") - - flag.BoolVar(&log.Verbose, "v", false, "verbose logging") - flag.BoolVar(&veryVerbose, "vv", false, "very verbose logging") - - flag.StringVar(&machines, "C", "", "the ip address and port of a existing machines in the cluster, sepearate by comma") - flag.StringVar(&machinesFile, "CF", "", "the file contains a list of existing machines in the cluster, seperate by comma") - - flag.StringVar(&argInfo.Name, "n", "default-name", "the node name (required)") - flag.StringVar(&argInfo.EtcdURL, "c", "127.0.0.1:4001", "the advertised public hostname:port for etcd client communication") - flag.StringVar(&argInfo.RaftURL, "s", "127.0.0.1:7001", "the advertised public hostname:port for raft server communication") - flag.StringVar(&argInfo.EtcdListenHost, "cl", "", "the listening hostname for etcd client communication (defaults to advertised ip)") - flag.StringVar(&argInfo.RaftListenHost, "sl", "", "the listening hostname for raft server communication (defaults to advertised ip)") - flag.StringVar(&argInfo.WebURL, "w", "", "the hostname:port of web interface") - - flag.StringVar(&argInfo.RaftTLS.CAFile, "serverCAFile", "", "the path of the CAFile") - flag.StringVar(&argInfo.RaftTLS.CertFile, "serverCert", "", "the cert file of the server") - flag.StringVar(&argInfo.RaftTLS.KeyFile, "serverKey", "", "the key file of the server") - - flag.StringVar(&argInfo.EtcdTLS.CAFile, "clientCAFile", "", "the path of the client CAFile") - flag.StringVar(&argInfo.EtcdTLS.CertFile, "clientCert", "", "the cert file of the client") - flag.StringVar(&argInfo.EtcdTLS.KeyFile, "clientKey", "", "the key file of the client") - - flag.StringVar(&dirPath, "d", ".", "the directory to store log and snapshot") - - flag.BoolVar(&force, "f", false, "force new node configuration if existing is found (WARNING: data loss!)") - - flag.BoolVar(&snapshot, "snapshot", false, "open or close snapshot") - - flag.IntVar(&maxSize, "m", 1024, "the max size of result buffer") - - flag.IntVar(&retryTimes, "r", 3, "the max retry attempts when trying to join a cluster") - - flag.IntVar(&maxClusterSize, "maxsize", 9, "the max size of the cluster") - - flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file") - - flag.StringVar(&cors, "cors", "", "whitelist origins for cross-origin resource sharing (e.g. '*' or 'http://localhost:8001,etc')") -} - -//------------------------------------------------------------------------------ -// -// Typedefs -// -//------------------------------------------------------------------------------ - -type Info struct { - Name string `json:"name"` - - RaftURL string `json:"raftURL"` - EtcdURL string `json:"etcdURL"` - WebURL string `json:"webURL"` - - RaftListenHost string `json:"raftListenHost"` - EtcdListenHost string `json:"etcdListenHost"` - - RaftTLS server.TLSInfo `json:"raftTLS"` - EtcdTLS server.TLSInfo `json:"etcdTLS"` -} - -//------------------------------------------------------------------------------ -// -// Functions -// -//------------------------------------------------------------------------------ - -//-------------------------------------- -// Main -//-------------------------------------- - func main() { - flag.Parse() + parseFlags() - if printVersion { - fmt.Println(server.ReleaseVersion) - os.Exit(0) + // Load configuration. + var config = server.NewConfig() + if err := config.Load(os.Args[1:]); err != nil { + log.Fatal("Configuration error:", err) } - if cpuprofile != "" { - runCPUProfile() - } - - if veryVerbose { + // Turn on logging. + if config.VeryVerbose { log.Verbose = true raft.SetLogLevel(raft.Debug) + } else if config.Verbose { + log.Verbose = true } - if machines != "" { - cluster = strings.Split(machines, ",") - } else if machinesFile != "" { - b, err := ioutil.ReadFile(machinesFile) - if err != nil { - log.Fatalf("Unable to read the given machines file: %s", err) - } - cluster = strings.Split(string(b), ",") - } - - // Check TLS arguments - raftTLSConfig, ok := tlsConfigFromInfo(argInfo.RaftTLS) - if !ok { - log.Fatal("Please specify cert and key file or cert and key file and CAFile or none of the three") - } - - etcdTLSConfig, ok := tlsConfigFromInfo(argInfo.EtcdTLS) - if !ok { - log.Fatal("Please specify cert and key file or cert and key file and CAFile or none of the three") - } - - argInfo.Name = strings.TrimSpace(argInfo.Name) - if argInfo.Name == "" { - log.Fatal("ERROR: server name required. e.g. '-n=server_name'") - } - - // Check host name arguments - argInfo.RaftURL = sanitizeURL(argInfo.RaftURL, raftTLSConfig.Scheme) - argInfo.EtcdURL = sanitizeURL(argInfo.EtcdURL, etcdTLSConfig.Scheme) - argInfo.WebURL = sanitizeURL(argInfo.WebURL, "http") - - argInfo.RaftListenHost = sanitizeListenHost(argInfo.RaftListenHost, argInfo.RaftURL) - argInfo.EtcdListenHost = sanitizeListenHost(argInfo.EtcdListenHost, argInfo.EtcdURL) - - // Read server info from file or grab it from user. - if err := os.MkdirAll(dirPath, 0744); err != nil { + // Create data directory if it doesn't already exist. + if err := os.MkdirAll(config.DataDir, 0744); err != nil { log.Fatalf("Unable to create path: %s", err) } - info := getInfo(dirPath) + // Load info object. + info, err := config.Info() + if err != nil { + log.Fatal("info:", err) + } + if info.Name == "" { + log.Fatal("ERROR: server name required. e.g. '-n=server_name'") + } - // Create etcd key-value store + // Retrieve TLS configuration. + tlsConfig, err := info.EtcdTLS.Config() + if err != nil { + log.Fatal("Client TLS:", err) + } + peerTLSConfig, err := info.RaftTLS.Config() + if err != nil { + log.Fatal("Peer TLS:", err) + } + + // Create etcd key-value store and registry. store := store.New() - - // Create a shared node registry. registry := server.NewRegistry(store) // Create peer server. - ps := server.NewPeerServer(info.Name, dirPath, info.RaftURL, info.RaftListenHost, &raftTLSConfig, &info.RaftTLS, registry, store) - ps.MaxClusterSize = maxClusterSize - ps.RetryTimes = retryTimes + ps := server.NewPeerServer(info.Name, config.DataDir, info.RaftURL, info.RaftListenHost, &peerTLSConfig, &info.RaftTLS, registry, store, config.SnapCount) + ps.MaxClusterSize = config.MaxClusterSize + ps.RetryTimes = config.MaxRetryAttempts - s := server.New(info.Name, info.EtcdURL, info.EtcdListenHost, &etcdTLSConfig, &info.EtcdTLS, ps, registry, store) - if err := s.AllowOrigins(cors); err != nil { + // Create client server. + s := server.New(info.Name, info.EtcdURL, info.EtcdListenHost, &tlsConfig, &info.EtcdTLS, ps, registry, store) + if err := s.AllowOrigins(config.Cors); err != nil { panic(err) } ps.SetServer(s) + // Run peer server in separate thread while the client server blocks. go func() { - log.Fatal(ps.ListenAndServe(snapshot, cluster)) + log.Fatal(ps.ListenAndServe(config.Snapshot, config.Machines)) }() log.Fatal(s.ListenAndServe()) } + +// Parses non-configuration flags. +func parseFlags() { + var versionFlag bool + var cpuprofile string + + f := flag.NewFlagSet(os.Args[0], -1) + f.SetOutput(ioutil.Discard) + f.BoolVar(&versionFlag, "version", false, "print the version and exit") + f.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file") + f.Parse(os.Args[1:]) + + // Print version if necessary. + if versionFlag { + fmt.Println(server.ReleaseVersion) + os.Exit(0) + } + + // Begin CPU profiling if specified. + if cpuprofile != "" { + f, err := os.Create(cpuprofile) + if err != nil { + log.Fatal(err) + } + pprof.StartCPUProfile(f) + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + sig := <-c + log.Infof("captured %v, stopping profiler and exiting..", sig) + pprof.StopCPUProfile() + os.Exit(1) + }() + } +} diff --git a/server/config.go b/server/config.go new file mode 100644 index 000000000..95ec59737 --- /dev/null +++ b/server/config.go @@ -0,0 +1,406 @@ +package server + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net" + "net/url" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + + "github.com/BurntSushi/toml" +) + +// The default location for the etcd configuration file. +const DefaultSystemConfigPath = "/etc/etcd/etcd.conf" + +// Config represents the server configuration. +type Config struct { + SystemPath string + + AdvertisedUrl string `toml:"advertised_url" env:"ETCD_ADVERTISED_URL"` + CAFile string `toml:"ca_file" env:"ETCD_CA_FILE"` + CertFile string `toml:"cert_file" env:"ETCD_CERT_FILE"` + Cors []string `toml:"cors" env:"ETCD_CORS"` + DataDir string `toml:"datadir" env:"ETCD_DATADIR"` + KeyFile string `toml:"key_file" env:"ETCD_KEY_FILE"` + ListenHost string `toml:"listen_host" env:"ETCD_LISTEN_HOST"` + Machines []string `toml:"machines" env:"ETCD_MACHINES"` + MachinesFile string `toml:"machines_file" env:"ETCD_MACHINES_FILE"` + MaxClusterSize int `toml:"max_cluster_size" env:"ETCD_MAX_CLUSTER_SIZE"` + MaxResultBuffer int `toml:"max_result_buffer" env:"ETCD_MAX_RESULT_BUFFER"` + MaxRetryAttempts int `toml:"max_retry_attempts" env:"ETCD_MAX_RETRY_ATTEMPTS"` + Name string `toml:"name" env:"ETCD_NAME"` + Snapshot bool `toml:"snapshot" env:"ETCD_SNAPSHOT"` + SnapCount int `toml:"snap_count" env:"ETCD_SNAPCOUNT"` + Verbose bool `toml:"verbose" env:"ETCD_VERBOSE"` + VeryVerbose bool `toml:"very_verbose" env:"ETCD_VERY_VERBOSE"` + WebURL string `toml:"web_url" env:"ETCD_WEB_URL"` + + Peer struct { + AdvertisedUrl string `toml:"advertised_url" env:"ETCD_PEER_ADVERTISED_URL"` + CAFile string `toml:"ca_file" env:"ETCD_PEER_CA_FILE"` + CertFile string `toml:"cert_file" env:"ETCD_PEER_CERT_FILE"` + KeyFile string `toml:"key_file" env:"ETCD_PEER_KEY_FILE"` + ListenHost string `toml:"listen_host" env:"ETCD_PEER_LISTEN_HOST"` + } +} + +// NewConfig returns a Config initialized with default values. +func NewConfig() *Config { + c := new(Config) + c.SystemPath = DefaultSystemConfigPath + c.AdvertisedUrl = "127.0.0.1:4001" + c.AdvertisedUrl = "127.0.0.1:4001" + c.DataDir = "." + c.MaxClusterSize = 9 + c.MaxResultBuffer = 1024 + c.MaxRetryAttempts = 3 + c.Peer.AdvertisedUrl = "127.0.0.1:7001" + c.SnapCount = 10000 + return c +} + +// Loads the configuration from the system config, command line config, +// environment variables, and finally command line arguments. +func (c *Config) Load(arguments []string) error { + var path string + f := flag.NewFlagSet("etcd", -1) + f.SetOutput(ioutil.Discard) + f.StringVar(&path, "config", "", "path to config file") + f.Parse(arguments) + + // Load from system file. + if err := c.LoadSystemFile(); err != nil { + return err + } + + // Load from config file specified in arguments. + if path != "" { + if err := c.LoadFile(path); err != nil { + return err + } + } + + // Load from the environment variables next. + if err := c.LoadEnv(); err != nil { + return err + } + + // Load from command line flags. + if err := c.LoadFlags(arguments); err != nil { + return err + } + + // Loads machines if a machine file was specified. + if err := c.LoadMachineFile(); err != nil { + return err + } + + // Sanitize all the input fields. + if err := c.Sanitize(); err != nil { + return fmt.Errorf("sanitize:", err) + } + + return nil +} + +// Loads from the system etcd configuration file if it exists. +func (c *Config) LoadSystemFile() error { + if _, err := os.Stat(c.SystemPath); os.IsNotExist(err) { + return nil + } + return c.LoadFile(c.SystemPath) +} + +// Loads configuration from a file. +func (c *Config) LoadFile(path string) error { + _, err := toml.DecodeFile(path, &c) + return err +} + +// LoadEnv loads the configuration via environment variables. +func (c *Config) LoadEnv() error { + if err := c.loadEnv(c); err != nil { + return err + } + if err := c.loadEnv(&c.Peer); err != nil { + return err + } + return nil +} + +func (c *Config) loadEnv(target interface{}) error { + value := reflect.Indirect(reflect.ValueOf(target)) + typ := value.Type() + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + + // Retrieve environment variable. + v := strings.TrimSpace(os.Getenv(field.Tag.Get("env"))) + if v == "" { + continue + } + + // Set the appropriate type. + switch field.Type.Kind() { + case reflect.Bool: + value.Field(i).SetBool(v != "0" && v != "false") + case reflect.Int: + newValue, err := strconv.ParseInt(v, 10, 0) + if err != nil { + return fmt.Errorf("Parse error: %s: %s", field.Tag.Get("env"), err) + } + value.Field(i).SetInt(newValue) + case reflect.String: + value.Field(i).SetString(v) + case reflect.Slice: + value.Field(i).Set(reflect.ValueOf(trimsplit(v, ","))) + } + } + return nil +} + +// Loads configuration from command line flags. +func (c *Config) LoadFlags(arguments []string) error { + var machines, cors string + var force bool + + f := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + + f.BoolVar(&force, "f", false, "force new node configuration if existing is found (WARNING: data loss!)") + + f.BoolVar(&c.Verbose, "v", c.Verbose, "verbose logging") + f.BoolVar(&c.VeryVerbose, "vv", c.Verbose, "very verbose logging") + + f.StringVar(&machines, "C", "", "the ip address and port of a existing machines in the cluster, sepearate by comma") + f.StringVar(&c.MachinesFile, "CF", c.MachinesFile, "the file contains a list of existing machines in the cluster, seperate by comma") + + f.StringVar(&c.Name, "n", c.Name, "the node name (required)") + f.StringVar(&c.AdvertisedUrl, "c", c.AdvertisedUrl, "the advertised public hostname:port for etcd client communication") + f.StringVar(&c.Peer.AdvertisedUrl, "s", c.Peer.AdvertisedUrl, "the advertised public hostname:port for raft server communication") + f.StringVar(&c.ListenHost, "cl", c.ListenHost, "the listening hostname for etcd client communication (defaults to advertised ip)") + f.StringVar(&c.Peer.ListenHost, "sl", c.Peer.ListenHost, "the listening hostname for raft server communication (defaults to advertised ip)") + f.StringVar(&c.WebURL, "w", c.WebURL, "the hostname:port of web interface") + + f.StringVar(&c.Peer.CAFile, "serverCAFile", c.Peer.CAFile, "the path of the CAFile") + f.StringVar(&c.Peer.CertFile, "serverCert", c.Peer.CertFile, "the cert file of the server") + f.StringVar(&c.Peer.KeyFile, "serverKey", c.Peer.KeyFile, "the key file of the server") + + f.StringVar(&c.CAFile, "clientCAFile", c.CAFile, "the path of the client CAFile") + f.StringVar(&c.CertFile, "clientCert", c.CertFile, "the cert file of the client") + f.StringVar(&c.KeyFile, "clientKey", c.KeyFile, "the key file of the client") + + f.StringVar(&c.DataDir, "d", c.DataDir, "the directory to store log and snapshot") + f.IntVar(&c.MaxResultBuffer, "m", c.MaxResultBuffer, "the max size of result buffer") + f.IntVar(&c.MaxRetryAttempts, "r", c.MaxRetryAttempts, "the max retry attempts when trying to join a cluster") + f.IntVar(&c.MaxClusterSize, "maxsize", c.MaxClusterSize, "the max size of the cluster") + f.StringVar(&cors, "cors", "", "whitelist origins for cross-origin resource sharing (e.g. '*' or 'http://localhost:8001,etc')") + + f.BoolVar(&c.Snapshot, "snapshot", c.Snapshot, "open or close snapshot") + f.IntVar(&c.SnapCount, "snapCount", c.SnapCount, "save the in memory logs and states to a snapshot file after snapCount transactions") + + // These flags are ignored since they were already parsed. + var path string + f.StringVar(&path, "config", "", "path to config file") + + f.Parse(arguments) + + // Convert some parameters to lists. + if machines != "" { + c.Machines = trimsplit(machines, ",") + } + if cors != "" { + c.Cors = trimsplit(cors, ",") + } + + // Force remove server configuration if specified. + if force { + c.Reset() + } + + return nil +} + +// LoadMachineFile loads the machines listed in the machine file. +func (c *Config) LoadMachineFile() error { + if c.MachinesFile == "" { + return nil + } + + b, err := ioutil.ReadFile(c.MachinesFile) + if err != nil { + return fmt.Errorf("Machines file error: %s", err) + } + c.Machines = trimsplit(string(b), ",") + + return nil +} + +// Reset removes all server configuration files. +func (c *Config) Reset() error { + if err := os.RemoveAll(filepath.Join(c.DataDir, "info")); err != nil { + return err + } + if err := os.RemoveAll(filepath.Join(c.DataDir, "log")); err != nil { + return err + } + if err := os.RemoveAll(filepath.Join(c.DataDir, "conf")); err != nil { + return err + } + if err := os.RemoveAll(filepath.Join(c.DataDir, "snapshot")); err != nil { + return err + } + + return nil +} + +// Reads the info file from the file system or initializes it based on the config. +func (c *Config) Info() (*Info, error) { + info := &Info{} + path := filepath.Join(c.DataDir, "info") + + // Open info file and read it out. + f, err := os.Open(path) + if err != nil && !os.IsNotExist(err) { + return nil, err + } else if f != nil { + defer f.Close() + if err := json.NewDecoder(f).Decode(&info); err != nil { + return nil, err + } + return info, nil + } + + // If the file doesn't exist then initialize it. + info.Name = strings.TrimSpace(c.Name) + info.EtcdURL = c.AdvertisedUrl + info.EtcdListenHost = c.ListenHost + info.RaftURL = c.Peer.AdvertisedUrl + info.RaftListenHost = c.Peer.ListenHost + info.WebURL = c.WebURL + info.EtcdTLS = c.TLSInfo() + info.RaftTLS = c.PeerTLSInfo() + + // Write to file. + f, err = os.Create(path) + if err != nil { + return nil, err + } + defer f.Close() + + if err := json.NewEncoder(f).Encode(info); err != nil { + return nil, err + } + + return info, nil +} + +// Sanitize cleans the input fields. +func (c *Config) Sanitize() error { + tlsConfig, err := c.TLSConfig() + if err != nil { + return err + } + + peerTlsConfig, err := c.PeerTLSConfig() + if err != nil { + return err + } + + // Sanitize the URLs first. + if c.AdvertisedUrl, err = sanitizeURL(c.AdvertisedUrl, tlsConfig.Scheme); err != nil { + return fmt.Errorf("Advertised URL: %s", err) + } + if c.ListenHost, err = sanitizeListenHost(c.ListenHost, c.AdvertisedUrl); err != nil { + return fmt.Errorf("Listen Host: %s", err) + } + if c.WebURL, err = sanitizeURL(c.WebURL, "http"); err != nil { + return fmt.Errorf("Web URL: %s", err) + } + if c.Peer.AdvertisedUrl, err = sanitizeURL(c.Peer.AdvertisedUrl, peerTlsConfig.Scheme); err != nil { + return fmt.Errorf("Peer Advertised URL: %s", err) + } + if c.Peer.ListenHost, err = sanitizeListenHost(c.Peer.ListenHost, c.Peer.AdvertisedUrl); err != nil { + return fmt.Errorf("Peer Listen Host: %s", err) + } + + return nil +} + +// TLSInfo retrieves a TLSInfo object for the client server. +func (c *Config) TLSInfo() TLSInfo { + return TLSInfo{ + CAFile: c.CAFile, + CertFile: c.CertFile, + KeyFile: c.KeyFile, + } +} + +// ClientTLSConfig generates the TLS configuration for the client server. +func (c *Config) TLSConfig() (TLSConfig, error) { + return c.TLSInfo().Config() +} + +// PeerTLSInfo retrieves a TLSInfo object for the peer server. +func (c *Config) PeerTLSInfo() TLSInfo { + return TLSInfo{ + CAFile: c.Peer.CAFile, + CertFile: c.Peer.CertFile, + KeyFile: c.Peer.KeyFile, + } +} + +// PeerTLSConfig generates the TLS configuration for the peer server. +func (c *Config) PeerTLSConfig() (TLSConfig, error) { + return c.PeerTLSInfo().Config() +} + +// sanitizeURL will cleanup a host string in the format hostname:port and +// attach a schema. +func sanitizeURL(host string, defaultScheme string) (string, error) { + // Blank URLs are fine input, just return it + if len(host) == 0 { + return host, nil + } + + p, err := url.Parse(host) + if err != nil { + return "", err + } + + // Make sure the host is in Host:Port format + _, _, err = net.SplitHostPort(host) + if err != nil { + return "", err + } + + p = &url.URL{Host: host, Scheme: defaultScheme} + return p.String(), nil +} + +// sanitizeListenHost cleans up the ListenHost parameter and appends a port +// if necessary based on the advertised port. +func sanitizeListenHost(listen string, advertised string) (string, error) { + aurl, err := url.Parse(advertised) + if err != nil { + return "", err + } + + ahost, aport, err := net.SplitHostPort(aurl.Host) + if err != nil { + return "", err + } + + // If the listen host isn't set use the advertised host + if listen == "" { + listen = ahost + } + + return net.JoinHostPort(listen, aport), nil +} diff --git a/server/config_test.go b/server/config_test.go new file mode 100644 index 000000000..720534888 --- /dev/null +++ b/server/config_test.go @@ -0,0 +1,479 @@ +package server + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/BurntSushi/toml" + "github.com/stretchr/testify/assert" +) + + +// Ensures that a configuration can be deserialized from TOML. +func TestConfigTOML(t *testing.T) { + content := ` + advertised_url = "127.0.0.1:4002" + ca_file = "/tmp/file.ca" + cert_file = "/tmp/file.cert" + cors = ["*"] + cpu_profile_file = "XXX" + datadir = "/tmp/data" + key_file = "/tmp/file.key" + listen_host = "127.0.0.1:4003" + machines = ["coreos.com:4001", "coreos.com:4002"] + machines_file = "/tmp/machines" + max_cluster_size = 10 + max_result_buffer = 512 + max_retry_attempts = 5 + name = "test-name" + snapshot = true + verbose = true + very_verbose = true + web_url = "/web" + + [peer] + advertised_url = "127.0.0.1:7002" + ca_file = "/tmp/peer/file.ca" + cert_file = "/tmp/peer/file.cert" + key_file = "/tmp/peer/file.key" + listen_host = "127.0.0.1:7003" + ` + c := NewConfig() + _, err := toml.Decode(content, &c) + assert.Nil(t, err, "") + assert.Equal(t, c.AdvertisedUrl, "127.0.0.1:4002", "") + assert.Equal(t, c.CAFile, "/tmp/file.ca", "") + assert.Equal(t, c.CertFile, "/tmp/file.cert", "") + assert.Equal(t, c.Cors, []string{"*"}, "") + assert.Equal(t, c.DataDir, "/tmp/data", "") + assert.Equal(t, c.KeyFile, "/tmp/file.key", "") + assert.Equal(t, c.ListenHost, "127.0.0.1:4003", "") + assert.Equal(t, c.Machines, []string{"coreos.com:4001", "coreos.com:4002"}, "") + assert.Equal(t, c.MachinesFile, "/tmp/machines", "") + assert.Equal(t, c.MaxClusterSize, 10, "") + assert.Equal(t, c.MaxResultBuffer, 512, "") + assert.Equal(t, c.MaxRetryAttempts, 5, "") + assert.Equal(t, c.Name, "test-name", "") + assert.Equal(t, c.Snapshot, true, "") + assert.Equal(t, c.Verbose, true, "") + assert.Equal(t, c.VeryVerbose, true, "") + assert.Equal(t, c.WebURL, "/web", "") + assert.Equal(t, c.Peer.AdvertisedUrl, "127.0.0.1:7002", "") + assert.Equal(t, c.Peer.CAFile, "/tmp/peer/file.ca", "") + assert.Equal(t, c.Peer.CertFile, "/tmp/peer/file.cert", "") + assert.Equal(t, c.Peer.KeyFile, "/tmp/peer/file.key", "") + assert.Equal(t, c.Peer.ListenHost, "127.0.0.1:7003", "") +} + +// Ensures that a configuration can be retrieved from environment variables. +func TestConfigEnv(t *testing.T) { + os.Setenv("ETCD_CA_FILE", "/tmp/file.ca") + os.Setenv("ETCD_CERT_FILE", "/tmp/file.cert") + os.Setenv("ETCD_CPU_PROFILE_FILE", "XXX") + os.Setenv("ETCD_CORS", "localhost:4001,localhost:4002") + os.Setenv("ETCD_DATADIR", "/tmp/data") + os.Setenv("ETCD_KEY_FILE", "/tmp/file.key") + os.Setenv("ETCD_LISTEN_HOST", "127.0.0.1:4003") + os.Setenv("ETCD_MACHINES", "coreos.com:4001,coreos.com:4002") + os.Setenv("ETCD_MACHINES_FILE", "/tmp/machines") + os.Setenv("ETCD_MAX_CLUSTER_SIZE", "10") + os.Setenv("ETCD_MAX_RESULT_BUFFER", "512") + os.Setenv("ETCD_MAX_RETRY_ATTEMPTS", "5") + os.Setenv("ETCD_NAME", "test-name") + os.Setenv("ETCD_SNAPSHOT", "true") + os.Setenv("ETCD_VERBOSE", "1") + os.Setenv("ETCD_VERY_VERBOSE", "yes") + os.Setenv("ETCD_WEB_URL", "/web") + os.Setenv("ETCD_PEER_ADVERTISED_URL", "127.0.0.1:7002") + os.Setenv("ETCD_PEER_CA_FILE", "/tmp/peer/file.ca") + os.Setenv("ETCD_PEER_CERT_FILE", "/tmp/peer/file.cert") + os.Setenv("ETCD_PEER_KEY_FILE", "/tmp/peer/file.key") + os.Setenv("ETCD_PEER_LISTEN_HOST", "127.0.0.1:7003") + + c := NewConfig() + c.LoadEnv() + assert.Equal(t, c.CAFile, "/tmp/file.ca", "") + assert.Equal(t, c.CertFile, "/tmp/file.cert", "") + assert.Equal(t, c.Cors, []string{"localhost:4001", "localhost:4002"}, "") + assert.Equal(t, c.DataDir, "/tmp/data", "") + assert.Equal(t, c.KeyFile, "/tmp/file.key", "") + assert.Equal(t, c.ListenHost, "127.0.0.1:4003", "") + assert.Equal(t, c.Machines, []string{"coreos.com:4001", "coreos.com:4002"}, "") + assert.Equal(t, c.MachinesFile, "/tmp/machines", "") + assert.Equal(t, c.MaxClusterSize, 10, "") + assert.Equal(t, c.MaxResultBuffer, 512, "") + assert.Equal(t, c.MaxRetryAttempts, 5, "") + assert.Equal(t, c.Name, "test-name", "") + assert.Equal(t, c.Snapshot, true, "") + assert.Equal(t, c.Verbose, true, "") + assert.Equal(t, c.VeryVerbose, true, "") + assert.Equal(t, c.WebURL, "/web", "") + assert.Equal(t, c.Peer.AdvertisedUrl, "127.0.0.1:7002", "") + assert.Equal(t, c.Peer.CAFile, "/tmp/peer/file.ca", "") + assert.Equal(t, c.Peer.CertFile, "/tmp/peer/file.cert", "") + assert.Equal(t, c.Peer.KeyFile, "/tmp/peer/file.key", "") + assert.Equal(t, c.Peer.ListenHost, "127.0.0.1:7003", "") +} + +// Ensures that a the advertised url can be parsed from the environment. +func TestConfigAdvertisedUrlEnv(t *testing.T) { + withEnv("ETCD_ADVERTISED_URL", "127.0.0.1:4002", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.AdvertisedUrl, "127.0.0.1:4002", "") + }) +} + +// Ensures that a the advertised flag can be parsed. +func TestConfigAdvertisedUrlFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-c", "127.0.0.1:4002"}), "") + assert.Equal(t, c.AdvertisedUrl, "127.0.0.1:4002", "") +} + +// Ensures that a the CA file can be parsed from the environment. +func TestConfigCAFileEnv(t *testing.T) { + withEnv("ETCD_CA_FILE", "/tmp/file.ca", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.CAFile, "/tmp/file.ca", "") + }) +} + +// Ensures that a the CA file flag can be parsed. +func TestConfigCAFileFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-clientCAFile", "/tmp/file.ca"}), "") + assert.Equal(t, c.CAFile, "/tmp/file.ca", "") +} + +// Ensures that a the CA file can be parsed from the environment. +func TestConfigCertFileEnv(t *testing.T) { + withEnv("ETCD_CERT_FILE", "/tmp/file.cert", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.CertFile, "/tmp/file.cert", "") + }) +} + +// Ensures that a the Cert file flag can be parsed. +func TestConfigCertFileFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-clientCert", "/tmp/file.cert"}), "") + assert.Equal(t, c.CertFile, "/tmp/file.cert", "") +} + +// Ensures that a the Key file can be parsed from the environment. +func TestConfigKeyFileEnv(t *testing.T) { + withEnv("ETCD_KEY_FILE", "/tmp/file.key", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.KeyFile, "/tmp/file.key", "") + }) +} + +// Ensures that a the Key file flag can be parsed. +func TestConfigKeyFileFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-clientKey", "/tmp/file.key"}), "") + assert.Equal(t, c.KeyFile, "/tmp/file.key", "") +} + +// Ensures that a the Listen Host can be parsed from the environment. +func TestConfigListenHostEnv(t *testing.T) { + withEnv("ETCD_LISTEN_HOST", "127.0.0.1:4003", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.ListenHost, "127.0.0.1:4003", "") + }) +} + +// Ensures that a the Listen Host file flag can be parsed. +func TestConfigListenHostFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-cl", "127.0.0.1:4003"}), "") + assert.Equal(t, c.ListenHost, "127.0.0.1:4003", "") +} + +// Ensures that the Machines can be parsed from the environment. +func TestConfigMachinesEnv(t *testing.T) { + withEnv("ETCD_MACHINES", "coreos.com:4001,coreos.com:4002", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Machines, []string{"coreos.com:4001", "coreos.com:4002"}, "") + }) +} + +// Ensures that a the Machines flag can be parsed. +func TestConfigMachinesFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-C", "coreos.com:4001,coreos.com:4002"}), "") + assert.Equal(t, c.Machines, []string{"coreos.com:4001", "coreos.com:4002"}, "") +} + +// Ensures that the Machines File can be parsed from the environment. +func TestConfigMachinesFileEnv(t *testing.T) { + withEnv("ETCD_MACHINES_FILE", "/tmp/machines", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.MachinesFile, "/tmp/machines", "") + }) +} + +// Ensures that a the Machines File flag can be parsed. +func TestConfigMachinesFileFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-CF", "/tmp/machines"}), "") + assert.Equal(t, c.MachinesFile, "/tmp/machines", "") +} + +// Ensures that the Max Cluster Size can be parsed from the environment. +func TestConfigMaxClusterSizeEnv(t *testing.T) { + withEnv("ETCD_MAX_CLUSTER_SIZE", "5", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.MaxClusterSize, 5, "") + }) +} + +// Ensures that a the Max Cluster Size flag can be parsed. +func TestConfigMaxClusterSizeFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-maxsize", "5"}), "") + assert.Equal(t, c.MaxClusterSize, 5, "") +} + +// Ensures that the Max Result Buffer can be parsed from the environment. +func TestConfigMaxResultBufferEnv(t *testing.T) { + withEnv("ETCD_MAX_RESULT_BUFFER", "512", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.MaxResultBuffer, 512, "") + }) +} + +// Ensures that a the Max Result Buffer flag can be parsed. +func TestConfigMaxResultBufferFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-m", "512"}), "") + assert.Equal(t, c.MaxResultBuffer, 512, "") +} + +// Ensures that the Max Retry Attempts can be parsed from the environment. +func TestConfigMaxRetryAttemptsEnv(t *testing.T) { + withEnv("ETCD_MAX_RETRY_ATTEMPTS", "10", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.MaxRetryAttempts, 10, "") + }) +} + +// Ensures that a the Max Retry Attempts flag can be parsed. +func TestConfigMaxRetryAttemptsFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-r", "10"}), "") + assert.Equal(t, c.MaxRetryAttempts, 10, "") +} + +// Ensures that the Name can be parsed from the environment. +func TestConfigNameEnv(t *testing.T) { + withEnv("ETCD_NAME", "test-name", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Name, "test-name", "") + }) +} + +// Ensures that a the Name flag can be parsed. +func TestConfigNameFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-n", "test-name"}), "") + assert.Equal(t, c.Name, "test-name", "") +} + +// Ensures that Snapshot can be parsed from the environment. +func TestConfigSnapshotEnv(t *testing.T) { + withEnv("ETCD_SNAPSHOT", "1", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Snapshot, true, "") + }) +} + +// Ensures that a the Snapshot flag can be parsed. +func TestConfigSnapshotFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-snapshot"}), "") + assert.Equal(t, c.Snapshot, true, "") +} + +// Ensures that Verbose can be parsed from the environment. +func TestConfigVerboseEnv(t *testing.T) { + withEnv("ETCD_VERBOSE", "true", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Verbose, true, "") + }) +} + +// Ensures that a the Verbose flag can be parsed. +func TestConfigVerboseFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-v"}), "") + assert.Equal(t, c.Verbose, true, "") +} + +// Ensures that Very Verbose can be parsed from the environment. +func TestConfigVeryVerboseEnv(t *testing.T) { + withEnv("ETCD_VERY_VERBOSE", "true", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.VeryVerbose, true, "") + }) +} + +// Ensures that a the Very Verbose flag can be parsed. +func TestConfigVeryVerboseFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-vv"}), "") + assert.Equal(t, c.VeryVerbose, true, "") +} + +// Ensures that Web URL can be parsed from the environment. +func TestConfigWebURLEnv(t *testing.T) { + withEnv("ETCD_WEB_URL", "/web", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.WebURL, "/web", "") + }) +} + +// Ensures that a the Web URL flag can be parsed. +func TestConfigWebURLFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-w", "/web"}), "") + assert.Equal(t, c.WebURL, "/web", "") +} + +// Ensures that the Peer Advertised URL can be parsed from the environment. +func TestConfigPeerAdvertisedUrlEnv(t *testing.T) { + withEnv("ETCD_PEER_ADVERTISED_URL", "localhost:7002", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Peer.AdvertisedUrl, "localhost:7002", "") + }) +} + +// Ensures that a the Peer Advertised URL flag can be parsed. +func TestConfigPeerAdvertisedUrlFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-s", "localhost:7002"}), "") + assert.Equal(t, c.Peer.AdvertisedUrl, "localhost:7002", "") +} + +// Ensures that the Peer CA File can be parsed from the environment. +func TestConfigPeerCAFileEnv(t *testing.T) { + withEnv("ETCD_PEER_CA_FILE", "/tmp/peer/file.ca", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Peer.CAFile, "/tmp/peer/file.ca", "") + }) +} + +// Ensures that a the Peer CA file flag can be parsed. +func TestConfigPeerCAFileFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-serverCAFile", "/tmp/peer/file.ca"}), "") + assert.Equal(t, c.Peer.CAFile, "/tmp/peer/file.ca", "") +} + +// Ensures that the Peer Cert File can be parsed from the environment. +func TestConfigPeerCertFileEnv(t *testing.T) { + withEnv("ETCD_PEER_CERT_FILE", "/tmp/peer/file.cert", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Peer.CertFile, "/tmp/peer/file.cert", "") + }) +} + +// Ensures that a the Cert file flag can be parsed. +func TestConfigPeerCertFileFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-serverCert", "/tmp/peer/file.cert"}), "") + assert.Equal(t, c.Peer.CertFile, "/tmp/peer/file.cert", "") +} + +// Ensures that the Peer Key File can be parsed from the environment. +func TestConfigPeerKeyFileEnv(t *testing.T) { + withEnv("ETCD_PEER_KEY_FILE", "/tmp/peer/file.key", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Peer.KeyFile, "/tmp/peer/file.key", "") + }) +} + +// Ensures that a the Peer Key file flag can be parsed. +func TestConfigPeerKeyFileFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-serverKey", "/tmp/peer/file.key"}), "") + assert.Equal(t, c.Peer.KeyFile, "/tmp/peer/file.key", "") +} + +// Ensures that the Peer Listen Host can be parsed from the environment. +func TestConfigPeerListenHostEnv(t *testing.T) { + withEnv("ETCD_PEER_LISTEN_HOST", "localhost:7004", func(c *Config) { + assert.Nil(t, c.LoadEnv(), "") + assert.Equal(t, c.Peer.ListenHost, "localhost:7004", "") + }) +} + +// Ensures that a the Peer Listen Host file flag can be parsed. +func TestConfigPeerListenHostFlag(t *testing.T) { + c := NewConfig() + assert.Nil(t, c.LoadFlags([]string{"-sl", "127.0.0.1:4003"}), "") + assert.Equal(t, c.Peer.ListenHost, "127.0.0.1:4003", "") +} + + +// Ensures that a system config field is overridden by a custom config field. +func TestConfigCustomConfigOverrideSystemConfig(t *testing.T) { + system := `advertised_url = "127.0.0.1:5000"` + custom := `advertised_url = "127.0.0.1:6000"` + withTempFile(system, func(p1 string) { + withTempFile(custom, func(p2 string) { + c := NewConfig() + c.SystemPath = p1 + assert.Nil(t, c.Load([]string{"-config", p2}), "") + assert.Equal(t, c.AdvertisedUrl, "http://127.0.0.1:6000", "") + }) + }) +} + +// Ensures that a custom config field is overridden by an environment variable. +func TestConfigEnvVarOverrideCustomConfig(t *testing.T) { + os.Setenv("ETCD_PEER_ADVERTISED_URL", "127.0.0.1:8000") + defer os.Setenv("ETCD_PEER_ADVERTISED_URL", "") + + custom := `[peer]`+"\n"+`advertised_url = "127.0.0.1:9000"` + withTempFile(custom, func(path string) { + c := NewConfig() + c.SystemPath = "" + assert.Nil(t, c.Load([]string{"-config", path}), "") + assert.Equal(t, c.Peer.AdvertisedUrl, "http://127.0.0.1:8000", "") + }) +} + +// Ensures that an environment variable field is overridden by a command line argument. +func TestConfigCLIArgsOverrideEnvVar(t *testing.T) { + os.Setenv("ETCD_ADVERTISED_URL", "127.0.0.1:1000") + defer os.Setenv("ETCD_ADVERTISED_URL", "") + + c := NewConfig() + c.SystemPath = "" + assert.Nil(t, c.Load([]string{"-c", "127.0.0.1:2000"}), "") + assert.Equal(t, c.AdvertisedUrl, "http://127.0.0.1:2000", "") +} + + +//-------------------------------------- +// Helpers +//-------------------------------------- + +// Sets up the environment with a given environment variable set. +func withEnv(key, value string, f func(c *Config)) { + os.Setenv(key, value) + defer os.Setenv(key, "") + c := NewConfig() + f(c) +} + +// Creates a temp file and calls a function with the context. +func withTempFile(content string, fn func(string)) { + f, _ := ioutil.TempFile("", "") + f.WriteString(content) + f.Close() + defer os.Remove(f.Name()) + fn(f.Name()) +} diff --git a/server/info.go b/server/info.go new file mode 100644 index 000000000..434f24a03 --- /dev/null +++ b/server/info.go @@ -0,0 +1,19 @@ +package server + +// Info describes the non-mutable state of the server upon initialization. +// These fields cannot be changed without deleting the server fields and +// reinitializing. +type Info struct { + Name string `json:"name"` + + RaftURL string `json:"raftURL"` + EtcdURL string `json:"etcdURL"` + WebURL string `json:"webURL"` + + RaftListenHost string `json:"raftListenHost"` + EtcdListenHost string `json:"etcdListenHost"` + + RaftTLS TLSInfo `json:"raftTLS"` + EtcdTLS TLSInfo `json:"etcdTLS"` +} + diff --git a/server/join_command.go b/server/join_command.go index 7bebbe704..49dab4f2e 100644 --- a/server/join_command.go +++ b/server/join_command.go @@ -14,15 +14,17 @@ func init() { // The JoinCommand adds a node to the cluster. type JoinCommand struct { - RaftVersion string `json:"raftVersion"` + MinVersion int `json:"minVersion"` + MaxVersion int `json:"maxVersion"` Name string `json:"name"` RaftURL string `json:"raftURL"` EtcdURL string `json:"etcdURL"` } -func NewJoinCommand(version, name, raftUrl, etcdUrl string) *JoinCommand { +func NewJoinCommand(minVersion int, maxVersion int, name, raftUrl, etcdUrl string) *JoinCommand { return &JoinCommand{ - RaftVersion: version, + MinVersion: minVersion, + MaxVersion: maxVersion, Name: name, RaftURL: raftUrl, EtcdURL: etcdUrl, @@ -56,7 +58,7 @@ func (c *JoinCommand) Apply(server raft.Server) (interface{}, error) { } // Add to shared machine registry. - ps.registry.Register(c.Name, c.RaftVersion, c.RaftURL, c.EtcdURL, server.CommitIndex(), server.Term()) + ps.registry.Register(c.Name, c.RaftURL, c.EtcdURL, server.CommitIndex(), server.Term()) // Add peer in raft err := server.AddPeer(c.Name, "") diff --git a/server/peer_server.go b/server/peer_server.go index 538d3ac70..bfa2ea741 100644 --- a/server/peer_server.go +++ b/server/peer_server.go @@ -10,12 +10,14 @@ import ( "net" "net/http" "net/url" + "strconv" "time" etcdErr "github.com/coreos/etcd/error" "github.com/coreos/etcd/log" "github.com/coreos/etcd/store" "github.com/coreos/go-raft" + "github.com/gorilla/mux" ) type PeerServer struct { @@ -51,7 +53,7 @@ type snapshotConf struct { writesThr uint64 } -func NewPeerServer(name string, path string, url string, listenHost string, tlsConf *TLSConfig, tlsInfo *TLSInfo, registry *Registry, store store.Store) *PeerServer { +func NewPeerServer(name string, path string, url string, listenHost string, tlsConf *TLSConfig, tlsInfo *TLSInfo, registry *Registry, store store.Store, snapCount int) *PeerServer { s := &PeerServer{ name: name, url: url, @@ -60,7 +62,7 @@ func NewPeerServer(name string, path string, url string, listenHost string, tlsC tlsInfo: tlsInfo, registry: registry, store: store, - snapConf: &snapshotConf{time.Second * 3, 0, 2}, + snapConf: &snapshotConf{time.Second * 3, 0, uint64(snapCount)}, followersStats: &raftFollowersStats{ Leader: name, Followers: make(map[string]*raftFollowerStats), @@ -209,7 +211,7 @@ func (s *PeerServer) SetServer(server *Server) { func (s *PeerServer) startAsLeader() { // leader need to join self as a peer for { - _, err := s.raftServer.Do(NewJoinCommand(PeerVersion, s.raftServer.Name(), s.url, s.server.URL())) + _, err := s.raftServer.Do(NewJoinCommand(store.MinVersion(), store.MaxVersion(), s.raftServer.Name(), s.url, s.server.URL())) if err == nil { break } @@ -235,25 +237,27 @@ func (s *PeerServer) startAsFollower(cluster []string) { func (s *PeerServer) startTransport(scheme string, tlsConf tls.Config) error { log.Infof("raft server [name %s, listen on %s, advertised url %s]", s.name, s.listenHost, s.url) - raftMux := http.NewServeMux() + router := mux.NewRouter() s.httpServer = &http.Server{ - Handler: raftMux, + Handler: router, TLSConfig: &tlsConf, Addr: s.listenHost, } // internal commands - raftMux.HandleFunc("/name", s.NameHttpHandler) - raftMux.HandleFunc("/version", s.RaftVersionHttpHandler) - raftMux.HandleFunc("/join", s.JoinHttpHandler) - raftMux.HandleFunc("/remove/", s.RemoveHttpHandler) - raftMux.HandleFunc("/vote", s.VoteHttpHandler) - raftMux.HandleFunc("/log", s.GetLogHttpHandler) - raftMux.HandleFunc("/log/append", s.AppendEntriesHttpHandler) - raftMux.HandleFunc("/snapshot", s.SnapshotHttpHandler) - raftMux.HandleFunc("/snapshotRecovery", s.SnapshotRecoveryHttpHandler) - raftMux.HandleFunc("/etcdURL", s.EtcdURLHttpHandler) + router.HandleFunc("/name", s.NameHttpHandler) + router.HandleFunc("/version", s.VersionHttpHandler) + router.HandleFunc("/version/{version:[0-9]+}/check", s.VersionCheckHttpHandler) + router.HandleFunc("/upgrade", s.UpgradeHttpHandler) + router.HandleFunc("/join", s.JoinHttpHandler) + router.HandleFunc("/remove/{name:.+}", s.RemoveHttpHandler) + router.HandleFunc("/vote", s.VoteHttpHandler) + router.HandleFunc("/log", s.GetLogHttpHandler) + router.HandleFunc("/log/append", s.AppendEntriesHttpHandler) + router.HandleFunc("/snapshot", s.SnapshotHttpHandler) + router.HandleFunc("/snapshotRecovery", s.SnapshotRecoveryHttpHandler) + router.HandleFunc("/etcdURL", s.EtcdURLHttpHandler) if scheme == "http" { return s.listenAndServe() @@ -263,21 +267,46 @@ func (s *PeerServer) startTransport(scheme string, tlsConf tls.Config) error { } -// getVersion fetches the raft version of a peer. This works for now but we -// will need to do something more sophisticated later when we allow mixed -// version clusters. -func getVersion(t *transporter, versionURL url.URL) (string, error) { +// getVersion fetches the peer version of a cluster. +func getVersion(t *transporter, versionURL url.URL) (int, error) { resp, req, err := t.Get(versionURL.String()) if err != nil { - return "", err + return 0, err } defer resp.Body.Close() t.CancelWhenTimeout(req) - body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, err + } - return string(body), nil + // Parse version number. + version, _ := strconv.Atoi(string(body)) + return version, nil +} + +// Upgradable checks whether all peers in a cluster support an upgrade to the next store version. +func (s *PeerServer) Upgradable() error { + nextVersion := s.store.Version() + 1 + for _, peerURL := range s.registry.PeerURLs(s.raftServer.Leader(), s.name) { + u, err := url.Parse(peerURL) + if err != nil { + return fmt.Errorf("PeerServer: Cannot parse URL: '%s' (%s)", peerURL, err) + } + + t, _ := s.raftServer.Transporter().(*transporter) + checkURL := (&url.URL{Host: u.Host, Scheme: s.tlsConf.Scheme, Path: fmt.Sprintf("/version/%d/check", nextVersion)}).String() + resp, _, err := t.Get(checkURL) + if err != nil { + return fmt.Errorf("PeerServer: Cannot check version compatibility: %s", u.Host) + } + if resp.StatusCode != 200 { + return fmt.Errorf("PeerServer: Version %d is not compatible with peer: %s", nextVersion, u.Host) + } + } + + return nil } func (s *PeerServer) joinCluster(cluster []string) bool { @@ -315,14 +344,11 @@ func (s *PeerServer) joinByMachine(server raft.Server, machine string, scheme st if err != nil { return fmt.Errorf("Error during join version check: %v", err) } - - // TODO: versioning of the internal protocol. See: - // Documentation/internatl-protocol-versioning.md - if version != PeerVersion { - return fmt.Errorf("Unable to join: internal version mismatch, entire cluster must be running identical versions of etcd") + if version < store.MinVersion() || version > store.MaxVersion() { + return fmt.Errorf("Unable to join: cluster version is %d; version compatibility is %d - %d", version, store.MinVersion(), store.MaxVersion()) } - json.NewEncoder(&b).Encode(NewJoinCommand(PeerVersion, server.Name(), s.url, s.server.URL())) + json.NewEncoder(&b).Encode(NewJoinCommand(store.MinVersion(), store.MaxVersion(), server.Name(), s.url, s.server.URL())) joinURL := url.URL{Host: machine, Scheme: scheme, Path: "/join"} @@ -347,7 +373,7 @@ func (s *PeerServer) joinByMachine(server raft.Server, machine string, scheme st if resp.StatusCode == http.StatusTemporaryRedirect { address := resp.Header.Get("Location") log.Debugf("Send Join Request to %s", address) - json.NewEncoder(&b).Encode(NewJoinCommand(PeerVersion, server.Name(), s.url, s.server.URL())) + json.NewEncoder(&b).Encode(NewJoinCommand(store.MinVersion(), store.MaxVersion(), server.Name(), s.url, s.server.URL())) resp, req, err = t.Post(address, &b) } else if resp.StatusCode == http.StatusBadRequest { diff --git a/server/peer_server_handlers.go b/server/peer_server_handlers.go index 26ce7a5cd..be665dbf5 100644 --- a/server/peer_server_handlers.go +++ b/server/peer_server_handlers.go @@ -3,10 +3,13 @@ package server import ( "encoding/json" "net/http" + "strconv" etcdErr "github.com/coreos/etcd/error" "github.com/coreos/etcd/log" + "github.com/coreos/etcd/store" "github.com/coreos/go-raft" + "github.com/gorilla/mux" ) // Get all the current logs @@ -133,9 +136,9 @@ func (ps *PeerServer) RemoveHttpHandler(w http.ResponseWriter, req *http.Request return } - nodeName := req.URL.Path[len("/remove/"):] + vars := mux.Vars(req) command := &RemoveCommand{ - Name: nodeName, + Name: vars["name"], } log.Debugf("[recv] Remove Request [%s]", command.Name) @@ -151,8 +154,40 @@ func (ps *PeerServer) NameHttpHandler(w http.ResponseWriter, req *http.Request) } // Response to the name request -func (ps *PeerServer) RaftVersionHttpHandler(w http.ResponseWriter, req *http.Request) { +func (ps *PeerServer) VersionHttpHandler(w http.ResponseWriter, req *http.Request) { log.Debugf("[recv] Get %s/version/ ", ps.url) w.WriteHeader(http.StatusOK) - w.Write([]byte(PeerVersion)) + w.Write([]byte(strconv.Itoa(ps.store.Version()))) +} + +// Checks whether a given version is supported. +func (ps *PeerServer) VersionCheckHttpHandler(w http.ResponseWriter, req *http.Request) { + log.Debugf("[recv] Get %s%s ", ps.url, req.URL.Path) + vars := mux.Vars(req) + version, _ := strconv.Atoi(vars["version"]) + if version >= store.MinVersion() && version <= store.MaxVersion() { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusForbidden) + } +} + +// Upgrades the current store version to the next version. +func (ps *PeerServer) UpgradeHttpHandler(w http.ResponseWriter, req *http.Request) { + log.Debugf("[recv] Get %s/version", ps.url) + + // Check if upgrade is possible for all nodes. + if err := ps.Upgradable(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Create an upgrade command from the current version. + c := ps.store.CommandFactory().CreateUpgradeCommand() + if err := ps.server.Dispatch(c, w, req); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) } diff --git a/server/registry.go b/server/registry.go index 75b0e98c9..05cccc10c 100644 --- a/server/registry.go +++ b/server/registry.go @@ -38,13 +38,13 @@ func NewRegistry(s store.Store) *Registry { } // Adds a node to the registry. -func (r *Registry) Register(name string, peerVersion string, peerURL string, url string, commitIndex uint64, term uint64) error { +func (r *Registry) Register(name string, peerURL string, url string, commitIndex uint64, term uint64) error { r.Lock() defer r.Unlock() // Write data to store. key := path.Join(RegistryKey, name) - value := fmt.Sprintf("raft=%s&etcd=%s&raftVersion=%s", peerURL, url, peerVersion) + value := fmt.Sprintf("raft=%s&etcd=%s", peerURL, url) _, err := r.store.Create(key, value, false, store.Permanent, commitIndex, term) log.Debugf("Register: %s", name) return err @@ -175,6 +175,5 @@ func (r *Registry) load(name string) { r.nodes[name] = &node{ url: m["etcd"][0], peerURL: m["raft"][0], - peerVersion: m["raftVersion"][0], } } diff --git a/server/server.go b/server/server.go index f6a8779a2..b88d5ac0a 100644 --- a/server/server.go +++ b/server/server.go @@ -15,6 +15,7 @@ import ( "github.com/coreos/etcd/server/v1" "github.com/coreos/etcd/server/v2" "github.com/coreos/etcd/store" + _ "github.com/coreos/etcd/store/v2" "github.com/coreos/go-raft" "github.com/gorilla/mux" ) @@ -283,10 +284,10 @@ func (s *Server) Dispatch(c raft.Command, w http.ResponseWriter, req *http.Reque } // Sets a comma-delimited list of origins that are allowed. -func (s *Server) AllowOrigins(origins string) error { +func (s *Server) AllowOrigins(origins []string) error { // Construct a lookup of all origins. m := make(map[string]bool) - for _, v := range strings.Split(origins, ",") { + for _, v := range origins { if v != "*" { if _, err := url.Parse(v); err != nil { return fmt.Errorf("Invalid CORS origin: %s", err) @@ -366,11 +367,7 @@ func (s *Server) SpeedTestHandler(w http.ResponseWriter, req *http.Request) erro for i := 0; i < count; i++ { go func() { for j := 0; j < 10; j++ { - c := &store.SetCommand{ - Key: "foo", - Value: "bar", - ExpireTime: time.Unix(0, 0), - } + c := s.Store().CommandFactory().CreateSetCommand("foo", "bar", time.Unix(0, 0)) s.peerServer.RaftServer().Do(c) } c <- true diff --git a/server/tls_config.go b/server/tls_config.go index 4b944626d..29b8fa5e4 100644 --- a/server/tls_config.go +++ b/server/tls_config.go @@ -4,6 +4,7 @@ import ( "crypto/tls" ) +// TLSConfig holds the TLS configuration. type TLSConfig struct { Scheme string Server tls.Config diff --git a/server/tls_info.go b/server/tls_info.go index 91936b090..1867ef44f 100644 --- a/server/tls_info.go +++ b/server/tls_info.go @@ -1,7 +1,76 @@ package server +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "io/ioutil" +) + +// TLSInfo holds the SSL certificates paths. type TLSInfo struct { CertFile string `json:"CertFile"` KeyFile string `json:"KeyFile"` CAFile string `json:"CAFile"` } + +// Generates a TLS configuration from the given files. +func (info TLSInfo) Config() (TLSConfig, error) { + var t TLSConfig + t.Scheme = "http" + + // If the user do not specify key file, cert file and CA file, the type will be HTTP + if info.KeyFile == "" && info.CertFile == "" && info.CAFile == "" { + return t, nil + } + + // Both the key and cert must be present. + if info.KeyFile == "" || info.CertFile == "" { + return t, errors.New("KeyFile and CertFile must both be present") + } + + tlsCert, err := tls.LoadX509KeyPair(info.CertFile, info.KeyFile) + if err != nil { + return t, err + } + + t.Scheme = "https" + t.Server.ClientAuth, t.Server.ClientCAs, err = newCertPool(info.CAFile) + if err != nil { + return t, err + } + + // The client should trust the RootCA that the Server uses since + // everyone is a peer in the network. + t.Client.Certificates = []tls.Certificate{tlsCert} + t.Client.RootCAs = t.Server.ClientCAs + + return t, nil +} + +// newCertPool creates x509 certPool and corresponding Auth Type. +// If the given CAfile is valid, add the cert into the pool and verify the clients' +// certs against the cert in the pool. +// If the given CAfile is empty, do not verify the clients' cert. +// If the given CAfile is not valid, fatal. +func newCertPool(CAFile string) (tls.ClientAuthType, *x509.CertPool, error) { + if CAFile == "" { + return tls.NoClientCert, nil, nil + } + pemByte, err := ioutil.ReadFile(CAFile) + if err != nil { + return 0, nil, err + } + + block, pemByte := pem.Decode(pemByte) + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return 0, nil, err + } + + certPool := x509.NewCertPool() + certPool.AddCert(cert) + + return tls.RequireAndVerifyClientCert, certPool, nil +} diff --git a/server/transporter.go b/server/transporter.go index 3c5002f1b..95545a827 100644 --- a/server/transporter.go +++ b/server/transporter.go @@ -18,13 +18,15 @@ import ( // This should not exceed 3 * RTT var dailTimeout = 3 * HeartbeatTimeout -// Timeout for setup internal raft http connection + receive response header -// This should not exceed 3 * RTT + RTT -var responseHeaderTimeout = 4 * HeartbeatTimeout +// Timeout for setup internal raft http connection + receive all post body +// The raft server will not send back response header until it received all the +// post body. +// This should not exceed dailTimeout + electionTimeout +var responseHeaderTimeout = 3*HeartbeatTimeout + ElectionTimeout // Timeout for receiving the response body from the server -// This should not exceed election timeout -var tranTimeout = ElectionTimeout +// This should not exceed heartbeatTimeout +var tranTimeout = HeartbeatTimeout // Transporter layer for communication between raft nodes type transporter struct { @@ -221,7 +223,7 @@ func (t *transporter) Get(urlStr string) (*http.Response, *http.Request, error) // Cancel the on fly HTTP transaction when timeout happens. func (t *transporter) CancelWhenTimeout(req *http.Request) { go func() { - time.Sleep(ElectionTimeout) + time.Sleep(tranTimeout) t.transport.CancelRequest(req) }() } diff --git a/server/transporter_test.go b/server/transporter_test.go index e83ea3400..989b2e250 100644 --- a/server/transporter_test.go +++ b/server/transporter_test.go @@ -1,5 +1,6 @@ package server +/* import ( "crypto/tls" "fmt" @@ -59,3 +60,4 @@ func TestTransporterTimeout(t *testing.T) { } } +*/ diff --git a/server/util.go b/server/util.go index 578c048cc..bae2eaa72 100644 --- a/server/util.go +++ b/server/util.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/url" + "strings" "github.com/coreos/etcd/log" ) @@ -31,3 +32,15 @@ func redirect(hostname string, w http.ResponseWriter, req *http.Request) { log.Debugf("Redirect to %s", redirectURL.String()) http.Redirect(w, req, redirectURL.String(), http.StatusTemporaryRedirect) } + +// trimsplit slices s into all substrings separated by sep and returns a +// slice of the substrings between the separator with all leading and trailing +// white space removed, as defined by Unicode. +func trimsplit(s, sep string) []string { + raw := strings.Split(s, ",") + trimmed := make([]string, 0) + for _, r := range raw { + trimmed = append(trimmed, strings.TrimSpace(r)) + } + return trimmed +} diff --git a/server/v1/delete_key_handler.go b/server/v1/delete_key_handler.go index 1288d6597..44fe459e4 100644 --- a/server/v1/delete_key_handler.go +++ b/server/v1/delete_key_handler.go @@ -1,7 +1,6 @@ package v1 import ( - "github.com/coreos/etcd/store" "github.com/gorilla/mux" "net/http" ) @@ -10,6 +9,6 @@ import ( func DeleteKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error { vars := mux.Vars(req) key := "/" + vars["key"] - c := &store.DeleteCommand{Key: key} + c := s.Store().CommandFactory().CreateDeleteCommand(key, false) return s.Dispatch(c, w, req) } diff --git a/server/v1/set_key_handler.go b/server/v1/set_key_handler.go index 5e29fafe6..7acfe7ecb 100644 --- a/server/v1/set_key_handler.go +++ b/server/v1/set_key_handler.go @@ -31,27 +31,16 @@ func SetKeyHandler(w http.ResponseWriter, req *http.Request, s Server) error { // If the "prevValue" is specified then test-and-set. Otherwise create a new key. var c raft.Command if prevValueArr, ok := req.Form["prevValue"]; ok { - if len(prevValueArr[0]) > 0 { // test against previous value - c = &store.CompareAndSwapCommand{ - Key: key, - Value: value, - PrevValue: prevValueArr[0], - ExpireTime: expireTime, - } + if len(prevValueArr[0]) > 0 { + // test against previous value + c = s.Store().CommandFactory().CreateCompareAndSwapCommand(key, value, prevValueArr[0], 0, expireTime) } else { - c = &store.CreateCommand{ // test against existence - Key: key, - Value: value, - ExpireTime: expireTime, - } + // test against existence + c = s.Store().CommandFactory().CreateCreateCommand(key, value, expireTime, false) } } else { - c = &store.SetCommand{ - Key: key, - Value: value, - ExpireTime: expireTime, - } + c = s.Store().CommandFactory().CreateSetCommand(key, value, expireTime) } return s.Dispatch(c, w, req) diff --git a/server/v2/delete_handler.go b/server/v2/delete_handler.go index 9012498cf..7afc02f04 100644 --- a/server/v2/delete_handler.go +++ b/server/v2/delete_handler.go @@ -3,18 +3,14 @@ package v2 import ( "net/http" - "github.com/coreos/etcd/store" "github.com/gorilla/mux" ) func DeleteHandler(w http.ResponseWriter, req *http.Request, s Server) error { vars := mux.Vars(req) key := "/" + vars["key"] + recursive := (req.FormValue("recursive") == "true") - c := &store.DeleteCommand{ - Key: key, - Recursive: (req.FormValue("recursive") == "true"), - } - + c := s.Store().CommandFactory().CreateDeleteCommand(key, recursive) return s.Dispatch(c, w, req) } diff --git a/server/v2/post_handler.go b/server/v2/post_handler.go index b55eddba4..4dc98b925 100644 --- a/server/v2/post_handler.go +++ b/server/v2/post_handler.go @@ -18,12 +18,6 @@ func PostHandler(w http.ResponseWriter, req *http.Request, s Server) error { return etcdErr.NewError(etcdErr.EcodeTTLNaN, "Create", store.UndefIndex, store.UndefTerm) } - c := &store.CreateCommand{ - Key: key, - Value: value, - ExpireTime: expireTime, - Unique: true, - } - + c := s.Store().CommandFactory().CreateCreateCommand(key, value, expireTime, true) return s.Dispatch(c, w, req) } diff --git a/server/v2/put_handler.go b/server/v2/put_handler.go index 536bbe216..3afb018b7 100644 --- a/server/v2/put_handler.go +++ b/server/v2/put_handler.go @@ -71,31 +71,17 @@ func PutHandler(w http.ResponseWriter, req *http.Request, s Server) error { } } - c = &store.CompareAndSwapCommand{ - Key: key, - Value: value, - PrevValue: prevValue, - PrevIndex: prevIndex, - } - + c = s.Store().CommandFactory().CreateCompareAndSwapCommand(key, value, prevValue, prevIndex, expireTime) return s.Dispatch(c, w, req) } func SetHandler(w http.ResponseWriter, req *http.Request, s Server, key, value string, expireTime time.Time) error { - c := &store.SetCommand{ - Key: key, - Value: value, - ExpireTime: expireTime, - } + c := s.Store().CommandFactory().CreateSetCommand(key, value, expireTime) return s.Dispatch(c, w, req) } func CreateHandler(w http.ResponseWriter, req *http.Request, s Server, key, value string, expireTime time.Time) error { - c := &store.CreateCommand{ - Key: key, - Value: value, - ExpireTime: expireTime, - } + c := s.Store().CommandFactory().CreateCreateCommand(key, value, expireTime, false) return s.Dispatch(c, w, req) } @@ -105,10 +91,6 @@ func UpdateHandler(w http.ResponseWriter, req *http.Request, s Server, key, valu return etcdErr.NewError(etcdErr.EcodeValueOrTTLRequired, "Update", store.UndefIndex, store.UndefTerm) } - c := &store.UpdateCommand{ - Key: key, - Value: value, - ExpireTime: expireTime, - } + c := s.Store().CommandFactory().CreateUpdateCommand(key, value, expireTime) return s.Dispatch(c, w, req) } diff --git a/server/v2/tests/delete_handler_test.go b/server/v2/tests/delete_handler_test.go new file mode 100644 index 000000000..ab710c5ac --- /dev/null +++ b/server/v2/tests/delete_handler_test.go @@ -0,0 +1,29 @@ +package v2 + +import ( + "fmt" + "net/url" + "testing" + + "github.com/coreos/etcd/server" + "github.com/coreos/etcd/tests" + "github.com/stretchr/testify/assert" +) + +// Ensures that a key is deleted. +// +// $ curl -X PUT localhost:4001/v2/keys/foo/bar -d value=XXX +// $ curl -X DELETE localhost:4001/v2/keys/foo/bar +// +func TestV2DeleteKey(t *testing.T) { + tests.RunServer(func(s *server.Server) { + v := url.Values{} + v.Set("value", "XXX") + resp, err := tests.PutForm(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/bar"), v) + tests.ReadBody(resp) + resp, err = tests.DeleteForm(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/bar"), url.Values{}) + body := tests.ReadBody(resp) + assert.Nil(t, err, "") + assert.Equal(t, string(body), `{"action":"delete","key":"/foo/bar","prevValue":"XXX","index":4,"term":0}`, "") + }) +} diff --git a/server/v2/tests/get_handler_test.go b/server/v2/tests/get_handler_test.go index a248d33f4..d6ceae58a 100644 --- a/server/v2/tests/get_handler_test.go +++ b/server/v2/tests/get_handler_test.go @@ -42,6 +42,7 @@ func TestV2GetKeyRecursively(t *testing.T) { tests.RunServer(func(s *server.Server) { v := url.Values{} v.Set("value", "XXX") + v.Set("ttl", "10") resp, _ := tests.PutForm(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/x"), v) tests.ReadBody(resp) @@ -60,6 +61,7 @@ func TestV2GetKeyRecursively(t *testing.T) { kv0 := body["kvs"].([]interface{})[0].(map[string]interface{}) assert.Equal(t, kv0["key"], "/foo/x", "") assert.Equal(t, kv0["value"], "XXX", "") + assert.Equal(t, kv0["ttl"], 10, "") kv1 := body["kvs"].([]interface{})[1].(map[string]interface{}) assert.Equal(t, kv1["key"], "/foo/y", "") @@ -105,7 +107,6 @@ func TestV2WatchKey(t *testing.T) { }) } - // Ensures that a watcher can wait for a value to be set after a given index. // // $ curl localhost:4001/v2/keys/foo/bar?wait=true&waitIndex=4 @@ -115,9 +116,11 @@ func TestV2WatchKey(t *testing.T) { func TestV2WatchKeyWithIndex(t *testing.T) { tests.RunServer(func(s *server.Server) { var body map[string]interface{} + c := make(chan bool) go func() { resp, _ := tests.Get(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/bar?wait=true&waitIndex=5")) body = tests.ReadBodyJSON(resp) + c <- true }() // Make sure response didn't fire early. @@ -141,6 +144,14 @@ func TestV2WatchKeyWithIndex(t *testing.T) { // A response should follow from the GET above. time.Sleep(1 * time.Millisecond) + + select { + case <-c: + + default: + t.Fatal("cannot get watch result") + } + assert.NotNil(t, body, "") assert.Equal(t, body["action"], "set", "") assert.Equal(t, body["key"], "/foo/bar", "") @@ -149,4 +160,3 @@ func TestV2WatchKeyWithIndex(t *testing.T) { assert.Equal(t, body["term"], 0, "") }) } - diff --git a/server/version.go b/server/version.go index 354bda3a5..1d0fc6a4b 100644 --- a/server/version.go +++ b/server/version.go @@ -1,8 +1,3 @@ package server const Version = "v2" - -// TODO: The release version (generated from the git tag) will be the raft -// protocol version for now. When things settle down we will fix it like the -// client API above. -const PeerVersion = ReleaseVersion diff --git a/store/command_factory.go b/store/command_factory.go new file mode 100644 index 000000000..9b52f42f9 --- /dev/null +++ b/store/command_factory.go @@ -0,0 +1,58 @@ +package store + +import ( + "fmt" + "time" + + "github.com/coreos/go-raft" +) + +// A lookup of factories by version. +var factories = make(map[int]CommandFactory) +var minVersion, maxVersion int + +// The CommandFactory provides a way to create different types of commands +// depending on the current version of the store. +type CommandFactory interface { + Version() int + CreateUpgradeCommand() raft.Command + CreateSetCommand(key string, value string, expireTime time.Time) raft.Command + CreateCreateCommand(key string, value string, expireTime time.Time, unique bool) raft.Command + CreateUpdateCommand(key string, value string, expireTime time.Time) raft.Command + CreateDeleteCommand(key string, recursive bool) raft.Command + CreateCompareAndSwapCommand(key string, value string, prevValue string, prevIndex uint64, expireTime time.Time) raft.Command +} + +// RegisterCommandFactory adds a command factory to the global registry. +func RegisterCommandFactory(factory CommandFactory) { + version := factory.Version() + + if GetCommandFactory(version) != nil { + panic(fmt.Sprintf("Command factory already registered for version: %d", factory.Version())) + } + + factories[version] = factory + + // Update compatibility versions. + if minVersion == 0 || version > minVersion { + minVersion = version + } + if maxVersion == 0 || version > maxVersion { + maxVersion = version + } +} + +// GetCommandFactory retrieves a command factory for a given command version. +func GetCommandFactory(version int) CommandFactory { + return factories[version] +} + +// MinVersion returns the minimum compatible store version. +func MinVersion() int { + return minVersion +} + +// MaxVersion returns the maximum compatible store version. +func MaxVersion() int { + return maxVersion +} diff --git a/store/node.go b/store/node.go index 2dfccc1ed..0905d4d1b 100644 --- a/store/node.go +++ b/store/node.go @@ -375,8 +375,8 @@ func (n *Node) UpdateTTL(expireTime time.Time) { } } + n.ExpireTime = expireTime if expireTime.Sub(Permanent) != 0 { - n.ExpireTime = expireTime n.Expire() } } diff --git a/store/store.go b/store/store.go index 7fa21a638..f752f6935 100644 --- a/store/store.go +++ b/store/store.go @@ -13,7 +13,12 @@ import ( etcdErr "github.com/coreos/etcd/error" ) +// The default version to set when the store is first initialized. +const defaultVersion = 2 + type Store interface { + Version() int + CommandFactory() CommandFactory Get(nodePath string, recursive, sorted bool, index uint64, term uint64) (*Event, error) Set(nodePath string, value string, expireTime time.Time, index uint64, term uint64) (*Event, error) Update(nodePath string, newValue string, expireTime time.Time, index uint64, term uint64) (*Event, error) @@ -30,12 +35,13 @@ type Store interface { } type store struct { - Root *Node - WatcherHub *watcherHub - Index uint64 - Term uint64 - Stats *Stats - worldLock sync.RWMutex // stop the world lock + Root *Node + WatcherHub *watcherHub + Index uint64 + Term uint64 + Stats *Stats + CurrentVersion int + worldLock sync.RWMutex // stop the world lock } func New() Store { @@ -44,13 +50,23 @@ func New() Store { func newStore() *store { s := new(store) + s.CurrentVersion = defaultVersion s.Root = newDir(s, "/", UndefIndex, UndefTerm, nil, "", Permanent) s.Stats = newStats() s.WatcherHub = newWatchHub(1000) - return s } +// Version retrieves current version of the store. +func (s *store) Version() int { + return s.CurrentVersion +} + +// CommandFactory retrieves the command factory for the current version of the store. +func (s *store) CommandFactory() CommandFactory { + return GetCommandFactory(s.Version()) +} + // Get function returns a get event. // If recursive is true, it will return all the content under the node path. // If sorted is true, it will sort the content by keys. @@ -450,6 +466,7 @@ func (s *store) Save() ([]byte, error) { clonedStore.Root = s.Root.Clone() clonedStore.WatcherHub = s.WatcherHub.clone() clonedStore.Stats = s.Stats.clone() + clonedStore.CurrentVersion = s.CurrentVersion s.worldLock.Unlock() diff --git a/store/v2/command_factory.go b/store/v2/command_factory.go new file mode 100644 index 000000000..4f0e7260c --- /dev/null +++ b/store/v2/command_factory.go @@ -0,0 +1,73 @@ +package v2 + +import ( + "time" + + "github.com/coreos/etcd/store" + "github.com/coreos/go-raft" +) + +func init() { + store.RegisterCommandFactory(&CommandFactory{}) +} + +// CommandFactory provides a pluggable way to create version 2 commands. +type CommandFactory struct { +} + +// Version returns the version of this factory. +func (f *CommandFactory) Version() int { + return 2 +} + +// CreateUpgradeCommand is a no-op since version 2 is the first version to support store versioning. +func (f *CommandFactory) CreateUpgradeCommand() raft.Command { + return &raft.NOPCommand{} +} + +// CreateSetCommand creates a version 2 command to set a key to a given value in the store. +func (f *CommandFactory) CreateSetCommand(key string, value string, expireTime time.Time) raft.Command { + return &SetCommand{ + Key: key, + Value: value, + ExpireTime: expireTime, + } +} + +// CreateCreateCommand creates a version 2 command to create a new key in the store. +func (f *CommandFactory) CreateCreateCommand(key string, value string, expireTime time.Time, unique bool) raft.Command { + return &CreateCommand{ + Key: key, + Value: value, + ExpireTime: expireTime, + Unique: unique, + } +} + +// CreateUpdateCommand creates a version 2 command to update a key to a given value in the store. +func (f *CommandFactory) CreateUpdateCommand(key string, value string, expireTime time.Time) raft.Command { + return &UpdateCommand{ + Key: key, + Value: value, + ExpireTime: expireTime, + } +} + +// CreateDeleteCommand creates a version 2 command to delete a key from the store. +func (f *CommandFactory) CreateDeleteCommand(key string, recursive bool) raft.Command { + return &DeleteCommand{ + Key: key, + Recursive: recursive, + } +} + +// CreateCompareAndSwapCommand creates a version 2 command to conditionally set a key in the store. +func (f *CommandFactory) CreateCompareAndSwapCommand(key string, value string, prevValue string, prevIndex uint64, expireTime time.Time) raft.Command { + return &CompareAndSwapCommand{ + Key: key, + Value: value, + PrevValue: prevValue, + PrevIndex: prevIndex, + ExpireTime: expireTime, + } +} diff --git a/store/compare_and_swap_command.go b/store/v2/compare_and_swap_command.go similarity index 91% rename from store/compare_and_swap_command.go rename to store/v2/compare_and_swap_command.go index 8ee119f36..1de79fb1f 100644 --- a/store/compare_and_swap_command.go +++ b/store/v2/compare_and_swap_command.go @@ -1,9 +1,10 @@ -package store +package v2 import ( "time" "github.com/coreos/etcd/log" + "github.com/coreos/etcd/store" "github.com/coreos/go-raft" ) @@ -27,7 +28,7 @@ func (c *CompareAndSwapCommand) CommandName() string { // Set the key-value pair if the current value of the key equals to the given prevValue func (c *CompareAndSwapCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(Store) + s, _ := server.StateMachine().(store.Store) e, err := s.CompareAndSwap(c.Key, c.PrevValue, c.PrevIndex, c.Value, c.ExpireTime, server.CommitIndex(), server.Term()) diff --git a/store/create_command.go b/store/v2/create_command.go similarity index 88% rename from store/create_command.go rename to store/v2/create_command.go index 6a2487cf0..e187d99f7 100644 --- a/store/create_command.go +++ b/store/v2/create_command.go @@ -1,9 +1,11 @@ -package store +package v2 import ( - "github.com/coreos/etcd/log" - "github.com/coreos/go-raft" "time" + + "github.com/coreos/etcd/log" + "github.com/coreos/etcd/store" + "github.com/coreos/go-raft" ) func init() { @@ -25,7 +27,7 @@ func (c *CreateCommand) CommandName() string { // Create node func (c *CreateCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(Store) + s, _ := server.StateMachine().(store.Store) e, err := s.Create(c.Key, c.Value, c.Unique, c.ExpireTime, server.CommitIndex(), server.Term()) diff --git a/store/delete_command.go b/store/v2/delete_command.go similarity index 87% rename from store/delete_command.go rename to store/v2/delete_command.go index 6ff3c5c6f..6bd48368f 100644 --- a/store/delete_command.go +++ b/store/v2/delete_command.go @@ -1,6 +1,7 @@ -package store +package v2 import ( + "github.com/coreos/etcd/store" "github.com/coreos/etcd/log" "github.com/coreos/go-raft" ) @@ -22,7 +23,7 @@ func (c *DeleteCommand) CommandName() string { // Delete the key func (c *DeleteCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(Store) + s, _ := server.StateMachine().(store.Store) e, err := s.Delete(c.Key, c.Recursive, server.CommitIndex(), server.Term()) diff --git a/store/set_command.go b/store/v2/set_command.go similarity index 88% rename from store/set_command.go rename to store/v2/set_command.go index 55635cd99..4f6ecf59f 100644 --- a/store/set_command.go +++ b/store/v2/set_command.go @@ -1,9 +1,11 @@ -package store +package v2 import ( - "github.com/coreos/etcd/log" - "github.com/coreos/go-raft" "time" + + "github.com/coreos/etcd/log" + "github.com/coreos/etcd/store" + "github.com/coreos/go-raft" ) func init() { @@ -24,7 +26,7 @@ func (c *SetCommand) CommandName() string { // Create node func (c *SetCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(Store) + s, _ := server.StateMachine().(store.Store) // create a new node or replace the old node. e, err := s.Set(c.Key, c.Value, c.ExpireTime, server.CommitIndex(), server.Term()) diff --git a/store/update_command.go b/store/v2/update_command.go similarity index 88% rename from store/update_command.go rename to store/v2/update_command.go index 8e353cdd0..d080ecced 100644 --- a/store/update_command.go +++ b/store/v2/update_command.go @@ -1,7 +1,8 @@ -package store +package v2 import ( "github.com/coreos/etcd/log" + "github.com/coreos/etcd/store" "github.com/coreos/go-raft" "time" ) @@ -24,7 +25,7 @@ func (c *UpdateCommand) CommandName() string { // Create node func (c *UpdateCommand) Apply(server raft.Server) (interface{}, error) { - s, _ := server.StateMachine().(Store) + s, _ := server.StateMachine().(store.Store) e, err := s.Update(c.Key, c.Value, c.ExpireTime, server.CommitIndex(), server.Term()) diff --git a/test.sh b/test.sh index 66f745ed4..5cc633975 100755 --- a/test.sh +++ b/test.sh @@ -8,6 +8,9 @@ set -e export GOPATH="${PWD}" # Unit tests +go test -i ./server +go test -v ./server + go test -i ./server/v2/tests go test -v ./server/v2/tests diff --git a/tests/functional/version_check_test.go b/tests/functional/version_check_test.go new file mode 100644 index 000000000..98a155390 --- /dev/null +++ b/tests/functional/version_check_test.go @@ -0,0 +1,46 @@ +package test + +import ( + "net/http" + "os" + "testing" + "time" +) + +// Ensure that a node can reply to a version check appropriately. +func TestVersionCheck(t *testing.T) { + procAttr := new(os.ProcAttr) + procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} + args := []string{"etcd", "-n=node1", "-f", "-d=/tmp/version_check"} + + process, err := os.StartProcess(EtcdBinPath, args, procAttr) + if err != nil { + t.Fatal("start process failed:" + err.Error()) + return + } + defer process.Kill() + + time.Sleep(time.Second) + + // Check a version too small. + resp, _ := http.Get("http://localhost:7001/version/1/check") + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatal("Invalid version check: ", resp.StatusCode) + } + + // Check a version too large. + resp, _ = http.Get("http://localhost:7001/version/3/check") + resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatal("Invalid version check: ", resp.StatusCode) + } + + // Check a version that's just right. + resp, _ = http.Get("http://localhost:7001/version/2/check") + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatal("Invalid version check: ", resp.StatusCode) + } +} + diff --git a/tests/http_utils.go b/tests/http_utils.go index b1395373d..ebe6f5273 100644 --- a/tests/http_utils.go +++ b/tests/http_utils.go @@ -55,6 +55,14 @@ func PutForm(url string, data url.Values) (*http.Response, error) { return Put(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) } +func Delete(url string, bodyType string, body io.Reader) (*http.Response, error) { + return send("DELETE", url, bodyType, body) +} + +func DeleteForm(url string, data url.Values) (*http.Response, error) { + return Delete(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} + func send(method string, url string, bodyType string, body io.Reader) (*http.Response, error) { c := NewHTTPClient() diff --git a/tests/server_utils.go b/tests/server_utils.go index adddf336b..e7ed347a3 100644 --- a/tests/server_utils.go +++ b/tests/server_utils.go @@ -5,14 +5,15 @@ import ( "os" "time" - "github.com/coreos/etcd/store" "github.com/coreos/etcd/server" + "github.com/coreos/etcd/store" ) const ( - testName = "ETCDTEST" + testName = "ETCDTEST" testClientURL = "localhost:4401" - testRaftURL = "localhost:7701" + testRaftURL = "localhost:7701" + testSnapCount = 10000 ) // Starts a server in a temporary directory. @@ -22,8 +23,8 @@ func RunServer(f func(*server.Server)) { store := store.New() registry := server.NewRegistry(store) - ps := server.NewPeerServer(testName, path, testRaftURL, testRaftURL, &server.TLSConfig{Scheme:"http"}, &server.TLSInfo{}, registry, store) - s := server.New(testName, testClientURL, testClientURL, &server.TLSConfig{Scheme:"http"}, &server.TLSInfo{}, ps, registry, store) + ps := server.NewPeerServer(testName, path, testRaftURL, testRaftURL, &server.TLSConfig{Scheme: "http"}, &server.TLSInfo{}, registry, store, testSnapCount) + s := server.New(testName, testClientURL, testClientURL, &server.TLSConfig{Scheme: "http"}, &server.TLSInfo{}, ps, registry, store) ps.SetServer(s) // Start up peer server. @@ -32,17 +33,17 @@ func RunServer(f func(*server.Server)) { c <- true ps.ListenAndServe(false, []string{}) }() - <- c + <-c // Start up etcd server. go func() { c <- true s.ListenAndServe() }() - <- c + <-c // Wait to make sure servers have started. - time.Sleep(5 * time.Millisecond) + time.Sleep(50 * time.Millisecond) // Execute the function passed in. f(s) diff --git a/third_party/deps b/third_party/deps index 0cb7ffea3..76464bdfc 100755 --- a/third_party/deps +++ b/third_party/deps @@ -1,4 +1,5 @@ packages=" + github.com/BurntSushi/toml github.com/coreos/go-raft github.com/coreos/go-etcd github.com/coreos/go-log/log diff --git a/third_party/github.com/BurntSushi/toml/.gitignore b/third_party/github.com/BurntSushi/toml/.gitignore new file mode 100644 index 000000000..55e90a1e5 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/.gitignore @@ -0,0 +1,4 @@ +TAGS +tags +.*.swp +tomlcheck/tomlcheck diff --git a/third_party/github.com/BurntSushi/toml/COMPATIBLE b/third_party/github.com/BurntSushi/toml/COMPATIBLE new file mode 100644 index 000000000..21e0938ca --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/COMPATIBLE @@ -0,0 +1,3 @@ +Compatible with TOML version +[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) + diff --git a/third_party/github.com/BurntSushi/toml/COPYING b/third_party/github.com/BurntSushi/toml/COPYING new file mode 100644 index 000000000..5a8e33254 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/third_party/github.com/BurntSushi/toml/Makefile b/third_party/github.com/BurntSushi/toml/Makefile new file mode 100644 index 000000000..e6adc3e9e --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/Makefile @@ -0,0 +1,14 @@ +install: + go install + +fmt: + gofmt -w *.go */*.go + colcheck *.go */*.go + +tags: + find ./ -name '*.go' -print0 | xargs -0 gotags > TAGS + +push: + git push origin master + git push github master + diff --git a/third_party/github.com/BurntSushi/toml/README.md b/third_party/github.com/BurntSushi/toml/README.md new file mode 100644 index 000000000..f8c847d66 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/README.md @@ -0,0 +1,163 @@ +# TOML parser for Go with reflection + +TOML stands for Tom's Obvious, Minimal Language. + +Spec: https://github.com/mojombo/toml + +Compatible with TOML version +[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) + +Documentation: http://godoc.org/github.com/BurntSushi/toml + +Installation: + +```bash +go get github.com/BurntSushi/toml +``` + +Try the toml validator: + +```bash +go get github.com/BurntSushi/toml/tomlv +tomlv some-toml-file.toml +``` + + +## Testing + +This package passes all tests in +[toml-test](https://github.com/BurntSushi/toml-test). + +## Examples + +This package works similarly to how the Go standard library handles `XML` +and `JSON`. Namely, data is loaded into Go values via reflection. + +For the simplest example, consider some TOML file as just a list of keys +and values: + +```toml +Age = 25 +Cats = [ "Cauchy", "Plato" ] +Pi = 3.14 +Perfection = [ 6, 28, 496, 8128 ] +DOB = 1987-07-05T05:45:00Z +``` + +Which could be defined in Go as: + +```go +type Config struct { + Age int + Cats []string + Pi float64 + Perfection []int + DOB time.Time // requires `import time` +} +``` + +And then decoded with: + +```go +var conf Config +if _, err := toml.Decode(tomlData, &conf); err != nil { + // handle error +} +``` + +You can also use struct tags if your struct field name doesn't map to a TOML +key value directly: + +```toml +some_key_NAME = "wat" +``` + +```go +type TOML struct { + ObscureKey string `toml:"some_key_NAME"` +} +``` + +## More complex usage + +Here's an example of how to load the example from the official spec page: + +```toml +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] +``` + +And the corresponding Go types are: + +```go +type tomlConfig struct { + Title string + Owner ownerInfo + DB database `toml:"database"` + Servers map[string]server + Clients clients +} + +type ownerInfo struct { + Name string + Org string `toml:"organization"` + Bio string + DOB time.Time +} + +type database struct { + Server string + Ports []int + ConnMax int `toml:"connection_max"` + Enabled bool +} + +type server struct { + IP string + DC string +} + +type clients struct { + Data [][]interface{} + Hosts []string +} +``` + +Note that a case insensitive match will be tried if an exact match can't be +found. + +A working example of the above can be found in `_examples/example.{go,toml}`. + diff --git a/third_party/github.com/BurntSushi/toml/_examples/example.go b/third_party/github.com/BurntSushi/toml/_examples/example.go new file mode 100644 index 000000000..c81d25a52 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/_examples/example.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + "time" + + "github.com/BurntSushi/toml" +) + +type tomlConfig struct { + Title string + Owner ownerInfo + DB database `toml:"database"` + Servers map[string]server + Clients clients +} + +type ownerInfo struct { + Name string + Org string `toml:"organization"` + Bio string + DOB time.Time +} + +type database struct { + Server string + Ports []int + ConnMax int `toml:"connection_max"` + Enabled bool +} + +type server struct { + IP string + DC string +} + +type clients struct { + Data [][]interface{} + Hosts []string +} + +func main() { + var config tomlConfig + if _, err := toml.DecodeFile("example.toml", &config); err != nil { + fmt.Println(err) + return + } + + fmt.Printf("Title: %s\n", config.Title) + fmt.Printf("Owner: %s (%s, %s), Born: %s\n", + config.Owner.Name, config.Owner.Org, config.Owner.Bio, config.Owner.DOB) + fmt.Printf("Database: %s %v (Max conn. %d), Enabled? %v\n", + config.DB.Server, config.DB.Ports, config.DB.ConnMax, config.DB.Enabled) + for serverName, server := range config.Servers { + fmt.Printf("Server: %s (%s, %s)\n", serverName, server.IP, server.DC) + } + fmt.Printf("Client data: %v\n", config.Clients.Data) + fmt.Printf("Client hosts: %v\n", config.Clients.Hosts) +} diff --git a/third_party/github.com/BurntSushi/toml/_examples/example.toml b/third_party/github.com/BurntSushi/toml/_examples/example.toml new file mode 100644 index 000000000..32c7a4faa --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/_examples/example.toml @@ -0,0 +1,35 @@ +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] diff --git a/third_party/github.com/BurntSushi/toml/_examples/hard.toml b/third_party/github.com/BurntSushi/toml/_examples/hard.toml new file mode 100644 index 000000000..26145d2b4 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/_examples/hard.toml @@ -0,0 +1,22 @@ +# Test file for TOML +# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate +# This part you'll really hate + +[the] +test_string = "You'll hate me after this - #" # " Annoying, isn't it? + + [the.hard] + test_array = [ "] ", " # "] # ] There you go, parse this! + test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ] + # You didn't think it'd as easy as chucking out the last #, did you? + another_test_string = " Same thing, but with a string #" + harder_test_string = " And when \"'s are in the string, along with # \"" # "and comments are there too" + # Things will get harder + + [the.hard.bit#] + what? = "You don't think some user won't do that?" + multi_line_array = [ + "]", + # ] Oh yes I did + ] + diff --git a/third_party/github.com/BurntSushi/toml/_examples/implicit.toml b/third_party/github.com/BurntSushi/toml/_examples/implicit.toml new file mode 100644 index 000000000..1dea5ceb4 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/_examples/implicit.toml @@ -0,0 +1,4 @@ +# [x] you +# [x.y] don't +# [x.y.z] need these +[x.y.z.w] # for this to work diff --git a/third_party/github.com/BurntSushi/toml/_examples/invalid-apples.toml b/third_party/github.com/BurntSushi/toml/_examples/invalid-apples.toml new file mode 100644 index 000000000..74e9e337e --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/_examples/invalid-apples.toml @@ -0,0 +1,6 @@ +# DO NOT WANT +[fruit] +type = "apple" + +[fruit.type] +apple = "yes" diff --git a/third_party/github.com/BurntSushi/toml/_examples/invalid.toml b/third_party/github.com/BurntSushi/toml/_examples/invalid.toml new file mode 100644 index 000000000..beb1dba54 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/_examples/invalid.toml @@ -0,0 +1,35 @@ +# This is an INVALID TOML document. Boom. +# Can you spot the error without help? + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T7:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it + +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] diff --git a/third_party/github.com/BurntSushi/toml/_examples/readme1.toml b/third_party/github.com/BurntSushi/toml/_examples/readme1.toml new file mode 100644 index 000000000..3e1261d4c --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/_examples/readme1.toml @@ -0,0 +1,5 @@ +Age = 25 +Cats = [ "Cauchy", "Plato" ] +Pi = 3.14 +Perfection = [ 6, 28, 496, 8128 ] +DOB = 1987-07-05T05:45:00Z diff --git a/third_party/github.com/BurntSushi/toml/_examples/readme2.toml b/third_party/github.com/BurntSushi/toml/_examples/readme2.toml new file mode 100644 index 000000000..b51cd9340 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/_examples/readme2.toml @@ -0,0 +1 @@ +some_key_NAME = "wat" diff --git a/third_party/github.com/BurntSushi/toml/decode.go b/third_party/github.com/BurntSushi/toml/decode.go new file mode 100644 index 000000000..ddb26cf2b --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/decode.go @@ -0,0 +1,429 @@ +package toml + +import ( + "fmt" + "io" + "io/ioutil" + "reflect" + "strings" + "time" +) + +var e = fmt.Errorf + +// Primitive is a TOML value that hasn't been decoded into a Go value. +// When using the various `Decode*` functions, the type `Primitive` may +// be given to any value, and its decoding will be delayed. +// +// A `Primitive` value can be decoded using the `PrimitiveDecode` function. +// +// The underlying representation of a `Primitive` value is subject to change. +// Do not rely on it. +// +// N.B. Primitive values are still parsed, so using them will only avoid +// the overhead of reflection. They can be useful when you don't know the +// exact type of TOML data until run time. +type Primitive interface{} + +// PrimitiveDecode is just like the other `Decode*` functions, except it +// decodes a TOML value that has already been parsed. Valid primitive values +// can *only* be obtained from values filled by the decoder functions, +// including `PrimitiveDecode`. (i.e., `v` may contain more `Primitive` +// values.) +// +// Meta data for primitive values is included in the meta data returned by +// the `Decode*` functions. +func PrimitiveDecode(primValue Primitive, v interface{}) error { + return unify(primValue, rvalue(v)) +} + +// Decode will decode the contents of `data` in TOML format into a pointer +// `v`. +// +// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be +// used interchangeably.) +// +// TOML datetimes correspond to Go `time.Time` values. +// +// All other TOML types (float, string, int, bool and array) correspond +// to the obvious Go types. +// +// TOML keys can map to either keys in a Go map or field names in a Go +// struct. The special `toml` struct tag may be used to map TOML keys to +// struct fields that don't match the key name exactly. (See the example.) +// A case insensitive match to struct names will be tried if an exact match +// can't be found. +// +// The mapping between TOML values and Go values is loose. That is, there +// may exist TOML values that cannot be placed into your representation, and +// there may be parts of your representation that do not correspond to +// TOML values. +// +// This decoder will not handle cyclic types. If a cyclic type is passed, +// `Decode` will not terminate. +func Decode(data string, v interface{}) (MetaData, error) { + p, err := parse(data) + if err != nil { + return MetaData{}, err + } + return MetaData{p.mapping, p.types, p.ordered}, unify(p.mapping, rvalue(v)) +} + +// DecodeFile is just like Decode, except it will automatically read the +// contents of the file at `fpath` and decode it for you. +func DecodeFile(fpath string, v interface{}) (MetaData, error) { + bs, err := ioutil.ReadFile(fpath) + if err != nil { + return MetaData{}, err + } + return Decode(string(bs), v) +} + +// DecodeReader is just like Decode, except it will consume all bytes +// from the reader and decode it for you. +func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { + bs, err := ioutil.ReadAll(r) + if err != nil { + return MetaData{}, err + } + return Decode(string(bs), v) +} + +// unify performs a sort of type unification based on the structure of `rv`, +// which is the client representation. +// +// Any type mismatch produces an error. Finding a type that we don't know +// how to handle produces an unsupported type error. +func unify(data interface{}, rv reflect.Value) error { + // Special case. Look for a `Primitive` value. + if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() { + return unifyAnything(data, rv) + } + + // Special case. Go's `time.Time` is a struct, which we don't want + // to confuse with a user struct. + if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { + return unifyDatetime(data, rv) + } + + k := rv.Kind() + + // laziness + if k >= reflect.Int && k <= reflect.Uint64 { + return unifyInt(data, rv) + } + switch k { + case reflect.Ptr: + elem := reflect.New(rv.Type().Elem()) + err := unify(data, reflect.Indirect(elem)) + if err != nil { + return err + } + rv.Set(elem) + return nil + case reflect.Struct: + return unifyStruct(data, rv) + case reflect.Map: + return unifyMap(data, rv) + case reflect.Slice: + return unifySlice(data, rv) + case reflect.String: + return unifyString(data, rv) + case reflect.Bool: + return unifyBool(data, rv) + case reflect.Interface: + // we only support empty interfaces. + if rv.NumMethod() > 0 { + return e("Unsupported type '%s'.", rv.Kind()) + } + return unifyAnything(data, rv) + case reflect.Float32: + fallthrough + case reflect.Float64: + return unifyFloat64(data, rv) + } + return e("Unsupported type '%s'.", rv.Kind()) +} + +func unifyStruct(mapping interface{}, rv reflect.Value) error { + tmap, ok := mapping.(map[string]interface{}) + if !ok { + return mismatch(rv, "map", mapping) + } + + rt := rv.Type() + for i := 0; i < rt.NumField(); i++ { + // A little tricky. We want to use the special `toml` name in the + // struct tag if it exists. In particular, we need to make sure that + // this struct field is in the current map before trying to unify it. + sft := rt.Field(i) + kname := sft.Tag.Get("toml") + if len(kname) == 0 { + kname = sft.Name + } + if datum, ok := insensitiveGet(tmap, kname); ok { + sf := indirect(rv.Field(i)) + + // Don't try to mess with unexported types and other such things. + if sf.CanSet() { + if err := unify(datum, sf); err != nil { + return e("Type mismatch for '%s.%s': %s", + rt.String(), sft.Name, err) + } + } else if len(sft.Tag.Get("toml")) > 0 { + // Bad user! No soup for you! + return e("Field '%s.%s' is unexported, and therefore cannot "+ + "be loaded with reflection.", rt.String(), sft.Name) + } + } + } + return nil +} + +func unifyMap(mapping interface{}, rv reflect.Value) error { + tmap, ok := mapping.(map[string]interface{}) + if !ok { + return badtype("map", mapping) + } + if rv.IsNil() { + rv.Set(reflect.MakeMap(rv.Type())) + } + for k, v := range tmap { + rvkey := indirect(reflect.New(rv.Type().Key())) + rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) + if err := unify(v, rvval); err != nil { + return err + } + + rvkey.SetString(k) + rv.SetMapIndex(rvkey, rvval) + } + return nil +} + +func unifySlice(data interface{}, rv reflect.Value) error { + datav := reflect.ValueOf(data) + if datav.Kind() != reflect.Slice { + return badtype("slice", data) + } + sliceLen := datav.Len() + if rv.IsNil() { + rv.Set(reflect.MakeSlice(rv.Type(), sliceLen, sliceLen)) + } + for i := 0; i < sliceLen; i++ { + v := datav.Index(i).Interface() + sliceval := indirect(rv.Index(i)) + if err := unify(v, sliceval); err != nil { + return err + } + } + return nil +} + +func unifyDatetime(data interface{}, rv reflect.Value) error { + if _, ok := data.(time.Time); ok { + rv.Set(reflect.ValueOf(data)) + return nil + } + return badtype("time.Time", data) +} + +func unifyString(data interface{}, rv reflect.Value) error { + if s, ok := data.(string); ok { + rv.SetString(s) + return nil + } + return badtype("string", data) +} + +func unifyFloat64(data interface{}, rv reflect.Value) error { + if num, ok := data.(float64); ok { + switch rv.Kind() { + case reflect.Float32: + fallthrough + case reflect.Float64: + rv.SetFloat(num) + default: + panic("bug") + } + return nil + } + return badtype("float", data) +} + +func unifyInt(data interface{}, rv reflect.Value) error { + if num, ok := data.(int64); ok { + switch rv.Kind() { + case reflect.Int: + fallthrough + case reflect.Int8: + fallthrough + case reflect.Int16: + fallthrough + case reflect.Int32: + fallthrough + case reflect.Int64: + rv.SetInt(int64(num)) + case reflect.Uint: + fallthrough + case reflect.Uint8: + fallthrough + case reflect.Uint16: + fallthrough + case reflect.Uint32: + fallthrough + case reflect.Uint64: + rv.SetUint(uint64(num)) + default: + panic("bug") + } + return nil + } + return badtype("integer", data) +} + +func unifyBool(data interface{}, rv reflect.Value) error { + if b, ok := data.(bool); ok { + rv.SetBool(b) + return nil + } + return badtype("integer", data) +} + +func unifyAnything(data interface{}, rv reflect.Value) error { + // too awesome to fail + rv.Set(reflect.ValueOf(data)) + return nil +} + +// rvalue returns a reflect.Value of `v`. All pointers are resolved. +func rvalue(v interface{}) reflect.Value { + return indirect(reflect.ValueOf(v)) +} + +// indirect returns the value pointed to by a pointer. +// Pointers are followed until the value is not a pointer. +// New values are allocated for each nil pointer. +func indirect(v reflect.Value) reflect.Value { + if v.Kind() != reflect.Ptr { + return v + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + return indirect(reflect.Indirect(v)) +} + +func tstring(rv reflect.Value) string { + return rv.Type().String() +} + +func badtype(expected string, data interface{}) error { + return e("Expected %s but found '%T'.", expected, data) +} + +func mismatch(user reflect.Value, expected string, data interface{}) error { + return e("Type mismatch for %s. Expected %s but found '%T'.", + tstring(user), expected, data) +} + +func insensitiveGet( + tmap map[string]interface{}, kname string) (interface{}, bool) { + + if datum, ok := tmap[kname]; ok { + return datum, true + } + for k, v := range tmap { + if strings.EqualFold(kname, k) { + return v, true + } + } + return nil, false +} + +// MetaData allows access to meta information about TOML data that may not +// be inferrable via reflection. In particular, whether a key has been defined +// and the TOML type of a key. +// +// (XXX: If TOML gets NULL values, that information will be added here too.) +type MetaData struct { + mapping map[string]interface{} + types map[string]tomlType + keys []Key +} + +// IsDefined returns true if the key given exists in the TOML data. The key +// should be specified hierarchially. e.g., +// +// // access the TOML key 'a.b.c' +// IsDefined("a", "b", "c") +// +// IsDefined will return false if an empty key given. Keys are case sensitive. +func (md MetaData) IsDefined(key ...string) bool { + var hashOrVal interface{} + var hash map[string]interface{} + var ok bool + + if len(key) == 0 { + return false + } + + hashOrVal = md.mapping + for _, k := range key { + if hash, ok = hashOrVal.(map[string]interface{}); !ok { + return false + } + if hashOrVal, ok = hash[k]; !ok { + return false + } + } + return true +} + +// Type returns a string representation of the type of the key specified. +// +// Type will return the empty string if given an empty key or a key that +// does not exist. Keys are case sensitive. +func (md MetaData) Type(key ...string) string { + fullkey := strings.Join(key, ".") + if typ, ok := md.types[fullkey]; ok { + return typ.typeString() + } + return "" +} + +// Key is the type of any TOML key, including key groups. Use (MetaData).Keys +// to get values of this type. +type Key []string + +func (k Key) String() string { + return strings.Join(k, ".") +} + +func (k Key) add(piece string) Key { + newKey := make(Key, len(k)) + copy(newKey, k) + return append(newKey, piece) +} + +// Keys returns a slice of every key in the TOML data, including key groups. +// Each key is itself a slice, where the first element is the top of the +// hierarchy and the last is the most specific. +// +// The list will have the same order as the keys appeared in the TOML data. +// +// All keys returned are non-empty. +func (md MetaData) Keys() []Key { + return md.keys +} + +func allKeys(m map[string]interface{}, context Key) []Key { + keys := make([]Key, 0, len(m)) + for k, v := range m { + keys = append(keys, context.add(k)) + if t, ok := v.(map[string]interface{}); ok { + keys = append(keys, allKeys(t, context.add(k))...) + } + } + return keys +} diff --git a/third_party/github.com/BurntSushi/toml/decode_test.go b/third_party/github.com/BurntSushi/toml/decode_test.go new file mode 100644 index 000000000..ad80adee3 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/decode_test.go @@ -0,0 +1,343 @@ +package toml + +import ( + "fmt" + "log" + "reflect" + "testing" + "time" +) + +func init() { + log.SetFlags(0) +} + +var testSimple = ` +age = 250 + +andrew = "gallant" +kait = "brady" +now = 1987-07-05T05:45:00Z +yesOrNo = true +pi = 3.14 +colors = [ + ["red", "green", "blue"], + ["cyan", "magenta", "yellow", "black"], +] + +[Annoying.Cats] +plato = "smelly" +cauchy = "stupido" + +` + +type kitties struct { + Plato string + Cauchy string +} + +type simple struct { + Age int + Colors [][]string + Pi float64 + YesOrNo bool + Now time.Time + Andrew string + Kait string + Annoying map[string]kitties +} + +func TestDecode(t *testing.T) { + var val simple + + md, err := Decode(testSimple, &val) + if err != nil { + t.Fatal(err) + } + + testf("Is 'Annoying.Cats.plato' defined? %v\n", + md.IsDefined("Annoying", "Cats", "plato")) + testf("Is 'Cats.Stinky' defined? %v\n", md.IsDefined("Cats", "Stinky")) + testf("Type of 'colors'? %s\n\n", md.Type("colors")) + + testf("%v\n", val) +} + +var tomlTableArrays = ` +[[albums]] +name = "Born to Run" + + [[albums.songs]] + name = "Jungleland" + + [[albums.songs]] + name = "Meeting Across the River" + +[[albums]] +name = "Born in the USA" + + [[albums.songs]] + name = "Glory Days" + + [[albums.songs]] + name = "Dancing in the Dark" +` + +type Music struct { + Albums []Album +} + +type Album struct { + Name string + Songs []Song +} + +type Song struct { + Name string +} + +func TestTableArrays(t *testing.T) { + expected := Music{[]Album{ + {"Born to Run", []Song{{"Jungleland"}, {"Meeting Across the River"}}}, + {"Born in the USA", []Song{{"Glory Days"}, {"Dancing in the Dark"}}}, + }} + var got Music + if _, err := Decode(tomlTableArrays, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, got) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, got) + } +} + +// Case insensitive matching tests. +// A bit more comprehensive than needed given the current implementation, +// but implementations change. +// Probably still missing demonstrations of some ugly corner cases regarding +// case insensitive matching and multiple fields. +var caseToml = ` +tOpString = "string" +tOpInt = 1 +tOpFloat = 1.1 +tOpBool = true +tOpdate = 2006-01-02T15:04:05Z +tOparray = [ "array" ] +Match = "i should be in Match only" +MatcH = "i should be in MatcH only" +Field = "neat" +FielD = "messy" +once = "just once" +[nEst.eD] +nEstedString = "another string" +` + +type Insensitive struct { + TopString string + TopInt int + TopFloat float64 + TopBool bool + TopDate time.Time + TopArray []string + Match string + MatcH string + Field string + Once string + OncE string + Nest InsensitiveNest +} + +type InsensitiveNest struct { + Ed InsensitiveEd +} + +type InsensitiveEd struct { + NestedString string +} + +func TestCase(t *testing.T) { + tme, err := time.Parse(time.RFC3339, time.RFC3339[:len(time.RFC3339)-5]) + if err != nil { + panic(err) + } + expected := Insensitive{ + TopString: "string", + TopInt: 1, + TopFloat: 1.1, + TopBool: true, + TopDate: tme, + TopArray: []string{"array"}, + MatcH: "i should be in MatcH only", + Match: "i should be in Match only", + Field: "neat", // encoding/json would store "messy" here + Once: "just once", + OncE: "just once", // wait, what? + Nest: InsensitiveNest{ + Ed: InsensitiveEd{NestedString: "another string"}, + }, + } + var got Insensitive + _, err = Decode(caseToml, &got) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, got) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, got) + } +} + +func TestPointers(t *testing.T) { + type Object struct { + Type string + Description string + } + + type Dict struct { + NamedObject map[string]*Object + BaseObject *Object + Strptr *string + Strptrs []*string + } + s1, s2, s3 := "blah", "abc", "def" + expected := &Dict{ + Strptr: &s1, + Strptrs: []*string{&s2, &s3}, + NamedObject: map[string]*Object{ + "foo": {"FOO", "fooooo!!!"}, + "bar": {"BAR", "ba-ba-ba-ba-barrrr!!!"}, + }, + BaseObject: &Object{"BASE", "da base"}, + } + + ex1 := ` +Strptr = "blah" +Strptrs = ["abc", "def"] + +[NamedObject.foo] +Type = "FOO" +Description = "fooooo!!!" + +[NamedObject.bar] +Type = "BAR" +Description = "ba-ba-ba-ba-barrrr!!!" + +[BaseObject] +Type = "BASE" +Description = "da base" +` + dict := new(Dict) + _, err := Decode(ex1, dict) + if err != nil { + t.Errorf("Decode error: %v", err) + } + if !reflect.DeepEqual(expected, dict) { + t.Fatalf("\n%#v\n!=\n%#v\n", expected, dict) + } +} + +func ExamplePrimitiveDecode() { + var md MetaData + var err error + + var tomlBlob = ` +ranking = ["Springsteen", "J Geils"] + +[bands.Springsteen] +started = 1973 +albums = ["Greetings", "WIESS", "Born to Run", "Darkness"] + +[bands.J Geils] +started = 1970 +albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"] +` + + type band struct { + Started int + Albums []string + } + + type classics struct { + Ranking []string + Bands map[string]Primitive + } + + // Do the initial decode. Reflection is delayed on Primitive values. + var music classics + if md, err = Decode(tomlBlob, &music); err != nil { + log.Fatal(err) + } + + // MetaData still includes information on Primitive values. + fmt.Printf("Is `bands.Springsteen` defined? %v\n", + md.IsDefined("bands", "Springsteen")) + + // Decode primitive data into Go values. + for _, artist := range music.Ranking { + // A band is a primitive value, so we need to decode it to get a + // real `band` value. + primValue := music.Bands[artist] + + var aBand band + if err = PrimitiveDecode(primValue, &aBand); err != nil { + log.Fatal(err) + } + fmt.Printf("%s started in %d.\n", artist, aBand.Started) + } + + // Output: + // Is `bands.Springsteen` defined? true + // Springsteen started in 1973. + // J Geils started in 1970. +} + +func ExampleDecode() { + var tomlBlob = ` +# Some comments. +[alpha] +ip = "10.0.0.1" + + [alpha.config] + Ports = [ 8001, 8002 ] + Location = "Toronto" + Created = 1987-07-05T05:45:00Z + +[beta] +ip = "10.0.0.2" + + [beta.config] + Ports = [ 9001, 9002 ] + Location = "New Jersey" + Created = 1887-01-05T05:55:00Z +` + + type serverConfig struct { + Ports []int + Location string + Created time.Time + } + + type server struct { + IP string `toml:"ip"` + Config serverConfig `toml:"config"` + } + + type servers map[string]server + + var config servers + if _, err := Decode(tomlBlob, &config); err != nil { + log.Fatal(err) + } + + for _, name := range []string{"alpha", "beta"} { + s := config[name] + fmt.Printf("Server: %s (ip: %s) in %s created on %s\n", + name, s.IP, s.Config.Location, + s.Config.Created.Format("2006-01-02")) + fmt.Printf("Ports: %v\n", s.Config.Ports) + } + + // // Output: + // Server: alpha (ip: 10.0.0.1) in Toronto created on 1987-07-05 + // Ports: [8001 8002] + // Server: beta (ip: 10.0.0.2) in New Jersey created on 1887-01-05 + // Ports: [9001 9002] +} diff --git a/third_party/github.com/BurntSushi/toml/doc.go b/third_party/github.com/BurntSushi/toml/doc.go new file mode 100644 index 000000000..1c2d7dffc --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/doc.go @@ -0,0 +1,10 @@ +/* +Package toml provides facilities for decoding TOML configuration files +via reflection. + +Specification: https://github.com/mojombo/toml + +Use github.com/BurntSushi/toml/tomlv to check whether a file is valid +TOML or not, with helpful error messages. +*/ +package toml diff --git a/third_party/github.com/BurntSushi/toml/encode.go b/third_party/github.com/BurntSushi/toml/encode.go new file mode 100644 index 000000000..7b080f414 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/encode.go @@ -0,0 +1,99 @@ +package toml + +// TODO: Build a decent encoder. +// Interestingly, this isn't as trivial as recursing down the type of the +// value given and outputting the corresponding TOML. In particular, multiple +// TOML types (especially if tuples are added) can map to a single Go type, so +// that the reverse correspondence isn't clear. +// +// One possible avenue is to choose a reasonable default (like structs map +// to hashes), but allow the user to override with struct tags. But this seems +// like a mess. +// +// The other possibility is to scrap an encoder altogether. After all, TOML +// is a configuration file format, and not a data exchange format. + +import ( + "bufio" + "fmt" + "io" + "reflect" + "strings" +) + +type encoder struct { + // A single indentation level. By default it is two spaces. + Indent string + + w *bufio.Writer +} + +func newEncoder(w io.Writer) *encoder { + return &encoder{ + w: bufio.NewWriter(w), + Indent: " ", + } +} + +func (enc *encoder) Encode(v interface{}) error { + rv := eindirect(reflect.ValueOf(v)) + if err := enc.encode(Key([]string{}), rv); err != nil { + return err + } + return enc.w.Flush() +} + +func (enc *encoder) encode(key Key, rv reflect.Value) error { + k := rv.Kind() + switch k { + case reflect.Struct: + return enc.eStruct(key, rv) + case reflect.String: + return enc.eString(key, rv) + } + return e("Unsupported type for key '%s': %s", key, k) +} + +func (enc *encoder) eStruct(key Key, rv reflect.Value) error { + rt := rv.Type() + for i := 0; i < rt.NumField(); i++ { + sft := rt.Field(i) + sf := rv.Field(i) + if err := enc.encode(key.add(sft.Name), sf); err != nil { + return err + } + } + return nil +} + +func (enc *encoder) eString(key Key, rv reflect.Value) error { + s := rv.String() + s = strings.NewReplacer( + "\t", "\\t", + "\n", "\\n", + "\r", "\\r", + "\"", "\\\"", + "\\", "\\\\", + ).Replace(s) + s = "\"" + s + "\"" + if err := enc.eKeyVal(key, s); err != nil { + return err + } + return nil +} + +func (enc *encoder) eKeyVal(key Key, value string) error { + out := fmt.Sprintf("%s%s = %s", + strings.Repeat(enc.Indent, len(key)-1), key[len(key)-1], value) + if _, err := fmt.Fprintln(enc.w, out); err != nil { + return err + } + return nil +} + +func eindirect(v reflect.Value) reflect.Value { + if v.Kind() != reflect.Ptr { + return v + } + return eindirect(reflect.Indirect(v)) +} diff --git a/third_party/github.com/BurntSushi/toml/encode_test.go b/third_party/github.com/BurntSushi/toml/encode_test.go new file mode 100644 index 000000000..3f7fad8b4 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/encode_test.go @@ -0,0 +1,25 @@ +package toml + +import ( + "bytes" + "testing" +) + +type encodeSimple struct { + Location string + // Ages []int + // DOB time.Time +} + +func TestEncode(t *testing.T) { + v := encodeSimple{ + Location: "Westborough, MA", + } + + buf := new(bytes.Buffer) + e := newEncoder(buf) + if err := e.Encode(v); err != nil { + t.Fatal(err) + } + testf(buf.String()) +} diff --git a/third_party/github.com/BurntSushi/toml/lex.go b/third_party/github.com/BurntSushi/toml/lex.go new file mode 100644 index 000000000..3dcae4923 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/lex.go @@ -0,0 +1,741 @@ +package toml + +import ( + "fmt" + "unicode/utf8" +) + +type itemType int + +const ( + itemError itemType = iota + itemNIL // used in the parser to indicate no type + itemEOF + itemText + itemString + itemBool + itemInteger + itemFloat + itemDatetime + itemArray // the start of an array + itemArrayEnd + itemTableStart + itemTableEnd + itemArrayTableStart + itemArrayTableEnd + itemKeyStart + itemCommentStart +) + +const ( + eof = 0 + tableStart = '[' + tableEnd = ']' + arrayTableStart = '[' + arrayTableEnd = ']' + tableSep = '.' + keySep = '=' + arrayStart = '[' + arrayEnd = ']' + arrayValTerm = ',' + commentStart = '#' + stringStart = '"' + stringEnd = '"' +) + +type stateFn func(lx *lexer) stateFn + +type lexer struct { + input string + start int + pos int + width int + line int + state stateFn + items chan item + + // A stack of state functions used to maintain context. + // The idea is to reuse parts of the state machine in various places. + // For example, values can appear at the top level or within arbitrarily + // nested arrays. The last state on the stack is used after a value has + // been lexed. Similarly for comments. + stack []stateFn +} + +type item struct { + typ itemType + val string + line int +} + +func (lx *lexer) nextItem() item { + for { + select { + case item := <-lx.items: + return item + default: + lx.state = lx.state(lx) + } + } + panic("not reached") +} + +func lex(input string) *lexer { + lx := &lexer{ + input: input, + state: lexTop, + line: 1, + items: make(chan item, 10), + stack: make([]stateFn, 0, 10), + } + return lx +} + +func (lx *lexer) push(state stateFn) { + lx.stack = append(lx.stack, state) +} + +func (lx *lexer) pop() stateFn { + if len(lx.stack) == 0 { + return lx.errorf("BUG in lexer: no states to pop.") + } + last := lx.stack[len(lx.stack)-1] + lx.stack = lx.stack[0 : len(lx.stack)-1] + return last +} + +func (lx *lexer) current() string { + return lx.input[lx.start:lx.pos] +} + +func (lx *lexer) emit(typ itemType) { + lx.items <- item{typ, lx.current(), lx.line} + lx.start = lx.pos +} + +func (lx *lexer) next() (r rune) { + if lx.pos >= len(lx.input) { + lx.width = 0 + return eof + } + + if lx.input[lx.pos] == '\n' { + lx.line++ + } + r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) + lx.pos += lx.width + return r +} + +// ignore skips over the pending input before this point. +func (lx *lexer) ignore() { + lx.start = lx.pos +} + +// backup steps back one rune. Can be called only once per call of next. +func (lx *lexer) backup() { + lx.pos -= lx.width + if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { + lx.line-- + } +} + +// accept consumes the next rune if it's equal to `valid`. +func (lx *lexer) accept(valid rune) bool { + if lx.next() == valid { + return true + } + lx.backup() + return false +} + +// peek returns but does not consume the next rune in the input. +func (lx *lexer) peek() rune { + r := lx.next() + lx.backup() + return r +} + +// errorf stops all lexing by emitting an error and returning `nil`. +// Note that any value that is a character is escaped if it's a special +// character (new lines, tabs, etc.). +func (lx *lexer) errorf(format string, values ...interface{}) stateFn { + for i, value := range values { + if v, ok := value.(rune); ok { + values[i] = escapeSpecial(v) + } + } + lx.items <- item{ + itemError, + fmt.Sprintf(format, values...), + lx.line, + } + return nil +} + +// lexTop consumes elements at the top level of TOML data. +func lexTop(lx *lexer) stateFn { + r := lx.next() + if isWhitespace(r) || isNL(r) { + return lexSkip(lx, lexTop) + } + + switch r { + case commentStart: + lx.push(lexTop) + return lexCommentStart + case tableStart: + return lexTableStart + case eof: + if lx.pos > lx.start { + return lx.errorf("Unexpected EOF.") + } + lx.emit(itemEOF) + return nil + } + + // At this point, the only valid item can be a key, so we back up + // and let the key lexer do the rest. + lx.backup() + lx.push(lexTopEnd) + return lexKeyStart +} + +// lexTopEnd is entered whenever a top-level item has been consumed. (A value +// or a table.) It must see only whitespace, and will turn back to lexTop +// upon a new line. If it sees EOF, it will quit the lexer successfully. +func lexTopEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case r == commentStart: + // a comment will read to a new line for us. + lx.push(lexTop) + return lexCommentStart + case isWhitespace(r): + return lexTopEnd + case isNL(r): + lx.ignore() + return lexTop + case r == eof: + lx.ignore() + return lexTop + } + return lx.errorf("Expected a top-level item to end with a new line, "+ + "comment or EOF, but got '%s' instead.", r) +} + +// lexTable lexes the beginning of a table. Namely, it makes sure that +// it starts with a character other than '.' and ']'. +// It assumes that '[' has already been consumed. +// It also handles the case that this is an item in an array of tables. +// e.g., '[[name]]'. +func lexTableStart(lx *lexer) stateFn { + if lx.peek() == arrayTableStart { + lx.next() + lx.emit(itemArrayTableStart) + lx.push(lexArrayTableEnd) + } else { + lx.emit(itemTableStart) + lx.push(lexTableEnd) + } + return lexTableNameStart +} + +func lexTableEnd(lx *lexer) stateFn { + lx.emit(itemTableEnd) + return lexTopEnd +} + +func lexArrayTableEnd(lx *lexer) stateFn { + if r := lx.next(); r != arrayTableEnd { + return lx.errorf("Expected end of table array name delimiter '%s', "+ + "but got '%s' instead.", arrayTableEnd, r) + } + lx.emit(itemArrayTableEnd) + return lexTopEnd +} + +func lexTableNameStart(lx *lexer) stateFn { + switch lx.next() { + case tableEnd: + return lx.errorf("Unexpected end of table. (Tables cannot " + + "be empty.)") + case tableSep: + return lx.errorf("Unexpected table separator. (Tables cannot " + + "be empty.)") + } + return lexTableName +} + +// lexTableName lexes the name of a table. It assumes that at least one +// valid character for the table has already been read. +func lexTableName(lx *lexer) stateFn { + switch lx.peek() { + case tableStart: + return lx.errorf("Table names cannot contain '%s' or '%s'.", + tableStart, tableEnd) + case tableEnd: + lx.emit(itemText) + lx.next() + return lx.pop() + case tableSep: + lx.emit(itemText) + lx.next() + lx.ignore() + return lexTableNameStart + } + lx.next() + return lexTableName +} + +// lexKeyStart consumes a key name up until the first non-whitespace character. +// lexKeyStart will ignore whitespace. +func lexKeyStart(lx *lexer) stateFn { + r := lx.peek() + switch { + case r == keySep: + return lx.errorf("Unexpected key separator '%s'.", keySep) + case isWhitespace(r) || isNL(r): + lx.next() + return lexSkip(lx, lexKeyStart) + } + + lx.ignore() + lx.emit(itemKeyStart) + lx.next() + return lexKey +} + +// lexKey consumes the text of a key. Assumes that the first character (which +// is not whitespace) has already been consumed. +func lexKey(lx *lexer) stateFn { + r := lx.peek() + + // XXX: Possible divergence from spec? + // "Keys start with the first non-whitespace character and end with the + // last non-whitespace character before the equals sign." + // Note here that whitespace is either a tab or a space. + // But we'll call it quits if we see a new line too. + if isWhitespace(r) || isNL(r) { + lx.emit(itemText) + return lexKeyEnd + } + + // Let's also call it quits if we see an equals sign. + if r == keySep { + lx.emit(itemText) + return lexKeyEnd + } + + lx.next() + return lexKey +} + +// lexKeyEnd consumes the end of a key (up to the key separator). +// Assumes that the first whitespace character after a key (or the '=' +// separator) has NOT been consumed. +func lexKeyEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexKeyEnd) + case r == keySep: + return lexSkip(lx, lexValue) + } + return lx.errorf("Expected key separator '%s', but got '%s' instead.", + keySep, r) +} + +// lexValue starts the consumption of a value anywhere a value is expected. +// lexValue will ignore whitespace. +// After a value is lexed, the last state on the next is popped and returned. +func lexValue(lx *lexer) stateFn { + // We allow whitespace to precede a value, but NOT new lines. + // In array syntax, the array states are responsible for ignoring new lines. + r := lx.next() + if isWhitespace(r) { + return lexSkip(lx, lexValue) + } + + switch { + case r == arrayStart: + lx.ignore() + lx.emit(itemArray) + return lexArrayValue + case r == stringStart: + lx.ignore() // ignore the '"' + return lexString + case r == 't': + return lexTrue + case r == 'f': + return lexFalse + case r == '-': + return lexNumberStart + case isDigit(r): + lx.backup() // avoid an extra state and use the same as above + return lexNumberOrDateStart + case r == '.': // special error case, be kind to users + return lx.errorf("Floats must start with a digit, not '.'.") + } + return lx.errorf("Expected value but found '%s' instead.", r) +} + +// lexArrayValue consumes one value in an array. It assumes that '[' or ',' +// have already been consumed. All whitespace and new lines are ignored. +func lexArrayValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexArrayValue) + case r == commentStart: + lx.push(lexArrayValue) + return lexCommentStart + case r == arrayValTerm: + return lx.errorf("Unexpected array value terminator '%s'.", + arrayValTerm) + case r == arrayEnd: + return lexArrayEnd + } + + lx.backup() + lx.push(lexArrayValueEnd) + return lexValue +} + +// lexArrayValueEnd consumes the cruft between values of an array. Namely, +// it ignores whitespace and expects either a ',' or a ']'. +func lexArrayValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r) || isNL(r): + return lexSkip(lx, lexArrayValueEnd) + case r == commentStart: + lx.push(lexArrayValueEnd) + return lexCommentStart + case r == arrayValTerm: + return lexArrayValue // move on to the next value + case r == arrayEnd: + return lexArrayEnd + } + return lx.errorf("Expected an array value terminator '%s' or an array "+ + "terminator '%s', but got '%s' instead.", arrayValTerm, arrayEnd, r) +} + +// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has +// just been consumed. +func lexArrayEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemArrayEnd) + return lx.pop() +} + +// lexString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. +func lexString(lx *lexer) stateFn { + r := lx.next() + switch { + case isNL(r): + return lx.errorf("Strings cannot contain new lines.") + case r == '\\': + return lexStringEscape + case r == stringEnd: + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexString +} + +// lexStringEscape consumes an escaped character. It assumes that the preceding +// '\\' has already been consumed. +func lexStringEscape(lx *lexer) stateFn { + r := lx.next() + switch r { + case 'b': + fallthrough + case 't': + fallthrough + case 'n': + fallthrough + case 'f': + fallthrough + case 'r': + fallthrough + case '"': + fallthrough + case '/': + fallthrough + case '\\': + return lexString + case 'u': + return lexStringUnicode + } + return lx.errorf("Invalid escape character '%s'. Only the following "+ + "escape characters are allowed: "+ + "\\b, \\t, \\n, \\f, \\r, \\\", \\/, \\\\, and \\uXXXX.", r) +} + +// lexStringBinary consumes two hexadecimal digits following '\x'. It assumes +// that the '\x' has already been consumed. +func lexStringUnicode(lx *lexer) stateFn { + var r rune + + for i := 0; i < 4; i++ { + r = lx.next() + if !isHexadecimal(r) { + return lx.errorf("Expected four hexadecimal digits after '\\x', "+ + "but got '%s' instead.", lx.current()) + } + } + return lexString +} + +// lexNumberOrDateStart consumes either a (positive) integer, float or datetime. +// It assumes that NO negative sign has been consumed. +func lexNumberOrDateStart(lx *lexer) stateFn { + r := lx.next() + if !isDigit(r) { + if r == '.' { + return lx.errorf("Floats must start with a digit, not '.'.") + } else { + return lx.errorf("Expected a digit but got '%s'.", r) + } + } + return lexNumberOrDate +} + +// lexNumberOrDate consumes either a (positive) integer, float or datetime. +func lexNumberOrDate(lx *lexer) stateFn { + r := lx.next() + switch { + case r == '-': + if lx.pos-lx.start != 5 { + return lx.errorf("All ISO8601 dates must be in full Zulu form.") + } + return lexDateAfterYear + case isDigit(r): + return lexNumberOrDate + case r == '.': + return lexFloatStart + } + + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. +// It assumes that "YYYY-" has already been consumed. +func lexDateAfterYear(lx *lexer) stateFn { + formats := []rune{ + // digits are '0'. + // everything else is direct equality. + '0', '0', '-', '0', '0', + 'T', + '0', '0', ':', '0', '0', ':', '0', '0', + 'Z', + } + for _, f := range formats { + r := lx.next() + if f == '0' { + if !isDigit(r) { + return lx.errorf("Expected digit in ISO8601 datetime, "+ + "but found '%s' instead.", r) + } + } else if f != r { + return lx.errorf("Expected '%s' in ISO8601 datetime, "+ + "but found '%s' instead.", f, r) + } + } + lx.emit(itemDatetime) + return lx.pop() +} + +// lexNumberStart consumes either an integer or a float. It assumes that a +// negative sign has already been read, but that *no* digits have been consumed. +// lexNumberStart will move to the appropriate integer or float states. +func lexNumberStart(lx *lexer) stateFn { + // we MUST see a digit. Even floats have to start with a digit. + r := lx.next() + if !isDigit(r) { + if r == '.' { + return lx.errorf("Floats must start with a digit, not '.'.") + } else { + return lx.errorf("Expected a digit but got '%s'.", r) + } + } + return lexNumber +} + +// lexNumber consumes an integer or a float after seeing the first digit. +func lexNumber(lx *lexer) stateFn { + r := lx.next() + switch { + case isDigit(r): + return lexNumber + case r == '.': + return lexFloatStart + } + + lx.backup() + lx.emit(itemInteger) + return lx.pop() +} + +// lexFloatStart starts the consumption of digits of a float after a '.'. +// Namely, at least one digit is required. +func lexFloatStart(lx *lexer) stateFn { + r := lx.next() + if !isDigit(r) { + return lx.errorf("Floats must have a digit after the '.', but got "+ + "'%s' instead.", r) + } + return lexFloat +} + +// lexFloat consumes the digits of a float after a '.'. +// Assumes that one digit has been consumed after a '.' already. +func lexFloat(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexFloat + } + + lx.backup() + lx.emit(itemFloat) + return lx.pop() +} + +// lexTrue consumes the "rue" in "true". It assumes that 't' has already +// been consumed. +func lexTrue(lx *lexer) stateFn { + if r := lx.next(); r != 'r' { + return lx.errorf("Expected 'tr', but found 't%s' instead.", r) + } + if r := lx.next(); r != 'u' { + return lx.errorf("Expected 'tru', but found 'tr%s' instead.", r) + } + if r := lx.next(); r != 'e' { + return lx.errorf("Expected 'true', but found 'tru%s' instead.", r) + } + lx.emit(itemBool) + return lx.pop() +} + +// lexFalse consumes the "alse" in "false". It assumes that 'f' has already +// been consumed. +func lexFalse(lx *lexer) stateFn { + if r := lx.next(); r != 'a' { + return lx.errorf("Expected 'fa', but found 'f%s' instead.", r) + } + if r := lx.next(); r != 'l' { + return lx.errorf("Expected 'fal', but found 'fa%s' instead.", r) + } + if r := lx.next(); r != 's' { + return lx.errorf("Expected 'fals', but found 'fal%s' instead.", r) + } + if r := lx.next(); r != 'e' { + return lx.errorf("Expected 'false', but found 'fals%s' instead.", r) + } + lx.emit(itemBool) + return lx.pop() +} + +// lexCommentStart begins the lexing of a comment. It will emit +// itemCommentStart and consume no characters, passing control to lexComment. +func lexCommentStart(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemCommentStart) + return lexComment +} + +// lexComment lexes an entire comment. It assumes that '#' has been consumed. +// It will consume *up to* the first new line character, and pass control +// back to the last state on the stack. +func lexComment(lx *lexer) stateFn { + r := lx.peek() + if isNL(r) || r == eof { + lx.emit(itemText) + return lx.pop() + } + lx.next() + return lexComment +} + +// lexSkip ignores all slurped input and moves on to the next state. +func lexSkip(lx *lexer, nextState stateFn) stateFn { + return func(lx *lexer) stateFn { + lx.ignore() + return nextState + } +} + +// isWhitespace returns true if `r` is a whitespace character according +// to the spec. +func isWhitespace(r rune) bool { + return r == '\t' || r == ' ' +} + +func isNL(r rune) bool { + return r == '\n' || r == '\r' +} + +func isDigit(r rune) bool { + return r >= '0' && r <= '9' +} + +func isHexadecimal(r rune) bool { + return (r >= '0' && r <= '9') || + (r >= 'a' && r <= 'f') || + (r >= 'A' && r <= 'F') +} + +func (itype itemType) String() string { + switch itype { + case itemError: + return "Error" + case itemNIL: + return "NIL" + case itemEOF: + return "EOF" + case itemText: + return "Text" + case itemString: + return "String" + case itemBool: + return "Bool" + case itemInteger: + return "Integer" + case itemFloat: + return "Float" + case itemDatetime: + return "DateTime" + case itemTableStart: + return "TableStart" + case itemTableEnd: + return "TableEnd" + case itemKeyStart: + return "KeyStart" + case itemArray: + return "Array" + case itemArrayEnd: + return "ArrayEnd" + case itemCommentStart: + return "CommentStart" + } + panic(fmt.Sprintf("BUG: Unknown type '%s'.", itype)) +} + +func (item item) String() string { + return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) +} + +func escapeSpecial(c rune) string { + switch c { + case '\n': + return "\\n" + } + return string(c) +} diff --git a/third_party/github.com/BurntSushi/toml/lex_test.go b/third_party/github.com/BurntSushi/toml/lex_test.go new file mode 100644 index 000000000..6cfa21088 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/lex_test.go @@ -0,0 +1,59 @@ +package toml + +import ( + "log" + "testing" +) + +func init() { + log.SetFlags(0) +} + +var testSmall = ` +# This is a TOML document. Boom. + +[owner] +[owner] # Whoa there. +andrew = "gallant # poopy" # weeeee +predicate = false +num = -5192 +f = -0.5192 +zulu = 1979-05-27T07:32:00Z +whoop = "poop" +arrs = [ + 1987-07-05T05:45:00Z, + 5, + "wat?", + "hehe \n\r kewl", + [6], [], + 5.0, + # sweetness +] # more comments +# hehe +` + +var testSmaller = ` +[a.b] # Do you ignore me? +andrew = "ga# ll\"ant" # what about me? +kait = "brady" +awesomeness = true +pi = 3.14 +dob = 1987-07-05T17:45:00Z +perfection = [ + [6, 28], + [496, 8128] +] +` + +func TestLexer(t *testing.T) { + lx := lex(testSmaller) + for { + item := lx.nextItem() + if item.typ == itemEOF { + break + } else if item.typ == itemError { + t.Fatal(item.val) + } + testf("%s\n", item) + } +} diff --git a/third_party/github.com/BurntSushi/toml/out_test.go b/third_party/github.com/BurntSushi/toml/out_test.go new file mode 100644 index 000000000..ab121e375 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/out_test.go @@ -0,0 +1,19 @@ +package toml + +import ( + "flag" + "fmt" +) + +var flagOut = false + +func init() { + flag.BoolVar(&flagOut, "out", flagOut, "Print debug output.") + flag.Parse() +} + +func testf(format string, v ...interface{}) { + if flagOut { + fmt.Printf(format, v...) + } +} diff --git a/third_party/github.com/BurntSushi/toml/parse.go b/third_party/github.com/BurntSushi/toml/parse.go new file mode 100644 index 000000000..57e4a70a9 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/parse.go @@ -0,0 +1,388 @@ +package toml + +import ( + "fmt" + "log" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type parser struct { + mapping map[string]interface{} + types map[string]tomlType + lx *lexer + + // A list of keys in the order that they appear in the TOML data. + ordered []Key + + // the full key for the current hash in scope + context Key + + // the base key name for everything except hashes + currentKey string + + // rough approximation of line number + approxLine int + + // A map of 'key.group.names' to whether they were created implicitly. + implicits map[string]bool +} + +type parseError string + +func (pe parseError) Error() string { + return string(pe) +} + +func parse(data string) (p *parser, err error) { + defer func() { + if r := recover(); r != nil { + var ok bool + if err, ok = r.(parseError); ok { + return + } + panic(r) + } + }() + + p = &parser{ + mapping: make(map[string]interface{}), + types: make(map[string]tomlType), + lx: lex(data), + ordered: make([]Key, 0), + implicits: make(map[string]bool), + } + for { + item := p.next() + if item.typ == itemEOF { + break + } + p.topLevel(item) + } + + return p, nil +} + +func (p *parser) panic(format string, v ...interface{}) { + msg := fmt.Sprintf("Near line %d, key '%s': %s", + p.approxLine, p.current(), fmt.Sprintf(format, v...)) + panic(parseError(msg)) +} + +func (p *parser) next() item { + it := p.lx.nextItem() + if it.typ == itemError { + p.panic("Near line %d: %s", it.line, it.val) + } + return it +} + +func (p *parser) bug(format string, v ...interface{}) { + log.Fatalf("BUG: %s\n\n", fmt.Sprintf(format, v...)) +} + +func (p *parser) expect(typ itemType) item { + it := p.next() + p.assertEqual(typ, it.typ) + return it +} + +func (p *parser) assertEqual(expected, got itemType) { + if expected != got { + p.bug("Expected '%s' but got '%s'.", expected, got) + } +} + +func (p *parser) topLevel(item item) { + switch item.typ { + case itemCommentStart: + p.approxLine = item.line + p.expect(itemText) + case itemTableStart: + kg := p.expect(itemText) + p.approxLine = kg.line + + key := make(Key, 0) + for ; kg.typ == itemText; kg = p.next() { + key = append(key, kg.val) + } + p.assertEqual(itemTableEnd, kg.typ) + + p.establishContext(key, false) + p.setType("", tomlHash) + p.ordered = append(p.ordered, key) + case itemArrayTableStart: + kg := p.expect(itemText) + p.approxLine = kg.line + + key := make(Key, 0) + for ; kg.typ == itemText; kg = p.next() { + key = append(key, kg.val) + } + p.assertEqual(itemArrayTableEnd, kg.typ) + + p.establishContext(key, true) + p.setType("", tomlArrayHash) + p.ordered = append(p.ordered, key) + case itemKeyStart: + kname := p.expect(itemText) + p.currentKey = kname.val + p.approxLine = kname.line + + val, typ := p.value(p.next()) + p.setValue(p.currentKey, val) + p.setType(p.currentKey, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + + p.currentKey = "" + default: + p.bug("Unexpected type at top level: %s", item.typ) + } +} + +// value translates an expected value from the lexer into a Go value wrapped +// as an empty interface. +func (p *parser) value(it item) (interface{}, tomlType) { + switch it.typ { + case itemString: + return p.replaceUnicode(replaceEscapes(it.val)), p.typeOfPrimitive(it) + case itemBool: + switch it.val { + case "true": + return true, p.typeOfPrimitive(it) + case "false": + return false, p.typeOfPrimitive(it) + } + p.bug("Expected boolean value, but got '%s'.", it.val) + case itemInteger: + num, err := strconv.ParseInt(it.val, 10, 64) + if err != nil { + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + + p.panic("Integer '%s' is out of the range of 64-bit "+ + "signed integers.", it.val) + } else { + p.bug("Expected integer value, but got '%s'.", it.val) + } + } + return num, p.typeOfPrimitive(it) + case itemFloat: + num, err := strconv.ParseFloat(it.val, 64) + if err != nil { + if e, ok := err.(*strconv.NumError); ok && + e.Err == strconv.ErrRange { + + p.panic("Float '%s' is out of the range of 64-bit "+ + "IEEE-754 floating-point numbers.", it.val) + } else { + p.bug("Expected float value, but got '%s'.", it.val) + } + } + return num, p.typeOfPrimitive(it) + case itemDatetime: + t, err := time.Parse("2006-01-02T15:04:05Z", it.val) + if err != nil { + p.bug("Expected Zulu formatted DateTime, but got '%s'.", it.val) + } + return t, p.typeOfPrimitive(it) + case itemArray: + array := make([]interface{}, 0) + types := make([]tomlType, 0) + + for it = p.next(); it.typ != itemArrayEnd; it = p.next() { + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + val, typ := p.value(it) + array = append(array, val) + types = append(types, typ) + } + return array, p.typeOfArray(types) + } + p.bug("Unexpected value type: %s", it.typ) + panic("unreachable") +} + +// establishContext sets the current context of the parser, +// where the context is the hash currently in scope. +// +// Establishing the context also makes sure that the key isn't a duplicate, and +// will create implicit hashes automatically. +func (p *parser) establishContext(key Key, array bool) { + var ok bool + + // Always start at the top level and drill down for our context. + hashContext := p.mapping + keyContext := make(Key, 0) + + // We only need implicit hashes for key[0:-1] + for _, k := range key[0 : len(key)-1] { + _, ok = hashContext[k] + keyContext = append(keyContext, k) + + // No key? Make an implicit hash and move on. + if !ok { + p.addImplicit(keyContext) + hashContext[k] = make(map[string]interface{}) + } + + // If the hash context is actually an array of tables, then set + // the hash context to the last element in that array. + // + // Otherwise, it better be a table, since this MUST be a key group (by + // virtue of it not being the last element in a key). + switch t := hashContext[k].(type) { + case []map[string]interface{}: + hashContext = t[len(t)-1] + case map[string]interface{}: + hashContext = t + default: + p.panic("Key '%s' was already created as a hash.", keyContext) + } + } + + p.context = keyContext + if array { + k := key[len(key)-1] + if _, ok := hashContext[k]; !ok { + hashContext[k] = make([]map[string]interface{}, 0, 5) + } + if hash, ok := hashContext[k].([]map[string]interface{}); ok { + hashContext[k] = append(hash, make(map[string]interface{})) + } else { + p.panic("Key '%s' was already created and cannot be used as "+ + "an array.", keyContext) + } + } else { + p.setValue(key[len(key)-1], make(map[string]interface{})) + } + p.context = append(p.context, key[len(key)-1]) +} + +// setValue sets the given key to the given value in the current context. +// It will make sure that the key hasn't already been defined, account for +// implicit key groups. +func (p *parser) setValue(key string, value interface{}) { + var tmpHash interface{} + var ok bool + + hash := p.mapping + keyContext := make(Key, 0) + for _, k := range p.context { + keyContext = append(keyContext, k) + if tmpHash, ok = hash[k]; !ok { + p.bug("Context for key '%s' has not been established.", keyContext) + } + switch t := tmpHash.(type) { + case []map[string]interface{}: + hash = t[len(t)-1] + case map[string]interface{}: + hash = t + default: + p.bug("Expected hash to have type 'map[string]interface{}', but "+ + "it has '%T' instead.", tmpHash) + } + } + keyContext = append(keyContext, key) + + if _, ok := hash[key]; ok { + // We need to do some fancy footwork here. If `hash[key]` was implcitly + // created AND `value` is a hash, then let this go through and stop + // tagging this table as implicit. + if p.isImplicit(keyContext) { + p.removeImplicit(keyContext) + return + } + + // Otherwise, we have a concrete key trying to override a previous + // key, which is *always* wrong. + p.panic("Key '%s' has already been defined.", keyContext) + } + hash[key] = value +} + +// setType sets the type of a particular value at a given key. +// It should be called immediately AFTER setValue. +func (p *parser) setType(key string, typ tomlType) { + keyContext := make(Key, 0, len(p.context)+1) + for _, k := range p.context { + keyContext = append(keyContext, k) + } + if len(key) > 0 { // allow type setting for hashes + keyContext = append(keyContext, key) + } + p.types[keyContext.String()] = typ +} + +// addImplicit sets the given Key as having been created implicitly. +func (p *parser) addImplicit(key Key) { + p.implicits[key.String()] = true +} + +// removeImplicit stops tagging the given key as having been implicitly created. +func (p *parser) removeImplicit(key Key) { + p.implicits[key.String()] = false +} + +// isImplicit returns true if the key group pointed to by the key was created +// implicitly. +func (p *parser) isImplicit(key Key) bool { + return p.implicits[key.String()] +} + +// current returns the full key name of the current context. +func (p *parser) current() string { + if len(p.currentKey) == 0 { + return p.context.String() + } + if len(p.context) == 0 { + return p.currentKey + } + return fmt.Sprintf("%s.%s", p.context, p.currentKey) +} + +func replaceEscapes(s string) string { + return strings.NewReplacer( + "\\b", "\u0008", + "\\t", "\u0009", + "\\n", "\u000A", + "\\f", "\u000C", + "\\r", "\u000D", + "\\\"", "\u0022", + "\\/", "\u002F", + "\\\\", "\u005C", + ).Replace(s) +} + +func (p *parser) replaceUnicode(s string) string { + indexEsc := func() int { + return strings.Index(s, "\\u") + } + for i := indexEsc(); i != -1; i = indexEsc() { + asciiBytes := s[i+2 : i+6] + s = strings.Replace(s, s[i:i+6], p.asciiEscapeToUnicode(asciiBytes), -1) + } + return s +} + +func (p *parser) asciiEscapeToUnicode(s string) string { + hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) + if err != nil { + p.bug("Could not parse '%s' as a hexadecimal number, but the "+ + "lexer claims it's OK: %s", s, err) + } + + // I honestly don't understand how this works. I can't seem to find + // a way to make this fail. I figured this would fail on invalid UTF-8 + // characters like U+DCFF, but it doesn't. + r := string(rune(hex)) + if !utf8.ValidString(r) { + p.panic("Escaped character '\\u%s' is not valid UTF-8.", s) + } + return string(r) +} diff --git a/third_party/github.com/BurntSushi/toml/parse_test.go b/third_party/github.com/BurntSushi/toml/parse_test.go new file mode 100644 index 000000000..d6f0d32fb --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/parse_test.go @@ -0,0 +1,61 @@ +package toml + +import ( + "strings" + "testing" +) + +var testParseSmall = ` +# This is a TOML document. Boom. + +wat = "chipper" + +[owner.andrew.gallant] +hmm = "hi" + +[owner] # Whoa there. +andreW = "gallant # poopy" # weeeee +predicate = false +num = -5192 +f = -0.5192 +zulu = 1979-05-27T07:32:00Z +whoop = "poop" +tests = [ [1, 2, 3], ["abc", "xyz"] ] +arrs = [ # hmm + # more comments are awesome. + 1987-07-05T05:45:00Z, + # say wat? + 1987-07-05T05:45:00Z, + 1987-07-05T05:45:00Z, + # sweetness +] # more comments +# hehe +` + +var testParseSmall2 = ` +[a] +better = 43 + +[a.b.c] +answer = 42 +` + +func TestParse(t *testing.T) { + m, err := parse(testParseSmall) + if err != nil { + t.Fatal(err) + } + printMap(m.mapping, 0) +} + +func printMap(m map[string]interface{}, depth int) { + for k, v := range m { + testf("%s%s\n", strings.Repeat(" ", depth), k) + switch subm := v.(type) { + case map[string]interface{}: + printMap(subm, depth+1) + default: + testf("%s%v\n", strings.Repeat(" ", depth+1), v) + } + } +} diff --git a/third_party/github.com/BurntSushi/toml/session.vim b/third_party/github.com/BurntSushi/toml/session.vim new file mode 100644 index 000000000..562164be0 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/session.vim @@ -0,0 +1 @@ +au BufWritePost *.go silent!make tags > /dev/null 2>&1 diff --git a/third_party/github.com/BurntSushi/toml/toml-test-go/COPYING b/third_party/github.com/BurntSushi/toml/toml-test-go/COPYING new file mode 100644 index 000000000..5a8e33254 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/toml-test-go/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/third_party/github.com/BurntSushi/toml/toml-test-go/README.md b/third_party/github.com/BurntSushi/toml/toml-test-go/README.md new file mode 100644 index 000000000..24421eb70 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/toml-test-go/README.md @@ -0,0 +1,14 @@ +# Implements the TOML test suite interface + +This is an implementation of the interface expected by +[toml-test](https://github.com/BurntSushi/toml-test) for my +[toml parser written in Go](https://github.com/BurntSushi/toml). +In particular, it maps TOML data on `stdin` to a JSON format on `stdout`. + + +Compatible with TOML version +[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) + +Compatible with `toml-test` version +[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0) + diff --git a/third_party/github.com/BurntSushi/toml/toml-test-go/main.go b/third_party/github.com/BurntSushi/toml/toml-test-go/main.go new file mode 100644 index 000000000..79a3c212f --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/toml-test-go/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path" + "time" + + "github.com/BurntSushi/toml" +) + +func init() { + log.SetFlags(0) + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s < toml-file\n", path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() != 0 { + flag.Usage() + } + + var tmp interface{} + if _, err := toml.DecodeReader(os.Stdin, &tmp); err != nil { + log.Fatalf("Error decoding TOML: %s", err) + } + + typedTmp := translate(tmp) + if err := json.NewEncoder(os.Stdout).Encode(typedTmp); err != nil { + log.Fatalf("Error encoding JSON: %s", err) + } +} + +func translate(tomlData interface{}) interface{} { + + switch orig := tomlData.(type) { + case map[string]interface{}: + typed := make(map[string]interface{}, len(orig)) + for k, v := range orig { + typed[k] = translate(v) + } + return typed + case []map[string]interface{}: + typed := make([]map[string]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v).(map[string]interface{}) + } + return typed + case []interface{}: + typed := make([]interface{}, len(orig)) + for i, v := range orig { + typed[i] = translate(v) + } + + // We don't really need to tag arrays, but let's be future proof. + // (If TOML ever supports tuples, we'll need this.) + return tag("array", typed) + case time.Time: + return tag("datetime", orig.Format("2006-01-02T15:04:05Z")) + case bool: + return tag("bool", fmt.Sprintf("%v", orig)) + case int64: + return tag("integer", fmt.Sprintf("%d", orig)) + case float64: + return tag("float", fmt.Sprintf("%v", orig)) + case string: + return tag("string", orig) + } + + panic(fmt.Sprintf("Unknown type: %T", tomlData)) +} + +func tag(typeName string, data interface{}) map[string]interface{} { + return map[string]interface{}{ + "type": typeName, + "value": data, + } +} diff --git a/third_party/github.com/BurntSushi/toml/tomlv/COPYING b/third_party/github.com/BurntSushi/toml/tomlv/COPYING new file mode 100644 index 000000000..5a8e33254 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/tomlv/COPYING @@ -0,0 +1,14 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + diff --git a/third_party/github.com/BurntSushi/toml/tomlv/README.md b/third_party/github.com/BurntSushi/toml/tomlv/README.md new file mode 100644 index 000000000..bcc3f47b1 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/tomlv/README.md @@ -0,0 +1,22 @@ +# TOML Validator + +If Go is installed, it's simple to try it out: + +```bash +go get github.com/BurntSushi/toml/tomlv +tomlv some-toml-file.toml +``` + +You can see the types of every key in a TOML file with: + +```bash +tomlv -types some-toml-file.toml +``` + +At the moment, only one error message is reported at a time. Error messages +include line numbers. No output means that the files given are valid TOML, or +there is a bug in `tomlv`. + +Compatible with TOML version +[v0.1.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.1.0.md) + diff --git a/third_party/github.com/BurntSushi/toml/tomlv/main.go b/third_party/github.com/BurntSushi/toml/tomlv/main.go new file mode 100644 index 000000000..a52086ef5 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/tomlv/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "path" + "strings" + "text/tabwriter" + + "github.com/BurntSushi/toml" +) + +var ( + flagTypes = false +) + +func init() { + log.SetFlags(0) + + flag.BoolVar(&flagTypes, "types", flagTypes, + "When set, the types of every defined key will be shown.") + + flag.Usage = usage + flag.Parse() +} + +func usage() { + log.Printf("Usage: %s toml-file [ toml-file ... ]\n", + path.Base(os.Args[0])) + flag.PrintDefaults() + + os.Exit(1) +} + +func main() { + if flag.NArg() < 1 { + flag.Usage() + } + for _, f := range flag.Args() { + var tmp interface{} + md, err := toml.DecodeFile(f, &tmp) + if err != nil { + log.Fatalf("Error in '%s': %s", f, err) + } + if flagTypes { + printTypes(md) + } + } +} + +func printTypes(md toml.MetaData) { + tabw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for _, key := range md.Keys() { + fmt.Fprintf(tabw, "%s%s\t%s\n", + strings.Repeat(" ", len(key)-1), key, md.Type(key...)) + } + tabw.Flush() +} diff --git a/third_party/github.com/BurntSushi/toml/type-check.go b/third_party/github.com/BurntSushi/toml/type-check.go new file mode 100644 index 000000000..22f188d42 --- /dev/null +++ b/third_party/github.com/BurntSushi/toml/type-check.go @@ -0,0 +1,78 @@ +package toml + +// tomlType represents any Go type that corresponds to a TOML type. +// While the first draft of the TOML spec has a simplistic type system that +// probably doesn't need this level of sophistication, we seem to be militating +// toward adding real composite types. +type tomlType interface { + typeString() string +} + +// typeEqual accepts any two types and returns true if they are equal. +func typeEqual(t1, t2 tomlType) bool { + return t1.typeString() == t2.typeString() +} + +type tomlBaseType string + +func (btype tomlBaseType) typeString() string { + return string(btype) +} + +func (btype tomlBaseType) String() string { + return btype.typeString() +} + +var ( + tomlInteger tomlBaseType = "Integer" + tomlFloat tomlBaseType = "Float" + tomlDatetime tomlBaseType = "Datetime" + tomlString tomlBaseType = "String" + tomlBool tomlBaseType = "Bool" + tomlArray tomlBaseType = "Array" + tomlHash tomlBaseType = "Hash" + tomlArrayHash tomlBaseType = "ArrayHash" +) + +// typeOfPrimitive returns a tomlType of any primitive value in TOML. +// Primitive values are: Integer, Float, Datetime, String and Bool. +// +// Passing a lexer item other than the following will cause a BUG message +// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime. +func (p *parser) typeOfPrimitive(lexItem item) tomlType { + switch lexItem.typ { + case itemInteger: + return tomlInteger + case itemFloat: + return tomlFloat + case itemDatetime: + return tomlDatetime + case itemString: + return tomlString + case itemBool: + return tomlBool + } + p.bug("Cannot infer primitive type of lex item '%s'.", lexItem) + panic("unreachable") +} + +// typeOfArray returns a tomlType for an array given a list of types of its +// values. +// +// In the current spec, if an array is homogeneous, then its type is always +// "Array". If the array is not homogeneous, an error is generated. +func (p *parser) typeOfArray(types []tomlType) tomlType { + // Empty arrays are cool. + if len(types) == 0 { + return tomlArray + } + + theType := types[0] + for _, t := range types[1:] { + if !typeEqual(theType, t) { + p.panic("Array contains values of type '%s' and '%s', but arrays "+ + "must be homogeneous.", theType, t) + } + } + return tomlArray +} diff --git a/util.go b/util.go deleted file mode 100644 index 089eba40c..000000000 --- a/util.go +++ /dev/null @@ -1,88 +0,0 @@ -package main - -import ( - "net" - "net/url" - "os" - "os/signal" - "runtime/pprof" - - "github.com/coreos/etcd/log" -) - -//-------------------------------------- -// HTTP Utilities -//-------------------------------------- - -// sanitizeURL will cleanup a host string in the format hostname:port and -// attach a schema. -func sanitizeURL(host string, defaultScheme string) string { - // Blank URLs are fine input, just return it - if len(host) == 0 { - return host - } - - p, err := url.Parse(host) - if err != nil { - log.Fatal(err) - } - - // Make sure the host is in Host:Port format - _, _, err = net.SplitHostPort(host) - if err != nil { - log.Fatal(err) - } - - p = &url.URL{Host: host, Scheme: defaultScheme} - - return p.String() -} - -// sanitizeListenHost cleans up the ListenHost parameter and appends a port -// if necessary based on the advertised port. -func sanitizeListenHost(listen string, advertised string) string { - aurl, err := url.Parse(advertised) - if err != nil { - log.Fatal(err) - } - - ahost, aport, err := net.SplitHostPort(aurl.Host) - if err != nil { - log.Fatal(err) - } - - // If the listen host isn't set use the advertised host - if listen == "" { - listen = ahost - } - - return net.JoinHostPort(listen, aport) -} - -func check(err error) { - if err != nil { - log.Fatal(err) - } -} - -//-------------------------------------- -// CPU profile -//-------------------------------------- -func runCPUProfile() { - - f, err := os.Create(cpuprofile) - if err != nil { - log.Fatal(err) - } - pprof.StartCPUProfile(f) - - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - go func() { - for sig := range c { - log.Infof("captured %v, stopping profiler and exiting..", sig) - pprof.StopCPUProfile() - os.Exit(1) - } - }() -}