2014-10-30 02:14:31 +03:00
/ *
Copyright 2014 CoreOS , Inc .
Licensed under the Apache License , Version 2.0 ( the "License" ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : //www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an "AS IS" BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
* /
package etcdmain
import (
"flag"
"fmt"
"log"
2014-11-04 02:17:21 +03:00
"net"
2014-10-30 02:14:31 +03:00
"net/http"
2014-11-10 23:55:35 +03:00
"net/url"
2014-10-30 02:14:31 +03:00
"os"
"strings"
2014-11-05 00:09:24 +03:00
"github.com/coreos/etcd/discovery"
2014-10-30 02:14:31 +03:00
"github.com/coreos/etcd/etcdserver"
"github.com/coreos/etcd/etcdserver/etcdhttp"
"github.com/coreos/etcd/pkg/cors"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/pkg/flags"
"github.com/coreos/etcd/pkg/transport"
"github.com/coreos/etcd/pkg/types"
"github.com/coreos/etcd/proxy"
2014-11-24 10:37:54 +03:00
"github.com/coreos/etcd/rafthttp"
2014-10-30 02:14:31 +03:00
"github.com/coreos/etcd/version"
)
const (
// the owner can make/remove files inside the directory
privateDirMode = 0700
2014-11-05 02:57:38 +03:00
proxyFlagOff = "off"
proxyFlagReadonly = "readonly"
proxyFlagOn = "on"
fallbackFlagExit = "exit"
fallbackFlagProxy = "proxy"
2014-11-06 22:09:33 +03:00
clusterStateFlagNew = "new"
clusterStateFlagExisting = "existing"
2014-10-30 02:14:31 +03:00
)
var (
2014-11-07 08:50:35 +03:00
fs = flag . NewFlagSet ( "etcd" , flag . ContinueOnError )
name = fs . String ( "name" , "default" , "Unique human-readable name for this node" )
dir = fs . String ( "data-dir" , "" , "Path to the data directory" )
durl = fs . String ( "discovery" , "" , "Discovery service used to bootstrap the cluster" )
dproxy = fs . String ( "discovery-proxy" , "" , "HTTP proxy to use for traffic to discovery service" )
snapCount = fs . Uint64 ( "snapshot-count" , etcdserver . DefaultSnapCount , "Number of committed transactions to trigger a snapshot" )
printVersion = fs . Bool ( "version" , false , "Print the version and exit" )
forceNewCluster = fs . Bool ( "force-new-cluster" , false , "Force to create a new one member cluster" )
2014-11-29 08:32:28 +03:00
maxSnapFiles = fs . Uint ( "max-snapshots" , 5 , "Maximum number of snapshot files to retain (0 is unlimited)" )
maxWalFiles = fs . Uint ( "max-wals" , 5 , "Maximum number of wal files to retain (0 is unlimited)" )
2014-10-30 02:14:31 +03:00
2014-10-30 23:43:38 +03:00
initialCluster = fs . String ( "initial-cluster" , "default=http://localhost:2380,default=http://localhost:7001" , "Initial cluster configuration for bootstrapping" )
initialClusterToken = fs . String ( "initial-cluster-token" , "etcd-cluster" , "Initial cluster token for the etcd cluster during bootstrap" )
2014-10-30 02:14:31 +03:00
2014-11-06 22:09:33 +03:00
corsInfo = & cors . CORSInfo { }
clientTLSInfo = transport . TLSInfo { }
peerTLSInfo = transport . TLSInfo { }
2014-11-05 02:57:38 +03:00
proxyFlag = flags . NewStringsFlag (
proxyFlagOff ,
proxyFlagReadonly ,
proxyFlagOn ,
)
fallbackFlag = flags . NewStringsFlag (
fallbackFlagExit ,
fallbackFlagProxy ,
)
2014-11-06 22:09:33 +03:00
clusterStateFlag = flags . NewStringsFlag (
clusterStateFlagNew ,
clusterStateFlagExisting ,
)
2014-10-30 02:14:31 +03:00
ignored = [ ] string {
"cluster-active-size" ,
"cluster-remove-delay" ,
"cluster-sync-interval" ,
"config" ,
"force" ,
"max-result-buffer" ,
"max-retry-attempts" ,
"peer-heartbeat-interval" ,
"peer-election-timeout" ,
"retry-interval" ,
"snapshot" ,
"v" ,
"vv" ,
}
)
func init ( ) {
2014-12-05 04:07:04 +03:00
fs . Usage = func ( ) {
fmt . Println ( usageline )
fmt . Println ( flagsline )
}
2014-11-06 22:09:33 +03:00
fs . Var ( clusterStateFlag , "initial-cluster-state" , "Initial cluster configuration for bootstrapping" )
if err := clusterStateFlag . Set ( clusterStateFlagNew ) ; err != nil {
2014-10-30 02:14:31 +03:00
// Should never happen.
2014-11-06 22:09:33 +03:00
log . Panicf ( "unexpected error setting up clusterStateFlag: %v" , err )
2014-10-30 02:14:31 +03:00
}
fs . Var ( flags . NewURLsValue ( "http://localhost:2380,http://localhost:7001" ) , "initial-advertise-peer-urls" , "List of this member's peer URLs to advertise to the rest of the cluster" )
fs . Var ( flags . NewURLsValue ( "http://localhost:2379,http://localhost:4001" ) , "advertise-client-urls" , "List of this member's client URLs to advertise to the rest of the cluster" )
fs . Var ( flags . NewURLsValue ( "http://localhost:2380,http://localhost:7001" ) , "listen-peer-urls" , "List of URLs to listen on for peer traffic" )
fs . Var ( flags . NewURLsValue ( "http://localhost:2379,http://localhost:4001" ) , "listen-client-urls" , "List of URLs to listen on for client traffic" )
fs . Var ( corsInfo , "cors" , "Comma-separated white list of origins for CORS (cross-origin resource sharing)." )
2014-11-05 02:57:38 +03:00
fs . Var ( proxyFlag , "proxy" , fmt . Sprintf ( "Valid values include %s" , strings . Join ( proxyFlag . Values , ", " ) ) )
if err := proxyFlag . Set ( proxyFlagOff ) ; err != nil {
2014-10-30 02:14:31 +03:00
// Should never happen.
log . Panicf ( "unexpected error setting up proxyFlag: %v" , err )
}
2014-11-05 02:57:38 +03:00
fs . Var ( fallbackFlag , "discovery-fallback" , fmt . Sprintf ( "Valid values include %s" , strings . Join ( fallbackFlag . Values , ", " ) ) )
if err := fallbackFlag . Set ( fallbackFlagProxy ) ; err != nil {
2014-11-05 00:09:24 +03:00
// Should never happen.
log . Panicf ( "unexpected error setting up discovery-fallback flag: %v" , err )
}
2014-10-30 02:14:31 +03:00
fs . StringVar ( & clientTLSInfo . CAFile , "ca-file" , "" , "Path to the client server TLS CA file." )
fs . StringVar ( & clientTLSInfo . CertFile , "cert-file" , "" , "Path to the client server TLS cert file." )
fs . StringVar ( & clientTLSInfo . KeyFile , "key-file" , "" , "Path to the client server TLS key file." )
fs . StringVar ( & peerTLSInfo . CAFile , "peer-ca-file" , "" , "Path to the peer server TLS CA file." )
fs . StringVar ( & peerTLSInfo . CertFile , "peer-cert-file" , "" , "Path to the peer server TLS cert file." )
fs . StringVar ( & peerTLSInfo . KeyFile , "peer-key-file" , "" , "Path to the peer server TLS key file." )
// backwards-compatibility with v0.4.6
fs . Var ( & flags . IPAddressPort { } , "addr" , "DEPRECATED: Use -advertise-client-urls instead." )
fs . Var ( & flags . IPAddressPort { } , "bind-addr" , "DEPRECATED: Use -listen-client-urls instead." )
fs . Var ( & flags . IPAddressPort { } , "peer-addr" , "DEPRECATED: Use -initial-advertise-peer-urls instead." )
fs . Var ( & flags . IPAddressPort { } , "peer-bind-addr" , "DEPRECATED: Use -listen-peer-urls instead." )
for _ , f := range ignored {
fs . Var ( & flags . IgnoredFlag { Name : f } , f , "" )
}
fs . Var ( & flags . DeprecatedFlag { Name : "peers" } , "peers" , "DEPRECATED: Use -initial-cluster instead" )
fs . Var ( & flags . DeprecatedFlag { Name : "peers-file" } , "peers-file" , "DEPRECATED: Use -initial-cluster instead" )
}
func Main ( ) {
2014-11-05 22:31:13 +03:00
perr := fs . Parse ( os . Args [ 1 : ] )
switch perr {
2014-10-30 02:14:31 +03:00
case nil :
case flag . ErrHelp :
os . Exit ( 0 )
default :
os . Exit ( 2 )
}
if * printVersion {
fmt . Println ( "etcd version" , version . Version )
os . Exit ( 0 )
}
2014-11-07 01:39:30 +03:00
err := flags . SetFlagsFromEnv ( fs )
if err != nil {
log . Fatalf ( "etcd: %v" , err )
}
2014-10-30 02:14:31 +03:00
2014-11-05 22:31:13 +03:00
shouldProxy := proxyFlag . String ( ) != proxyFlagOff
2014-11-14 10:03:34 +03:00
var stopped <- chan struct { }
2014-11-05 22:31:13 +03:00
if ! shouldProxy {
2014-11-14 10:03:34 +03:00
stopped , err = startEtcd ( )
2014-11-05 22:31:13 +03:00
if err == discovery . ErrFullCluster && fallbackFlag . String ( ) == fallbackFlagProxy {
log . Printf ( "etcd: discovery cluster full, falling back to %s" , fallbackFlagProxy )
shouldProxy = true
2014-11-05 00:09:24 +03:00
}
}
2014-11-05 22:31:13 +03:00
if shouldProxy {
err = startProxy ( )
}
if err != nil {
log . Fatalf ( "etcd: %v" , err )
2014-10-30 02:14:31 +03:00
}
2014-11-14 10:03:34 +03:00
<- stopped
2014-10-30 02:14:31 +03:00
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
2014-11-14 10:03:34 +03:00
func startEtcd ( ) ( <- chan struct { } , error ) {
2014-11-11 00:45:29 +03:00
apurls , err := flags . URLsFromFlags ( fs , "initial-advertise-peer-urls" , "addr" , peerTLSInfo )
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-11-11 00:45:29 +03:00
}
cls , err := setupCluster ( apurls )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , fmt . Errorf ( "error setting up initial cluster: %v" , err )
2014-10-30 02:14:31 +03:00
}
if * dir == "" {
* dir = fmt . Sprintf ( "%v.etcd" , * name )
2014-11-10 23:33:31 +03:00
log . Printf ( "no data-dir provided, using default data-dir ./%s" , * dir )
2014-10-30 02:14:31 +03:00
}
if err := os . MkdirAll ( * dir , privateDirMode ) ; err != nil {
2014-11-14 10:03:34 +03:00
return nil , fmt . Errorf ( "cannot create data directory: %v" , err )
2014-10-30 02:14:31 +03:00
}
if err := fileutil . IsDirWriteable ( * dir ) ; err != nil {
2014-11-14 10:03:34 +03:00
return nil , fmt . Errorf ( "cannot write to data directory: %v" , err )
2014-10-30 02:14:31 +03:00
}
2014-11-24 10:37:54 +03:00
pt , err := transport . NewTimeoutTransport ( peerTLSInfo , rafthttp . ConnReadTimeout , rafthttp . ConnWriteTimeout )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-10-30 02:14:31 +03:00
}
acurls , err := flags . URLsFromFlags ( fs , "advertise-client-urls" , "addr" , clientTLSInfo )
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-10-30 02:14:31 +03:00
}
lpurls , err := flags . URLsFromFlags ( fs , "listen-peer-urls" , "peer-bind-addr" , peerTLSInfo )
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-10-30 02:14:31 +03:00
}
2014-11-13 07:11:04 +03:00
if ! peerTLSInfo . Empty ( ) {
log . Printf ( "etcd: peerTLS: %s" , peerTLSInfo )
}
2014-11-04 02:17:21 +03:00
plns := make ( [ ] net . Listener , 0 )
2014-10-30 02:14:31 +03:00
for _ , u := range lpurls {
2014-11-05 00:09:24 +03:00
var l net . Listener
2014-11-24 10:37:54 +03:00
l , err = transport . NewTimeoutListener ( u . Host , u . Scheme , peerTLSInfo , rafthttp . ConnReadTimeout , rafthttp . ConnWriteTimeout )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-10-30 02:14:31 +03:00
}
urlStr := u . String ( )
2014-11-04 02:17:21 +03:00
log . Print ( "etcd: listening for peers on " , urlStr )
2014-11-05 00:09:24 +03:00
defer func ( ) {
if err != nil {
l . Close ( )
log . Print ( "etcd: stopping listening for peers on " , urlStr )
}
} ( )
2014-11-04 02:17:21 +03:00
plns = append ( plns , l )
2014-10-30 02:14:31 +03:00
}
lcurls , err := flags . URLsFromFlags ( fs , "listen-client-urls" , "bind-addr" , clientTLSInfo )
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-10-30 02:14:31 +03:00
}
2014-11-13 07:11:04 +03:00
if ! clientTLSInfo . Empty ( ) {
log . Printf ( "etcd: clientTLS: %s" , clientTLSInfo )
}
2014-11-04 02:17:21 +03:00
clns := make ( [ ] net . Listener , 0 )
2014-10-30 02:14:31 +03:00
for _ , u := range lcurls {
2014-11-05 00:09:24 +03:00
var l net . Listener
2014-11-11 22:51:57 +03:00
l , err = transport . NewListener ( u . Host , u . Scheme , clientTLSInfo )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-10-30 02:14:31 +03:00
}
urlStr := u . String ( )
2014-11-04 02:17:21 +03:00
log . Print ( "etcd: listening for client requests on " , urlStr )
2014-11-05 00:09:24 +03:00
defer func ( ) {
if err != nil {
l . Close ( )
log . Print ( "etcd: stopping listening for client requests on " , urlStr )
}
} ( )
2014-11-04 02:17:21 +03:00
clns = append ( clns , l )
}
cfg := & etcdserver . ServerConfig {
2014-11-07 08:50:35 +03:00
Name : * name ,
ClientURLs : acurls ,
2014-11-11 00:45:29 +03:00
PeerURLs : apurls ,
2014-11-07 08:50:35 +03:00
DataDir : * dir ,
SnapCount : * snapCount ,
2014-11-29 08:32:28 +03:00
MaxSnapFiles : * maxSnapFiles ,
MaxWALFiles : * maxWalFiles ,
2014-11-07 08:50:35 +03:00
Cluster : cls ,
DiscoveryURL : * durl ,
DiscoveryProxy : * dproxy ,
NewCluster : clusterStateFlag . String ( ) == clusterStateFlagNew ,
ForceNewCluster : * forceNewCluster ,
2014-11-07 21:31:19 +03:00
Transport : pt ,
2014-11-04 02:17:21 +03:00
}
2014-11-05 00:09:24 +03:00
var s * etcdserver . EtcdServer
s , err = etcdserver . NewServer ( cfg )
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-11-05 00:09:24 +03:00
}
2014-11-04 02:17:21 +03:00
s . Start ( )
2014-11-13 07:11:04 +03:00
if corsInfo . String ( ) != "" {
log . Printf ( "etcd: cors = %s" , corsInfo )
}
2014-11-04 02:17:21 +03:00
ch := & cors . CORSHandler {
Handler : etcdhttp . NewClientHandler ( s ) ,
Info : corsInfo ,
}
ph := etcdhttp . NewPeerHandler ( s )
// Start the peer server in a goroutine
for _ , l := range plns {
go func ( l net . Listener ) {
log . Fatal ( http . Serve ( l , ph ) )
} ( l )
}
// Start a client server goroutine for each listen address
for _ , l := range clns {
go func ( l net . Listener ) {
2014-10-30 02:14:31 +03:00
log . Fatal ( http . Serve ( l , ch ) )
2014-11-04 02:17:21 +03:00
} ( l )
2014-10-30 02:14:31 +03:00
}
2014-11-14 10:03:34 +03:00
return s . StopNotify ( ) , nil
2014-10-30 02:14:31 +03:00
}
// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
2014-11-05 00:09:24 +03:00
func startProxy ( ) error {
2014-11-11 00:45:29 +03:00
apurls , err := flags . URLsFromFlags ( fs , "initial-advertise-peer-urls" , "addr" , peerTLSInfo )
if err != nil {
return err
}
cls , err := setupCluster ( apurls )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-05 00:09:24 +03:00
return fmt . Errorf ( "error setting up initial cluster: %v" , err )
}
if * durl != "" {
2014-11-07 03:35:24 +03:00
s , err := discovery . GetCluster ( * durl , * dproxy )
2014-11-05 00:09:24 +03:00
if err != nil {
return err
}
if cls , err = etcdserver . NewClusterFromString ( * durl , s ) ; err != nil {
return err
}
2014-10-30 02:14:31 +03:00
}
pt , err := transport . NewTransport ( clientTLSInfo )
if err != nil {
2014-11-05 00:09:24 +03:00
return err
2014-10-30 02:14:31 +03:00
}
// TODO(jonboulle): update peerURLs dynamically (i.e. when updating
// clientURLs) instead of just using the initial fixed list here
peerURLs := cls . PeerURLs ( )
uf := func ( ) [ ] string {
cls , err := etcdserver . GetClusterFromPeers ( peerURLs )
if err != nil {
2014-11-05 00:09:24 +03:00
log . Printf ( "proxy: %v" , err )
2014-10-30 02:14:31 +03:00
return [ ] string { }
}
return cls . ClientURLs ( )
}
ph := proxy . NewHandler ( pt , uf )
ph = & cors . CORSHandler {
Handler : ph ,
Info : corsInfo ,
}
2014-11-05 02:57:38 +03:00
if proxyFlag . String ( ) == proxyFlagReadonly {
2014-10-30 02:14:31 +03:00
ph = proxy . NewReadonlyHandler ( ph )
}
lcurls , err := flags . URLsFromFlags ( fs , "listen-client-urls" , "bind-addr" , clientTLSInfo )
if err != nil {
2014-11-05 00:09:24 +03:00
return err
2014-10-30 02:14:31 +03:00
}
// Start a proxy server goroutine for each listen address
for _ , u := range lcurls {
2014-11-11 22:51:57 +03:00
l , err := transport . NewListener ( u . Host , u . Scheme , clientTLSInfo )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-05 00:09:24 +03:00
return err
2014-10-30 02:14:31 +03:00
}
host := u . Host
go func ( ) {
2014-11-05 00:09:24 +03:00
log . Print ( "proxy: listening for client requests on " , host )
2014-10-30 02:14:31 +03:00
log . Fatal ( http . Serve ( l , ph ) )
} ( )
}
2014-11-05 00:09:24 +03:00
return nil
2014-10-30 02:14:31 +03:00
}
2014-11-10 23:55:35 +03:00
// setupCluster sets up an initial cluster definition for bootstrap or discovery.
2014-11-11 00:45:29 +03:00
func setupCluster ( apurls [ ] url . URL ) ( * etcdserver . Cluster , error ) {
2014-10-30 02:14:31 +03:00
set := make ( map [ string ] bool )
fs . Visit ( func ( f * flag . Flag ) {
set [ f . Name ] = true
} )
if set [ "discovery" ] && set [ "initial-cluster" ] {
return nil , fmt . Errorf ( "both discovery and bootstrap-config are set" )
}
var cls * etcdserver . Cluster
2014-11-11 00:45:29 +03:00
var err error
2014-10-30 02:14:31 +03:00
switch {
case set [ "discovery" ] :
2014-11-10 23:55:35 +03:00
// If using discovery, generate a temporary cluster based on
// self's advertised peer URLs
2014-10-30 02:14:31 +03:00
clusterStr := genClusterString ( * name , apurls )
cls , err = etcdserver . NewClusterFromString ( * durl , clusterStr )
case set [ "initial-cluster" ] :
fallthrough
default :
// We're statically configured, and cluster has appropriately been set.
2014-10-30 23:43:38 +03:00
cls , err = etcdserver . NewClusterFromString ( * initialClusterToken , * initialCluster )
2014-10-30 02:14:31 +03:00
}
return cls , err
}
func genClusterString ( name string , urls types . URLs ) string {
addrs := make ( [ ] string , 0 )
for _ , u := range urls {
addrs = append ( addrs , fmt . Sprintf ( "%v=%v" , name , u . String ( ) ) )
}
return strings . Join ( addrs , "," )
}