2015-01-25 06:19:16 +03:00
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2014-10-30 02:14:31 +03:00
2016-03-08 23:31:09 +03:00
// TODO: support arm64
// +build amd64
2014-10-30 02:14:31 +03:00
package etcdmain
import (
2015-01-31 02:18:26 +03:00
"encoding/json"
2014-10-30 02:14:31 +03:00
"fmt"
2015-01-31 02:18:26 +03:00
"io/ioutil"
2014-11-04 02:17:21 +03:00
"net"
2014-10-30 02:14:31 +03:00
"net/http"
2016-01-07 10:15:27 +03:00
_ "net/http/pprof"
2014-10-30 02:14:31 +03:00
"os"
2015-01-31 02:18:26 +03:00
"path"
"reflect"
2015-05-27 02:49:02 +03:00
"runtime"
2015-09-19 01:18:29 +03:00
"strings"
2015-01-06 22:01:11 +03:00
"time"
2014-10-30 02:14:31 +03:00
2015-07-30 11:20:38 +03:00
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/coreos/go-systemd/daemon"
systemdutil "github.com/coreos/etcd/Godeps/_workspace/src/github.com/coreos/go-systemd/util"
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/coreos/pkg/capnslog"
"github.com/coreos/etcd/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus"
2014-11-05 00:09:24 +03:00
"github.com/coreos/etcd/discovery"
2014-10-30 02:14:31 +03:00
"github.com/coreos/etcd/etcdserver"
2015-08-08 15:58:29 +03:00
"github.com/coreos/etcd/etcdserver/api/v3rpc"
2014-10-30 02:14:31 +03:00
"github.com/coreos/etcd/etcdserver/etcdhttp"
"github.com/coreos/etcd/pkg/cors"
2015-03-18 00:43:20 +03:00
"github.com/coreos/etcd/pkg/fileutil"
2015-12-22 03:14:58 +03:00
pkgioutil "github.com/coreos/etcd/pkg/ioutil"
2015-02-13 12:23:38 +03:00
"github.com/coreos/etcd/pkg/osutil"
2015-08-04 21:36:24 +03:00
runtimeutil "github.com/coreos/etcd/pkg/runtime"
2014-10-30 02:14:31 +03:00
"github.com/coreos/etcd/pkg/transport"
"github.com/coreos/etcd/pkg/types"
"github.com/coreos/etcd/proxy"
2014-11-24 10:37:54 +03:00
"github.com/coreos/etcd/rafthttp"
2015-08-21 00:48:53 +03:00
"github.com/coreos/etcd/version"
2014-10-30 02:14:31 +03:00
)
2015-03-18 00:43:20 +03:00
type dirType string
2015-06-11 02:19:06 +03:00
var plog = capnslog . NewPackageLogger ( "github.com/coreos/etcd" , "etcdmain" )
2015-04-28 21:00:23 +03:00
2014-10-30 02:14:31 +03:00
const (
// the owner can make/remove files inside the directory
privateDirMode = 0700
2015-08-04 21:36:24 +03:00
// internal fd usage includes disk usage and transport usage.
// To read/write snapshot, snap pkg needs 1. In normal case, wal pkg needs
// at most 2 to read/lock/write WALs. One case that it needs to 2 is to
// read all logs after some snapshot index, which locates at the end of
// the second last and the head of the last. For purging, it needs to read
// directory, so it needs 1. For fd monitor, it needs 1.
// For transport, rafthttp builds two long-polling connections and at most
// four temporary connections with each member. There are at most 9 members
// in a cluster, so it should reserve 96.
// For the safety, we set the total reserved number to 150.
reservedInternalFDNum = 150
2014-10-30 02:14:31 +03:00
)
2015-03-18 00:43:20 +03:00
var (
dirMember = dirType ( "member" )
dirProxy = dirType ( "proxy" )
dirEmpty = dirType ( "empty" )
)
2014-10-30 02:14:31 +03:00
func Main ( ) {
2014-12-20 01:47:07 +03:00
cfg := NewConfig ( )
err := cfg . Parse ( os . Args [ 1 : ] )
2014-11-07 01:39:30 +03:00
if err != nil {
2015-06-11 19:57:50 +03:00
plog . Errorf ( "error verifying flags, %v. See 'etcd --help'." , err )
2015-04-26 18:15:19 +03:00
switch err {
case errUnsetAdvertiseClientURLsFlag :
2015-06-11 19:57:50 +03:00
plog . Errorf ( "When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client." )
2015-04-26 18:15:19 +03:00
}
os . Exit ( 1 )
2014-11-07 01:39:30 +03:00
}
2015-04-28 21:00:23 +03:00
setupLogging ( cfg )
2014-10-30 02:14:31 +03:00
2014-11-14 10:03:34 +03:00
var stopped <- chan struct { }
2014-12-20 01:47:07 +03:00
2015-08-21 00:48:53 +03:00
plog . Infof ( "etcd Version: %s\n" , version . Version )
plog . Infof ( "Git SHA: %s\n" , version . GitSHA )
plog . Infof ( "Go Version: %s\n" , runtime . Version ( ) )
plog . Infof ( "Go OS/Arch: %s/%s\n" , runtime . GOOS , runtime . GOARCH )
2015-08-25 22:08:52 +03:00
GoMaxProcs := runtime . GOMAXPROCS ( 0 )
2015-06-11 19:57:50 +03:00
plog . Infof ( "setting maximum number of CPUs to %d, total number of available CPUs is %d" , GoMaxProcs , runtime . NumCPU ( ) )
2015-05-27 02:49:02 +03:00
2015-04-27 05:49:01 +03:00
// TODO: check whether fields are set instead of whether fields have default value
2015-03-19 02:59:41 +03:00
if cfg . name != defaultName && cfg . initialCluster == initialClusterFromName ( defaultName ) {
cfg . initialCluster = initialClusterFromName ( cfg . name )
}
2015-03-18 00:43:20 +03:00
if cfg . dir == "" {
cfg . dir = fmt . Sprintf ( "%v.etcd" , cfg . name )
2015-06-11 19:57:50 +03:00
plog . Warningf ( "no data-dir provided, using default data-dir ./%s" , cfg . dir )
2015-03-18 00:43:20 +03:00
}
which := identifyDataDirOrDie ( cfg . dir )
if which != dirEmpty {
2015-06-11 19:57:50 +03:00
plog . Noticef ( "the server is already initialized as %v before, starting as etcd %v..." , which , which )
2015-07-19 04:53:57 +03:00
switch which {
case dirMember :
stopped , err = startEtcd ( cfg )
case dirProxy :
err = startProxy ( cfg )
default :
plog . Panicf ( "unhandled dir type %v" , which )
}
} else {
shouldProxy := cfg . isProxy ( )
if ! shouldProxy {
stopped , err = startEtcd ( cfg )
2015-10-28 00:39:38 +03:00
if derr , ok := err . ( * etcdserver . DiscoveryError ) ; ok && derr . Err == discovery . ErrFullCluster {
if cfg . shouldFallbackToProxy ( ) {
plog . Noticef ( "discovery cluster full, falling back to %s" , fallbackFlagProxy )
shouldProxy = true
}
2015-07-19 04:53:57 +03:00
}
}
if shouldProxy {
err = startProxy ( cfg )
2014-11-05 00:09:24 +03:00
}
}
2015-07-19 04:53:57 +03:00
2014-11-05 22:31:13 +03:00
if err != nil {
2015-10-28 00:39:38 +03:00
if derr , ok := err . ( * etcdserver . DiscoveryError ) ; ok {
switch derr . Err {
case discovery . ErrDuplicateID :
plog . Errorf ( "member %q has previously registered with discovery service token (%s)." , cfg . name , cfg . durl )
plog . Errorf ( "But etcd could not find valid cluster configuration in the given data dir (%s)." , cfg . dir )
plog . Infof ( "Please check the given data dir path if the previous bootstrap succeeded" )
plog . Infof ( "or use a new discovery token if the previous bootstrap failed." )
case discovery . ErrDuplicateName :
plog . Errorf ( "member with duplicated name has registered with discovery service token(%s)." , cfg . durl )
plog . Errorf ( "please check (cURL) the discovery token for more information." )
plog . Errorf ( "please do not reuse the discovery token and generate a new one to bootstrap the cluster." )
default :
2015-09-25 00:48:18 +03:00
plog . Errorf ( "%v" , err )
plog . Infof ( "discovery token %s was used, but failed to bootstrap the cluster." , cfg . durl )
plog . Infof ( "please generate a new discovery token and try to bootstrap again." )
}
2015-10-28 00:39:38 +03:00
os . Exit ( 1 )
2014-12-12 02:42:54 +03:00
}
2015-10-28 00:39:38 +03:00
if strings . Contains ( err . Error ( ) , "include" ) && strings . Contains ( err . Error ( ) , "--initial-cluster" ) {
plog . Infof ( "%v" , err )
if cfg . initialCluster == initialClusterFromName ( cfg . name ) {
plog . Infof ( "forgot to set --initial-cluster flag?" )
}
if types . URLs ( cfg . apurls ) . String ( ) == defaultInitialAdvertisePeerURLs {
plog . Infof ( "forgot to set --initial-advertise-peer-urls flag?" )
}
if cfg . initialCluster == initialClusterFromName ( cfg . name ) && len ( cfg . durl ) == 0 {
plog . Infof ( "if you want to use discovery service, please set --discovery flag." )
}
os . Exit ( 1 )
}
plog . Fatalf ( "%v" , err )
2014-10-30 02:14:31 +03:00
}
2014-12-20 01:47:07 +03:00
2015-02-13 12:23:38 +03:00
osutil . HandleInterrupts ( )
2015-07-30 11:20:38 +03:00
if systemdutil . IsRunningSystemd ( ) {
// At this point, the initialization of etcd is done.
// The listeners are listening on the TCP ports and ready
// for accepting connections.
// The http server is probably ready for serving incoming
// connections. If it is not, the connection might be pending
// for less than one second.
err := daemon . SdNotify ( "READY=1" )
if err != nil {
2015-10-02 23:39:41 +03:00
plog . Errorf ( "failed to notify systemd for readiness: %v" , err )
if err == daemon . SdNotifyNoSocket {
plog . Errorf ( "forgot to set Type=notify in systemd service file?" )
}
2015-07-30 11:20:38 +03:00
}
}
2014-11-14 10:03:34 +03:00
<- stopped
2015-02-13 12:23:38 +03:00
osutil . Exit ( 0 )
2014-10-30 02:14:31 +03:00
}
// startEtcd launches the etcd server and HTTP handlers for client/server communication.
2014-12-20 01:47:07 +03:00
func startEtcd ( cfg * config ) ( <- chan struct { } , error ) {
2015-09-01 21:24:02 +03:00
urlsmap , token , err := getPeerURLsMapAndToken ( cfg , "etcd" )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , fmt . Errorf ( "error setting up initial cluster: %v" , err )
2014-10-30 02:14:31 +03:00
}
2014-12-20 01:47:07 +03:00
if ! cfg . peerTLSInfo . Empty ( ) {
2015-06-11 19:57:50 +03:00
plog . Infof ( "peerTLS: %s" , cfg . peerTLSInfo )
2014-11-13 07:11:04 +03:00
}
2014-11-04 02:17:21 +03:00
plns := make ( [ ] net . Listener , 0 )
2014-12-20 01:47:07 +03:00
for _ , u := range cfg . lpurls {
2015-07-16 02:44:00 +03:00
if u . Scheme == "http" && ! cfg . peerTLSInfo . Empty ( ) {
plog . Warningf ( "The scheme of peer url %s is http while peer key/cert files are presented. Ignored peer key/cert files." , u . String ( ) )
}
2014-11-05 00:09:24 +03:00
var l net . Listener
2015-11-04 02:13:50 +03:00
l , err = rafthttp . NewListener ( u , cfg . peerTLSInfo )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-10-30 02:14:31 +03:00
}
urlStr := u . String ( )
2015-06-11 19:57:50 +03:00
plog . Info ( "listening for peers on " , urlStr )
2014-11-05 00:09:24 +03:00
defer func ( ) {
if err != nil {
l . Close ( )
2015-06-11 19:57:50 +03:00
plog . Info ( "stopping listening for peers on " , urlStr )
2014-11-05 00:09:24 +03:00
}
} ( )
2014-11-04 02:17:21 +03:00
plns = append ( plns , l )
2014-10-30 02:14:31 +03:00
}
2014-12-20 01:47:07 +03:00
if ! cfg . clientTLSInfo . Empty ( ) {
2015-06-11 19:57:50 +03:00
plog . Infof ( "clientTLS: %s" , cfg . clientTLSInfo )
2014-11-13 07:11:04 +03:00
}
2014-11-04 02:17:21 +03:00
clns := make ( [ ] net . Listener , 0 )
2014-12-20 01:47:07 +03:00
for _ , u := range cfg . lcurls {
2015-07-16 02:44:00 +03:00
if u . Scheme == "http" && ! cfg . clientTLSInfo . Empty ( ) {
plog . Warningf ( "The scheme of client url %s is http while client key/cert files are presented. Ignored client key/cert files." , u . String ( ) )
}
2014-11-05 00:09:24 +03:00
var l net . Listener
2016-01-07 08:59:38 +03:00
l , err = net . Listen ( "tcp" , u . Host )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-10-30 02:14:31 +03:00
}
2016-01-07 08:59:38 +03:00
2015-08-04 21:36:24 +03:00
if fdLimit , err := runtimeutil . FDLimit ( ) ; err == nil {
if fdLimit <= reservedInternalFDNum {
plog . Fatalf ( "file descriptor limit[%d] of etcd process is too low, and should be set higher than %d to ensure internal usage" , fdLimit , reservedInternalFDNum )
}
2016-01-08 21:10:05 +03:00
l = transport . LimitListener ( l , int ( fdLimit - reservedInternalFDNum ) )
2015-08-04 21:36:24 +03:00
}
2014-10-30 02:14:31 +03:00
2016-01-07 08:59:38 +03:00
// Do not wrap around this listener if TLS Info is set.
// HTTPS server expects TLS Conn created by TLSListener.
l , err = transport . NewKeepAliveListener ( l , u . Scheme , cfg . clientTLSInfo )
if err != nil {
return nil , err
}
2014-10-30 02:14:31 +03:00
urlStr := u . String ( )
2015-06-11 19:57:50 +03:00
plog . Info ( "listening for client requests on " , urlStr )
2014-11-05 00:09:24 +03:00
defer func ( ) {
if err != nil {
l . Close ( )
2015-06-11 19:57:50 +03:00
plog . Info ( "stopping listening for client requests on " , urlStr )
2014-11-05 00:09:24 +03:00
}
} ( )
2014-11-04 02:17:21 +03:00
clns = append ( clns , l )
}
2015-08-08 15:58:29 +03:00
var v3l net . Listener
if cfg . v3demo {
2015-09-13 08:52:51 +03:00
v3l , err = net . Listen ( "tcp" , cfg . gRPCAddr )
2015-08-08 15:58:29 +03:00
if err != nil {
plog . Fatal ( err )
}
2015-09-13 08:52:51 +03:00
plog . Infof ( "listening for client rpc on %s" , cfg . gRPCAddr )
2015-08-08 15:58:29 +03:00
}
2014-12-20 01:47:07 +03:00
srvcfg := & etcdserver . ServerConfig {
2016-02-25 06:35:15 +03:00
Name : cfg . name ,
ClientURLs : cfg . acurls ,
PeerURLs : cfg . apurls ,
DataDir : cfg . dir ,
DedicatedWALDir : cfg . walDir ,
SnapCount : cfg . snapCount ,
MaxSnapFiles : cfg . maxSnapFiles ,
MaxWALFiles : cfg . maxWalFiles ,
InitialPeerURLsMap : urlsmap ,
InitialClusterToken : token ,
DiscoveryURL : cfg . durl ,
DiscoveryProxy : cfg . dproxy ,
NewCluster : cfg . isNewCluster ( ) ,
ForceNewCluster : cfg . forceNewCluster ,
PeerTLSInfo : cfg . peerTLSInfo ,
TickMs : cfg . TickMs ,
ElectionTicks : cfg . electionTicks ( ) ,
V3demo : cfg . v3demo ,
AutoCompactionRetention : cfg . autoCompactionRetention ,
StrictReconfigCheck : cfg . strictReconfigCheck ,
EnablePprof : cfg . enablePprof ,
2014-11-04 02:17:21 +03:00
}
2014-11-05 00:09:24 +03:00
var s * etcdserver . EtcdServer
2014-12-20 01:47:07 +03:00
s , err = etcdserver . NewServer ( srvcfg )
2014-11-05 00:09:24 +03:00
if err != nil {
2014-11-14 10:03:34 +03:00
return nil , err
2014-11-05 00:09:24 +03:00
}
2014-11-04 02:17:21 +03:00
s . Start ( )
2015-02-13 12:23:38 +03:00
osutil . RegisterInterruptHandler ( s . Stop )
2014-11-04 02:17:21 +03:00
2014-12-20 01:47:07 +03:00
if cfg . corsInfo . String ( ) != "" {
2015-06-11 19:57:50 +03:00
plog . Infof ( "cors = %s" , cfg . corsInfo )
2014-11-13 07:11:04 +03:00
}
2014-11-04 02:17:21 +03:00
ch := & cors . CORSHandler {
2015-08-12 07:22:05 +03:00
Handler : etcdhttp . NewClientHandler ( s , srvcfg . ReqTimeout ( ) ) ,
2014-12-20 01:47:07 +03:00
Info : cfg . corsInfo ,
2014-11-04 02:17:21 +03:00
}
2016-01-20 08:09:09 +03:00
ph := etcdhttp . NewPeerHandler ( s )
2014-11-04 02:17:21 +03:00
// Start the peer server in a goroutine
for _ , l := range plns {
go func ( l net . Listener ) {
2015-06-11 02:19:06 +03:00
plog . Fatal ( serveHTTP ( l , ph , 5 * time . Minute ) )
2014-11-04 02:17:21 +03:00
} ( l )
}
// Start a client server goroutine for each listen address
for _ , l := range clns {
go func ( l net . Listener ) {
2015-01-07 03:17:56 +03:00
// read timeout does not work with http close notify
// TODO: https://github.com/golang/go/issues/9524
2015-06-11 02:19:06 +03:00
plog . Fatal ( serveHTTP ( l , ch , 0 ) )
2014-11-04 02:17:21 +03:00
} ( l )
2014-10-30 02:14:31 +03:00
}
2015-08-08 15:58:29 +03:00
if cfg . v3demo {
// set up v3 demo rpc
2016-01-29 22:06:27 +03:00
tls := & cfg . clientTLSInfo
if cfg . clientTLSInfo . Empty ( ) {
tls = nil
}
grpcServer , err := v3rpc . Server ( s , tls )
2016-01-29 03:48:25 +03:00
if err != nil {
s . Stop ( )
<- s . StopNotify ( )
return nil , err
}
2015-11-17 20:03:56 +03:00
go func ( ) { plog . Fatal ( grpcServer . Serve ( v3l ) ) } ( )
2015-08-08 15:58:29 +03:00
}
2014-11-14 10:03:34 +03:00
return s . StopNotify ( ) , nil
2014-10-30 02:14:31 +03:00
}
// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
2014-12-20 01:47:07 +03:00
func startProxy ( cfg * config ) error {
2015-06-12 21:35:38 +03:00
pt , err := transport . NewTimeoutTransport ( cfg . peerTLSInfo , time . Duration ( cfg . proxyDialTimeoutMs ) * time . Millisecond , time . Duration ( cfg . proxyReadTimeoutMs ) * time . Millisecond , time . Duration ( cfg . proxyWriteTimeoutMs ) * time . Millisecond )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-05 00:09:24 +03:00
return err
2014-10-30 02:14:31 +03:00
}
2015-08-26 07:59:56 +03:00
pt . MaxIdleConnsPerHost = proxy . DefaultMaxIdleConnsPerHost
2014-10-30 02:14:31 +03:00
2015-06-12 21:35:38 +03:00
tr , err := transport . NewTimeoutTransport ( cfg . peerTLSInfo , time . Duration ( cfg . proxyDialTimeoutMs ) * time . Millisecond , time . Duration ( cfg . proxyReadTimeoutMs ) * time . Millisecond , time . Duration ( cfg . proxyWriteTimeoutMs ) * time . Millisecond )
2015-01-26 18:31:11 +03:00
if err != nil {
return err
}
2015-02-03 01:48:55 +03:00
cfg . dir = path . Join ( cfg . dir , "proxy" )
2015-01-31 02:18:26 +03:00
err = os . MkdirAll ( cfg . dir , 0700 )
if err != nil {
return err
}
var peerURLs [ ] string
clusterfile := path . Join ( cfg . dir , "cluster" )
b , err := ioutil . ReadFile ( clusterfile )
switch {
case err == nil :
2015-07-10 22:52:24 +03:00
if cfg . durl != "" {
2015-08-07 01:53:24 +03:00
plog . Warningf ( "discovery token ignored since the proxy has already been initialized. Valid cluster file found at %q" , clusterfile )
2015-07-10 22:52:24 +03:00
}
2016-01-14 22:21:02 +03:00
if cfg . dnsCluster != "" {
plog . Warningf ( "DNS SRV discovery ignored since the proxy has already been initialized. Valid cluster file found at %q" , clusterfile )
}
2015-01-31 02:18:26 +03:00
urls := struct { PeerURLs [ ] string } { }
2015-12-12 16:12:05 +03:00
err = json . Unmarshal ( b , & urls )
2015-01-31 02:18:26 +03:00
if err != nil {
return err
}
peerURLs = urls . PeerURLs
2015-08-07 01:53:24 +03:00
plog . Infof ( "proxy: using peer urls %v from cluster file %q" , peerURLs , clusterfile )
2015-01-31 02:18:26 +03:00
case os . IsNotExist ( err ) :
2016-01-14 22:21:02 +03:00
urlsmap , _ , err := getPeerURLsMapAndToken ( cfg , "proxy" )
if err != nil {
return fmt . Errorf ( "error setting up initial cluster: %v" , err )
}
2015-07-10 22:52:24 +03:00
if cfg . durl != "" {
s , err := discovery . GetCluster ( cfg . durl , cfg . dproxy )
if err != nil {
return err
}
if urlsmap , err = types . NewURLsMap ( s ) ; err != nil {
return err
}
}
2015-04-27 05:49:01 +03:00
peerURLs = urlsmap . URLs ( )
2015-06-11 19:57:50 +03:00
plog . Infof ( "proxy: using peer urls %v " , peerURLs )
2015-01-31 02:18:26 +03:00
default :
return err
}
2015-04-27 05:49:01 +03:00
clientURLs := [ ] string { }
2014-10-30 02:14:31 +03:00
uf := func ( ) [ ] string {
2015-02-14 06:05:29 +03:00
gcls , err := etcdserver . GetClusterFromRemotePeers ( peerURLs , tr )
2016-02-01 08:42:39 +03:00
// TODO: remove the 2nd check when we fix GetClusterFromRemotePeers
// GetClusterFromRemotePeers should not return nil error with an invalid empty cluster
2014-10-30 02:14:31 +03:00
if err != nil {
2015-06-11 19:57:50 +03:00
plog . Warningf ( "proxy: %v" , err )
2014-10-30 02:14:31 +03:00
return [ ] string { }
}
2015-02-03 01:48:55 +03:00
if len ( gcls . Members ( ) ) == 0 {
2015-04-27 05:49:01 +03:00
return clientURLs
2015-02-03 01:48:55 +03:00
}
2015-04-27 05:49:01 +03:00
clientURLs = gcls . ClientURLs ( )
2015-01-31 02:18:26 +03:00
2015-04-27 05:49:01 +03:00
urls := struct { PeerURLs [ ] string } { gcls . PeerURLs ( ) }
2015-01-31 02:18:26 +03:00
b , err := json . Marshal ( urls )
if err != nil {
2015-06-11 19:57:50 +03:00
plog . Warningf ( "proxy: error on marshal peer urls %s" , err )
2015-04-27 05:49:01 +03:00
return clientURLs
2015-01-31 02:18:26 +03:00
}
2015-12-22 03:14:58 +03:00
err = pkgioutil . WriteAndSyncFile ( clusterfile + ".bak" , b , 0600 )
2015-01-31 02:18:26 +03:00
if err != nil {
2015-06-11 19:57:50 +03:00
plog . Warningf ( "proxy: error on writing urls %s" , err )
2015-04-27 05:49:01 +03:00
return clientURLs
2015-01-31 02:18:26 +03:00
}
err = os . Rename ( clusterfile + ".bak" , clusterfile )
if err != nil {
2015-06-11 19:57:50 +03:00
plog . Warningf ( "proxy: error on updating clusterfile %s" , err )
2015-04-27 05:49:01 +03:00
return clientURLs
2015-01-31 02:18:26 +03:00
}
2015-04-27 05:49:01 +03:00
if ! reflect . DeepEqual ( gcls . PeerURLs ( ) , peerURLs ) {
2015-06-11 19:57:50 +03:00
plog . Noticef ( "proxy: updated peer urls in cluster file from %v to %v" , peerURLs , gcls . PeerURLs ( ) )
2015-01-31 02:18:26 +03:00
}
2015-04-27 05:49:01 +03:00
peerURLs = gcls . PeerURLs ( )
2015-01-31 02:18:26 +03:00
2015-04-27 05:49:01 +03:00
return clientURLs
2014-10-30 02:14:31 +03:00
}
2015-06-12 21:35:38 +03:00
ph := proxy . NewHandler ( pt , uf , time . Duration ( cfg . proxyFailureWaitMs ) * time . Millisecond , time . Duration ( cfg . proxyRefreshIntervalMs ) * time . Millisecond )
2014-10-30 02:14:31 +03:00
ph = & cors . CORSHandler {
Handler : ph ,
2014-12-20 01:47:07 +03:00
Info : cfg . corsInfo ,
2014-10-30 02:14:31 +03:00
}
2014-12-20 01:47:07 +03:00
if cfg . isReadonlyProxy ( ) {
2014-10-30 02:14:31 +03:00
ph = proxy . NewReadonlyHandler ( ph )
}
// Start a proxy server goroutine for each listen address
2014-12-20 01:47:07 +03:00
for _ , u := range cfg . lcurls {
l , err := transport . NewListener ( u . Host , u . Scheme , cfg . clientTLSInfo )
2014-10-30 02:14:31 +03:00
if err != nil {
2014-11-05 00:09:24 +03:00
return err
2014-10-30 02:14:31 +03:00
}
2015-09-09 00:19:23 +03:00
host := u . String ( )
2014-10-30 02:14:31 +03:00
go func ( ) {
2015-06-11 19:57:50 +03:00
plog . Info ( "proxy: listening for client requests on " , host )
2015-06-17 16:32:13 +03:00
mux := http . NewServeMux ( )
mux . Handle ( "/metrics" , prometheus . Handler ( ) )
mux . Handle ( "/" , ph )
plog . Fatal ( http . Serve ( l , mux ) )
2014-10-30 02:14:31 +03:00
} ( )
}
2014-11-05 00:09:24 +03:00
return nil
2014-10-30 02:14:31 +03:00
}
2015-04-27 05:49:01 +03:00
// getPeerURLsMapAndToken sets up an initial peer URLsMap and cluster token for bootstrap or discovery.
2015-09-01 21:24:02 +03:00
func getPeerURLsMapAndToken ( cfg * config , which string ) ( urlsmap types . URLsMap , token string , err error ) {
2014-10-30 02:14:31 +03:00
switch {
2014-12-20 01:47:07 +03:00
case cfg . durl != "" :
2015-04-27 05:49:01 +03:00
urlsmap = types . URLsMap { }
2014-11-10 23:55:35 +03:00
// If using discovery, generate a temporary cluster based on
// self's advertised peer URLs
2015-04-27 05:49:01 +03:00
urlsmap [ cfg . name ] = cfg . apurls
token = cfg . durl
2014-12-20 01:47:07 +03:00
case cfg . dnsCluster != "" :
2015-04-27 05:49:01 +03:00
var clusterStr string
clusterStr , token , err = discovery . SRVGetCluster ( cfg . name , cfg . dnsCluster , cfg . initialClusterToken , cfg . apurls )
2014-12-16 03:26:42 +03:00
if err != nil {
2015-04-27 05:49:01 +03:00
return nil , "" , err
2014-12-16 03:26:42 +03:00
}
2015-04-27 05:49:01 +03:00
urlsmap , err = types . NewURLsMap ( clusterStr )
2015-09-01 21:24:02 +03:00
// only etcd member must belong to the discovered cluster.
// proxy does not need to belong to the discovered cluster.
if which == "etcd" {
if _ , ok := urlsmap [ cfg . name ] ; ! ok {
return nil , "" , fmt . Errorf ( "cannot find local etcd member %q in SRV records" , cfg . name )
}
2015-08-06 21:38:53 +03:00
}
2014-10-30 02:14:31 +03:00
default :
// We're statically configured, and cluster has appropriately been set.
2015-04-27 05:49:01 +03:00
urlsmap , err = types . NewURLsMap ( cfg . initialCluster )
token = cfg . initialClusterToken
2014-10-30 02:14:31 +03:00
}
2015-04-27 05:49:01 +03:00
return urlsmap , token , err
2014-10-30 02:14:31 +03:00
}
2015-03-18 00:43:20 +03:00
// identifyDataDirOrDie returns the type of the data dir.
// Dies if the datadir is invalid.
func identifyDataDirOrDie ( dir string ) dirType {
names , err := fileutil . ReadDir ( dir )
if err != nil {
if os . IsNotExist ( err ) {
return dirEmpty
}
2015-06-11 02:19:06 +03:00
plog . Fatalf ( "error listing data dir: %s" , dir )
2015-03-18 00:43:20 +03:00
}
var m , p bool
for _ , name := range names {
switch dirType ( name ) {
case dirMember :
m = true
case dirProxy :
p = true
default :
2015-06-11 19:57:50 +03:00
plog . Warningf ( "found invalid file/dir %s under data dir %s (Ignore this if you are upgrading etcd)" , name , dir )
2015-03-18 00:43:20 +03:00
}
}
if m && p {
2015-06-11 02:19:06 +03:00
plog . Fatal ( "invalid datadir. Both member and proxy directories exist." )
2015-03-18 00:43:20 +03:00
}
if m {
return dirMember
}
if p {
return dirProxy
}
return dirEmpty
}
2015-04-28 21:00:23 +03:00
func setupLogging ( cfg * config ) {
capnslog . SetGlobalLogLevel ( capnslog . INFO )
if cfg . debug {
capnslog . SetGlobalLogLevel ( capnslog . DEBUG )
}
2015-04-28 23:29:58 +03:00
if cfg . logPkgLevels != "" {
2015-04-28 21:00:23 +03:00
repoLog := capnslog . MustRepoLogger ( "github.com/coreos/etcd" )
2015-04-28 23:29:58 +03:00
settings , err := repoLog . ParseLogLevelConfig ( cfg . logPkgLevels )
2015-04-28 21:00:23 +03:00
if err != nil {
2015-06-11 19:57:50 +03:00
plog . Warningf ( "couldn't parse log level string: %s, continuing with default levels" , err . Error ( ) )
2015-04-28 21:00:23 +03:00
return
}
repoLog . SetLogLevel ( settings )
}
}