go.etcd.io/etcd@v3.3.27+incompatible/etcdmain/etcd.go (about) 1 // Copyright 2015 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package etcdmain 16 17 import ( 18 "encoding/json" 19 "fmt" 20 "io/ioutil" 21 "net/http" 22 "os" 23 "path/filepath" 24 "reflect" 25 "runtime" 26 "strings" 27 "time" 28 29 "github.com/coreos/etcd/discovery" 30 "github.com/coreos/etcd/embed" 31 "github.com/coreos/etcd/etcdserver" 32 "github.com/coreos/etcd/etcdserver/api/etcdhttp" 33 "github.com/coreos/etcd/pkg/cors" 34 "github.com/coreos/etcd/pkg/fileutil" 35 pkgioutil "github.com/coreos/etcd/pkg/ioutil" 36 "github.com/coreos/etcd/pkg/osutil" 37 "github.com/coreos/etcd/pkg/transport" 38 "github.com/coreos/etcd/pkg/types" 39 "github.com/coreos/etcd/proxy/httpproxy" 40 "github.com/coreos/etcd/version" 41 42 "github.com/coreos/pkg/capnslog" 43 "google.golang.org/grpc" 44 ) 45 46 type dirType string 47 48 var plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "etcdmain") 49 50 var ( 51 dirMember = dirType("member") 52 dirProxy = dirType("proxy") 53 dirEmpty = dirType("empty") 54 ) 55 56 func startEtcdOrProxyV2() { 57 grpc.EnableTracing = false 58 59 cfg := newConfig() 60 defaultInitialCluster := cfg.ec.InitialCluster 61 62 err := cfg.parse(os.Args[1:]) 63 if err != nil { 64 plog.Errorf("error verifying flags, %v. See 'etcd --help'.", err) 65 switch err { 66 case embed.ErrUnsetAdvertiseClientURLsFlag: 67 plog.Errorf("When listening on specific address(es), this etcd process must advertise accessible url(s) to each connected client.") 68 } 69 os.Exit(1) 70 } 71 cfg.ec.SetupLogging() 72 73 var stopped <-chan struct{} 74 var errc <-chan error 75 76 plog.Infof("etcd Version: %s\n", version.Version) 77 plog.Infof("Git SHA: %s\n", version.GitSHA) 78 plog.Infof("Go Version: %s\n", runtime.Version()) 79 plog.Infof("Go OS/Arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) 80 81 GoMaxProcs := runtime.GOMAXPROCS(0) 82 plog.Infof("setting maximum number of CPUs to %d, total number of available CPUs is %d", GoMaxProcs, runtime.NumCPU()) 83 84 defaultHost, dhErr := (&cfg.ec).UpdateDefaultClusterFromName(defaultInitialCluster) 85 if defaultHost != "" { 86 plog.Infof("advertising using detected default host %q", defaultHost) 87 } 88 if dhErr != nil { 89 plog.Noticef("failed to detect default host (%v)", dhErr) 90 } 91 92 if cfg.ec.Dir == "" { 93 cfg.ec.Dir = fmt.Sprintf("%v.etcd", cfg.ec.Name) 94 plog.Warningf("no data-dir provided, using default data-dir ./%s", cfg.ec.Dir) 95 } 96 97 which := identifyDataDirOrDie(cfg.ec.Dir) 98 if which != dirEmpty { 99 plog.Noticef("the server is already initialized as %v before, starting as etcd %v...", which, which) 100 switch which { 101 case dirMember: 102 stopped, errc, err = startEtcd(&cfg.ec) 103 case dirProxy: 104 err = startProxy(cfg) 105 default: 106 plog.Panicf("unhandled dir type %v", which) 107 } 108 } else { 109 shouldProxy := cfg.isProxy() 110 if !shouldProxy { 111 stopped, errc, err = startEtcd(&cfg.ec) 112 if derr, ok := err.(*etcdserver.DiscoveryError); ok && derr.Err == discovery.ErrFullCluster { 113 if cfg.shouldFallbackToProxy() { 114 plog.Noticef("discovery cluster full, falling back to %s", fallbackFlagProxy) 115 shouldProxy = true 116 } 117 } 118 } 119 if shouldProxy { 120 err = startProxy(cfg) 121 } 122 } 123 124 if err != nil { 125 if derr, ok := err.(*etcdserver.DiscoveryError); ok { 126 switch derr.Err { 127 case discovery.ErrDuplicateID: 128 plog.Errorf("member %q has previously registered with discovery service token (%s).", cfg.ec.Name, cfg.ec.Durl) 129 plog.Errorf("But etcd could not find valid cluster configuration in the given data dir (%s).", cfg.ec.Dir) 130 plog.Infof("Please check the given data dir path if the previous bootstrap succeeded") 131 plog.Infof("or use a new discovery token if the previous bootstrap failed.") 132 case discovery.ErrDuplicateName: 133 plog.Errorf("member with duplicated name has registered with discovery service token(%s).", cfg.ec.Durl) 134 plog.Errorf("please check (cURL) the discovery token for more information.") 135 plog.Errorf("please do not reuse the discovery token and generate a new one to bootstrap the cluster.") 136 default: 137 plog.Errorf("%v", err) 138 plog.Infof("discovery token %s was used, but failed to bootstrap the cluster.", cfg.ec.Durl) 139 plog.Infof("please generate a new discovery token and try to bootstrap again.") 140 } 141 os.Exit(1) 142 } 143 144 if strings.Contains(err.Error(), "include") && strings.Contains(err.Error(), "--initial-cluster") { 145 plog.Infof("%v", err) 146 if cfg.ec.InitialCluster == cfg.ec.InitialClusterFromName(cfg.ec.Name) { 147 plog.Infof("forgot to set --initial-cluster flag?") 148 } 149 if types.URLs(cfg.ec.APUrls).String() == embed.DefaultInitialAdvertisePeerURLs { 150 plog.Infof("forgot to set --initial-advertise-peer-urls flag?") 151 } 152 if cfg.ec.InitialCluster == cfg.ec.InitialClusterFromName(cfg.ec.Name) && len(cfg.ec.Durl) == 0 { 153 plog.Infof("if you want to use discovery service, please set --discovery flag.") 154 } 155 os.Exit(1) 156 } 157 plog.Fatalf("%v", err) 158 } 159 160 osutil.HandleInterrupts() 161 162 // At this point, the initialization of etcd is done. 163 // The listeners are listening on the TCP ports and ready 164 // for accepting connections. The etcd instance should be 165 // joined with the cluster and ready to serve incoming 166 // connections. 167 notifySystemd() 168 169 select { 170 case lerr := <-errc: 171 // fatal out on listener errors 172 plog.Fatal(lerr) 173 case <-stopped: 174 } 175 176 osutil.Exit(0) 177 } 178 179 // startEtcd runs StartEtcd in addition to hooks needed for standalone etcd. 180 func startEtcd(cfg *embed.Config) (<-chan struct{}, <-chan error, error) { 181 e, err := embed.StartEtcd(cfg) 182 if err != nil { 183 return nil, nil, err 184 } 185 osutil.RegisterInterruptHandler(e.Close) 186 select { 187 case <-e.Server.ReadyNotify(): // wait for e.Server to join the cluster 188 case <-e.Server.StopNotify(): // publish aborted from 'ErrStopped' 189 } 190 return e.Server.StopNotify(), e.Err(), nil 191 } 192 193 // startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes. 194 func startProxy(cfg *config) error { 195 plog.Notice("proxy: this proxy supports v2 API only!") 196 197 clientTLSInfo := cfg.ec.ClientTLSInfo 198 if clientTLSInfo.Empty() { 199 // Support old proxy behavior of defaulting to PeerTLSInfo 200 // for both client and peer connections. 201 clientTLSInfo = cfg.ec.PeerTLSInfo 202 } 203 clientTLSInfo.InsecureSkipVerify = cfg.ec.ClientAutoTLS 204 cfg.ec.PeerTLSInfo.InsecureSkipVerify = cfg.ec.PeerAutoTLS 205 206 pt, err := transport.NewTimeoutTransport(clientTLSInfo, time.Duration(cfg.cp.ProxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.cp.ProxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.cp.ProxyWriteTimeoutMs)*time.Millisecond) 207 if err != nil { 208 return err 209 } 210 pt.MaxIdleConnsPerHost = httpproxy.DefaultMaxIdleConnsPerHost 211 212 if err = cfg.ec.PeerSelfCert(); err != nil { 213 plog.Fatalf("could not get certs (%v)", err) 214 } 215 tr, err := transport.NewTimeoutTransport(cfg.ec.PeerTLSInfo, time.Duration(cfg.cp.ProxyDialTimeoutMs)*time.Millisecond, time.Duration(cfg.cp.ProxyReadTimeoutMs)*time.Millisecond, time.Duration(cfg.cp.ProxyWriteTimeoutMs)*time.Millisecond) 216 if err != nil { 217 return err 218 } 219 220 cfg.ec.Dir = filepath.Join(cfg.ec.Dir, "proxy") 221 err = fileutil.TouchDirAll(cfg.ec.Dir) 222 if err != nil { 223 return err 224 } 225 226 var peerURLs []string 227 clusterfile := filepath.Join(cfg.ec.Dir, "cluster") 228 229 b, err := ioutil.ReadFile(clusterfile) 230 switch { 231 case err == nil: 232 if cfg.ec.Durl != "" { 233 plog.Warningf("discovery token ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile) 234 } 235 if cfg.ec.DNSCluster != "" { 236 plog.Warningf("DNS SRV discovery ignored since the proxy has already been initialized. Valid cluster file found at %q", clusterfile) 237 } 238 urls := struct{ PeerURLs []string }{} 239 err = json.Unmarshal(b, &urls) 240 if err != nil { 241 return err 242 } 243 peerURLs = urls.PeerURLs 244 plog.Infof("proxy: using peer urls %v from cluster file %q", peerURLs, clusterfile) 245 case os.IsNotExist(err): 246 var urlsmap types.URLsMap 247 urlsmap, _, err = cfg.ec.PeerURLsMapAndToken("proxy") 248 if err != nil { 249 return fmt.Errorf("error setting up initial cluster: %v", err) 250 } 251 252 if cfg.ec.Durl != "" { 253 var s string 254 s, err = discovery.GetCluster(cfg.ec.Durl, cfg.ec.Dproxy) 255 if err != nil { 256 return err 257 } 258 if urlsmap, err = types.NewURLsMap(s); err != nil { 259 return err 260 } 261 } 262 peerURLs = urlsmap.URLs() 263 plog.Infof("proxy: using peer urls %v ", peerURLs) 264 default: 265 return err 266 } 267 268 clientURLs := []string{} 269 uf := func() []string { 270 gcls, gerr := etcdserver.GetClusterFromRemotePeers(peerURLs, tr) 271 272 if gerr != nil { 273 plog.Warningf("proxy: %v", gerr) 274 return []string{} 275 } 276 277 clientURLs = gcls.ClientURLs() 278 279 urls := struct{ PeerURLs []string }{gcls.PeerURLs()} 280 b, jerr := json.Marshal(urls) 281 if jerr != nil { 282 plog.Warningf("proxy: error on marshal peer urls %s", jerr) 283 return clientURLs 284 } 285 286 err = pkgioutil.WriteAndSyncFile(clusterfile+".bak", b, 0600) 287 if err != nil { 288 plog.Warningf("proxy: error on writing urls %s", err) 289 return clientURLs 290 } 291 err = os.Rename(clusterfile+".bak", clusterfile) 292 if err != nil { 293 plog.Warningf("proxy: error on updating clusterfile %s", err) 294 return clientURLs 295 } 296 if !reflect.DeepEqual(gcls.PeerURLs(), peerURLs) { 297 plog.Noticef("proxy: updated peer urls in cluster file from %v to %v", peerURLs, gcls.PeerURLs()) 298 } 299 peerURLs = gcls.PeerURLs() 300 301 return clientURLs 302 } 303 ph := httpproxy.NewHandler(pt, uf, time.Duration(cfg.cp.ProxyFailureWaitMs)*time.Millisecond, time.Duration(cfg.cp.ProxyRefreshIntervalMs)*time.Millisecond) 304 ph = &cors.CORSHandler{ 305 Handler: ph, 306 Info: cfg.ec.CorsInfo, 307 } 308 309 if cfg.isReadonlyProxy() { 310 ph = httpproxy.NewReadonlyHandler(ph) 311 } 312 313 // setup self signed certs when serving https 314 cHosts, cTLS := []string{}, false 315 for _, u := range cfg.ec.LCUrls { 316 cHosts = append(cHosts, u.Host) 317 cTLS = cTLS || u.Scheme == "https" 318 } 319 for _, u := range cfg.ec.ACUrls { 320 cHosts = append(cHosts, u.Host) 321 cTLS = cTLS || u.Scheme == "https" 322 } 323 listenerTLS := cfg.ec.ClientTLSInfo 324 if cfg.ec.ClientAutoTLS && cTLS { 325 listenerTLS, err = transport.SelfCert(filepath.Join(cfg.ec.Dir, "clientCerts"), cHosts) 326 if err != nil { 327 plog.Fatalf("proxy: could not initialize self-signed client certs (%v)", err) 328 } 329 } 330 331 // Start a proxy server goroutine for each listen address 332 for _, u := range cfg.ec.LCUrls { 333 l, err := transport.NewListener(u.Host, u.Scheme, &listenerTLS) 334 if err != nil { 335 return err 336 } 337 338 host := u.String() 339 go func() { 340 plog.Info("proxy: listening for client requests on ", host) 341 mux := http.NewServeMux() 342 etcdhttp.HandlePrometheus(mux) // v2 proxy just uses the same port 343 mux.Handle("/", ph) 344 plog.Fatal(http.Serve(l, mux)) 345 }() 346 } 347 return nil 348 } 349 350 // identifyDataDirOrDie returns the type of the data dir. 351 // Dies if the datadir is invalid. 352 func identifyDataDirOrDie(dir string) dirType { 353 names, err := fileutil.ReadDir(dir) 354 if err != nil { 355 if os.IsNotExist(err) { 356 return dirEmpty 357 } 358 plog.Fatalf("error listing data dir: %s", dir) 359 } 360 361 var m, p bool 362 for _, name := range names { 363 switch dirType(name) { 364 case dirMember: 365 m = true 366 case dirProxy: 367 p = true 368 default: 369 plog.Warningf("found invalid file/dir %s under data dir %s (Ignore this if you are upgrading etcd)", name, dir) 370 } 371 } 372 373 if m && p { 374 plog.Fatal("invalid datadir. Both member and proxy directories exist.") 375 } 376 if m { 377 return dirMember 378 } 379 if p { 380 return dirProxy 381 } 382 return dirEmpty 383 } 384 385 func checkSupportArch() { 386 // TODO qualify arm64 387 if runtime.GOARCH == "amd64" || runtime.GOARCH == "ppc64le" { 388 return 389 } 390 // unsupported arch only configured via environment variable 391 // so unset here to not parse through flag 392 defer os.Unsetenv("ETCD_UNSUPPORTED_ARCH") 393 if env, ok := os.LookupEnv("ETCD_UNSUPPORTED_ARCH"); ok && env == runtime.GOARCH { 394 plog.Warningf("running etcd on unsupported architecture %q since ETCD_UNSUPPORTED_ARCH is set", env) 395 return 396 } 397 plog.Errorf("etcd on unsupported platform without ETCD_UNSUPPORTED_ARCH=%s set.", runtime.GOARCH) 398 os.Exit(1) 399 }