go.etcd.io/etcd@v3.3.27+incompatible/tests/e2e/cluster_test.go (about) 1 // Copyright 2016 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package e2e 16 17 import ( 18 "fmt" 19 "io/ioutil" 20 "net/url" 21 "os" 22 "strings" 23 ) 24 25 const etcdProcessBasePort = 20000 26 27 type clientConnType int 28 29 const ( 30 clientNonTLS clientConnType = iota 31 clientTLS 32 clientTLSAndNonTLS 33 ) 34 35 var ( 36 configNoTLS = etcdProcessClusterConfig{ 37 clusterSize: 3, 38 initialToken: "new", 39 } 40 configAutoTLS = etcdProcessClusterConfig{ 41 clusterSize: 3, 42 isPeerTLS: true, 43 isPeerAutoTLS: true, 44 initialToken: "new", 45 } 46 configTLS = etcdProcessClusterConfig{ 47 clusterSize: 3, 48 clientTLS: clientTLS, 49 isPeerTLS: true, 50 initialToken: "new", 51 } 52 configClientTLS = etcdProcessClusterConfig{ 53 clusterSize: 3, 54 clientTLS: clientTLS, 55 initialToken: "new", 56 } 57 configClientBoth = etcdProcessClusterConfig{ 58 clusterSize: 1, 59 clientTLS: clientTLSAndNonTLS, 60 initialToken: "new", 61 } 62 configClientAutoTLS = etcdProcessClusterConfig{ 63 clusterSize: 1, 64 isClientAutoTLS: true, 65 clientTLS: clientTLS, 66 initialToken: "new", 67 } 68 configPeerTLS = etcdProcessClusterConfig{ 69 clusterSize: 3, 70 isPeerTLS: true, 71 initialToken: "new", 72 } 73 configClientTLSCertAuth = etcdProcessClusterConfig{ 74 clusterSize: 1, 75 clientTLS: clientTLS, 76 initialToken: "new", 77 clientCertAuthEnabled: true, 78 } 79 configClientTLSCertAuthWithNoCN = etcdProcessClusterConfig{ 80 clusterSize: 1, 81 clientTLS: clientTLS, 82 initialToken: "new", 83 clientCertAuthEnabled: true, 84 noCN: true, 85 } 86 configJWT = etcdProcessClusterConfig{ 87 clusterSize: 1, 88 initialToken: "new", 89 authTokenOpts: "jwt,pub-key=../../integration/fixtures/server.crt,priv-key=../../integration/fixtures/server.key.insecure,sign-method=RS256,ttl=1s", 90 } 91 ) 92 93 func configStandalone(cfg etcdProcessClusterConfig) *etcdProcessClusterConfig { 94 ret := cfg 95 ret.clusterSize = 1 96 return &ret 97 } 98 99 type etcdProcessCluster struct { 100 cfg *etcdProcessClusterConfig 101 procs []etcdProcess 102 } 103 104 type etcdProcessClusterConfig struct { 105 execPath string 106 dataDirPath string 107 keepDataDir bool 108 109 clusterSize int 110 111 baseScheme string 112 basePort int 113 114 metricsURLScheme string 115 116 snapshotCount int // default is 10000 117 118 clientTLS clientConnType 119 clientCertAuthEnabled bool 120 isPeerTLS bool 121 isPeerAutoTLS bool 122 isClientAutoTLS bool 123 isClientCRL bool 124 noCN bool 125 126 cipherSuites []string 127 128 forceNewCluster bool 129 initialToken string 130 quotaBackendBytes int64 131 noStrictReconfig bool 132 enableV2 bool 133 initialCorruptCheck bool 134 authTokenOpts string 135 } 136 137 // newEtcdProcessCluster launches a new cluster from etcd processes, returning 138 // a new etcdProcessCluster once all nodes are ready to accept client requests. 139 func newEtcdProcessCluster(cfg *etcdProcessClusterConfig) (*etcdProcessCluster, error) { 140 etcdCfgs := cfg.etcdServerProcessConfigs() 141 epc := &etcdProcessCluster{ 142 cfg: cfg, 143 procs: make([]etcdProcess, cfg.clusterSize), 144 } 145 146 // launch etcd processes 147 for i := range etcdCfgs { 148 proc, err := newEtcdProcess(etcdCfgs[i]) 149 if err != nil { 150 epc.Close() 151 return nil, err 152 } 153 epc.procs[i] = proc 154 } 155 156 if err := epc.Start(); err != nil { 157 return nil, err 158 } 159 return epc, nil 160 } 161 162 func (cfg *etcdProcessClusterConfig) clientScheme() string { 163 if cfg.clientTLS == clientTLS { 164 return "https" 165 } 166 return "http" 167 } 168 169 func (cfg *etcdProcessClusterConfig) peerScheme() string { 170 peerScheme := cfg.baseScheme 171 if peerScheme == "" { 172 peerScheme = "http" 173 } 174 if cfg.isPeerTLS { 175 peerScheme += "s" 176 } 177 return peerScheme 178 } 179 180 func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs() []*etcdServerProcessConfig { 181 if cfg.basePort == 0 { 182 cfg.basePort = etcdProcessBasePort 183 } 184 if cfg.execPath == "" { 185 cfg.execPath = binPath 186 } 187 if cfg.snapshotCount == 0 { 188 cfg.snapshotCount = 10000 189 } 190 191 etcdCfgs := make([]*etcdServerProcessConfig, cfg.clusterSize) 192 initialCluster := make([]string, cfg.clusterSize) 193 for i := 0; i < cfg.clusterSize; i++ { 194 var curls []string 195 var curl, curltls string 196 port := cfg.basePort + 5*i 197 curlHost := fmt.Sprintf("localhost:%d", port) 198 199 switch cfg.clientTLS { 200 case clientNonTLS, clientTLS: 201 curl = (&url.URL{Scheme: cfg.clientScheme(), Host: curlHost}).String() 202 curls = []string{curl} 203 case clientTLSAndNonTLS: 204 curl = (&url.URL{Scheme: "http", Host: curlHost}).String() 205 curltls = (&url.URL{Scheme: "https", Host: curlHost}).String() 206 curls = []string{curl, curltls} 207 } 208 209 purl := url.URL{Scheme: cfg.peerScheme(), Host: fmt.Sprintf("localhost:%d", port+1)} 210 name := fmt.Sprintf("testname%d", i) 211 dataDirPath := cfg.dataDirPath 212 if cfg.dataDirPath == "" { 213 var derr error 214 dataDirPath, derr = ioutil.TempDir("", name+".etcd") 215 if derr != nil { 216 panic(fmt.Sprintf("could not get tempdir for datadir: %s", derr)) 217 } 218 } 219 initialCluster[i] = fmt.Sprintf("%s=%s", name, purl.String()) 220 221 args := []string{ 222 "--name", name, 223 "--listen-client-urls", strings.Join(curls, ","), 224 "--advertise-client-urls", strings.Join(curls, ","), 225 "--listen-peer-urls", purl.String(), 226 "--initial-advertise-peer-urls", purl.String(), 227 "--initial-cluster-token", cfg.initialToken, 228 "--data-dir", dataDirPath, 229 "--snapshot-count", fmt.Sprintf("%d", cfg.snapshotCount), 230 } 231 args = addV2Args(args) 232 if cfg.forceNewCluster { 233 args = append(args, "--force-new-cluster") 234 } 235 if cfg.quotaBackendBytes > 0 { 236 args = append(args, 237 "--quota-backend-bytes", fmt.Sprintf("%d", cfg.quotaBackendBytes), 238 ) 239 } 240 if cfg.noStrictReconfig { 241 args = append(args, "--strict-reconfig-check=false") 242 } 243 if cfg.enableV2 { 244 args = append(args, "--enable-v2") 245 } 246 if cfg.initialCorruptCheck { 247 args = append(args, "--experimental-initial-corrupt-check") 248 } 249 var murl string 250 if cfg.metricsURLScheme != "" { 251 murl = (&url.URL{ 252 Scheme: cfg.metricsURLScheme, 253 Host: fmt.Sprintf("localhost:%d", port+2), 254 }).String() 255 args = append(args, "--listen-metrics-urls", murl) 256 } 257 258 args = append(args, cfg.tlsArgs()...) 259 260 if cfg.authTokenOpts != "" { 261 args = append(args, "--auth-token", cfg.authTokenOpts) 262 } 263 264 etcdCfgs[i] = &etcdServerProcessConfig{ 265 execPath: cfg.execPath, 266 args: args, 267 tlsArgs: cfg.tlsArgs(), 268 dataDirPath: dataDirPath, 269 keepDataDir: cfg.keepDataDir, 270 name: name, 271 purl: purl, 272 acurl: curl, 273 murl: murl, 274 initialToken: cfg.initialToken, 275 } 276 } 277 278 initialClusterArgs := []string{"--initial-cluster", strings.Join(initialCluster, ",")} 279 for i := range etcdCfgs { 280 etcdCfgs[i].initialCluster = strings.Join(initialCluster, ",") 281 etcdCfgs[i].args = append(etcdCfgs[i].args, initialClusterArgs...) 282 } 283 284 return etcdCfgs 285 } 286 287 func (cfg *etcdProcessClusterConfig) tlsArgs() (args []string) { 288 if cfg.clientTLS != clientNonTLS { 289 if cfg.isClientAutoTLS { 290 args = append(args, "--auto-tls") 291 } else { 292 tlsClientArgs := []string{ 293 "--cert-file", certPath, 294 "--key-file", privateKeyPath, 295 "--trusted-ca-file", caPath, 296 } 297 args = append(args, tlsClientArgs...) 298 299 if cfg.clientCertAuthEnabled { 300 args = append(args, "--client-cert-auth") 301 } 302 } 303 } 304 305 if cfg.isPeerTLS { 306 if cfg.isPeerAutoTLS { 307 args = append(args, "--peer-auto-tls") 308 } else { 309 tlsPeerArgs := []string{ 310 "--peer-cert-file", certPath, 311 "--peer-key-file", privateKeyPath, 312 "--peer-trusted-ca-file", caPath, 313 } 314 args = append(args, tlsPeerArgs...) 315 } 316 } 317 318 if cfg.isClientCRL { 319 args = append(args, "--client-crl-file", crlPath, "--client-cert-auth") 320 } 321 322 if len(cfg.cipherSuites) > 0 { 323 args = append(args, "--cipher-suites", strings.Join(cfg.cipherSuites, ",")) 324 } 325 326 return args 327 } 328 329 func (epc *etcdProcessCluster) EndpointsV2() []string { 330 return epc.endpoints(func(ep etcdProcess) []string { return ep.EndpointsV2() }) 331 } 332 333 func (epc *etcdProcessCluster) EndpointsV3() []string { 334 return epc.endpoints(func(ep etcdProcess) []string { return ep.EndpointsV3() }) 335 } 336 337 func (epc *etcdProcessCluster) endpoints(f func(ep etcdProcess) []string) (ret []string) { 338 for _, p := range epc.procs { 339 ret = append(ret, f(p)...) 340 } 341 return ret 342 } 343 344 func (epc *etcdProcessCluster) Start() error { 345 return epc.start(func(ep etcdProcess) error { return ep.Start() }) 346 } 347 348 func (epc *etcdProcessCluster) Restart() error { 349 return epc.start(func(ep etcdProcess) error { return ep.Restart() }) 350 } 351 352 func (epc *etcdProcessCluster) start(f func(ep etcdProcess) error) error { 353 readyC := make(chan error, len(epc.procs)) 354 for i := range epc.procs { 355 go func(n int) { readyC <- f(epc.procs[n]) }(i) 356 } 357 for range epc.procs { 358 if err := <-readyC; err != nil { 359 epc.Close() 360 return err 361 } 362 } 363 return nil 364 } 365 366 func (epc *etcdProcessCluster) Stop() (err error) { 367 for _, p := range epc.procs { 368 if p == nil { 369 continue 370 } 371 if curErr := p.Stop(); curErr != nil { 372 if err != nil { 373 err = fmt.Errorf("%v; %v", err, curErr) 374 } else { 375 err = curErr 376 } 377 } 378 } 379 return err 380 } 381 382 func (epc *etcdProcessCluster) Close() error { 383 err := epc.Stop() 384 for _, p := range epc.procs { 385 // p is nil when newEtcdProcess fails in the middle 386 // Close still gets called to clean up test data 387 if p == nil { 388 continue 389 } 390 if cerr := p.Close(); cerr != nil { 391 err = cerr 392 } 393 } 394 return err 395 } 396 397 func (epc *etcdProcessCluster) WithStopSignal(sig os.Signal) (ret os.Signal) { 398 for _, p := range epc.procs { 399 ret = p.WithStopSignal(sig) 400 } 401 return ret 402 }