github.com/manicqin/nomad@v0.9.5/nomad/server_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "os" 7 "path" 8 "strings" 9 "testing" 10 "time" 11 12 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 13 "github.com/hashicorp/nomad/helper/testlog" 14 "github.com/hashicorp/nomad/helper/uuid" 15 "github.com/hashicorp/nomad/nomad/mock" 16 "github.com/hashicorp/nomad/nomad/structs" 17 "github.com/hashicorp/nomad/nomad/structs/config" 18 "github.com/hashicorp/nomad/testutil" 19 "github.com/stretchr/testify/assert" 20 "github.com/stretchr/testify/require" 21 ) 22 23 func tmpDir(t *testing.T) string { 24 t.Helper() 25 dir, err := ioutil.TempDir("", "nomad") 26 if err != nil { 27 t.Fatalf("err: %v", err) 28 } 29 return dir 30 } 31 32 func TestServer_RPC(t *testing.T) { 33 t.Parallel() 34 35 s1, cleanupS1 := TestServer(t, nil) 36 defer cleanupS1() 37 38 var out struct{} 39 if err := s1.RPC("Status.Ping", struct{}{}, &out); err != nil { 40 t.Fatalf("err: %v", err) 41 } 42 } 43 44 func TestServer_RPC_TLS(t *testing.T) { 45 t.Parallel() 46 47 const ( 48 cafile = "../helper/tlsutil/testdata/ca.pem" 49 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 50 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 51 ) 52 dir := tmpDir(t) 53 defer os.RemoveAll(dir) 54 55 s1, cleanupS1 := TestServer(t, func(c *Config) { 56 c.Region = "regionFoo" 57 c.BootstrapExpect = 3 58 c.DevMode = false 59 c.DevDisableBootstrap = true 60 c.DataDir = path.Join(dir, "node1") 61 c.TLSConfig = &config.TLSConfig{ 62 EnableHTTP: true, 63 EnableRPC: true, 64 VerifyServerHostname: true, 65 CAFile: cafile, 66 CertFile: foocert, 67 KeyFile: fookey, 68 } 69 }) 70 defer cleanupS1() 71 72 s2, cleanupS2 := TestServer(t, func(c *Config) { 73 c.Region = "regionFoo" 74 c.BootstrapExpect = 3 75 c.DevMode = false 76 c.DevDisableBootstrap = true 77 c.DataDir = path.Join(dir, "node2") 78 c.TLSConfig = &config.TLSConfig{ 79 EnableHTTP: true, 80 EnableRPC: true, 81 VerifyServerHostname: true, 82 CAFile: cafile, 83 CertFile: foocert, 84 KeyFile: fookey, 85 } 86 }) 87 defer cleanupS2() 88 89 s3, cleanupS3 := TestServer(t, func(c *Config) { 90 c.Region = "regionFoo" 91 c.BootstrapExpect = 3 92 c.DevMode = false 93 c.DevDisableBootstrap = true 94 c.DataDir = path.Join(dir, "node3") 95 c.TLSConfig = &config.TLSConfig{ 96 EnableHTTP: true, 97 EnableRPC: true, 98 VerifyServerHostname: true, 99 CAFile: cafile, 100 CertFile: foocert, 101 KeyFile: fookey, 102 } 103 }) 104 defer cleanupS3() 105 106 TestJoin(t, s1, s2, s3) 107 testutil.WaitForLeader(t, s1.RPC) 108 109 // Part of a server joining is making an RPC request, so just by testing 110 // that there is a leader we verify that the RPCs are working over TLS. 111 } 112 113 func TestServer_RPC_MixedTLS(t *testing.T) { 114 t.Parallel() 115 116 const ( 117 cafile = "../helper/tlsutil/testdata/ca.pem" 118 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 119 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 120 ) 121 dir := tmpDir(t) 122 defer os.RemoveAll(dir) 123 124 s1, cleanupS1 := TestServer(t, func(c *Config) { 125 c.Region = "regionFoo" 126 c.BootstrapExpect = 3 127 c.DevMode = false 128 c.DevDisableBootstrap = true 129 c.DataDir = path.Join(dir, "node1") 130 c.TLSConfig = &config.TLSConfig{ 131 EnableHTTP: true, 132 EnableRPC: true, 133 VerifyServerHostname: true, 134 CAFile: cafile, 135 CertFile: foocert, 136 KeyFile: fookey, 137 } 138 }) 139 defer cleanupS1() 140 141 s2, cleanupS2 := TestServer(t, func(c *Config) { 142 c.Region = "regionFoo" 143 c.BootstrapExpect = 3 144 c.DevMode = false 145 c.DevDisableBootstrap = true 146 c.DataDir = path.Join(dir, "node2") 147 c.TLSConfig = &config.TLSConfig{ 148 EnableHTTP: true, 149 EnableRPC: true, 150 VerifyServerHostname: true, 151 CAFile: cafile, 152 CertFile: foocert, 153 KeyFile: fookey, 154 } 155 }) 156 defer cleanupS2() 157 158 s3, cleanupS3 := TestServer(t, func(c *Config) { 159 c.Region = "regionFoo" 160 c.BootstrapExpect = 3 161 c.DevMode = false 162 c.DevDisableBootstrap = true 163 c.DataDir = path.Join(dir, "node3") 164 }) 165 defer cleanupS3() 166 167 TestJoin(t, s1, s2, s3) 168 169 // Ensure that we do not form a quorum 170 start := time.Now() 171 for { 172 if time.Now().After(start.Add(2 * time.Second)) { 173 break 174 } 175 176 args := &structs.GenericRequest{} 177 var leader string 178 err := s1.RPC("Status.Leader", args, &leader) 179 if err == nil || leader != "" { 180 t.Fatalf("Got leader or no error: %q %v", leader, err) 181 } 182 } 183 } 184 185 func TestServer_Regions(t *testing.T) { 186 t.Parallel() 187 188 // Make the servers 189 s1, cleanupS1 := TestServer(t, func(c *Config) { 190 c.Region = "region1" 191 }) 192 defer cleanupS1() 193 194 s2, cleanupS2 := TestServer(t, func(c *Config) { 195 c.Region = "region2" 196 }) 197 defer cleanupS2() 198 199 // Join them together 200 s2Addr := fmt.Sprintf("127.0.0.1:%d", 201 s2.config.SerfConfig.MemberlistConfig.BindPort) 202 if n, err := s1.Join([]string{s2Addr}); err != nil || n != 1 { 203 t.Fatalf("Failed joining: %v (%d joined)", err, n) 204 } 205 206 // Try listing the regions 207 testutil.WaitForResult(func() (bool, error) { 208 out := s1.Regions() 209 if len(out) != 2 || out[0] != "region1" || out[1] != "region2" { 210 return false, fmt.Errorf("unexpected regions: %v", out) 211 } 212 return true, nil 213 }, func(err error) { 214 t.Fatalf("err: %v", err) 215 }) 216 } 217 218 func TestServer_Reload_Vault(t *testing.T) { 219 t.Parallel() 220 221 s1, cleanupS1 := TestServer(t, func(c *Config) { 222 c.Region = "global" 223 }) 224 defer cleanupS1() 225 226 if s1.vault.Running() { 227 t.Fatalf("Vault client should not be running") 228 } 229 230 tr := true 231 config := DefaultConfig() 232 config.VaultConfig.Enabled = &tr 233 config.VaultConfig.Token = uuid.Generate() 234 235 if err := s1.Reload(config); err != nil { 236 t.Fatalf("Reload failed: %v", err) 237 } 238 239 if !s1.vault.Running() { 240 t.Fatalf("Vault client should be running") 241 } 242 } 243 244 func connectionReset(msg string) bool { 245 return strings.Contains(msg, "EOF") || strings.Contains(msg, "connection reset by peer") 246 } 247 248 // Tests that the server will successfully reload its network connections, 249 // upgrading from plaintext to TLS if the server's TLS configuration changes. 250 func TestServer_Reload_TLSConnections_PlaintextToTLS(t *testing.T) { 251 t.Parallel() 252 assert := assert.New(t) 253 254 const ( 255 cafile = "../helper/tlsutil/testdata/ca.pem" 256 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 257 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 258 ) 259 dir := tmpDir(t) 260 defer os.RemoveAll(dir) 261 262 s1, cleanupS1 := TestServer(t, func(c *Config) { 263 c.DataDir = path.Join(dir, "nodeA") 264 }) 265 defer cleanupS1() 266 267 // assert that the server started in plaintext mode 268 assert.Equal(s1.config.TLSConfig.CertFile, "") 269 270 newTLSConfig := &config.TLSConfig{ 271 EnableHTTP: true, 272 EnableRPC: true, 273 VerifyServerHostname: true, 274 CAFile: cafile, 275 CertFile: foocert, 276 KeyFile: fookey, 277 } 278 279 err := s1.reloadTLSConnections(newTLSConfig) 280 assert.Nil(err) 281 assert.True(s1.config.TLSConfig.CertificateInfoIsEqual(newTLSConfig)) 282 283 codec := rpcClient(t, s1) 284 285 node := mock.Node() 286 req := &structs.NodeRegisterRequest{ 287 Node: node, 288 WriteRequest: structs.WriteRequest{Region: "global"}, 289 } 290 291 var resp structs.GenericResponse 292 err = msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp) 293 assert.NotNil(err) 294 assert.True(connectionReset(err.Error())) 295 } 296 297 // Tests that the server will successfully reload its network connections, 298 // downgrading from TLS to plaintext if the server's TLS configuration changes. 299 func TestServer_Reload_TLSConnections_TLSToPlaintext_RPC(t *testing.T) { 300 t.Parallel() 301 assert := assert.New(t) 302 303 const ( 304 cafile = "../helper/tlsutil/testdata/ca.pem" 305 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 306 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 307 ) 308 309 dir := tmpDir(t) 310 defer os.RemoveAll(dir) 311 312 s1, cleanupS1 := TestServer(t, func(c *Config) { 313 c.DataDir = path.Join(dir, "nodeB") 314 c.TLSConfig = &config.TLSConfig{ 315 EnableHTTP: true, 316 EnableRPC: true, 317 VerifyServerHostname: true, 318 CAFile: cafile, 319 CertFile: foocert, 320 KeyFile: fookey, 321 } 322 }) 323 defer cleanupS1() 324 325 newTLSConfig := &config.TLSConfig{} 326 327 err := s1.reloadTLSConnections(newTLSConfig) 328 assert.Nil(err) 329 assert.True(s1.config.TLSConfig.CertificateInfoIsEqual(newTLSConfig)) 330 331 codec := rpcClient(t, s1) 332 333 node := mock.Node() 334 req := &structs.NodeRegisterRequest{ 335 Node: node, 336 WriteRequest: structs.WriteRequest{Region: "global"}, 337 } 338 339 var resp structs.GenericResponse 340 err = msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp) 341 assert.Nil(err) 342 } 343 344 // Tests that the server will successfully reload its network connections, 345 // downgrading only RPC connections 346 func TestServer_Reload_TLSConnections_TLSToPlaintext_OnlyRPC(t *testing.T) { 347 t.Parallel() 348 assert := assert.New(t) 349 350 const ( 351 cafile = "../helper/tlsutil/testdata/ca.pem" 352 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 353 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 354 ) 355 356 dir := tmpDir(t) 357 defer os.RemoveAll(dir) 358 359 s1, cleanupS1 := TestServer(t, func(c *Config) { 360 c.DataDir = path.Join(dir, "nodeB") 361 c.TLSConfig = &config.TLSConfig{ 362 EnableHTTP: true, 363 EnableRPC: true, 364 VerifyServerHostname: true, 365 CAFile: cafile, 366 CertFile: foocert, 367 KeyFile: fookey, 368 } 369 }) 370 defer cleanupS1() 371 372 newTLSConfig := &config.TLSConfig{ 373 EnableHTTP: true, 374 EnableRPC: false, 375 VerifyServerHostname: true, 376 CAFile: cafile, 377 CertFile: foocert, 378 KeyFile: fookey, 379 } 380 381 err := s1.reloadTLSConnections(newTLSConfig) 382 assert.Nil(err) 383 assert.True(s1.config.TLSConfig.CertificateInfoIsEqual(newTLSConfig)) 384 385 codec := rpcClient(t, s1) 386 387 node := mock.Node() 388 req := &structs.NodeRegisterRequest{ 389 Node: node, 390 WriteRequest: structs.WriteRequest{Region: "global"}, 391 } 392 393 var resp structs.GenericResponse 394 err = msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp) 395 assert.Nil(err) 396 } 397 398 // Tests that the server will successfully reload its network connections, 399 // upgrading only RPC connections 400 func TestServer_Reload_TLSConnections_PlaintextToTLS_OnlyRPC(t *testing.T) { 401 t.Parallel() 402 assert := assert.New(t) 403 404 const ( 405 cafile = "../helper/tlsutil/testdata/ca.pem" 406 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 407 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 408 ) 409 410 dir := tmpDir(t) 411 defer os.RemoveAll(dir) 412 413 s1, cleanupS1 := TestServer(t, func(c *Config) { 414 c.DataDir = path.Join(dir, "nodeB") 415 c.TLSConfig = &config.TLSConfig{ 416 EnableHTTP: true, 417 EnableRPC: false, 418 VerifyServerHostname: true, 419 CAFile: cafile, 420 CertFile: foocert, 421 KeyFile: fookey, 422 } 423 }) 424 defer cleanupS1() 425 426 newTLSConfig := &config.TLSConfig{ 427 EnableHTTP: true, 428 EnableRPC: true, 429 VerifyServerHostname: true, 430 CAFile: cafile, 431 CertFile: foocert, 432 KeyFile: fookey, 433 } 434 435 err := s1.reloadTLSConnections(newTLSConfig) 436 assert.Nil(err) 437 assert.True(s1.config.TLSConfig.EnableRPC) 438 assert.True(s1.config.TLSConfig.CertificateInfoIsEqual(newTLSConfig)) 439 440 codec := rpcClient(t, s1) 441 442 node := mock.Node() 443 req := &structs.NodeRegisterRequest{ 444 Node: node, 445 WriteRequest: structs.WriteRequest{Region: "global"}, 446 } 447 448 var resp structs.GenericResponse 449 err = msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp) 450 assert.NotNil(err) 451 assert.True(connectionReset(err.Error())) 452 } 453 454 // Test that Raft connections are reloaded as expected when a Nomad server is 455 // upgraded from plaintext to TLS 456 func TestServer_Reload_TLSConnections_Raft(t *testing.T) { 457 t.Parallel() 458 assert := assert.New(t) 459 460 const ( 461 cafile = "../../helper/tlsutil/testdata/ca.pem" 462 foocert = "../../helper/tlsutil/testdata/nomad-foo.pem" 463 fookey = "../../helper/tlsutil/testdata/nomad-foo-key.pem" 464 barcert = "../dev/tls_cluster/certs/nomad.pem" 465 barkey = "../dev/tls_cluster/certs/nomad-key.pem" 466 ) 467 dir := tmpDir(t) 468 defer os.RemoveAll(dir) 469 470 s1, cleanupS1 := TestServer(t, func(c *Config) { 471 c.BootstrapExpect = 2 472 c.DevMode = false 473 c.DevDisableBootstrap = true 474 c.DataDir = path.Join(dir, "node1") 475 c.NodeName = "node1" 476 c.Region = "regionFoo" 477 }) 478 defer cleanupS1() 479 480 s2, cleanupS2 := TestServer(t, func(c *Config) { 481 c.BootstrapExpect = 2 482 c.DevMode = false 483 c.DevDisableBootstrap = true 484 c.DataDir = path.Join(dir, "node2") 485 c.NodeName = "node2" 486 c.Region = "regionFoo" 487 }) 488 defer cleanupS2() 489 490 TestJoin(t, s1, s2) 491 servers := []*Server{s1, s2} 492 493 testutil.WaitForLeader(t, s1.RPC) 494 495 newTLSConfig := &config.TLSConfig{ 496 EnableHTTP: true, 497 VerifyHTTPSClient: true, 498 CAFile: cafile, 499 CertFile: foocert, 500 KeyFile: fookey, 501 } 502 503 err := s1.reloadTLSConnections(newTLSConfig) 504 assert.Nil(err) 505 506 { 507 for _, serv := range servers { 508 testutil.WaitForResult(func() (bool, error) { 509 args := &structs.GenericRequest{} 510 var leader string 511 err := serv.RPC("Status.Leader", args, &leader) 512 if leader != "" && err != nil { 513 return false, fmt.Errorf("Should not have found leader but got %s", leader) 514 } 515 return true, nil 516 }, func(err error) { 517 t.Fatalf("err: %v", err) 518 }) 519 } 520 } 521 522 secondNewTLSConfig := &config.TLSConfig{ 523 EnableHTTP: true, 524 VerifyHTTPSClient: true, 525 CAFile: cafile, 526 CertFile: barcert, 527 KeyFile: barkey, 528 } 529 530 // Now, transition the other server to TLS, which should restore their 531 // ability to communicate. 532 err = s2.reloadTLSConnections(secondNewTLSConfig) 533 assert.Nil(err) 534 535 testutil.WaitForLeader(t, s2.RPC) 536 } 537 538 func TestServer_InvalidSchedulers(t *testing.T) { 539 t.Parallel() 540 require := require.New(t) 541 542 // Set the config to not have the core scheduler 543 config := DefaultConfig() 544 logger := testlog.HCLogger(t) 545 s := &Server{ 546 config: config, 547 logger: logger, 548 } 549 550 config.EnabledSchedulers = []string{"batch"} 551 err := s.setupWorkers() 552 require.NotNil(err) 553 require.Contains(err.Error(), "scheduler not enabled") 554 555 // Set the config to have an unknown scheduler 556 config.EnabledSchedulers = []string{"batch", structs.JobTypeCore, "foo"} 557 err = s.setupWorkers() 558 require.NotNil(err) 559 require.Contains(err.Error(), "foo") 560 }