github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/nomad/server_test.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "os" 7 "path" 8 "strings" 9 "testing" 10 "time" 11 12 msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" 13 "github.com/hashicorp/nomad/helper/testlog" 14 "github.com/hashicorp/nomad/helper/uuid" 15 "github.com/hashicorp/nomad/nomad/mock" 16 "github.com/hashicorp/nomad/nomad/structs" 17 "github.com/hashicorp/nomad/nomad/structs/config" 18 "github.com/hashicorp/nomad/testutil" 19 "github.com/stretchr/testify/assert" 20 "github.com/stretchr/testify/require" 21 ) 22 23 func tmpDir(t *testing.T) string { 24 t.Helper() 25 dir, err := ioutil.TempDir("", "nomad") 26 if err != nil { 27 t.Fatalf("err: %v", err) 28 } 29 return dir 30 } 31 32 func TestServer_RPC(t *testing.T) { 33 t.Parallel() 34 35 s1, cleanupS1 := TestServer(t, nil) 36 defer cleanupS1() 37 38 var out struct{} 39 if err := s1.RPC("Status.Ping", struct{}{}, &out); err != nil { 40 t.Fatalf("err: %v", err) 41 } 42 } 43 44 func TestServer_RPC_TLS(t *testing.T) { 45 t.Parallel() 46 47 const ( 48 cafile = "../helper/tlsutil/testdata/ca.pem" 49 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 50 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 51 ) 52 dir := tmpDir(t) 53 defer os.RemoveAll(dir) 54 55 s1, cleanupS1 := TestServer(t, func(c *Config) { 56 c.Region = "regionFoo" 57 c.BootstrapExpect = 3 58 c.DevMode = false 59 c.DataDir = path.Join(dir, "node1") 60 c.TLSConfig = &config.TLSConfig{ 61 EnableHTTP: true, 62 EnableRPC: true, 63 VerifyServerHostname: true, 64 CAFile: cafile, 65 CertFile: foocert, 66 KeyFile: fookey, 67 } 68 }) 69 defer cleanupS1() 70 71 s2, cleanupS2 := TestServer(t, func(c *Config) { 72 c.Region = "regionFoo" 73 c.BootstrapExpect = 3 74 c.DevMode = false 75 c.DataDir = path.Join(dir, "node2") 76 c.TLSConfig = &config.TLSConfig{ 77 EnableHTTP: true, 78 EnableRPC: true, 79 VerifyServerHostname: true, 80 CAFile: cafile, 81 CertFile: foocert, 82 KeyFile: fookey, 83 } 84 }) 85 defer cleanupS2() 86 87 s3, cleanupS3 := TestServer(t, func(c *Config) { 88 c.Region = "regionFoo" 89 c.BootstrapExpect = 3 90 c.DevMode = false 91 c.DataDir = path.Join(dir, "node3") 92 c.TLSConfig = &config.TLSConfig{ 93 EnableHTTP: true, 94 EnableRPC: true, 95 VerifyServerHostname: true, 96 CAFile: cafile, 97 CertFile: foocert, 98 KeyFile: fookey, 99 } 100 }) 101 defer cleanupS3() 102 103 TestJoin(t, s1, s2, s3) 104 testutil.WaitForLeader(t, s1.RPC) 105 106 // Part of a server joining is making an RPC request, so just by testing 107 // that there is a leader we verify that the RPCs are working over TLS. 108 } 109 110 func TestServer_RPC_MixedTLS(t *testing.T) { 111 t.Parallel() 112 113 const ( 114 cafile = "../helper/tlsutil/testdata/ca.pem" 115 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 116 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 117 ) 118 dir := tmpDir(t) 119 defer os.RemoveAll(dir) 120 121 s1, cleanupS1 := TestServer(t, func(c *Config) { 122 c.Region = "regionFoo" 123 c.BootstrapExpect = 3 124 c.DevMode = false 125 c.DataDir = path.Join(dir, "node1") 126 c.TLSConfig = &config.TLSConfig{ 127 EnableHTTP: true, 128 EnableRPC: true, 129 VerifyServerHostname: true, 130 CAFile: cafile, 131 CertFile: foocert, 132 KeyFile: fookey, 133 } 134 }) 135 defer cleanupS1() 136 137 s2, cleanupS2 := TestServer(t, func(c *Config) { 138 c.Region = "regionFoo" 139 c.BootstrapExpect = 3 140 c.DevMode = false 141 c.DataDir = path.Join(dir, "node2") 142 c.TLSConfig = &config.TLSConfig{ 143 EnableHTTP: true, 144 EnableRPC: true, 145 VerifyServerHostname: true, 146 CAFile: cafile, 147 CertFile: foocert, 148 KeyFile: fookey, 149 } 150 }) 151 defer cleanupS2() 152 153 s3, cleanupS3 := TestServer(t, func(c *Config) { 154 c.Region = "regionFoo" 155 c.BootstrapExpect = 3 156 c.DevMode = false 157 c.DataDir = path.Join(dir, "node3") 158 }) 159 defer cleanupS3() 160 161 TestJoin(t, s1, s2, s3) 162 163 // Ensure that we do not form a quorum 164 start := time.Now() 165 for { 166 if time.Now().After(start.Add(2 * time.Second)) { 167 break 168 } 169 170 args := &structs.GenericRequest{} 171 var leader string 172 err := s1.RPC("Status.Leader", args, &leader) 173 if err == nil || leader != "" { 174 t.Fatalf("Got leader or no error: %q %v", leader, err) 175 } 176 } 177 } 178 179 func TestServer_Regions(t *testing.T) { 180 t.Parallel() 181 182 // Make the servers 183 s1, cleanupS1 := TestServer(t, func(c *Config) { 184 c.Region = "region1" 185 }) 186 defer cleanupS1() 187 188 s2, cleanupS2 := TestServer(t, func(c *Config) { 189 c.Region = "region2" 190 }) 191 defer cleanupS2() 192 193 // Join them together 194 s2Addr := fmt.Sprintf("127.0.0.1:%d", 195 s2.config.SerfConfig.MemberlistConfig.BindPort) 196 if n, err := s1.Join([]string{s2Addr}); err != nil || n != 1 { 197 t.Fatalf("Failed joining: %v (%d joined)", err, n) 198 } 199 200 // Try listing the regions 201 testutil.WaitForResult(func() (bool, error) { 202 out := s1.Regions() 203 if len(out) != 2 || out[0] != "region1" || out[1] != "region2" { 204 return false, fmt.Errorf("unexpected regions: %v", out) 205 } 206 return true, nil 207 }, func(err error) { 208 t.Fatalf("err: %v", err) 209 }) 210 } 211 212 func TestServer_Reload_Vault(t *testing.T) { 213 t.Parallel() 214 215 s1, cleanupS1 := TestServer(t, func(c *Config) { 216 c.Region = "global" 217 }) 218 defer cleanupS1() 219 220 if s1.vault.Running() { 221 t.Fatalf("Vault client should not be running") 222 } 223 224 tr := true 225 config := DefaultConfig() 226 config.VaultConfig.Enabled = &tr 227 config.VaultConfig.Token = uuid.Generate() 228 229 if err := s1.Reload(config); err != nil { 230 t.Fatalf("Reload failed: %v", err) 231 } 232 233 if !s1.vault.Running() { 234 t.Fatalf("Vault client should be running") 235 } 236 } 237 238 func connectionReset(msg string) bool { 239 return strings.Contains(msg, "EOF") || strings.Contains(msg, "connection reset by peer") 240 } 241 242 // Tests that the server will successfully reload its network connections, 243 // upgrading from plaintext to TLS if the server's TLS configuration changes. 244 func TestServer_Reload_TLSConnections_PlaintextToTLS(t *testing.T) { 245 t.Parallel() 246 assert := assert.New(t) 247 248 const ( 249 cafile = "../helper/tlsutil/testdata/ca.pem" 250 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 251 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 252 ) 253 dir := tmpDir(t) 254 defer os.RemoveAll(dir) 255 256 s1, cleanupS1 := TestServer(t, func(c *Config) { 257 c.DataDir = path.Join(dir, "nodeA") 258 }) 259 defer cleanupS1() 260 261 // assert that the server started in plaintext mode 262 assert.Equal(s1.config.TLSConfig.CertFile, "") 263 264 newTLSConfig := &config.TLSConfig{ 265 EnableHTTP: true, 266 EnableRPC: true, 267 VerifyServerHostname: true, 268 CAFile: cafile, 269 CertFile: foocert, 270 KeyFile: fookey, 271 } 272 273 err := s1.reloadTLSConnections(newTLSConfig) 274 assert.Nil(err) 275 assert.True(s1.config.TLSConfig.CertificateInfoIsEqual(newTLSConfig)) 276 277 codec := rpcClient(t, s1) 278 279 node := mock.Node() 280 req := &structs.NodeRegisterRequest{ 281 Node: node, 282 WriteRequest: structs.WriteRequest{Region: "global"}, 283 } 284 285 var resp structs.GenericResponse 286 err = msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp) 287 assert.NotNil(err) 288 assert.True(connectionReset(err.Error())) 289 } 290 291 // Tests that the server will successfully reload its network connections, 292 // downgrading from TLS to plaintext if the server's TLS configuration changes. 293 func TestServer_Reload_TLSConnections_TLSToPlaintext_RPC(t *testing.T) { 294 t.Parallel() 295 assert := assert.New(t) 296 297 const ( 298 cafile = "../helper/tlsutil/testdata/ca.pem" 299 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 300 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 301 ) 302 303 dir := tmpDir(t) 304 defer os.RemoveAll(dir) 305 306 s1, cleanupS1 := TestServer(t, func(c *Config) { 307 c.DataDir = path.Join(dir, "nodeB") 308 c.TLSConfig = &config.TLSConfig{ 309 EnableHTTP: true, 310 EnableRPC: true, 311 VerifyServerHostname: true, 312 CAFile: cafile, 313 CertFile: foocert, 314 KeyFile: fookey, 315 } 316 }) 317 defer cleanupS1() 318 319 newTLSConfig := &config.TLSConfig{} 320 321 err := s1.reloadTLSConnections(newTLSConfig) 322 assert.Nil(err) 323 assert.True(s1.config.TLSConfig.CertificateInfoIsEqual(newTLSConfig)) 324 325 codec := rpcClient(t, s1) 326 327 node := mock.Node() 328 req := &structs.NodeRegisterRequest{ 329 Node: node, 330 WriteRequest: structs.WriteRequest{Region: "global"}, 331 } 332 333 var resp structs.GenericResponse 334 err = msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp) 335 assert.Nil(err) 336 } 337 338 // Tests that the server will successfully reload its network connections, 339 // downgrading only RPC connections 340 func TestServer_Reload_TLSConnections_TLSToPlaintext_OnlyRPC(t *testing.T) { 341 t.Parallel() 342 assert := assert.New(t) 343 344 const ( 345 cafile = "../helper/tlsutil/testdata/ca.pem" 346 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 347 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 348 ) 349 350 dir := tmpDir(t) 351 defer os.RemoveAll(dir) 352 353 s1, cleanupS1 := TestServer(t, func(c *Config) { 354 c.DataDir = path.Join(dir, "nodeB") 355 c.TLSConfig = &config.TLSConfig{ 356 EnableHTTP: true, 357 EnableRPC: true, 358 VerifyServerHostname: true, 359 CAFile: cafile, 360 CertFile: foocert, 361 KeyFile: fookey, 362 } 363 }) 364 defer cleanupS1() 365 366 newTLSConfig := &config.TLSConfig{ 367 EnableHTTP: true, 368 EnableRPC: false, 369 VerifyServerHostname: true, 370 CAFile: cafile, 371 CertFile: foocert, 372 KeyFile: fookey, 373 } 374 375 err := s1.reloadTLSConnections(newTLSConfig) 376 assert.Nil(err) 377 assert.True(s1.config.TLSConfig.CertificateInfoIsEqual(newTLSConfig)) 378 379 codec := rpcClient(t, s1) 380 381 node := mock.Node() 382 req := &structs.NodeRegisterRequest{ 383 Node: node, 384 WriteRequest: structs.WriteRequest{Region: "global"}, 385 } 386 387 var resp structs.GenericResponse 388 err = msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp) 389 assert.Nil(err) 390 } 391 392 // Tests that the server will successfully reload its network connections, 393 // upgrading only RPC connections 394 func TestServer_Reload_TLSConnections_PlaintextToTLS_OnlyRPC(t *testing.T) { 395 t.Parallel() 396 assert := assert.New(t) 397 398 const ( 399 cafile = "../helper/tlsutil/testdata/ca.pem" 400 foocert = "../helper/tlsutil/testdata/nomad-foo.pem" 401 fookey = "../helper/tlsutil/testdata/nomad-foo-key.pem" 402 ) 403 404 dir := tmpDir(t) 405 defer os.RemoveAll(dir) 406 407 s1, cleanupS1 := TestServer(t, func(c *Config) { 408 c.DataDir = path.Join(dir, "nodeB") 409 c.TLSConfig = &config.TLSConfig{ 410 EnableHTTP: true, 411 EnableRPC: false, 412 VerifyServerHostname: true, 413 CAFile: cafile, 414 CertFile: foocert, 415 KeyFile: fookey, 416 } 417 }) 418 defer cleanupS1() 419 420 newTLSConfig := &config.TLSConfig{ 421 EnableHTTP: true, 422 EnableRPC: true, 423 VerifyServerHostname: true, 424 CAFile: cafile, 425 CertFile: foocert, 426 KeyFile: fookey, 427 } 428 429 err := s1.reloadTLSConnections(newTLSConfig) 430 assert.Nil(err) 431 assert.True(s1.config.TLSConfig.EnableRPC) 432 assert.True(s1.config.TLSConfig.CertificateInfoIsEqual(newTLSConfig)) 433 434 codec := rpcClient(t, s1) 435 436 node := mock.Node() 437 req := &structs.NodeRegisterRequest{ 438 Node: node, 439 WriteRequest: structs.WriteRequest{Region: "global"}, 440 } 441 442 var resp structs.GenericResponse 443 err = msgpackrpc.CallWithCodec(codec, "Node.Register", req, &resp) 444 assert.NotNil(err) 445 assert.True(connectionReset(err.Error())) 446 } 447 448 // Test that Raft connections are reloaded as expected when a Nomad server is 449 // upgraded from plaintext to TLS 450 func TestServer_Reload_TLSConnections_Raft(t *testing.T) { 451 t.Parallel() 452 assert := assert.New(t) 453 454 const ( 455 cafile = "../../helper/tlsutil/testdata/ca.pem" 456 foocert = "../../helper/tlsutil/testdata/nomad-foo.pem" 457 fookey = "../../helper/tlsutil/testdata/nomad-foo-key.pem" 458 barcert = "../dev/tls_cluster/certs/nomad.pem" 459 barkey = "../dev/tls_cluster/certs/nomad-key.pem" 460 ) 461 dir := tmpDir(t) 462 defer os.RemoveAll(dir) 463 464 s1, cleanupS1 := TestServer(t, func(c *Config) { 465 c.BootstrapExpect = 2 466 c.DevMode = false 467 c.DataDir = path.Join(dir, "node1") 468 c.NodeName = "node1" 469 c.Region = "regionFoo" 470 }) 471 defer cleanupS1() 472 473 s2, cleanupS2 := TestServer(t, func(c *Config) { 474 c.BootstrapExpect = 2 475 c.DevMode = false 476 c.DataDir = path.Join(dir, "node2") 477 c.NodeName = "node2" 478 c.Region = "regionFoo" 479 }) 480 defer cleanupS2() 481 482 TestJoin(t, s1, s2) 483 servers := []*Server{s1, s2} 484 485 testutil.WaitForLeader(t, s1.RPC) 486 487 newTLSConfig := &config.TLSConfig{ 488 EnableHTTP: true, 489 VerifyHTTPSClient: true, 490 CAFile: cafile, 491 CertFile: foocert, 492 KeyFile: fookey, 493 } 494 495 err := s1.reloadTLSConnections(newTLSConfig) 496 assert.Nil(err) 497 498 { 499 for _, serv := range servers { 500 testutil.WaitForResult(func() (bool, error) { 501 args := &structs.GenericRequest{} 502 var leader string 503 err := serv.RPC("Status.Leader", args, &leader) 504 if leader != "" && err != nil { 505 return false, fmt.Errorf("Should not have found leader but got %s", leader) 506 } 507 return true, nil 508 }, func(err error) { 509 t.Fatalf("err: %v", err) 510 }) 511 } 512 } 513 514 secondNewTLSConfig := &config.TLSConfig{ 515 EnableHTTP: true, 516 VerifyHTTPSClient: true, 517 CAFile: cafile, 518 CertFile: barcert, 519 KeyFile: barkey, 520 } 521 522 // Now, transition the other server to TLS, which should restore their 523 // ability to communicate. 524 err = s2.reloadTLSConnections(secondNewTLSConfig) 525 assert.Nil(err) 526 527 testutil.WaitForLeader(t, s2.RPC) 528 } 529 530 func TestServer_InvalidSchedulers(t *testing.T) { 531 t.Parallel() 532 require := require.New(t) 533 534 // Set the config to not have the core scheduler 535 config := DefaultConfig() 536 logger := testlog.HCLogger(t) 537 s := &Server{ 538 config: config, 539 logger: logger, 540 } 541 542 config.EnabledSchedulers = []string{"batch"} 543 err := s.setupWorkers() 544 require.NotNil(err) 545 require.Contains(err.Error(), "scheduler not enabled") 546 547 // Set the config to have an unknown scheduler 548 config.EnabledSchedulers = []string{"batch", structs.JobTypeCore, "foo"} 549 err = s.setupWorkers() 550 require.NotNil(err) 551 require.Contains(err.Error(), "foo") 552 } 553 554 func TestServer_RPCNameAndRegionValidation(t *testing.T) { 555 t.Parallel() 556 for _, tc := range []struct { 557 name string 558 region string 559 expected bool 560 }{ 561 // OK 562 {name: "client.global.nomad", region: "global", expected: true}, 563 {name: "server.global.nomad", region: "global", expected: true}, 564 {name: "server.other.nomad", region: "global", expected: true}, 565 {name: "server.other.region.nomad", region: "other.region", expected: true}, 566 567 // Bad 568 {name: "client.other.nomad", region: "global", expected: false}, 569 {name: "client.global.nomad.other", region: "global", expected: false}, 570 {name: "server.global.nomad.other", region: "global", expected: false}, 571 {name: "other.global.nomad", region: "global", expected: false}, 572 {name: "server.nomad", region: "global", expected: false}, 573 {name: "localhost", region: "global", expected: false}, 574 } { 575 assert.Equal(t, tc.expected, validateRPCRegionPeer(tc.name, tc.region), 576 "expected %q in region %q to validate as %v", 577 tc.name, tc.region, tc.expected) 578 } 579 }