github.com/bigcommerce/nomad@v0.9.3-bc/client/alloc_endpoint_test.go (about) 1 package client 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "net" 8 "runtime" 9 "strings" 10 "testing" 11 "time" 12 13 "github.com/hashicorp/nomad/acl" 14 "github.com/hashicorp/nomad/client/config" 15 cstructs "github.com/hashicorp/nomad/client/structs" 16 "github.com/hashicorp/nomad/helper/pluginutils/catalog" 17 "github.com/hashicorp/nomad/helper/uuid" 18 "github.com/hashicorp/nomad/nomad" 19 "github.com/hashicorp/nomad/nomad/mock" 20 "github.com/hashicorp/nomad/nomad/structs" 21 nstructs "github.com/hashicorp/nomad/nomad/structs" 22 nconfig "github.com/hashicorp/nomad/nomad/structs/config" 23 "github.com/hashicorp/nomad/plugins/drivers" 24 "github.com/hashicorp/nomad/testutil" 25 "github.com/stretchr/testify/require" 26 "github.com/ugorji/go/codec" 27 "golang.org/x/sys/unix" 28 ) 29 30 func TestAllocations_Restart(t *testing.T) { 31 t.Parallel() 32 require := require.New(t) 33 client, cleanup := TestClient(t, nil) 34 defer cleanup() 35 36 a := mock.Alloc() 37 a.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver" 38 a.Job.TaskGroups[0].RestartPolicy = &nstructs.RestartPolicy{ 39 Attempts: 0, 40 Mode: nstructs.RestartPolicyModeFail, 41 } 42 a.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 43 "run_for": "10ms", 44 } 45 require.Nil(client.addAlloc(a, "")) 46 47 // Try with bad alloc 48 req := &nstructs.AllocRestartRequest{} 49 var resp nstructs.GenericResponse 50 err := client.ClientRPC("Allocations.Restart", &req, &resp) 51 require.Error(err) 52 53 // Try with good alloc 54 req.AllocID = a.ID 55 56 testutil.WaitForResult(func() (bool, error) { 57 var resp2 nstructs.GenericResponse 58 err := client.ClientRPC("Allocations.Restart", &req, &resp2) 59 if err != nil && strings.Contains(err.Error(), "not running") { 60 return false, err 61 } 62 63 return true, nil 64 }, func(err error) { 65 t.Fatalf("err: %v", err) 66 }) 67 } 68 69 func TestAllocations_Restart_ACL(t *testing.T) { 70 t.Parallel() 71 require := require.New(t) 72 server, addr, root := testACLServer(t, nil) 73 defer server.Shutdown() 74 75 client, cleanup := TestClient(t, func(c *config.Config) { 76 c.Servers = []string{addr} 77 c.ACLEnabled = true 78 }) 79 defer cleanup() 80 81 // Try request without a token and expect failure 82 { 83 req := &nstructs.AllocRestartRequest{} 84 var resp nstructs.GenericResponse 85 err := client.ClientRPC("Allocations.Restart", &req, &resp) 86 require.NotNil(err) 87 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 88 } 89 90 // Try request with an invalid token and expect failure 91 { 92 token := mock.CreatePolicyAndToken(t, server.State(), 1005, "invalid", mock.NamespacePolicy(nstructs.DefaultNamespace, "", []string{})) 93 req := &nstructs.AllocRestartRequest{} 94 req.AuthToken = token.SecretID 95 96 var resp nstructs.GenericResponse 97 err := client.ClientRPC("Allocations.Restart", &req, &resp) 98 99 require.NotNil(err) 100 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 101 } 102 103 // Try request with a valid token 104 { 105 policyHCL := mock.NamespacePolicy(nstructs.DefaultNamespace, "", []string{acl.NamespaceCapabilityAllocLifecycle}) 106 token := mock.CreatePolicyAndToken(t, server.State(), 1007, "valid", policyHCL) 107 require.NotNil(token) 108 req := &nstructs.AllocRestartRequest{} 109 req.AuthToken = token.SecretID 110 req.Namespace = nstructs.DefaultNamespace 111 var resp nstructs.GenericResponse 112 err := client.ClientRPC("Allocations.Restart", &req, &resp) 113 require.True(nstructs.IsErrUnknownAllocation(err), "Expected unknown alloc, found: %v", err) 114 } 115 116 // Try request with a management token 117 { 118 req := &nstructs.AllocRestartRequest{} 119 req.AuthToken = root.SecretID 120 var resp nstructs.GenericResponse 121 err := client.ClientRPC("Allocations.Restart", &req, &resp) 122 require.True(nstructs.IsErrUnknownAllocation(err), "Expected unknown alloc, found: %v", err) 123 } 124 } 125 126 func TestAllocations_GarbageCollectAll(t *testing.T) { 127 t.Parallel() 128 require := require.New(t) 129 client, cleanup := TestClient(t, nil) 130 defer cleanup() 131 132 req := &nstructs.NodeSpecificRequest{} 133 var resp nstructs.GenericResponse 134 require.Nil(client.ClientRPC("Allocations.GarbageCollectAll", &req, &resp)) 135 } 136 137 func TestAllocations_GarbageCollectAll_ACL(t *testing.T) { 138 t.Parallel() 139 require := require.New(t) 140 server, addr, root := testACLServer(t, nil) 141 defer server.Shutdown() 142 143 client, cleanup := TestClient(t, func(c *config.Config) { 144 c.Servers = []string{addr} 145 c.ACLEnabled = true 146 }) 147 defer cleanup() 148 149 // Try request without a token and expect failure 150 { 151 req := &nstructs.NodeSpecificRequest{} 152 var resp nstructs.GenericResponse 153 err := client.ClientRPC("Allocations.GarbageCollectAll", &req, &resp) 154 require.NotNil(err) 155 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 156 } 157 158 // Try request with an invalid token and expect failure 159 { 160 token := mock.CreatePolicyAndToken(t, server.State(), 1005, "invalid", mock.NodePolicy(acl.PolicyDeny)) 161 req := &nstructs.NodeSpecificRequest{} 162 req.AuthToken = token.SecretID 163 164 var resp nstructs.GenericResponse 165 err := client.ClientRPC("Allocations.GarbageCollectAll", &req, &resp) 166 167 require.NotNil(err) 168 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 169 } 170 171 // Try request with a valid token 172 { 173 token := mock.CreatePolicyAndToken(t, server.State(), 1007, "valid", mock.NodePolicy(acl.PolicyWrite)) 174 req := &nstructs.NodeSpecificRequest{} 175 req.AuthToken = token.SecretID 176 var resp nstructs.GenericResponse 177 require.Nil(client.ClientRPC("Allocations.GarbageCollectAll", &req, &resp)) 178 } 179 180 // Try request with a management token 181 { 182 req := &nstructs.NodeSpecificRequest{} 183 req.AuthToken = root.SecretID 184 var resp nstructs.GenericResponse 185 require.Nil(client.ClientRPC("Allocations.GarbageCollectAll", &req, &resp)) 186 } 187 } 188 189 func TestAllocations_GarbageCollect(t *testing.T) { 190 t.Parallel() 191 require := require.New(t) 192 client, cleanup := TestClient(t, func(c *config.Config) { 193 c.GCDiskUsageThreshold = 100.0 194 }) 195 defer cleanup() 196 197 a := mock.Alloc() 198 a.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver" 199 a.Job.TaskGroups[0].RestartPolicy = &nstructs.RestartPolicy{ 200 Attempts: 0, 201 Mode: nstructs.RestartPolicyModeFail, 202 } 203 a.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 204 "run_for": "10ms", 205 } 206 require.Nil(client.addAlloc(a, "")) 207 208 // Try with bad alloc 209 req := &nstructs.AllocSpecificRequest{} 210 var resp nstructs.GenericResponse 211 err := client.ClientRPC("Allocations.GarbageCollect", &req, &resp) 212 require.NotNil(err) 213 214 // Try with good alloc 215 req.AllocID = a.ID 216 testutil.WaitForResult(func() (bool, error) { 217 // Check if has been removed first 218 if ar, ok := client.allocs[a.ID]; !ok || ar.IsDestroyed() { 219 return true, nil 220 } 221 222 var resp2 nstructs.GenericResponse 223 err := client.ClientRPC("Allocations.GarbageCollect", &req, &resp2) 224 return err == nil, err 225 }, func(err error) { 226 t.Fatalf("err: %v", err) 227 }) 228 } 229 230 func TestAllocations_GarbageCollect_ACL(t *testing.T) { 231 t.Parallel() 232 require := require.New(t) 233 server, addr, root := testACLServer(t, nil) 234 defer server.Shutdown() 235 236 client, cleanup := TestClient(t, func(c *config.Config) { 237 c.Servers = []string{addr} 238 c.ACLEnabled = true 239 }) 240 defer cleanup() 241 242 // Try request without a token and expect failure 243 { 244 req := &nstructs.AllocSpecificRequest{} 245 var resp nstructs.GenericResponse 246 err := client.ClientRPC("Allocations.GarbageCollect", &req, &resp) 247 require.NotNil(err) 248 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 249 } 250 251 // Try request with an invalid token and expect failure 252 { 253 token := mock.CreatePolicyAndToken(t, server.State(), 1005, "invalid", mock.NodePolicy(acl.PolicyDeny)) 254 req := &nstructs.AllocSpecificRequest{} 255 req.AuthToken = token.SecretID 256 257 var resp nstructs.GenericResponse 258 err := client.ClientRPC("Allocations.GarbageCollect", &req, &resp) 259 260 require.NotNil(err) 261 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 262 } 263 264 // Try request with a valid token 265 { 266 token := mock.CreatePolicyAndToken(t, server.State(), 1005, "test-valid", 267 mock.NamespacePolicy(nstructs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})) 268 req := &nstructs.AllocSpecificRequest{} 269 req.AuthToken = token.SecretID 270 req.Namespace = nstructs.DefaultNamespace 271 272 var resp nstructs.GenericResponse 273 err := client.ClientRPC("Allocations.GarbageCollect", &req, &resp) 274 require.True(nstructs.IsErrUnknownAllocation(err)) 275 } 276 277 // Try request with a management token 278 { 279 req := &nstructs.AllocSpecificRequest{} 280 req.AuthToken = root.SecretID 281 282 var resp nstructs.GenericResponse 283 err := client.ClientRPC("Allocations.GarbageCollect", &req, &resp) 284 require.True(nstructs.IsErrUnknownAllocation(err)) 285 } 286 } 287 288 func TestAllocations_Signal(t *testing.T) { 289 t.Parallel() 290 291 client, cleanup := TestClient(t, nil) 292 defer cleanup() 293 294 a := mock.Alloc() 295 require.Nil(t, client.addAlloc(a, "")) 296 297 // Try with bad alloc 298 req := &nstructs.AllocSignalRequest{} 299 var resp nstructs.GenericResponse 300 err := client.ClientRPC("Allocations.Signal", &req, &resp) 301 require.NotNil(t, err) 302 require.True(t, nstructs.IsErrUnknownAllocation(err)) 303 304 // Try with good alloc 305 req.AllocID = a.ID 306 307 var resp2 nstructs.GenericResponse 308 err = client.ClientRPC("Allocations.Signal", &req, &resp2) 309 310 require.Error(t, err, "Expected error, got: %s, resp: %#+v", err, resp2) 311 require.Equal(t, "1 error(s) occurred:\n\n* Failed to signal task: web, err: Task not running", err.Error()) 312 } 313 314 func TestAllocations_Signal_ACL(t *testing.T) { 315 t.Parallel() 316 require := require.New(t) 317 server, addr, root := testACLServer(t, nil) 318 defer server.Shutdown() 319 320 client, cleanup := TestClient(t, func(c *config.Config) { 321 c.Servers = []string{addr} 322 c.ACLEnabled = true 323 }) 324 defer cleanup() 325 326 // Try request without a token and expect failure 327 { 328 req := &nstructs.AllocSignalRequest{} 329 var resp nstructs.GenericResponse 330 err := client.ClientRPC("Allocations.Signal", &req, &resp) 331 require.NotNil(err) 332 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 333 } 334 335 // Try request with an invalid token and expect failure 336 { 337 token := mock.CreatePolicyAndToken(t, server.State(), 1005, "invalid", mock.NodePolicy(acl.PolicyDeny)) 338 req := &nstructs.AllocSignalRequest{} 339 req.AuthToken = token.SecretID 340 341 var resp nstructs.GenericResponse 342 err := client.ClientRPC("Allocations.Signal", &req, &resp) 343 344 require.NotNil(err) 345 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 346 } 347 348 // Try request with a valid token 349 { 350 token := mock.CreatePolicyAndToken(t, server.State(), 1005, "test-valid", 351 mock.NamespacePolicy(nstructs.DefaultNamespace, "", []string{acl.NamespaceCapabilityAllocLifecycle})) 352 req := &nstructs.AllocSignalRequest{} 353 req.AuthToken = token.SecretID 354 req.Namespace = nstructs.DefaultNamespace 355 356 var resp nstructs.GenericResponse 357 err := client.ClientRPC("Allocations.Signal", &req, &resp) 358 require.True(nstructs.IsErrUnknownAllocation(err)) 359 } 360 361 // Try request with a management token 362 { 363 req := &nstructs.AllocSignalRequest{} 364 req.AuthToken = root.SecretID 365 366 var resp nstructs.GenericResponse 367 err := client.ClientRPC("Allocations.Signal", &req, &resp) 368 require.True(nstructs.IsErrUnknownAllocation(err)) 369 } 370 } 371 372 func TestAllocations_Stats(t *testing.T) { 373 t.Parallel() 374 require := require.New(t) 375 client, cleanup := TestClient(t, nil) 376 defer cleanup() 377 378 a := mock.Alloc() 379 require.Nil(client.addAlloc(a, "")) 380 381 // Try with bad alloc 382 req := &cstructs.AllocStatsRequest{} 383 var resp cstructs.AllocStatsResponse 384 err := client.ClientRPC("Allocations.Stats", &req, &resp) 385 require.NotNil(err) 386 387 // Try with good alloc 388 req.AllocID = a.ID 389 testutil.WaitForResult(func() (bool, error) { 390 var resp2 cstructs.AllocStatsResponse 391 err := client.ClientRPC("Allocations.Stats", &req, &resp2) 392 if err != nil { 393 return false, err 394 } 395 if resp2.Stats == nil { 396 return false, fmt.Errorf("invalid stats object") 397 } 398 399 return true, nil 400 }, func(err error) { 401 t.Fatalf("err: %v", err) 402 }) 403 } 404 405 func TestAllocations_Stats_ACL(t *testing.T) { 406 t.Parallel() 407 require := require.New(t) 408 server, addr, root := testACLServer(t, nil) 409 defer server.Shutdown() 410 411 client, cleanup := TestClient(t, func(c *config.Config) { 412 c.Servers = []string{addr} 413 c.ACLEnabled = true 414 }) 415 defer cleanup() 416 417 // Try request without a token and expect failure 418 { 419 req := &cstructs.AllocStatsRequest{} 420 var resp cstructs.AllocStatsResponse 421 err := client.ClientRPC("Allocations.Stats", &req, &resp) 422 require.NotNil(err) 423 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 424 } 425 426 // Try request with an invalid token and expect failure 427 { 428 token := mock.CreatePolicyAndToken(t, server.State(), 1005, "invalid", mock.NodePolicy(acl.PolicyDeny)) 429 req := &cstructs.AllocStatsRequest{} 430 req.AuthToken = token.SecretID 431 432 var resp cstructs.AllocStatsResponse 433 err := client.ClientRPC("Allocations.Stats", &req, &resp) 434 435 require.NotNil(err) 436 require.EqualError(err, nstructs.ErrPermissionDenied.Error()) 437 } 438 439 // Try request with a valid token 440 { 441 token := mock.CreatePolicyAndToken(t, server.State(), 1005, "test-valid", 442 mock.NamespacePolicy(nstructs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})) 443 req := &cstructs.AllocStatsRequest{} 444 req.AuthToken = token.SecretID 445 req.Namespace = nstructs.DefaultNamespace 446 447 var resp cstructs.AllocStatsResponse 448 err := client.ClientRPC("Allocations.Stats", &req, &resp) 449 require.True(nstructs.IsErrUnknownAllocation(err)) 450 } 451 452 // Try request with a management token 453 { 454 req := &cstructs.AllocStatsRequest{} 455 req.AuthToken = root.SecretID 456 457 var resp cstructs.AllocStatsResponse 458 err := client.ClientRPC("Allocations.Stats", &req, &resp) 459 require.True(nstructs.IsErrUnknownAllocation(err)) 460 } 461 } 462 463 func TestAlloc_ExecStreaming(t *testing.T) { 464 t.Parallel() 465 require := require.New(t) 466 467 // Start a server and client 468 s := nomad.TestServer(t, nil) 469 defer s.Shutdown() 470 testutil.WaitForLeader(t, s.RPC) 471 472 c, cleanup := TestClient(t, func(c *config.Config) { 473 c.Servers = []string{s.GetConfig().RPCAddr.String()} 474 }) 475 defer cleanup() 476 477 expectedStdout := "Hello from the other side\n" 478 expectedStderr := "Hello from the other side\n" 479 job := mock.BatchJob() 480 job.TaskGroups[0].Count = 1 481 job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 482 "run_for": "20s", 483 "exec_command": map[string]interface{}{ 484 "run_for": "1ms", 485 "stdout_string": expectedStdout, 486 "stderr_string": expectedStderr, 487 "exit_code": 3, 488 }, 489 } 490 491 // Wait for client to be running job 492 testutil.WaitForRunning(t, s.RPC, job) 493 494 // Get the allocation ID 495 args := nstructs.AllocListRequest{} 496 args.Region = "global" 497 resp := nstructs.AllocListResponse{} 498 require.NoError(s.RPC("Alloc.List", &args, &resp)) 499 require.Len(resp.Allocations, 1) 500 allocID := resp.Allocations[0].ID 501 502 // Make the request 503 req := &cstructs.AllocExecRequest{ 504 AllocID: allocID, 505 Task: job.TaskGroups[0].Tasks[0].Name, 506 Tty: true, 507 Cmd: []string{"placeholder command"}, 508 QueryOptions: nstructs.QueryOptions{Region: "global"}, 509 } 510 511 // Get the handler 512 handler, err := c.StreamingRpcHandler("Allocations.Exec") 513 require.Nil(err) 514 515 // Create a pipe 516 p1, p2 := net.Pipe() 517 defer p1.Close() 518 defer p2.Close() 519 520 errCh := make(chan error) 521 frames := make(chan *drivers.ExecTaskStreamingResponseMsg) 522 523 // Start the handler 524 go handler(p2) 525 go decodeFrames(t, p1, frames, errCh) 526 527 // Send the request 528 encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) 529 require.Nil(encoder.Encode(req)) 530 531 timeout := time.After(3 * time.Second) 532 533 exitCode := -1 534 receivedStdout := "" 535 receivedStderr := "" 536 537 OUTER: 538 for { 539 select { 540 case <-timeout: 541 // time out report 542 require.Equal(expectedStdout, receivedStderr, "didn't receive expected stdout") 543 require.Equal(expectedStderr, receivedStderr, "didn't receive expected stderr") 544 require.Equal(3, exitCode, "failed to get exit code") 545 require.FailNow("timed out") 546 case err := <-errCh: 547 require.NoError(err) 548 case f := <-frames: 549 switch { 550 case f.Stdout != nil && len(f.Stdout.Data) != 0: 551 receivedStdout += string(f.Stdout.Data) 552 case f.Stderr != nil && len(f.Stderr.Data) != 0: 553 receivedStderr += string(f.Stderr.Data) 554 case f.Exited && f.Result != nil: 555 exitCode = int(f.Result.ExitCode) 556 default: 557 t.Logf("received unrelevant frame: %v", f) 558 } 559 560 if expectedStdout == receivedStdout && expectedStderr == receivedStderr && exitCode == 3 { 561 break OUTER 562 } 563 } 564 } 565 } 566 567 func TestAlloc_ExecStreaming_NoAllocation(t *testing.T) { 568 t.Parallel() 569 require := require.New(t) 570 571 // Start a server and client 572 s := nomad.TestServer(t, nil) 573 defer s.Shutdown() 574 testutil.WaitForLeader(t, s.RPC) 575 576 c, cleanup := TestClient(t, func(c *config.Config) { 577 c.Servers = []string{s.GetConfig().RPCAddr.String()} 578 }) 579 defer cleanup() 580 581 // Make the request 582 req := &cstructs.AllocExecRequest{ 583 AllocID: uuid.Generate(), 584 Task: "testtask", 585 Tty: true, 586 Cmd: []string{"placeholder command"}, 587 QueryOptions: nstructs.QueryOptions{Region: "global"}, 588 } 589 590 // Get the handler 591 handler, err := c.StreamingRpcHandler("Allocations.Exec") 592 require.Nil(err) 593 594 // Create a pipe 595 p1, p2 := net.Pipe() 596 defer p1.Close() 597 defer p2.Close() 598 599 errCh := make(chan error) 600 frames := make(chan *drivers.ExecTaskStreamingResponseMsg) 601 602 // Start the handler 603 go handler(p2) 604 go decodeFrames(t, p1, frames, errCh) 605 606 // Send the request 607 encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) 608 require.Nil(encoder.Encode(req)) 609 610 timeout := time.After(3 * time.Second) 611 612 select { 613 case <-timeout: 614 require.FailNow("timed out") 615 case err := <-errCh: 616 require.True(nstructs.IsErrUnknownAllocation(err), "expected no allocation error but found: %v", err) 617 case f := <-frames: 618 require.Fail("received unexpected frame", "frame: %#v", f) 619 } 620 } 621 622 func TestAlloc_ExecStreaming_DisableRemoteExec(t *testing.T) { 623 t.Parallel() 624 require := require.New(t) 625 626 // Start a server and client 627 s := nomad.TestServer(t, nil) 628 defer s.Shutdown() 629 testutil.WaitForLeader(t, s.RPC) 630 631 c, cleanup := TestClient(t, func(c *config.Config) { 632 c.Servers = []string{s.GetConfig().RPCAddr.String()} 633 c.DisableRemoteExec = true 634 }) 635 defer cleanup() 636 637 // Make the request 638 req := &cstructs.AllocExecRequest{ 639 AllocID: uuid.Generate(), 640 Task: "testtask", 641 Tty: true, 642 Cmd: []string{"placeholder command"}, 643 QueryOptions: nstructs.QueryOptions{Region: "global"}, 644 } 645 646 // Get the handler 647 handler, err := c.StreamingRpcHandler("Allocations.Exec") 648 require.Nil(err) 649 650 // Create a pipe 651 p1, p2 := net.Pipe() 652 defer p1.Close() 653 defer p2.Close() 654 655 errCh := make(chan error) 656 frames := make(chan *drivers.ExecTaskStreamingResponseMsg) 657 658 // Start the handler 659 go handler(p2) 660 go decodeFrames(t, p1, frames, errCh) 661 662 // Send the request 663 encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) 664 require.Nil(encoder.Encode(req)) 665 666 timeout := time.After(3 * time.Second) 667 668 select { 669 case <-timeout: 670 require.FailNow("timed out") 671 case err := <-errCh: 672 require.True(nstructs.IsErrPermissionDenied(err), "expected permission denied error but found: %v", err) 673 case f := <-frames: 674 require.Fail("received unexpected frame", "frame: %#v", f) 675 } 676 } 677 678 func TestAlloc_ExecStreaming_ACL_Basic(t *testing.T) { 679 t.Parallel() 680 require := require.New(t) 681 682 // Start a server and client 683 s, root := nomad.TestACLServer(t, nil) 684 defer s.Shutdown() 685 testutil.WaitForLeader(t, s.RPC) 686 687 client, cleanup := TestClient(t, func(c *config.Config) { 688 c.ACLEnabled = true 689 c.Servers = []string{s.GetConfig().RPCAddr.String()} 690 }) 691 defer cleanup() 692 693 // Create a bad token 694 policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) 695 tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) 696 697 policyGood := mock.NamespacePolicy(structs.DefaultNamespace, "", 698 []string{acl.NamespaceCapabilityAllocExec, acl.NamespaceCapabilityReadFS}) 699 tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood) 700 701 cases := []struct { 702 Name string 703 Token string 704 ExpectedError string 705 }{ 706 { 707 Name: "bad token", 708 Token: tokenBad.SecretID, 709 ExpectedError: structs.ErrPermissionDenied.Error(), 710 }, 711 { 712 Name: "good token", 713 Token: tokenGood.SecretID, 714 ExpectedError: structs.ErrUnknownAllocationPrefix, 715 }, 716 { 717 Name: "root token", 718 Token: root.SecretID, 719 ExpectedError: structs.ErrUnknownAllocationPrefix, 720 }, 721 } 722 723 for _, c := range cases { 724 t.Run(c.Name, func(t *testing.T) { 725 726 // Make the request 727 req := &cstructs.AllocExecRequest{ 728 AllocID: uuid.Generate(), 729 Task: "testtask", 730 Tty: true, 731 Cmd: []string{"placeholder command"}, 732 QueryOptions: nstructs.QueryOptions{ 733 Region: "global", 734 AuthToken: c.Token, 735 Namespace: nstructs.DefaultNamespace, 736 }, 737 } 738 739 // Get the handler 740 handler, err := client.StreamingRpcHandler("Allocations.Exec") 741 require.Nil(err) 742 743 // Create a pipe 744 p1, p2 := net.Pipe() 745 defer p1.Close() 746 defer p2.Close() 747 748 errCh := make(chan error) 749 frames := make(chan *drivers.ExecTaskStreamingResponseMsg) 750 751 // Start the handler 752 go handler(p2) 753 go decodeFrames(t, p1, frames, errCh) 754 755 // Send the request 756 encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) 757 require.Nil(encoder.Encode(req)) 758 759 select { 760 case <-time.After(3 * time.Second): 761 require.FailNow("timed out") 762 case err := <-errCh: 763 require.Contains(err.Error(), c.ExpectedError) 764 case f := <-frames: 765 require.Fail("received unexpected frame", "frame: %#v", f) 766 } 767 }) 768 } 769 } 770 771 // TestAlloc_ExecStreaming_ACL_WithIsolation_Image asserts that token only needs 772 // alloc-exec acl policy when image isolation is used 773 func TestAlloc_ExecStreaming_ACL_WithIsolation_Image(t *testing.T) { 774 t.Parallel() 775 isolation := drivers.FSIsolationImage 776 777 // Start a server and client 778 s, root := nomad.TestACLServer(t, nil) 779 defer s.Shutdown() 780 testutil.WaitForLeader(t, s.RPC) 781 782 client, cleanup := TestClient(t, func(c *config.Config) { 783 c.ACLEnabled = true 784 c.Servers = []string{s.GetConfig().RPCAddr.String()} 785 786 pluginConfig := []*nconfig.PluginConfig{ 787 { 788 Name: "mock_driver", 789 Config: map[string]interface{}{ 790 "fs_isolation": string(isolation), 791 }, 792 }, 793 } 794 795 c.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", map[string]string{}, pluginConfig) 796 }) 797 defer cleanup() 798 799 // Create a bad token 800 policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) 801 tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) 802 803 policyAllocExec := mock.NamespacePolicy(structs.DefaultNamespace, "", 804 []string{acl.NamespaceCapabilityAllocExec}) 805 tokenAllocExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyAllocExec) 806 807 policyAllocNodeExec := mock.NamespacePolicy(structs.DefaultNamespace, "", 808 []string{acl.NamespaceCapabilityAllocExec, acl.NamespaceCapabilityAllocNodeExec}) 809 tokenAllocNodeExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyAllocNodeExec) 810 811 job := mock.BatchJob() 812 job.TaskGroups[0].Count = 1 813 job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 814 "run_for": "20s", 815 "exec_command": map[string]interface{}{ 816 "run_for": "1ms", 817 "stdout_string": "some output", 818 }, 819 } 820 821 // Wait for client to be running job 822 testutil.WaitForRunningWithToken(t, s.RPC, job, root.SecretID) 823 824 // Get the allocation ID 825 args := nstructs.AllocListRequest{} 826 args.Region = "global" 827 args.AuthToken = root.SecretID 828 args.Namespace = nstructs.DefaultNamespace 829 resp := nstructs.AllocListResponse{} 830 require.NoError(t, s.RPC("Alloc.List", &args, &resp)) 831 require.Len(t, resp.Allocations, 1) 832 allocID := resp.Allocations[0].ID 833 834 cases := []struct { 835 Name string 836 Token string 837 ExpectedError string 838 }{ 839 { 840 Name: "bad token", 841 Token: tokenBad.SecretID, 842 ExpectedError: structs.ErrPermissionDenied.Error(), 843 }, 844 { 845 Name: "alloc-exec token", 846 Token: tokenAllocExec.SecretID, 847 ExpectedError: "", 848 }, 849 { 850 Name: "alloc-node-exec token", 851 Token: tokenAllocNodeExec.SecretID, 852 ExpectedError: "", 853 }, 854 { 855 Name: "root token", 856 Token: root.SecretID, 857 ExpectedError: "", 858 }, 859 } 860 861 for _, c := range cases { 862 t.Run(c.Name, func(t *testing.T) { 863 864 // Make the request 865 req := &cstructs.AllocExecRequest{ 866 AllocID: allocID, 867 Task: job.TaskGroups[0].Tasks[0].Name, 868 Tty: true, 869 Cmd: []string{"placeholder command"}, 870 QueryOptions: nstructs.QueryOptions{ 871 Region: "global", 872 AuthToken: c.Token, 873 Namespace: nstructs.DefaultNamespace, 874 }, 875 } 876 877 // Get the handler 878 handler, err := client.StreamingRpcHandler("Allocations.Exec") 879 require.Nil(t, err) 880 881 // Create a pipe 882 p1, p2 := net.Pipe() 883 defer p1.Close() 884 defer p2.Close() 885 886 errCh := make(chan error) 887 frames := make(chan *drivers.ExecTaskStreamingResponseMsg) 888 889 // Start the handler 890 go handler(p2) 891 go decodeFrames(t, p1, frames, errCh) 892 893 // Send the request 894 encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) 895 require.Nil(t, encoder.Encode(req)) 896 897 select { 898 case <-time.After(3 * time.Second): 899 case err := <-errCh: 900 if c.ExpectedError == "" { 901 require.NoError(t, err) 902 } else { 903 require.Contains(t, err.Error(), c.ExpectedError) 904 } 905 case f := <-frames: 906 // we are good if we don't expect an error 907 if c.ExpectedError != "" { 908 require.Fail(t, "unexpected frame", "frame: %#v", f) 909 } 910 } 911 }) 912 } 913 } 914 915 // TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot asserts that token only needs 916 // alloc-exec acl policy when chroot isolation is used 917 func TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot(t *testing.T) { 918 t.Parallel() 919 920 if runtime.GOOS != "linux" || unix.Geteuid() != 0 { 921 t.Skip("chroot isolation requires linux root") 922 } 923 924 isolation := drivers.FSIsolationChroot 925 926 // Start a server and client 927 s, root := nomad.TestACLServer(t, nil) 928 defer s.Shutdown() 929 testutil.WaitForLeader(t, s.RPC) 930 931 client, cleanup := TestClient(t, func(c *config.Config) { 932 c.ACLEnabled = true 933 c.Servers = []string{s.GetConfig().RPCAddr.String()} 934 935 pluginConfig := []*nconfig.PluginConfig{ 936 { 937 Name: "mock_driver", 938 Config: map[string]interface{}{ 939 "fs_isolation": string(isolation), 940 }, 941 }, 942 } 943 944 c.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", map[string]string{}, pluginConfig) 945 }) 946 defer cleanup() 947 948 // Create a bad token 949 policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) 950 tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) 951 952 policyAllocExec := mock.NamespacePolicy(structs.DefaultNamespace, "", 953 []string{acl.NamespaceCapabilityAllocExec}) 954 tokenAllocExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "alloc-exec", policyAllocExec) 955 956 policyAllocNodeExec := mock.NamespacePolicy(structs.DefaultNamespace, "", 957 []string{acl.NamespaceCapabilityAllocExec, acl.NamespaceCapabilityAllocNodeExec}) 958 tokenAllocNodeExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "alloc-node-exec", policyAllocNodeExec) 959 960 job := mock.BatchJob() 961 job.TaskGroups[0].Count = 1 962 job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 963 "run_for": "20s", 964 "exec_command": map[string]interface{}{ 965 "run_for": "1ms", 966 "stdout_string": "some output", 967 }, 968 } 969 970 // Wait for client to be running job 971 testutil.WaitForRunningWithToken(t, s.RPC, job, root.SecretID) 972 973 // Get the allocation ID 974 args := nstructs.AllocListRequest{} 975 args.Region = "global" 976 args.AuthToken = root.SecretID 977 args.Namespace = nstructs.DefaultNamespace 978 resp := nstructs.AllocListResponse{} 979 require.NoError(t, s.RPC("Alloc.List", &args, &resp)) 980 require.Len(t, resp.Allocations, 1) 981 allocID := resp.Allocations[0].ID 982 983 cases := []struct { 984 Name string 985 Token string 986 ExpectedError string 987 }{ 988 { 989 Name: "bad token", 990 Token: tokenBad.SecretID, 991 ExpectedError: structs.ErrPermissionDenied.Error(), 992 }, 993 { 994 Name: "alloc-exec token", 995 Token: tokenAllocExec.SecretID, 996 ExpectedError: "", 997 }, 998 { 999 Name: "alloc-node-exec token", 1000 Token: tokenAllocNodeExec.SecretID, 1001 ExpectedError: "", 1002 }, 1003 { 1004 Name: "root token", 1005 Token: root.SecretID, 1006 ExpectedError: "", 1007 }, 1008 } 1009 1010 for _, c := range cases { 1011 t.Run(c.Name, func(t *testing.T) { 1012 1013 // Make the request 1014 req := &cstructs.AllocExecRequest{ 1015 AllocID: allocID, 1016 Task: job.TaskGroups[0].Tasks[0].Name, 1017 Tty: true, 1018 Cmd: []string{"placeholder command"}, 1019 QueryOptions: nstructs.QueryOptions{ 1020 Region: "global", 1021 AuthToken: c.Token, 1022 Namespace: nstructs.DefaultNamespace, 1023 }, 1024 } 1025 1026 // Get the handler 1027 handler, err := client.StreamingRpcHandler("Allocations.Exec") 1028 require.Nil(t, err) 1029 1030 // Create a pipe 1031 p1, p2 := net.Pipe() 1032 defer p1.Close() 1033 defer p2.Close() 1034 1035 errCh := make(chan error) 1036 frames := make(chan *drivers.ExecTaskStreamingResponseMsg) 1037 1038 // Start the handler 1039 go handler(p2) 1040 go decodeFrames(t, p1, frames, errCh) 1041 1042 // Send the request 1043 encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) 1044 require.Nil(t, encoder.Encode(req)) 1045 1046 select { 1047 case <-time.After(3 * time.Second): 1048 case err := <-errCh: 1049 if c.ExpectedError == "" { 1050 require.NoError(t, err) 1051 } else { 1052 require.Contains(t, err.Error(), c.ExpectedError) 1053 } 1054 case f := <-frames: 1055 // we are good if we don't expect an error 1056 if c.ExpectedError != "" { 1057 require.Fail(t, "unexpected frame", "frame: %#v", f) 1058 } 1059 } 1060 }) 1061 } 1062 } 1063 1064 // TestAlloc_ExecStreaming_ACL_WithIsolation_None asserts that token needs 1065 // alloc-node-exec acl policy as well when no isolation is used 1066 func TestAlloc_ExecStreaming_ACL_WithIsolation_None(t *testing.T) { 1067 t.Parallel() 1068 isolation := drivers.FSIsolationNone 1069 1070 // Start a server and client 1071 s, root := nomad.TestACLServer(t, nil) 1072 defer s.Shutdown() 1073 testutil.WaitForLeader(t, s.RPC) 1074 1075 client, cleanup := TestClient(t, func(c *config.Config) { 1076 c.ACLEnabled = true 1077 c.Servers = []string{s.GetConfig().RPCAddr.String()} 1078 1079 pluginConfig := []*nconfig.PluginConfig{ 1080 { 1081 Name: "mock_driver", 1082 Config: map[string]interface{}{ 1083 "fs_isolation": string(isolation), 1084 }, 1085 }, 1086 } 1087 1088 c.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", map[string]string{}, pluginConfig) 1089 }) 1090 defer cleanup() 1091 1092 // Create a bad token 1093 policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) 1094 tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) 1095 1096 policyAllocExec := mock.NamespacePolicy(structs.DefaultNamespace, "", 1097 []string{acl.NamespaceCapabilityAllocExec}) 1098 tokenAllocExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "alloc-exec", policyAllocExec) 1099 1100 policyAllocNodeExec := mock.NamespacePolicy(structs.DefaultNamespace, "", 1101 []string{acl.NamespaceCapabilityAllocExec, acl.NamespaceCapabilityAllocNodeExec}) 1102 tokenAllocNodeExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "alloc-node-exec", policyAllocNodeExec) 1103 1104 job := mock.BatchJob() 1105 job.TaskGroups[0].Count = 1 1106 job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ 1107 "run_for": "20s", 1108 "exec_command": map[string]interface{}{ 1109 "run_for": "1ms", 1110 "stdout_string": "some output", 1111 }, 1112 } 1113 1114 // Wait for client to be running job 1115 testutil.WaitForRunningWithToken(t, s.RPC, job, root.SecretID) 1116 1117 // Get the allocation ID 1118 args := nstructs.AllocListRequest{} 1119 args.Region = "global" 1120 args.AuthToken = root.SecretID 1121 args.Namespace = nstructs.DefaultNamespace 1122 resp := nstructs.AllocListResponse{} 1123 require.NoError(t, s.RPC("Alloc.List", &args, &resp)) 1124 require.Len(t, resp.Allocations, 1) 1125 allocID := resp.Allocations[0].ID 1126 1127 cases := []struct { 1128 Name string 1129 Token string 1130 ExpectedError string 1131 }{ 1132 { 1133 Name: "bad token", 1134 Token: tokenBad.SecretID, 1135 ExpectedError: structs.ErrPermissionDenied.Error(), 1136 }, 1137 { 1138 Name: "alloc-exec token", 1139 Token: tokenAllocExec.SecretID, 1140 ExpectedError: structs.ErrPermissionDenied.Error(), 1141 }, 1142 { 1143 Name: "alloc-node-exec token", 1144 Token: tokenAllocNodeExec.SecretID, 1145 ExpectedError: "", 1146 }, 1147 { 1148 Name: "root token", 1149 Token: root.SecretID, 1150 ExpectedError: "", 1151 }, 1152 } 1153 1154 for _, c := range cases { 1155 t.Run(c.Name, func(t *testing.T) { 1156 1157 // Make the request 1158 req := &cstructs.AllocExecRequest{ 1159 AllocID: allocID, 1160 Task: job.TaskGroups[0].Tasks[0].Name, 1161 Tty: true, 1162 Cmd: []string{"placeholder command"}, 1163 QueryOptions: nstructs.QueryOptions{ 1164 Region: "global", 1165 AuthToken: c.Token, 1166 Namespace: nstructs.DefaultNamespace, 1167 }, 1168 } 1169 1170 // Get the handler 1171 handler, err := client.StreamingRpcHandler("Allocations.Exec") 1172 require.Nil(t, err) 1173 1174 // Create a pipe 1175 p1, p2 := net.Pipe() 1176 defer p1.Close() 1177 defer p2.Close() 1178 1179 errCh := make(chan error) 1180 frames := make(chan *drivers.ExecTaskStreamingResponseMsg) 1181 1182 // Start the handler 1183 go handler(p2) 1184 go decodeFrames(t, p1, frames, errCh) 1185 1186 // Send the request 1187 encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) 1188 require.Nil(t, encoder.Encode(req)) 1189 1190 select { 1191 case <-time.After(3 * time.Second): 1192 case err := <-errCh: 1193 if c.ExpectedError == "" { 1194 require.NoError(t, err) 1195 } else { 1196 require.Contains(t, err.Error(), c.ExpectedError) 1197 } 1198 case f := <-frames: 1199 // we are good if we don't expect an error 1200 if c.ExpectedError != "" { 1201 require.Fail(t, "unexpected frame", "frame: %#v", f) 1202 } 1203 } 1204 }) 1205 } 1206 } 1207 1208 func decodeFrames(t *testing.T, p1 net.Conn, frames chan<- *drivers.ExecTaskStreamingResponseMsg, errCh chan<- error) { 1209 // Start the decoder 1210 decoder := codec.NewDecoder(p1, nstructs.MsgpackHandle) 1211 1212 for { 1213 var msg cstructs.StreamErrWrapper 1214 if err := decoder.Decode(&msg); err != nil { 1215 if err == io.EOF || strings.Contains(err.Error(), "closed") { 1216 return 1217 } 1218 t.Logf("received error decoding: %#v", err) 1219 1220 errCh <- fmt.Errorf("error decoding: %v", err) 1221 return 1222 } 1223 1224 if msg.Error != nil { 1225 errCh <- msg.Error 1226 continue 1227 } 1228 1229 var frame drivers.ExecTaskStreamingResponseMsg 1230 if err := json.Unmarshal(msg.Payload, &frame); err != nil { 1231 errCh <- err 1232 return 1233 } 1234 t.Logf("received message: %#v", msg) 1235 frames <- &frame 1236 } 1237 }