github.com/djenriquez/nomad-1@v0.8.1/client/fs_endpoint_test.go (about) 1 package client 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "log" 9 "math" 10 "net" 11 "os" 12 "path/filepath" 13 "reflect" 14 "strings" 15 "testing" 16 "time" 17 18 "github.com/hashicorp/nomad/acl" 19 "github.com/hashicorp/nomad/client/allocdir" 20 "github.com/hashicorp/nomad/client/config" 21 sframer "github.com/hashicorp/nomad/client/lib/streamframer" 22 cstructs "github.com/hashicorp/nomad/client/structs" 23 "github.com/hashicorp/nomad/helper/uuid" 24 "github.com/hashicorp/nomad/nomad" 25 "github.com/hashicorp/nomad/nomad/mock" 26 "github.com/hashicorp/nomad/nomad/structs" 27 "github.com/hashicorp/nomad/testutil" 28 "github.com/stretchr/testify/require" 29 "github.com/ugorji/go/codec" 30 ) 31 32 // tempAllocDir returns a new alloc dir that is rooted in a temp dir. The caller 33 // should destroy the temp dir. 34 func tempAllocDir(t testing.TB) *allocdir.AllocDir { 35 dir, err := ioutil.TempDir("", "") 36 if err != nil { 37 t.Fatalf("TempDir() failed: %v", err) 38 } 39 40 if err := os.Chmod(dir, 0777); err != nil { 41 t.Fatalf("failed to chmod dir: %v", err) 42 } 43 44 return allocdir.NewAllocDir(log.New(os.Stderr, "", log.LstdFlags), dir) 45 } 46 47 type nopWriteCloser struct { 48 io.Writer 49 } 50 51 func (n nopWriteCloser) Close() error { 52 return nil 53 } 54 55 func TestFS_Stat_NoAlloc(t *testing.T) { 56 t.Parallel() 57 require := require.New(t) 58 59 // Start a client 60 c := TestClient(t, nil) 61 defer c.Shutdown() 62 63 // Make the request with bad allocation id 64 req := &cstructs.FsStatRequest{ 65 AllocID: uuid.Generate(), 66 Path: "foo", 67 QueryOptions: structs.QueryOptions{Region: "global"}, 68 } 69 70 var resp cstructs.FsStatResponse 71 err := c.ClientRPC("FileSystem.Stat", req, &resp) 72 require.NotNil(err) 73 require.True(structs.IsErrUnknownAllocation(err)) 74 } 75 76 func TestFS_Stat(t *testing.T) { 77 t.Parallel() 78 require := require.New(t) 79 80 // Start a client 81 c := TestClient(t, nil) 82 defer c.Shutdown() 83 84 // Create and add an alloc 85 a := mock.Alloc() 86 c.addAlloc(a, "") 87 88 // Wait for the client to start it 89 testutil.WaitForResult(func() (bool, error) { 90 ar, ok := c.allocs[a.ID] 91 if !ok { 92 return false, fmt.Errorf("alloc doesn't exist") 93 } 94 95 return len(ar.tasks) != 0, fmt.Errorf("tasks not running") 96 }, func(err error) { 97 t.Fatal(err) 98 }) 99 100 // Make the request with bad allocation id 101 req := &cstructs.FsStatRequest{ 102 AllocID: a.ID, 103 Path: "/", 104 QueryOptions: structs.QueryOptions{Region: "global"}, 105 } 106 107 var resp cstructs.FsStatResponse 108 err := c.ClientRPC("FileSystem.Stat", req, &resp) 109 require.Nil(err) 110 require.NotNil(resp.Info) 111 require.True(resp.Info.IsDir) 112 } 113 114 func TestFS_Stat_ACL(t *testing.T) { 115 t.Parallel() 116 require := require.New(t) 117 118 // Start a server 119 s, root := nomad.TestACLServer(t, nil) 120 defer s.Shutdown() 121 testutil.WaitForLeader(t, s.RPC) 122 123 client := TestClient(t, func(c *config.Config) { 124 c.ACLEnabled = true 125 c.Servers = []string{s.GetConfig().RPCAddr.String()} 126 }) 127 defer client.Shutdown() 128 129 // Create a bad token 130 policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) 131 tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) 132 133 policyGood := mock.NamespacePolicy(structs.DefaultNamespace, "", 134 []string{acl.NamespaceCapabilityReadLogs, acl.NamespaceCapabilityReadFS}) 135 tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood) 136 137 cases := []struct { 138 Name string 139 Token string 140 ExpectedError string 141 }{ 142 { 143 Name: "bad token", 144 Token: tokenBad.SecretID, 145 ExpectedError: structs.ErrPermissionDenied.Error(), 146 }, 147 { 148 Name: "good token", 149 Token: tokenGood.SecretID, 150 ExpectedError: structs.ErrUnknownAllocationPrefix, 151 }, 152 { 153 Name: "root token", 154 Token: root.SecretID, 155 ExpectedError: structs.ErrUnknownAllocationPrefix, 156 }, 157 } 158 159 for _, c := range cases { 160 t.Run(c.Name, func(t *testing.T) { 161 // Make the request with bad allocation id 162 req := &cstructs.FsStatRequest{ 163 AllocID: uuid.Generate(), 164 Path: "/", 165 QueryOptions: structs.QueryOptions{ 166 Region: "global", 167 AuthToken: c.Token, 168 Namespace: structs.DefaultNamespace, 169 }, 170 } 171 172 var resp cstructs.FsStatResponse 173 err := client.ClientRPC("FileSystem.Stat", req, &resp) 174 require.NotNil(err) 175 require.Contains(err.Error(), c.ExpectedError) 176 }) 177 } 178 } 179 180 func TestFS_List_NoAlloc(t *testing.T) { 181 t.Parallel() 182 require := require.New(t) 183 184 // Start a client 185 c := TestClient(t, nil) 186 defer c.Shutdown() 187 188 // Make the request with bad allocation id 189 req := &cstructs.FsListRequest{ 190 AllocID: uuid.Generate(), 191 Path: "foo", 192 QueryOptions: structs.QueryOptions{Region: "global"}, 193 } 194 195 var resp cstructs.FsListResponse 196 err := c.ClientRPC("FileSystem.List", req, &resp) 197 require.NotNil(err) 198 require.True(structs.IsErrUnknownAllocation(err)) 199 } 200 201 func TestFS_List(t *testing.T) { 202 t.Parallel() 203 require := require.New(t) 204 205 // Start a client 206 c := TestClient(t, nil) 207 defer c.Shutdown() 208 209 // Create and add an alloc 210 a := mock.Alloc() 211 c.addAlloc(a, "") 212 213 // Wait for the client to start it 214 testutil.WaitForResult(func() (bool, error) { 215 ar, ok := c.allocs[a.ID] 216 if !ok { 217 return false, fmt.Errorf("alloc doesn't exist") 218 } 219 220 return len(ar.tasks) != 0, fmt.Errorf("tasks not running") 221 }, func(err error) { 222 t.Fatal(err) 223 }) 224 225 // Make the request 226 req := &cstructs.FsListRequest{ 227 AllocID: a.ID, 228 Path: "/", 229 QueryOptions: structs.QueryOptions{Region: "global"}, 230 } 231 232 var resp cstructs.FsListResponse 233 err := c.ClientRPC("FileSystem.List", req, &resp) 234 require.Nil(err) 235 require.NotEmpty(resp.Files) 236 require.True(resp.Files[0].IsDir) 237 } 238 239 func TestFS_List_ACL(t *testing.T) { 240 t.Parallel() 241 require := require.New(t) 242 243 // Start a server 244 s, root := nomad.TestACLServer(t, nil) 245 defer s.Shutdown() 246 testutil.WaitForLeader(t, s.RPC) 247 248 client := TestClient(t, func(c *config.Config) { 249 c.ACLEnabled = true 250 c.Servers = []string{s.GetConfig().RPCAddr.String()} 251 }) 252 defer client.Shutdown() 253 254 // Create a bad token 255 policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) 256 tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) 257 258 policyGood := mock.NamespacePolicy(structs.DefaultNamespace, "", 259 []string{acl.NamespaceCapabilityReadLogs, acl.NamespaceCapabilityReadFS}) 260 tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood) 261 262 cases := []struct { 263 Name string 264 Token string 265 ExpectedError string 266 }{ 267 { 268 Name: "bad token", 269 Token: tokenBad.SecretID, 270 ExpectedError: structs.ErrPermissionDenied.Error(), 271 }, 272 { 273 Name: "good token", 274 Token: tokenGood.SecretID, 275 ExpectedError: structs.ErrUnknownAllocationPrefix, 276 }, 277 { 278 Name: "root token", 279 Token: root.SecretID, 280 ExpectedError: structs.ErrUnknownAllocationPrefix, 281 }, 282 } 283 284 for _, c := range cases { 285 t.Run(c.Name, func(t *testing.T) { 286 // Make the request with bad allocation id 287 req := &cstructs.FsListRequest{ 288 AllocID: uuid.Generate(), 289 Path: "/", 290 QueryOptions: structs.QueryOptions{ 291 Region: "global", 292 AuthToken: c.Token, 293 Namespace: structs.DefaultNamespace, 294 }, 295 } 296 297 var resp cstructs.FsListResponse 298 err := client.ClientRPC("FileSystem.List", req, &resp) 299 require.NotNil(err) 300 require.Contains(err.Error(), c.ExpectedError) 301 }) 302 } 303 } 304 305 func TestFS_Stream_NoAlloc(t *testing.T) { 306 t.Parallel() 307 require := require.New(t) 308 309 // Start a client 310 c := TestClient(t, nil) 311 defer c.Shutdown() 312 313 // Make the request with bad allocation id 314 req := &cstructs.FsStreamRequest{ 315 AllocID: uuid.Generate(), 316 Path: "foo", 317 Origin: "start", 318 QueryOptions: structs.QueryOptions{Region: "global"}, 319 } 320 321 // Get the handler 322 handler, err := c.StreamingRpcHandler("FileSystem.Stream") 323 require.Nil(err) 324 325 // Create a pipe 326 p1, p2 := net.Pipe() 327 defer p1.Close() 328 defer p2.Close() 329 330 errCh := make(chan error) 331 streamMsg := make(chan *cstructs.StreamErrWrapper) 332 333 // Start the handler 334 go handler(p2) 335 336 // Start the decoder 337 go func() { 338 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 339 for { 340 var msg cstructs.StreamErrWrapper 341 if err := decoder.Decode(&msg); err != nil { 342 if err == io.EOF || strings.Contains(err.Error(), "closed") { 343 return 344 } 345 errCh <- fmt.Errorf("error decoding: %v", err) 346 } 347 348 streamMsg <- &msg 349 } 350 }() 351 352 // Send the request 353 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 354 require.Nil(encoder.Encode(req)) 355 356 timeout := time.After(3 * time.Second) 357 358 OUTER: 359 for { 360 select { 361 case <-timeout: 362 t.Fatal("timeout") 363 case err := <-errCh: 364 t.Fatal(err) 365 case msg := <-streamMsg: 366 t.Logf("Got msg %+v", msg) 367 if msg.Error == nil { 368 continue 369 } 370 371 if structs.IsErrUnknownAllocation(msg.Error) { 372 break OUTER 373 } else { 374 t.Fatalf("bad error: %v", err) 375 } 376 } 377 } 378 } 379 380 func TestFS_Stream_ACL(t *testing.T) { 381 t.Parallel() 382 require := require.New(t) 383 384 // Start a server 385 s, root := nomad.TestACLServer(t, nil) 386 defer s.Shutdown() 387 testutil.WaitForLeader(t, s.RPC) 388 389 client := TestClient(t, func(c *config.Config) { 390 c.ACLEnabled = true 391 c.Servers = []string{s.GetConfig().RPCAddr.String()} 392 }) 393 defer client.Shutdown() 394 395 // Create a bad token 396 policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityReadFS}) 397 tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) 398 399 policyGood := mock.NamespacePolicy(structs.DefaultNamespace, "", 400 []string{acl.NamespaceCapabilityReadLogs, acl.NamespaceCapabilityReadFS}) 401 tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood) 402 403 cases := []struct { 404 Name string 405 Token string 406 ExpectedError string 407 }{ 408 { 409 Name: "bad token", 410 Token: tokenBad.SecretID, 411 ExpectedError: structs.ErrPermissionDenied.Error(), 412 }, 413 { 414 Name: "good token", 415 Token: tokenGood.SecretID, 416 ExpectedError: structs.ErrUnknownAllocationPrefix, 417 }, 418 { 419 Name: "root token", 420 Token: root.SecretID, 421 ExpectedError: structs.ErrUnknownAllocationPrefix, 422 }, 423 } 424 425 for _, c := range cases { 426 t.Run(c.Name, func(t *testing.T) { 427 // Make the request with bad allocation id 428 req := &cstructs.FsStreamRequest{ 429 AllocID: uuid.Generate(), 430 Path: "foo", 431 Origin: "start", 432 QueryOptions: structs.QueryOptions{ 433 Namespace: structs.DefaultNamespace, 434 Region: "global", 435 AuthToken: c.Token, 436 }, 437 } 438 439 // Get the handler 440 handler, err := client.StreamingRpcHandler("FileSystem.Stream") 441 require.Nil(err) 442 443 // Create a pipe 444 p1, p2 := net.Pipe() 445 defer p1.Close() 446 defer p2.Close() 447 448 errCh := make(chan error) 449 streamMsg := make(chan *cstructs.StreamErrWrapper) 450 451 // Start the handler 452 go handler(p2) 453 454 // Start the decoder 455 go func() { 456 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 457 for { 458 var msg cstructs.StreamErrWrapper 459 if err := decoder.Decode(&msg); err != nil { 460 if err == io.EOF || strings.Contains(err.Error(), "closed") { 461 return 462 } 463 errCh <- fmt.Errorf("error decoding: %v", err) 464 } 465 466 streamMsg <- &msg 467 } 468 }() 469 470 // Send the request 471 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 472 require.Nil(encoder.Encode(req)) 473 474 timeout := time.After(5 * time.Second) 475 476 OUTER: 477 for { 478 select { 479 case <-timeout: 480 t.Fatal("timeout") 481 case err := <-errCh: 482 t.Fatal(err) 483 case msg := <-streamMsg: 484 if msg.Error == nil { 485 continue 486 } 487 488 if strings.Contains(msg.Error.Error(), c.ExpectedError) { 489 break OUTER 490 } else { 491 t.Fatalf("Bad error: %v", msg.Error) 492 } 493 } 494 } 495 }) 496 } 497 } 498 499 func TestFS_Stream(t *testing.T) { 500 t.Parallel() 501 require := require.New(t) 502 503 // Start a server and client 504 s := nomad.TestServer(t, nil) 505 defer s.Shutdown() 506 testutil.WaitForLeader(t, s.RPC) 507 508 c := TestClient(t, func(c *config.Config) { 509 c.Servers = []string{s.GetConfig().RPCAddr.String()} 510 }) 511 defer c.Shutdown() 512 513 // Force an allocation onto the node 514 expected := "Hello from the other side" 515 a := mock.Alloc() 516 a.Job.Type = structs.JobTypeBatch 517 a.NodeID = c.NodeID() 518 a.Job.TaskGroups[0].Count = 1 519 a.Job.TaskGroups[0].Tasks[0] = &structs.Task{ 520 Name: "web", 521 Driver: "mock_driver", 522 Config: map[string]interface{}{ 523 "run_for": "2s", 524 "stdout_string": expected, 525 }, 526 LogConfig: structs.DefaultLogConfig(), 527 Resources: &structs.Resources{ 528 CPU: 500, 529 MemoryMB: 256, 530 }, 531 } 532 533 // Wait for the client to connect 534 testutil.WaitForResult(func() (bool, error) { 535 node, err := s.State().NodeByID(nil, c.NodeID()) 536 if err != nil { 537 return false, err 538 } 539 if node == nil { 540 return false, fmt.Errorf("unknown node") 541 } 542 543 return node.Status == structs.NodeStatusReady, fmt.Errorf("bad node status") 544 }, func(err error) { 545 t.Fatal(err) 546 }) 547 548 // Upsert the allocation 549 state := s.State() 550 require.Nil(state.UpsertJob(999, a.Job)) 551 require.Nil(state.UpsertAllocs(1003, []*structs.Allocation{a})) 552 553 // Wait for the client to run the allocation 554 testutil.WaitForResult(func() (bool, error) { 555 alloc, err := state.AllocByID(nil, a.ID) 556 if err != nil { 557 return false, err 558 } 559 if alloc == nil { 560 return false, fmt.Errorf("unknown alloc") 561 } 562 if alloc.ClientStatus != structs.AllocClientStatusComplete { 563 return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus) 564 } 565 566 return true, nil 567 }, func(err error) { 568 t.Fatalf("Alloc on node %q not finished: %v", c.NodeID(), err) 569 }) 570 571 // Make the request 572 req := &cstructs.FsStreamRequest{ 573 AllocID: a.ID, 574 Path: "alloc/logs/web.stdout.0", 575 PlainText: true, 576 QueryOptions: structs.QueryOptions{Region: "global"}, 577 } 578 579 // Get the handler 580 handler, err := c.StreamingRpcHandler("FileSystem.Stream") 581 require.Nil(err) 582 583 // Create a pipe 584 p1, p2 := net.Pipe() 585 defer p1.Close() 586 defer p2.Close() 587 588 // Wrap the pipe so we can check it is closed 589 pipeChecker := &ReadWriteCloseChecker{ReadWriteCloser: p2} 590 591 errCh := make(chan error) 592 streamMsg := make(chan *cstructs.StreamErrWrapper) 593 594 // Start the handler 595 go handler(pipeChecker) 596 597 // Start the decoder 598 go func() { 599 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 600 for { 601 var msg cstructs.StreamErrWrapper 602 if err := decoder.Decode(&msg); err != nil { 603 if err == io.EOF || strings.Contains(err.Error(), "closed") { 604 return 605 } 606 errCh <- fmt.Errorf("error decoding: %v", err) 607 } 608 609 streamMsg <- &msg 610 } 611 }() 612 613 // Send the request 614 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 615 require.Nil(encoder.Encode(req)) 616 617 timeout := time.After(3 * time.Second) 618 received := "" 619 OUTER: 620 for { 621 select { 622 case <-timeout: 623 t.Fatal("timeout") 624 case err := <-errCh: 625 t.Fatal(err) 626 case msg := <-streamMsg: 627 if msg.Error != nil { 628 t.Fatalf("Got error: %v", msg.Error.Error()) 629 } 630 631 // Add the payload 632 received += string(msg.Payload) 633 if received == expected { 634 break OUTER 635 } 636 } 637 } 638 639 testutil.WaitForResult(func() (bool, error) { 640 return pipeChecker.Closed, nil 641 }, func(err error) { 642 t.Fatal("Pipe not closed") 643 }) 644 } 645 646 type ReadWriteCloseChecker struct { 647 io.ReadWriteCloser 648 Closed bool 649 } 650 651 func (r *ReadWriteCloseChecker) Close() error { 652 r.Closed = true 653 return r.ReadWriteCloser.Close() 654 } 655 656 func TestFS_Stream_Follow(t *testing.T) { 657 t.Parallel() 658 require := require.New(t) 659 660 // Start a server and client 661 s := nomad.TestServer(t, nil) 662 defer s.Shutdown() 663 testutil.WaitForLeader(t, s.RPC) 664 665 c := TestClient(t, func(c *config.Config) { 666 c.Servers = []string{s.GetConfig().RPCAddr.String()} 667 }) 668 defer c.Shutdown() 669 670 // Force an allocation onto the node 671 expectedBase := "Hello from the other side" 672 repeat := 10 673 674 a := mock.Alloc() 675 a.Job.Type = structs.JobTypeBatch 676 a.NodeID = c.NodeID() 677 a.Job.TaskGroups[0].Count = 1 678 a.Job.TaskGroups[0].Tasks[0] = &structs.Task{ 679 Name: "web", 680 Driver: "mock_driver", 681 Config: map[string]interface{}{ 682 "run_for": "20s", 683 "stdout_string": expectedBase, 684 "stdout_repeat": repeat, 685 "stdout_repeat_duration": 200 * time.Millisecond, 686 }, 687 LogConfig: structs.DefaultLogConfig(), 688 Resources: &structs.Resources{ 689 CPU: 500, 690 MemoryMB: 256, 691 }, 692 } 693 694 // Wait for the client to connect 695 testutil.WaitForResult(func() (bool, error) { 696 node, err := s.State().NodeByID(nil, c.NodeID()) 697 if err != nil { 698 return false, err 699 } 700 if node == nil { 701 return false, fmt.Errorf("unknown node") 702 } 703 704 return node.Status == structs.NodeStatusReady, fmt.Errorf("bad node status") 705 }, func(err error) { 706 t.Fatal(err) 707 }) 708 709 // Upsert the allocation 710 state := s.State() 711 require.Nil(state.UpsertJob(999, a.Job)) 712 require.Nil(state.UpsertAllocs(1003, []*structs.Allocation{a})) 713 714 // Wait for the client to run the allocation 715 testutil.WaitForResult(func() (bool, error) { 716 alloc, err := state.AllocByID(nil, a.ID) 717 if err != nil { 718 return false, err 719 } 720 if alloc == nil { 721 return false, fmt.Errorf("unknown alloc") 722 } 723 if alloc.ClientStatus != structs.AllocClientStatusRunning { 724 return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus) 725 } 726 727 return true, nil 728 }, func(err error) { 729 t.Fatalf("Alloc on node %q not running: %v", c.NodeID(), err) 730 }) 731 732 // Make the request 733 req := &cstructs.FsStreamRequest{ 734 AllocID: a.ID, 735 Path: "alloc/logs/web.stdout.0", 736 PlainText: true, 737 Follow: true, 738 QueryOptions: structs.QueryOptions{Region: "global"}, 739 } 740 741 // Get the handler 742 handler, err := c.StreamingRpcHandler("FileSystem.Stream") 743 require.Nil(err) 744 745 // Create a pipe 746 p1, p2 := net.Pipe() 747 defer p1.Close() 748 defer p2.Close() 749 750 errCh := make(chan error) 751 streamMsg := make(chan *cstructs.StreamErrWrapper) 752 753 // Start the handler 754 go handler(p2) 755 756 // Start the decoder 757 go func() { 758 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 759 for { 760 var msg cstructs.StreamErrWrapper 761 if err := decoder.Decode(&msg); err != nil { 762 if err == io.EOF || strings.Contains(err.Error(), "closed") { 763 return 764 } 765 errCh <- fmt.Errorf("error decoding: %v", err) 766 } 767 768 streamMsg <- &msg 769 } 770 }() 771 772 // Send the request 773 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 774 require.Nil(encoder.Encode(req)) 775 776 timeout := time.After(20 * time.Second) 777 expected := strings.Repeat(expectedBase, repeat+1) 778 received := "" 779 OUTER: 780 for { 781 select { 782 case <-timeout: 783 t.Fatal("timeout") 784 case err := <-errCh: 785 t.Fatal(err) 786 case msg := <-streamMsg: 787 if msg.Error != nil { 788 t.Fatalf("Got error: %v", msg.Error.Error()) 789 } 790 791 // Add the payload 792 received += string(msg.Payload) 793 if received == expected { 794 break OUTER 795 } 796 } 797 } 798 } 799 800 func TestFS_Stream_Limit(t *testing.T) { 801 t.Parallel() 802 require := require.New(t) 803 804 // Start a server and client 805 s := nomad.TestServer(t, nil) 806 defer s.Shutdown() 807 testutil.WaitForLeader(t, s.RPC) 808 809 c := TestClient(t, func(c *config.Config) { 810 c.Servers = []string{s.GetConfig().RPCAddr.String()} 811 }) 812 defer c.Shutdown() 813 814 // Force an allocation onto the node 815 var limit int64 = 5 816 full := "Hello from the other side" 817 expected := full[:limit] 818 a := mock.Alloc() 819 a.Job.Type = structs.JobTypeBatch 820 a.NodeID = c.NodeID() 821 a.Job.TaskGroups[0].Count = 1 822 a.Job.TaskGroups[0].Tasks[0] = &structs.Task{ 823 Name: "web", 824 Driver: "mock_driver", 825 Config: map[string]interface{}{ 826 "run_for": "2s", 827 "stdout_string": full, 828 }, 829 LogConfig: structs.DefaultLogConfig(), 830 Resources: &structs.Resources{ 831 CPU: 500, 832 MemoryMB: 256, 833 }, 834 } 835 836 // Wait for the client to connect 837 testutil.WaitForResult(func() (bool, error) { 838 node, err := s.State().NodeByID(nil, c.NodeID()) 839 if err != nil { 840 return false, err 841 } 842 if node == nil { 843 return false, fmt.Errorf("unknown node") 844 } 845 846 return node.Status == structs.NodeStatusReady, fmt.Errorf("bad node status") 847 }, func(err error) { 848 t.Fatal(err) 849 }) 850 851 // Upsert the allocation 852 state := s.State() 853 require.Nil(state.UpsertJob(999, a.Job)) 854 require.Nil(state.UpsertAllocs(1003, []*structs.Allocation{a})) 855 856 // Wait for the client to run the allocation 857 testutil.WaitForResult(func() (bool, error) { 858 alloc, err := state.AllocByID(nil, a.ID) 859 if err != nil { 860 return false, err 861 } 862 if alloc == nil { 863 return false, fmt.Errorf("unknown alloc") 864 } 865 if alloc.ClientStatus != structs.AllocClientStatusComplete { 866 return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus) 867 } 868 869 return true, nil 870 }, func(err error) { 871 t.Fatalf("Alloc on node %q not finished: %v", c.NodeID(), err) 872 }) 873 874 // Make the request 875 req := &cstructs.FsStreamRequest{ 876 AllocID: a.ID, 877 Path: "alloc/logs/web.stdout.0", 878 PlainText: true, 879 Limit: limit, 880 QueryOptions: structs.QueryOptions{Region: "global"}, 881 } 882 883 // Get the handler 884 handler, err := c.StreamingRpcHandler("FileSystem.Stream") 885 require.Nil(err) 886 887 // Create a pipe 888 p1, p2 := net.Pipe() 889 defer p1.Close() 890 defer p2.Close() 891 892 errCh := make(chan error) 893 streamMsg := make(chan *cstructs.StreamErrWrapper) 894 895 // Start the handler 896 go handler(p2) 897 898 // Start the decoder 899 go func() { 900 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 901 for { 902 var msg cstructs.StreamErrWrapper 903 if err := decoder.Decode(&msg); err != nil { 904 if err == io.EOF || strings.Contains(err.Error(), "closed") { 905 return 906 } 907 errCh <- fmt.Errorf("error decoding: %v", err) 908 } 909 910 streamMsg <- &msg 911 } 912 }() 913 914 // Send the request 915 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 916 require.Nil(encoder.Encode(req)) 917 918 timeout := time.After(3 * time.Second) 919 received := "" 920 OUTER: 921 for { 922 select { 923 case <-timeout: 924 t.Fatal("timeout") 925 case err := <-errCh: 926 t.Fatal(err) 927 case msg := <-streamMsg: 928 if msg.Error != nil { 929 t.Fatalf("Got error: %v", msg.Error.Error()) 930 } 931 932 // Add the payload 933 received += string(msg.Payload) 934 if received == expected { 935 break OUTER 936 } 937 } 938 } 939 } 940 941 func TestFS_Logs_NoAlloc(t *testing.T) { 942 t.Parallel() 943 require := require.New(t) 944 945 // Start a client 946 c := TestClient(t, nil) 947 defer c.Shutdown() 948 949 // Make the request with bad allocation id 950 req := &cstructs.FsLogsRequest{ 951 AllocID: uuid.Generate(), 952 Task: "foo", 953 LogType: "stdout", 954 Origin: "start", 955 QueryOptions: structs.QueryOptions{Region: "global"}, 956 } 957 958 // Get the handler 959 handler, err := c.StreamingRpcHandler("FileSystem.Logs") 960 require.Nil(err) 961 962 // Create a pipe 963 p1, p2 := net.Pipe() 964 defer p1.Close() 965 defer p2.Close() 966 967 errCh := make(chan error) 968 streamMsg := make(chan *cstructs.StreamErrWrapper) 969 970 // Start the handler 971 go handler(p2) 972 973 // Start the decoder 974 go func() { 975 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 976 for { 977 var msg cstructs.StreamErrWrapper 978 if err := decoder.Decode(&msg); err != nil { 979 if err == io.EOF || strings.Contains(err.Error(), "closed") { 980 return 981 } 982 errCh <- fmt.Errorf("error decoding: %v", err) 983 } 984 985 streamMsg <- &msg 986 } 987 }() 988 989 // Send the request 990 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 991 require.Nil(encoder.Encode(req)) 992 993 timeout := time.After(3 * time.Second) 994 995 OUTER: 996 for { 997 select { 998 case <-timeout: 999 t.Fatal("timeout") 1000 case err := <-errCh: 1001 t.Fatal(err) 1002 case msg := <-streamMsg: 1003 t.Logf("Got msg %+v", msg) 1004 if msg.Error == nil { 1005 continue 1006 } 1007 1008 if structs.IsErrUnknownAllocation(msg.Error) { 1009 break OUTER 1010 } else { 1011 t.Fatalf("bad error: %v", err) 1012 } 1013 } 1014 } 1015 } 1016 1017 func TestFS_Logs_ACL(t *testing.T) { 1018 t.Parallel() 1019 require := require.New(t) 1020 1021 // Start a server 1022 s, root := nomad.TestACLServer(t, nil) 1023 defer s.Shutdown() 1024 testutil.WaitForLeader(t, s.RPC) 1025 1026 client := TestClient(t, func(c *config.Config) { 1027 c.ACLEnabled = true 1028 c.Servers = []string{s.GetConfig().RPCAddr.String()} 1029 }) 1030 defer client.Shutdown() 1031 1032 // Create a bad token 1033 policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityReadFS}) 1034 tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) 1035 1036 policyGood := mock.NamespacePolicy(structs.DefaultNamespace, "", 1037 []string{acl.NamespaceCapabilityReadLogs, acl.NamespaceCapabilityReadFS}) 1038 tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood) 1039 1040 cases := []struct { 1041 Name string 1042 Token string 1043 ExpectedError string 1044 }{ 1045 { 1046 Name: "bad token", 1047 Token: tokenBad.SecretID, 1048 ExpectedError: structs.ErrPermissionDenied.Error(), 1049 }, 1050 { 1051 Name: "good token", 1052 Token: tokenGood.SecretID, 1053 ExpectedError: structs.ErrUnknownAllocationPrefix, 1054 }, 1055 { 1056 Name: "root token", 1057 Token: root.SecretID, 1058 ExpectedError: structs.ErrUnknownAllocationPrefix, 1059 }, 1060 } 1061 1062 for _, c := range cases { 1063 t.Run(c.Name, func(t *testing.T) { 1064 // Make the request with bad allocation id 1065 req := &cstructs.FsLogsRequest{ 1066 AllocID: uuid.Generate(), 1067 Task: "foo", 1068 LogType: "stdout", 1069 Origin: "start", 1070 QueryOptions: structs.QueryOptions{ 1071 Namespace: structs.DefaultNamespace, 1072 Region: "global", 1073 AuthToken: c.Token, 1074 }, 1075 } 1076 1077 // Get the handler 1078 handler, err := client.StreamingRpcHandler("FileSystem.Logs") 1079 require.Nil(err) 1080 1081 // Create a pipe 1082 p1, p2 := net.Pipe() 1083 defer p1.Close() 1084 defer p2.Close() 1085 1086 errCh := make(chan error) 1087 streamMsg := make(chan *cstructs.StreamErrWrapper) 1088 1089 // Start the handler 1090 go handler(p2) 1091 1092 // Start the decoder 1093 go func() { 1094 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 1095 for { 1096 var msg cstructs.StreamErrWrapper 1097 if err := decoder.Decode(&msg); err != nil { 1098 if err == io.EOF || strings.Contains(err.Error(), "closed") { 1099 return 1100 } 1101 errCh <- fmt.Errorf("error decoding: %v", err) 1102 } 1103 1104 streamMsg <- &msg 1105 } 1106 }() 1107 1108 // Send the request 1109 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 1110 require.Nil(encoder.Encode(req)) 1111 1112 timeout := time.After(5 * time.Second) 1113 1114 OUTER: 1115 for { 1116 select { 1117 case <-timeout: 1118 t.Fatal("timeout") 1119 case err := <-errCh: 1120 t.Fatal(err) 1121 case msg := <-streamMsg: 1122 if msg.Error == nil { 1123 continue 1124 } 1125 1126 if strings.Contains(msg.Error.Error(), c.ExpectedError) { 1127 break OUTER 1128 } else { 1129 t.Fatalf("Bad error: %v", msg.Error) 1130 } 1131 } 1132 } 1133 }) 1134 } 1135 } 1136 1137 func TestFS_Logs(t *testing.T) { 1138 t.Parallel() 1139 require := require.New(t) 1140 1141 // Start a server and client 1142 s := nomad.TestServer(t, nil) 1143 defer s.Shutdown() 1144 testutil.WaitForLeader(t, s.RPC) 1145 1146 c := TestClient(t, func(c *config.Config) { 1147 c.Servers = []string{s.GetConfig().RPCAddr.String()} 1148 }) 1149 defer c.Shutdown() 1150 1151 // Force an allocation onto the node 1152 expected := "Hello from the other side" 1153 a := mock.Alloc() 1154 a.Job.Type = structs.JobTypeBatch 1155 a.NodeID = c.NodeID() 1156 a.Job.TaskGroups[0].Count = 1 1157 a.Job.TaskGroups[0].Tasks[0] = &structs.Task{ 1158 Name: "web", 1159 Driver: "mock_driver", 1160 Config: map[string]interface{}{ 1161 "run_for": "2s", 1162 "stdout_string": expected, 1163 }, 1164 LogConfig: structs.DefaultLogConfig(), 1165 Resources: &structs.Resources{ 1166 CPU: 500, 1167 MemoryMB: 256, 1168 }, 1169 } 1170 1171 // Wait for the client to connect 1172 testutil.WaitForResult(func() (bool, error) { 1173 node, err := s.State().NodeByID(nil, c.NodeID()) 1174 if err != nil { 1175 return false, err 1176 } 1177 if node == nil { 1178 return false, fmt.Errorf("unknown node") 1179 } 1180 1181 return node.Status == structs.NodeStatusReady, fmt.Errorf("bad node status") 1182 }, func(err error) { 1183 t.Fatal(err) 1184 }) 1185 1186 // Upsert the allocation 1187 state := s.State() 1188 require.Nil(state.UpsertJob(999, a.Job)) 1189 require.Nil(state.UpsertAllocs(1003, []*structs.Allocation{a})) 1190 1191 // Wait for the client to run the allocation 1192 testutil.WaitForResult(func() (bool, error) { 1193 alloc, err := state.AllocByID(nil, a.ID) 1194 if err != nil { 1195 return false, err 1196 } 1197 if alloc == nil { 1198 return false, fmt.Errorf("unknown alloc") 1199 } 1200 if alloc.ClientStatus != structs.AllocClientStatusComplete { 1201 return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus) 1202 } 1203 1204 return true, nil 1205 }, func(err error) { 1206 t.Fatalf("Alloc on node %q not finished: %v", c.NodeID(), err) 1207 }) 1208 1209 // Make the request 1210 req := &cstructs.FsLogsRequest{ 1211 AllocID: a.ID, 1212 Task: a.Job.TaskGroups[0].Tasks[0].Name, 1213 LogType: "stdout", 1214 Origin: "start", 1215 PlainText: true, 1216 QueryOptions: structs.QueryOptions{Region: "global"}, 1217 } 1218 1219 // Get the handler 1220 handler, err := c.StreamingRpcHandler("FileSystem.Logs") 1221 require.Nil(err) 1222 1223 // Create a pipe 1224 p1, p2 := net.Pipe() 1225 defer p1.Close() 1226 defer p2.Close() 1227 1228 errCh := make(chan error) 1229 streamMsg := make(chan *cstructs.StreamErrWrapper) 1230 1231 // Start the handler 1232 go handler(p2) 1233 1234 // Start the decoder 1235 go func() { 1236 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 1237 for { 1238 var msg cstructs.StreamErrWrapper 1239 if err := decoder.Decode(&msg); err != nil { 1240 if err == io.EOF || strings.Contains(err.Error(), "closed") { 1241 return 1242 } 1243 errCh <- fmt.Errorf("error decoding: %v", err) 1244 } 1245 1246 streamMsg <- &msg 1247 } 1248 }() 1249 1250 // Send the request 1251 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 1252 require.Nil(encoder.Encode(req)) 1253 1254 timeout := time.After(3 * time.Second) 1255 received := "" 1256 OUTER: 1257 for { 1258 select { 1259 case <-timeout: 1260 t.Fatal("timeout") 1261 case err := <-errCh: 1262 t.Fatal(err) 1263 case msg := <-streamMsg: 1264 if msg.Error != nil { 1265 t.Fatalf("Got error: %v", msg.Error.Error()) 1266 } 1267 1268 // Add the payload 1269 received += string(msg.Payload) 1270 if received == expected { 1271 break OUTER 1272 } 1273 } 1274 } 1275 } 1276 1277 func TestFS_Logs_Follow(t *testing.T) { 1278 t.Parallel() 1279 require := require.New(t) 1280 1281 // Start a server and client 1282 s := nomad.TestServer(t, nil) 1283 defer s.Shutdown() 1284 testutil.WaitForLeader(t, s.RPC) 1285 1286 c := TestClient(t, func(c *config.Config) { 1287 c.Servers = []string{s.GetConfig().RPCAddr.String()} 1288 }) 1289 defer c.Shutdown() 1290 1291 // Force an allocation onto the node 1292 expectedBase := "Hello from the other side" 1293 repeat := 10 1294 1295 a := mock.Alloc() 1296 a.Job.Type = structs.JobTypeBatch 1297 a.NodeID = c.NodeID() 1298 a.Job.TaskGroups[0].Count = 1 1299 a.Job.TaskGroups[0].Tasks[0] = &structs.Task{ 1300 Name: "web", 1301 Driver: "mock_driver", 1302 Config: map[string]interface{}{ 1303 "run_for": "20s", 1304 "stdout_string": expectedBase, 1305 "stdout_repeat": repeat, 1306 "stdout_repeat_duration": 200 * time.Millisecond, 1307 }, 1308 LogConfig: structs.DefaultLogConfig(), 1309 Resources: &structs.Resources{ 1310 CPU: 500, 1311 MemoryMB: 256, 1312 }, 1313 } 1314 1315 // Wait for the client to connect 1316 testutil.WaitForResult(func() (bool, error) { 1317 node, err := s.State().NodeByID(nil, c.NodeID()) 1318 if err != nil { 1319 return false, err 1320 } 1321 if node == nil { 1322 return false, fmt.Errorf("unknown node") 1323 } 1324 1325 return node.Status == structs.NodeStatusReady, fmt.Errorf("bad node status") 1326 }, func(err error) { 1327 t.Fatal(err) 1328 }) 1329 1330 // Upsert the allocation 1331 state := s.State() 1332 require.Nil(state.UpsertJob(999, a.Job)) 1333 require.Nil(state.UpsertAllocs(1003, []*structs.Allocation{a})) 1334 1335 // Wait for the client to run the allocation 1336 testutil.WaitForResult(func() (bool, error) { 1337 alloc, err := state.AllocByID(nil, a.ID) 1338 if err != nil { 1339 return false, err 1340 } 1341 if alloc == nil { 1342 return false, fmt.Errorf("unknown alloc") 1343 } 1344 if alloc.ClientStatus != structs.AllocClientStatusRunning { 1345 return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus) 1346 } 1347 1348 return true, nil 1349 }, func(err error) { 1350 t.Fatalf("Alloc on node %q not running: %v", c.NodeID(), err) 1351 }) 1352 1353 // Make the request 1354 req := &cstructs.FsLogsRequest{ 1355 AllocID: a.ID, 1356 Task: a.Job.TaskGroups[0].Tasks[0].Name, 1357 LogType: "stdout", 1358 Origin: "start", 1359 PlainText: true, 1360 Follow: true, 1361 QueryOptions: structs.QueryOptions{Region: "global"}, 1362 } 1363 1364 // Get the handler 1365 handler, err := c.StreamingRpcHandler("FileSystem.Logs") 1366 require.Nil(err) 1367 1368 // Create a pipe 1369 p1, p2 := net.Pipe() 1370 defer p1.Close() 1371 defer p2.Close() 1372 1373 errCh := make(chan error) 1374 streamMsg := make(chan *cstructs.StreamErrWrapper) 1375 1376 // Start the handler 1377 go handler(p2) 1378 1379 // Start the decoder 1380 go func() { 1381 decoder := codec.NewDecoder(p1, structs.MsgpackHandle) 1382 for { 1383 var msg cstructs.StreamErrWrapper 1384 if err := decoder.Decode(&msg); err != nil { 1385 if err == io.EOF || strings.Contains(err.Error(), "closed") { 1386 return 1387 } 1388 errCh <- fmt.Errorf("error decoding: %v", err) 1389 } 1390 1391 streamMsg <- &msg 1392 } 1393 }() 1394 1395 // Send the request 1396 encoder := codec.NewEncoder(p1, structs.MsgpackHandle) 1397 require.Nil(encoder.Encode(req)) 1398 1399 timeout := time.After(20 * time.Second) 1400 expected := strings.Repeat(expectedBase, repeat+1) 1401 received := "" 1402 OUTER: 1403 for { 1404 select { 1405 case <-timeout: 1406 t.Fatal("timeout") 1407 case err := <-errCh: 1408 t.Fatal(err) 1409 case msg := <-streamMsg: 1410 if msg.Error != nil { 1411 t.Fatalf("Got error: %v", msg.Error.Error()) 1412 } 1413 1414 // Add the payload 1415 received += string(msg.Payload) 1416 if received == expected { 1417 break OUTER 1418 } 1419 } 1420 } 1421 } 1422 1423 func TestFS_findClosest(t *testing.T) { 1424 task := "foo" 1425 entries := []*cstructs.AllocFileInfo{ 1426 { 1427 Name: "foo.stdout.0", 1428 Size: 100, 1429 }, 1430 { 1431 Name: "foo.stdout.1", 1432 Size: 100, 1433 }, 1434 { 1435 Name: "foo.stdout.2", 1436 Size: 100, 1437 }, 1438 { 1439 Name: "foo.stdout.3", 1440 Size: 100, 1441 }, 1442 { 1443 Name: "foo.stderr.0", 1444 Size: 100, 1445 }, 1446 { 1447 Name: "foo.stderr.1", 1448 Size: 100, 1449 }, 1450 { 1451 Name: "foo.stderr.2", 1452 Size: 100, 1453 }, 1454 } 1455 1456 cases := []struct { 1457 Entries []*cstructs.AllocFileInfo 1458 DesiredIdx int64 1459 DesiredOffset int64 1460 Task string 1461 LogType string 1462 ExpectedFile string 1463 ExpectedIdx int64 1464 ExpectedOffset int64 1465 Error bool 1466 }{ 1467 // Test error cases 1468 { 1469 Entries: nil, 1470 DesiredIdx: 0, 1471 Task: task, 1472 LogType: "stdout", 1473 Error: true, 1474 }, 1475 { 1476 Entries: entries[0:3], 1477 DesiredIdx: 0, 1478 Task: task, 1479 LogType: "stderr", 1480 Error: true, 1481 }, 1482 1483 // Test beginning cases 1484 { 1485 Entries: entries, 1486 DesiredIdx: 0, 1487 Task: task, 1488 LogType: "stdout", 1489 ExpectedFile: entries[0].Name, 1490 ExpectedIdx: 0, 1491 }, 1492 { 1493 // Desired offset should be ignored at edges 1494 Entries: entries, 1495 DesiredIdx: 0, 1496 DesiredOffset: -100, 1497 Task: task, 1498 LogType: "stdout", 1499 ExpectedFile: entries[0].Name, 1500 ExpectedIdx: 0, 1501 ExpectedOffset: 0, 1502 }, 1503 { 1504 // Desired offset should be ignored at edges 1505 Entries: entries, 1506 DesiredIdx: 1, 1507 DesiredOffset: -1000, 1508 Task: task, 1509 LogType: "stdout", 1510 ExpectedFile: entries[0].Name, 1511 ExpectedIdx: 0, 1512 ExpectedOffset: 0, 1513 }, 1514 { 1515 Entries: entries, 1516 DesiredIdx: 0, 1517 Task: task, 1518 LogType: "stderr", 1519 ExpectedFile: entries[4].Name, 1520 ExpectedIdx: 0, 1521 }, 1522 { 1523 Entries: entries, 1524 DesiredIdx: 0, 1525 Task: task, 1526 LogType: "stdout", 1527 ExpectedFile: entries[0].Name, 1528 ExpectedIdx: 0, 1529 }, 1530 1531 // Test middle cases 1532 { 1533 Entries: entries, 1534 DesiredIdx: 1, 1535 Task: task, 1536 LogType: "stdout", 1537 ExpectedFile: entries[1].Name, 1538 ExpectedIdx: 1, 1539 }, 1540 { 1541 Entries: entries, 1542 DesiredIdx: 1, 1543 DesiredOffset: 10, 1544 Task: task, 1545 LogType: "stdout", 1546 ExpectedFile: entries[1].Name, 1547 ExpectedIdx: 1, 1548 ExpectedOffset: 10, 1549 }, 1550 { 1551 Entries: entries, 1552 DesiredIdx: 1, 1553 DesiredOffset: 110, 1554 Task: task, 1555 LogType: "stdout", 1556 ExpectedFile: entries[2].Name, 1557 ExpectedIdx: 2, 1558 ExpectedOffset: 10, 1559 }, 1560 { 1561 Entries: entries, 1562 DesiredIdx: 1, 1563 Task: task, 1564 LogType: "stderr", 1565 ExpectedFile: entries[5].Name, 1566 ExpectedIdx: 1, 1567 }, 1568 // Test end cases 1569 { 1570 Entries: entries, 1571 DesiredIdx: math.MaxInt64, 1572 Task: task, 1573 LogType: "stdout", 1574 ExpectedFile: entries[3].Name, 1575 ExpectedIdx: 3, 1576 }, 1577 { 1578 Entries: entries, 1579 DesiredIdx: math.MaxInt64, 1580 DesiredOffset: math.MaxInt64, 1581 Task: task, 1582 LogType: "stdout", 1583 ExpectedFile: entries[3].Name, 1584 ExpectedIdx: 3, 1585 ExpectedOffset: 100, 1586 }, 1587 { 1588 Entries: entries, 1589 DesiredIdx: math.MaxInt64, 1590 DesiredOffset: -10, 1591 Task: task, 1592 LogType: "stdout", 1593 ExpectedFile: entries[3].Name, 1594 ExpectedIdx: 3, 1595 ExpectedOffset: 90, 1596 }, 1597 { 1598 Entries: entries, 1599 DesiredIdx: math.MaxInt64, 1600 Task: task, 1601 LogType: "stderr", 1602 ExpectedFile: entries[6].Name, 1603 ExpectedIdx: 2, 1604 }, 1605 } 1606 1607 for i, c := range cases { 1608 entry, idx, offset, err := findClosest(c.Entries, c.DesiredIdx, c.DesiredOffset, c.Task, c.LogType) 1609 if err != nil { 1610 if !c.Error { 1611 t.Fatalf("case %d: Unexpected error: %v", i, err) 1612 } 1613 continue 1614 } 1615 1616 if entry.Name != c.ExpectedFile { 1617 t.Fatalf("case %d: Got file %q; want %q", i, entry.Name, c.ExpectedFile) 1618 } 1619 if idx != c.ExpectedIdx { 1620 t.Fatalf("case %d: Got index %d; want %d", i, idx, c.ExpectedIdx) 1621 } 1622 if offset != c.ExpectedOffset { 1623 t.Fatalf("case %d: Got offset %d; want %d", i, offset, c.ExpectedOffset) 1624 } 1625 } 1626 } 1627 1628 func TestFS_streamFile_NoFile(t *testing.T) { 1629 t.Parallel() 1630 require := require.New(t) 1631 c := TestClient(t, nil) 1632 defer c.Shutdown() 1633 1634 ad := tempAllocDir(t) 1635 defer os.RemoveAll(ad.AllocDir) 1636 1637 frames := make(chan *sframer.StreamFrame, 32) 1638 framer := sframer.NewStreamFramer(frames, streamHeartbeatRate, streamBatchWindow, streamFrameSize) 1639 framer.Run() 1640 defer framer.Destroy() 1641 1642 err := c.endpoints.FileSystem.streamFile( 1643 context.Background(), 0, "foo", 0, ad, framer, nil) 1644 require.NotNil(err) 1645 require.Contains(err.Error(), "no such file") 1646 } 1647 1648 func TestFS_streamFile_Modify(t *testing.T) { 1649 t.Parallel() 1650 1651 c := TestClient(t, nil) 1652 defer c.Shutdown() 1653 1654 // Get a temp alloc dir 1655 ad := tempAllocDir(t) 1656 defer os.RemoveAll(ad.AllocDir) 1657 1658 // Create a file in the temp dir 1659 streamFile := "stream_file" 1660 f, err := os.Create(filepath.Join(ad.AllocDir, streamFile)) 1661 if err != nil { 1662 t.Fatalf("Failed to create file: %v", err) 1663 } 1664 defer f.Close() 1665 1666 data := []byte("helloworld") 1667 1668 // Start the reader 1669 resultCh := make(chan struct{}) 1670 frames := make(chan *sframer.StreamFrame, 4) 1671 go func() { 1672 var collected []byte 1673 for { 1674 frame := <-frames 1675 if frame.IsHeartbeat() { 1676 continue 1677 } 1678 1679 collected = append(collected, frame.Data...) 1680 if reflect.DeepEqual(data, collected) { 1681 resultCh <- struct{}{} 1682 return 1683 } 1684 } 1685 }() 1686 1687 // Write a few bytes 1688 if _, err := f.Write(data[:3]); err != nil { 1689 t.Fatalf("write failed: %v", err) 1690 } 1691 1692 framer := sframer.NewStreamFramer(frames, streamHeartbeatRate, streamBatchWindow, streamFrameSize) 1693 framer.Run() 1694 defer framer.Destroy() 1695 1696 // Start streaming 1697 go func() { 1698 if err := c.endpoints.FileSystem.streamFile( 1699 context.Background(), 0, streamFile, 0, ad, framer, nil); err != nil { 1700 t.Fatalf("stream() failed: %v", err) 1701 } 1702 }() 1703 1704 // Sleep a little before writing more. This lets us check if the watch 1705 // is working. 1706 time.Sleep(1 * time.Duration(testutil.TestMultiplier()) * time.Second) 1707 if _, err := f.Write(data[3:]); err != nil { 1708 t.Fatalf("write failed: %v", err) 1709 } 1710 1711 select { 1712 case <-resultCh: 1713 case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * streamBatchWindow): 1714 t.Fatalf("failed to send new data") 1715 } 1716 } 1717 1718 func TestFS_streamFile_Truncate(t *testing.T) { 1719 t.Parallel() 1720 c := TestClient(t, nil) 1721 defer c.Shutdown() 1722 1723 // Get a temp alloc dir 1724 ad := tempAllocDir(t) 1725 defer os.RemoveAll(ad.AllocDir) 1726 1727 // Create a file in the temp dir 1728 data := []byte("helloworld") 1729 streamFile := "stream_file" 1730 streamFilePath := filepath.Join(ad.AllocDir, streamFile) 1731 f, err := os.Create(streamFilePath) 1732 if err != nil { 1733 t.Fatalf("Failed to create file: %v", err) 1734 } 1735 defer f.Close() 1736 1737 // Start the reader 1738 truncateCh := make(chan struct{}) 1739 dataPostTruncCh := make(chan struct{}) 1740 frames := make(chan *sframer.StreamFrame, 4) 1741 go func() { 1742 var collected []byte 1743 for { 1744 frame := <-frames 1745 if frame.IsHeartbeat() { 1746 continue 1747 } 1748 1749 if frame.FileEvent == truncateEvent { 1750 close(truncateCh) 1751 } 1752 1753 collected = append(collected, frame.Data...) 1754 if reflect.DeepEqual(data, collected) { 1755 close(dataPostTruncCh) 1756 return 1757 } 1758 } 1759 }() 1760 1761 // Write a few bytes 1762 if _, err := f.Write(data[:3]); err != nil { 1763 t.Fatalf("write failed: %v", err) 1764 } 1765 1766 framer := sframer.NewStreamFramer(frames, streamHeartbeatRate, streamBatchWindow, streamFrameSize) 1767 framer.Run() 1768 defer framer.Destroy() 1769 1770 // Start streaming 1771 go func() { 1772 if err := c.endpoints.FileSystem.streamFile( 1773 context.Background(), 0, streamFile, 0, ad, framer, nil); err != nil { 1774 t.Fatalf("stream() failed: %v", err) 1775 } 1776 }() 1777 1778 // Sleep a little before truncating. This lets us check if the watch 1779 // is working. 1780 time.Sleep(1 * time.Duration(testutil.TestMultiplier()) * time.Second) 1781 if err := f.Truncate(0); err != nil { 1782 t.Fatalf("truncate failed: %v", err) 1783 } 1784 if err := f.Sync(); err != nil { 1785 t.Fatalf("sync failed: %v", err) 1786 } 1787 if err := f.Close(); err != nil { 1788 t.Fatalf("failed to close file: %v", err) 1789 } 1790 1791 f2, err := os.OpenFile(streamFilePath, os.O_RDWR, 0) 1792 if err != nil { 1793 t.Fatalf("failed to reopen file: %v", err) 1794 } 1795 defer f2.Close() 1796 if _, err := f2.Write(data[3:5]); err != nil { 1797 t.Fatalf("write failed: %v", err) 1798 } 1799 1800 select { 1801 case <-truncateCh: 1802 case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * streamBatchWindow): 1803 t.Fatalf("did not receive truncate") 1804 } 1805 1806 // Sleep a little before writing more. This lets us check if the watch 1807 // is working. 1808 time.Sleep(1 * time.Duration(testutil.TestMultiplier()) * time.Second) 1809 if _, err := f2.Write(data[5:]); err != nil { 1810 t.Fatalf("write failed: %v", err) 1811 } 1812 1813 select { 1814 case <-dataPostTruncCh: 1815 case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * streamBatchWindow): 1816 t.Fatalf("did not receive post truncate data") 1817 } 1818 } 1819 1820 func TestFS_streamImpl_Delete(t *testing.T) { 1821 t.Parallel() 1822 1823 c := TestClient(t, nil) 1824 defer c.Shutdown() 1825 1826 // Get a temp alloc dir 1827 ad := tempAllocDir(t) 1828 defer os.RemoveAll(ad.AllocDir) 1829 1830 // Create a file in the temp dir 1831 data := []byte("helloworld") 1832 streamFile := "stream_file" 1833 streamFilePath := filepath.Join(ad.AllocDir, streamFile) 1834 f, err := os.Create(streamFilePath) 1835 if err != nil { 1836 t.Fatalf("Failed to create file: %v", err) 1837 } 1838 defer f.Close() 1839 1840 // Start the reader 1841 deleteCh := make(chan struct{}) 1842 frames := make(chan *sframer.StreamFrame, 4) 1843 go func() { 1844 for { 1845 frame := <-frames 1846 if frame.IsHeartbeat() { 1847 continue 1848 } 1849 1850 if frame.FileEvent == deleteEvent { 1851 close(deleteCh) 1852 return 1853 } 1854 } 1855 }() 1856 1857 // Write a few bytes 1858 if _, err := f.Write(data[:3]); err != nil { 1859 t.Fatalf("write failed: %v", err) 1860 } 1861 1862 framer := sframer.NewStreamFramer(frames, streamHeartbeatRate, streamBatchWindow, streamFrameSize) 1863 framer.Run() 1864 defer framer.Destroy() 1865 1866 // Start streaming 1867 go func() { 1868 if err := c.endpoints.FileSystem.streamFile( 1869 context.Background(), 0, streamFile, 0, ad, framer, nil); err != nil { 1870 t.Fatalf("stream() failed: %v", err) 1871 } 1872 }() 1873 1874 // Sleep a little before deleting. This lets us check if the watch 1875 // is working. 1876 time.Sleep(1 * time.Duration(testutil.TestMultiplier()) * time.Second) 1877 if err := os.Remove(streamFilePath); err != nil { 1878 t.Fatalf("delete failed: %v", err) 1879 } 1880 1881 select { 1882 case <-deleteCh: 1883 case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * streamBatchWindow): 1884 t.Fatalf("did not receive delete") 1885 } 1886 } 1887 1888 func TestFS_logsImpl_NoFollow(t *testing.T) { 1889 t.Parallel() 1890 1891 c := TestClient(t, nil) 1892 defer c.Shutdown() 1893 1894 // Get a temp alloc dir and create the log dir 1895 ad := tempAllocDir(t) 1896 defer os.RemoveAll(ad.AllocDir) 1897 1898 logDir := filepath.Join(ad.SharedDir, allocdir.LogDirName) 1899 if err := os.MkdirAll(logDir, 0777); err != nil { 1900 t.Fatalf("Failed to make log dir: %v", err) 1901 } 1902 1903 // Create a series of log files in the temp dir 1904 task := "foo" 1905 logType := "stdout" 1906 expected := []byte("012") 1907 for i := 0; i < 3; i++ { 1908 logFile := fmt.Sprintf("%s.%s.%d", task, logType, i) 1909 logFilePath := filepath.Join(logDir, logFile) 1910 err := ioutil.WriteFile(logFilePath, expected[i:i+1], 777) 1911 if err != nil { 1912 t.Fatalf("Failed to create file: %v", err) 1913 } 1914 } 1915 1916 // Start the reader 1917 resultCh := make(chan struct{}) 1918 frames := make(chan *sframer.StreamFrame, 4) 1919 var received []byte 1920 go func() { 1921 for { 1922 frame, ok := <-frames 1923 if !ok { 1924 return 1925 } 1926 1927 if frame.IsHeartbeat() { 1928 continue 1929 } 1930 1931 received = append(received, frame.Data...) 1932 if reflect.DeepEqual(received, expected) { 1933 close(resultCh) 1934 return 1935 } 1936 } 1937 }() 1938 1939 // Start streaming logs 1940 go func() { 1941 if err := c.endpoints.FileSystem.logsImpl( 1942 context.Background(), false, false, 0, 1943 OriginStart, task, logType, ad, frames); err != nil { 1944 t.Fatalf("logs() failed: %v", err) 1945 } 1946 }() 1947 1948 select { 1949 case <-resultCh: 1950 case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * streamBatchWindow): 1951 t.Fatalf("did not receive data: got %q", string(received)) 1952 } 1953 } 1954 1955 func TestFS_logsImpl_Follow(t *testing.T) { 1956 t.Parallel() 1957 1958 c := TestClient(t, nil) 1959 defer c.Shutdown() 1960 1961 // Get a temp alloc dir and create the log dir 1962 ad := tempAllocDir(t) 1963 defer os.RemoveAll(ad.AllocDir) 1964 1965 logDir := filepath.Join(ad.SharedDir, allocdir.LogDirName) 1966 if err := os.MkdirAll(logDir, 0777); err != nil { 1967 t.Fatalf("Failed to make log dir: %v", err) 1968 } 1969 1970 // Create a series of log files in the temp dir 1971 task := "foo" 1972 logType := "stdout" 1973 expected := []byte("012345") 1974 initialWrites := 3 1975 1976 writeToFile := func(index int, data []byte) { 1977 logFile := fmt.Sprintf("%s.%s.%d", task, logType, index) 1978 logFilePath := filepath.Join(logDir, logFile) 1979 err := ioutil.WriteFile(logFilePath, data, 777) 1980 if err != nil { 1981 t.Fatalf("Failed to create file: %v", err) 1982 } 1983 } 1984 for i := 0; i < initialWrites; i++ { 1985 writeToFile(i, expected[i:i+1]) 1986 } 1987 1988 // Start the reader 1989 firstResultCh := make(chan struct{}) 1990 fullResultCh := make(chan struct{}) 1991 frames := make(chan *sframer.StreamFrame, 4) 1992 var received []byte 1993 go func() { 1994 for { 1995 frame, ok := <-frames 1996 if !ok { 1997 return 1998 } 1999 2000 if frame.IsHeartbeat() { 2001 continue 2002 } 2003 2004 received = append(received, frame.Data...) 2005 if reflect.DeepEqual(received, expected[:initialWrites]) { 2006 close(firstResultCh) 2007 } else if reflect.DeepEqual(received, expected) { 2008 close(fullResultCh) 2009 return 2010 } 2011 } 2012 }() 2013 2014 // Start streaming logs 2015 go c.endpoints.FileSystem.logsImpl( 2016 context.Background(), true, false, 0, 2017 OriginStart, task, logType, ad, frames) 2018 2019 select { 2020 case <-firstResultCh: 2021 case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * streamBatchWindow): 2022 t.Fatalf("did not receive data: got %q", string(received)) 2023 } 2024 2025 // We got the first chunk of data, write out the rest to the next file 2026 // at an index much ahead to check that it is following and detecting 2027 // skips 2028 skipTo := initialWrites + 10 2029 writeToFile(skipTo, expected[initialWrites:]) 2030 2031 select { 2032 case <-fullResultCh: 2033 case <-time.After(10 * time.Duration(testutil.TestMultiplier()) * streamBatchWindow): 2034 t.Fatalf("did not receive data: got %q", string(received)) 2035 } 2036 }