github.com/observiq/carbon@v0.9.11-0.20200820160507-1b872e368a5e/operator/builtin/input/file/file_test.go (about) 1 package file 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "math/rand" 8 "os" 9 "path/filepath" 10 "runtime" 11 "strconv" 12 "testing" 13 "time" 14 15 "github.com/observiq/carbon/entry" 16 "github.com/observiq/carbon/operator" 17 "github.com/observiq/carbon/testutil" 18 "github.com/stretchr/testify/mock" 19 "github.com/stretchr/testify/require" 20 ) 21 22 func newTestFileSource(t *testing.T) (*InputOperator, chan *entry.Entry) { 23 mockOutput := testutil.NewMockOperator("output") 24 receivedEntries := make(chan *entry.Entry, 1000) 25 mockOutput.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { 26 receivedEntries <- args.Get(1).(*entry.Entry) 27 }) 28 29 cfg := NewInputConfig("testfile") 30 cfg.PollInterval = operator.Duration{Duration: 50 * time.Millisecond} 31 cfg.StartAt = "beginning" 32 cfg.Include = []string{"should-be-overwritten"} 33 34 pg, err := cfg.Build(testutil.NewBuildContext(t)) 35 if err != nil { 36 t.Fatalf("Error building operator: %s", err) 37 } 38 source := pg.(*InputOperator) 39 source.OutputOperators = []operator.Operator{mockOutput} 40 41 return source, receivedEntries 42 } 43 44 func TestFileSource_Build(t *testing.T) { 45 t.Parallel() 46 mockOutput := testutil.NewMockOperator("mock") 47 48 basicConfig := func() *InputConfig { 49 cfg := NewInputConfig("testfile") 50 cfg.OutputIDs = []string{"mock"} 51 cfg.Include = []string{"/var/log/testpath.*"} 52 cfg.Exclude = []string{"/var/log/testpath.ex*"} 53 cfg.PollInterval = operator.Duration{Duration: 10 * time.Millisecond} 54 return cfg 55 } 56 57 cases := []struct { 58 name string 59 modifyBaseConfig func(*InputConfig) 60 errorRequirement require.ErrorAssertionFunc 61 validate func(*testing.T, *InputOperator) 62 }{ 63 { 64 "Basic", 65 func(f *InputConfig) { return }, 66 require.NoError, 67 func(t *testing.T, f *InputOperator) { 68 require.Equal(t, f.OutputOperators[0], mockOutput) 69 require.Equal(t, f.Include, []string{"/var/log/testpath.*"}) 70 require.Equal(t, f.FilePathField, entry.NewNilField()) 71 require.Equal(t, f.FileNameField, entry.NewLabelField("file_name")) 72 require.Equal(t, f.PollInterval, 10*time.Millisecond) 73 }, 74 }, 75 { 76 "BadIncludeGlob", 77 func(f *InputConfig) { 78 f.Include = []string{"["} 79 }, 80 require.Error, 81 nil, 82 }, 83 { 84 "BadExcludeGlob", 85 func(f *InputConfig) { 86 f.Include = []string{"["} 87 }, 88 require.Error, 89 nil, 90 }, 91 { 92 "MultilineConfiguredStartAndEndPatterns", 93 func(f *InputConfig) { 94 f.Multiline = &MultilineConfig{ 95 LineEndPattern: "Exists", 96 LineStartPattern: "Exists", 97 } 98 }, 99 require.Error, 100 nil, 101 }, 102 { 103 "MultilineConfiguredStartPattern", 104 func(f *InputConfig) { 105 f.Multiline = &MultilineConfig{ 106 LineStartPattern: "START.*", 107 } 108 }, 109 require.NoError, 110 func(t *testing.T, f *InputOperator) {}, 111 }, 112 { 113 "MultilineConfiguredEndPattern", 114 func(f *InputConfig) { 115 f.Multiline = &MultilineConfig{ 116 LineEndPattern: "END.*", 117 } 118 }, 119 require.NoError, 120 func(t *testing.T, f *InputOperator) {}, 121 }, 122 } 123 124 for _, tc := range cases { 125 t.Run(tc.name, func(t *testing.T) { 126 cfg := basicConfig() 127 tc.modifyBaseConfig(cfg) 128 129 plg, err := cfg.Build(testutil.NewBuildContext(t)) 130 tc.errorRequirement(t, err) 131 if err != nil { 132 return 133 } 134 135 err = plg.SetOutputs([]operator.Operator{mockOutput}) 136 require.NoError(t, err) 137 138 fileInput := plg.(*InputOperator) 139 tc.validate(t, fileInput) 140 }) 141 } 142 } 143 144 func TestFileSource_CleanStop(t *testing.T) { 145 t.Parallel() 146 t.Skip(`Skipping due to goroutine leak in opencensus. 147 See this issue for details: https://github.com/census-instrumentation/opencensus-go/issues/1191#issuecomment-610440163`) 148 // defer goleak.VerifyNone(t) 149 150 source, _ := newTestFileSource(t) 151 152 tempDir := testutil.NewTempDir(t) 153 154 tempFile, err := ioutil.TempFile(tempDir, "") 155 require.NoError(t, err) 156 157 source.Include = []string{tempFile.Name()} 158 159 err = source.Start() 160 require.NoError(t, err) 161 source.Stop() 162 } 163 164 func TestFileSource_AddFields(t *testing.T) { 165 t.Parallel() 166 source, logReceived := newTestFileSource(t) 167 tempDir := testutil.NewTempDir(t) 168 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 169 source.FilePathField = entry.NewLabelField("path") 170 source.FileNameField = entry.NewLabelField("file_name") 171 172 // Create a file, then start 173 temp, err := ioutil.TempFile(tempDir, "") 174 require.NoError(t, err) 175 defer temp.Close() 176 177 _, err = temp.WriteString("testlog\n") 178 require.NoError(t, err) 179 180 err = source.Start() 181 require.NoError(t, err) 182 defer source.Stop() 183 184 select { 185 case e := <-logReceived: 186 require.Equal(t, filepath.Base(temp.Name()), e.Labels["file_name"]) 187 require.Equal(t, temp.Name(), e.Labels["path"]) 188 case <-time.After(time.Second): 189 require.FailNow(t, "Timed out waiting for message") 190 } 191 } 192 193 func TestFileSource_ReadExistingLogs(t *testing.T) { 194 t.Parallel() 195 source, logReceived := newTestFileSource(t) 196 tempDir := testutil.NewTempDir(t) 197 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 198 199 // Create a file, then start 200 temp, err := ioutil.TempFile(tempDir, "") 201 require.NoError(t, err) 202 defer temp.Close() 203 204 _, err = temp.WriteString("testlog\n") 205 require.NoError(t, err) 206 207 err = source.Start() 208 require.NoError(t, err) 209 defer source.Stop() 210 211 waitForMessage(t, logReceived, "testlog") 212 } 213 214 func TestFileSource_IncludeFile(t *testing.T) { 215 t.Parallel() 216 source, logReceived := newTestFileSource(t) 217 tempDir := testutil.NewTempDir(t) 218 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 219 source.FileNameField = entry.NewLabelField("file_name") 220 source.FilePathField = entry.NewLabelField("file_path") 221 222 // Create a file, then start 223 temp, err := ioutil.TempFile(tempDir, "") 224 require.NoError(t, err) 225 defer temp.Close() 226 227 _, err = temp.WriteString("testlog\n") 228 require.NoError(t, err) 229 230 err = source.Start() 231 require.NoError(t, err) 232 defer source.Stop() 233 234 select { 235 case e := <-logReceived: 236 require.Equal(t, e.Labels["file_name"], filepath.Base(temp.Name())) 237 require.Equal(t, e.Labels["file_path"], temp.Name()) 238 case <-time.After(time.Second): 239 require.FailNow(t, "Timed out waiting for message") 240 } 241 } 242 243 func TestFileSource_ReadNewLogs(t *testing.T) { 244 t.Parallel() 245 source, logReceived := newTestFileSource(t) 246 tempDir := testutil.NewTempDir(t) 247 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 248 249 // Start first, then create a new file 250 err := source.Start() 251 require.NoError(t, err) 252 defer source.Stop() 253 254 temp, err := ioutil.TempFile(tempDir, "") 255 require.NoError(t, err) 256 defer temp.Close() 257 258 _, err = temp.WriteString("testlog\n") 259 require.NoError(t, err) 260 261 waitForMessage(t, logReceived, "testlog") 262 } 263 264 func TestFileSource_ReadExistingAndNewLogs(t *testing.T) { 265 t.Parallel() 266 source, logReceived := newTestFileSource(t) 267 tempDir := testutil.NewTempDir(t) 268 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 269 270 temp, err := ioutil.TempFile(tempDir, "") 271 require.NoError(t, err) 272 defer temp.Close() 273 274 _, err = temp.WriteString("testlog1\n") 275 require.NoError(t, err) 276 277 err = source.Start() 278 require.NoError(t, err) 279 defer source.Stop() 280 281 _, err = temp.WriteString("testlog2\n") 282 require.NoError(t, err) 283 284 waitForMessage(t, logReceived, "testlog1") 285 waitForMessage(t, logReceived, "testlog2") 286 } 287 288 func TestFileSource_StartAtEnd(t *testing.T) { 289 t.Parallel() 290 source, logReceived := newTestFileSource(t) 291 tempDir := testutil.NewTempDir(t) 292 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 293 source.startAtBeginning = false 294 295 temp, err := ioutil.TempFile(tempDir, "") 296 require.NoError(t, err) 297 defer temp.Close() 298 299 _, err = temp.WriteString("testlog1\n") 300 require.NoError(t, err) 301 302 err = source.Start() 303 require.NoError(t, err) 304 defer source.Stop() 305 306 // Wait until file has been read the first time 307 time.Sleep(200 * time.Millisecond) 308 309 _, err = temp.WriteString("testlog2\n") 310 require.NoError(t, err) 311 temp.Close() 312 313 waitForMessage(t, logReceived, "testlog2") 314 } 315 316 func TestFileSource_StartAtEndNewFile(t *testing.T) { 317 t.Parallel() 318 source, logReceived := newTestFileSource(t) 319 tempDir := testutil.NewTempDir(t) 320 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 321 source.startAtBeginning = false 322 323 err := source.Start() 324 require.NoError(t, err) 325 defer source.Stop() 326 327 // Wait for the first check to complete 328 time.Sleep(200 * time.Millisecond) 329 330 temp, err := ioutil.TempFile(tempDir, "") 331 require.NoError(t, err) 332 defer temp.Close() 333 334 _, err = temp.WriteString("testlog1\ntestlog2\n") 335 require.NoError(t, err) 336 337 waitForMessage(t, logReceived, "testlog1") 338 waitForMessage(t, logReceived, "testlog2") 339 } 340 341 func TestFileSource_MultiFileSimple(t *testing.T) { 342 t.Parallel() 343 source, logReceived := newTestFileSource(t) 344 tempDir := testutil.NewTempDir(t) 345 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 346 347 temp1, err := ioutil.TempFile(tempDir, "") 348 require.NoError(t, err) 349 temp2, err := ioutil.TempFile(tempDir, "") 350 require.NoError(t, err) 351 352 _, err = temp1.WriteString("testlog1\n") 353 require.NoError(t, err) 354 _, err = temp2.WriteString("testlog2\n") 355 require.NoError(t, err) 356 357 err = source.Start() 358 require.NoError(t, err) 359 defer source.Stop() 360 361 waitForMessages(t, logReceived, []string{"testlog1", "testlog2"}) 362 } 363 364 func TestFileSource_MoveFile(t *testing.T) { 365 if runtime.GOOS == "windows" { 366 t.Skip("Moving files while open is unsupported on Windows") 367 } 368 t.Parallel() 369 source, logReceived := newTestFileSource(t) 370 tempDir := testutil.NewTempDir(t) 371 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 372 373 temp1, err := ioutil.TempFile(tempDir, "") 374 require.NoError(t, err) 375 376 _, err = temp1.WriteString("testlog1\n") 377 require.NoError(t, err) 378 temp1.Close() 379 380 err = source.Start() 381 require.NoError(t, err) 382 defer source.Stop() 383 384 waitForMessage(t, logReceived, "testlog1") 385 time.Sleep(200 * time.Millisecond) 386 387 i := 0 388 for { 389 err = os.Rename(temp1.Name(), fmt.Sprintf("%s.2", temp1.Name())) 390 if err != nil { 391 if i < 3 { 392 t.Error(err) 393 i++ 394 time.Sleep(10 * time.Millisecond) 395 continue 396 } else { 397 require.NoError(t, err) 398 } 399 } 400 401 break 402 } 403 404 expectNoMessages(t, logReceived) 405 } 406 407 func TestFileSource_TruncateThenWrite(t *testing.T) { 408 t.Parallel() 409 source, logReceived := newTestFileSource(t) 410 tempDir := testutil.NewTempDir(t) 411 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 412 413 temp1, err := ioutil.TempFile(tempDir, "") 414 require.NoError(t, err) 415 416 _, err = temp1.WriteString("testlog1\n") 417 require.NoError(t, err) 418 _, err = temp1.WriteString("testlog2\n") 419 require.NoError(t, err) 420 421 err = source.Start() 422 require.NoError(t, err) 423 defer source.Stop() 424 425 waitForMessage(t, logReceived, "testlog1") 426 waitForMessage(t, logReceived, "testlog2") 427 428 err = temp1.Truncate(0) 429 require.NoError(t, err) 430 temp1.Seek(0, 0) 431 432 _, err = temp1.WriteString("testlog3\n") 433 require.NoError(t, err) 434 435 waitForMessage(t, logReceived, "testlog3") 436 expectNoMessages(t, logReceived) 437 } 438 439 func TestFileSource_CopyTruncateWriteBoth(t *testing.T) { 440 t.Parallel() 441 source, logReceived := newTestFileSource(t) 442 tempDir := testutil.NewTempDir(t) 443 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 444 445 temp1, err := ioutil.TempFile(tempDir, "") 446 require.NoError(t, err) 447 defer temp1.Close() 448 449 _, err = temp1.WriteString("testlog1\n") 450 require.NoError(t, err) 451 _, err = temp1.WriteString("testlog2\n") 452 require.NoError(t, err) 453 454 err = source.Start() 455 require.NoError(t, err) 456 defer source.Stop() 457 458 waitForMessage(t, logReceived, "testlog1") 459 waitForMessage(t, logReceived, "testlog2") 460 461 time.Sleep(50 * time.Millisecond) 462 463 temp2, err := ioutil.TempFile(tempDir, "") 464 require.NoError(t, err) 465 defer temp2.Close() 466 467 _, err = io.Copy(temp1, temp2) 468 require.NoError(t, err) 469 470 // Truncate original file 471 err = temp1.Truncate(0) 472 temp1.Seek(0, 0) 473 require.NoError(t, err) 474 475 // Write to original and new file 476 _, err = temp1.WriteString("testlog3\n") 477 require.NoError(t, err) 478 479 waitForMessage(t, logReceived, "testlog3") 480 481 _, err = temp2.WriteString("testlog4\n") 482 require.NoError(t, err) 483 484 waitForMessage(t, logReceived, "testlog4") 485 } 486 487 func TestFileSource_OffsetsAfterRestart(t *testing.T) { 488 t.Parallel() 489 source, logReceived := newTestFileSource(t) 490 tempDir := testutil.NewTempDir(t) 491 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 492 493 temp1, err := ioutil.TempFile(tempDir, "") 494 require.NoError(t, err) 495 496 // Write to a file 497 _, err = temp1.WriteString("testlog1\n") 498 require.NoError(t, err) 499 500 // Start the source 501 err = source.Start() 502 require.NoError(t, err) 503 defer source.Stop() 504 505 waitForMessage(t, logReceived, "testlog1") 506 507 // Restart the source 508 err = source.Stop() 509 require.NoError(t, err) 510 err = source.Start() 511 require.NoError(t, err) 512 513 // Write a new log 514 _, err = temp1.WriteString("testlog2\n") 515 require.NoError(t, err) 516 517 waitForMessage(t, logReceived, "testlog2") 518 } 519 520 func TestFileSource_OffsetsAfterRestart_BigFiles(t *testing.T) { 521 t.Parallel() 522 source, logReceived := newTestFileSource(t) 523 tempDir := testutil.NewTempDir(t) 524 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 525 526 log1 := stringWithLength(1000) 527 log2 := stringWithLength(1000) 528 529 temp1, err := ioutil.TempFile(tempDir, "") 530 require.NoError(t, err) 531 532 // Write to a file 533 _, err = temp1.WriteString(log1 + "\n") 534 require.NoError(t, err) 535 536 // Start the source 537 err = source.Start() 538 require.NoError(t, err) 539 540 waitForMessage(t, logReceived, log1) 541 542 // Restart the source 543 err = source.Stop() 544 require.NoError(t, err) 545 err = source.Start() 546 require.NoError(t, err) 547 defer source.Stop() 548 549 _, err = temp1.WriteString(log2 + "\n") 550 require.NoError(t, err) 551 552 waitForMessage(t, logReceived, log2) 553 } 554 555 func TestFileSource_OffsetsAfterRestart_BigFilesWrittenWhileOff(t *testing.T) { 556 t.Parallel() 557 source, logReceived := newTestFileSource(t) 558 tempDir := testutil.NewTempDir(t) 559 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 560 561 log1 := stringWithLength(1000) 562 log2 := stringWithLength(1000) 563 564 temp1, err := ioutil.TempFile(tempDir, "") 565 require.NoError(t, err) 566 567 // Write to a file 568 _, err = temp1.WriteString(log1 + "\n") 569 require.NoError(t, err) 570 571 // Start the source 572 err = source.Start() 573 require.NoError(t, err) 574 575 waitForMessage(t, logReceived, log1) 576 577 // Restart the source 578 err = source.Stop() 579 require.NoError(t, err) 580 581 _, err = temp1.WriteString(log2 + "\n") 582 require.NoError(t, err) 583 584 err = source.Start() 585 require.NoError(t, err) 586 defer source.Stop() 587 588 waitForMessage(t, logReceived, log2) 589 } 590 591 func TestFileSource_FileMovedWhileOff_BigFiles(t *testing.T) { 592 t.Parallel() 593 source, logReceived := newTestFileSource(t) 594 tempDir := testutil.NewTempDir(t) 595 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 596 597 log1 := stringWithLength(1000) 598 log2 := stringWithLength(1000) 599 600 temp1, err := ioutil.TempFile(tempDir, "") 601 require.NoError(t, err) 602 603 // Write to a file 604 _, err = temp1.WriteString(log1 + "\n") 605 require.NoError(t, err) 606 607 // Start the source 608 err = source.Start() 609 require.NoError(t, err) 610 611 waitForMessage(t, logReceived, log1) 612 613 // Stop the source, then rename and write a new log 614 err = source.Stop() 615 require.NoError(t, err) 616 617 _, err = temp1.WriteString(log2 + "\n") 618 require.NoError(t, err) 619 temp1.Close() 620 621 err = os.Rename(temp1.Name(), fmt.Sprintf("%s2", temp1.Name())) 622 require.NoError(t, err) 623 624 err = source.Start() 625 require.NoError(t, err) 626 defer source.Stop() 627 628 waitForMessage(t, logReceived, log2) 629 } 630 631 func TestFileSource_FileMovedWhileOff_SmallFiles(t *testing.T) { 632 t.Parallel() 633 source, logReceived := newTestFileSource(t) 634 tempDir := testutil.NewTempDir(t) 635 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 636 637 log1 := stringWithLength(10) 638 log2 := stringWithLength(10) 639 640 temp1, err := ioutil.TempFile(tempDir, "") 641 require.NoError(t, err) 642 643 // Write to a file 644 _, err = temp1.WriteString(log1 + "\n") 645 require.NoError(t, err) 646 647 // Start the source 648 err = source.Start() 649 require.NoError(t, err) 650 651 waitForMessage(t, logReceived, log1) 652 time.Sleep(50 * time.Millisecond) 653 654 // Restart the source 655 err = source.Stop() 656 require.NoError(t, err) 657 658 _, err = temp1.WriteString(log2 + "\n") 659 require.NoError(t, err) 660 temp1.Close() 661 662 err = os.Rename(temp1.Name(), fmt.Sprintf("%s2", temp1.Name())) 663 require.NoError(t, err) 664 665 err = source.Start() 666 require.NoError(t, err) 667 defer source.Stop() 668 669 waitForMessage(t, logReceived, log2) 670 } 671 672 func TestFileSource_ManyLogsDelivered(t *testing.T) { 673 t.Parallel() 674 source, logReceived := newTestFileSource(t) 675 tempDir := testutil.NewTempDir(t) 676 source.Include = []string{fmt.Sprintf("%s/*", tempDir)} 677 678 temp1, err := ioutil.TempFile(tempDir, "") 679 require.NoError(t, err) 680 681 count := 1000 682 expectedMessages := make([]string, 0, count) 683 for i := 0; i < count; i++ { 684 expectedMessages = append(expectedMessages, strconv.Itoa(i)) 685 } 686 687 // Start the source 688 err = source.Start() 689 require.NoError(t, err) 690 defer source.Stop() 691 692 // Write lots of logs 693 for _, message := range expectedMessages { 694 temp1.WriteString(message + "\n") 695 } 696 697 // Expect each of them to come through 698 for _, message := range expectedMessages { 699 waitForMessage(t, logReceived, message) 700 } 701 702 expectNoMessages(t, logReceived) 703 } 704 705 func stringWithLength(length int) string { 706 charset := "abcdefghijklmnopqrstuvwxyz" 707 b := make([]byte, length) 708 for i := range b { 709 b[i] = charset[rand.Intn(len(charset))] 710 } 711 return string(b) 712 } 713 714 func waitForMessage(t *testing.T, c chan *entry.Entry, expected string) { 715 select { 716 case e := <-c: 717 require.Equal(t, expected, e.Record.(string)) 718 case <-time.After(time.Second): 719 require.FailNow(t, "Timed out waiting for message") 720 } 721 } 722 723 func waitForMessages(t *testing.T, c chan *entry.Entry, expected []string) { 724 receivedMessages := make([]string, 0, 100) 725 LOOP: 726 for { 727 select { 728 case e := <-c: 729 receivedMessages = append(receivedMessages, e.Record.(string)) 730 if len(receivedMessages) == len(expected) { 731 break LOOP 732 } 733 case <-time.After(time.Second): 734 require.FailNow(t, "Timed out waiting for expected messages") 735 } 736 } 737 738 require.ElementsMatch(t, expected, receivedMessages) 739 } 740 741 func expectNoMessages(t *testing.T, c chan *entry.Entry) { 742 select { 743 case e := <-c: 744 require.FailNow(t, "Received unexpected message", "Message: %s", e.Record.(string)) 745 case <-time.After(200 * time.Millisecond): 746 } 747 } 748 749 func TestEncodings(t *testing.T) { 750 t.Parallel() 751 cases := []struct { 752 name string 753 contents []byte 754 encoding string 755 expected [][]byte 756 }{ 757 { 758 "Nop", 759 []byte{0xc5, '\n'}, 760 "", 761 [][]byte{{0xc5}}, 762 }, 763 { 764 "InvalidUTFReplacement", 765 []byte{0xc5, '\n'}, 766 "utf8", 767 [][]byte{{0xef, 0xbf, 0xbd}}, 768 }, 769 { 770 "ValidUTF8", 771 []byte("foo\n"), 772 "utf8", 773 [][]byte{[]byte("foo")}, 774 }, 775 { 776 "ChineseCharacter", 777 []byte{230, 138, 152, '\n'}, // 折\n 778 "utf8", 779 [][]byte{{230, 138, 152}}, 780 }, 781 { 782 "SmileyFaceUTF16", 783 []byte{216, 61, 222, 0, 0, 10}, // 😀\n 784 "utf-16be", 785 [][]byte{{240, 159, 152, 128}}, 786 }, 787 { 788 "SmileyFaceNewlineUTF16", 789 []byte{216, 61, 222, 0, 0, 10, 0, 102, 0, 111, 0, 111}, // 😀\nfoo 790 "utf-16be", 791 [][]byte{{240, 159, 152, 128}, {102, 111, 111}}, 792 }, 793 { 794 "SmileyFaceNewlineUTF16LE", 795 []byte{61, 216, 0, 222, 10, 0, 102, 0, 111, 0, 111, 0}, // 😀\nfoo 796 "utf-16le", 797 [][]byte{{240, 159, 152, 128}, {102, 111, 111}}, 798 }, 799 { 800 "ChineseCharacterBig5", 801 []byte{167, 233, 10}, // 折\n 802 "big5", 803 [][]byte{{230, 138, 152}}, 804 }, 805 } 806 807 for _, tc := range cases { 808 t.Run(tc.name, func(t *testing.T) { 809 tempDir := testutil.NewTempDir(t) 810 path := filepath.Join(tempDir, "in.log") 811 err := ioutil.WriteFile(path, tc.contents, 0777) 812 require.NoError(t, err) 813 814 source, receivedEntries := newTestFileSource(t) 815 source.Include = []string{path} 816 source.encoding, err = lookupEncoding(tc.encoding) 817 require.NoError(t, err) 818 source.SplitFunc, err = NewNewlineSplitFunc(source.encoding) 819 require.NoError(t, err) 820 require.NotNil(t, source.encoding) 821 822 err = source.Start() 823 require.NoError(t, err) 824 defer source.Stop() 825 826 for _, expected := range tc.expected { 827 select { 828 case entry := <-receivedEntries: 829 require.Equal(t, expected, []byte(entry.Record.(string))) 830 case <-time.After(time.Second): 831 require.FailNow(t, "Timed out waiting for entry to be read") 832 } 833 } 834 }) 835 } 836 } 837 838 type fileInputBenchmark struct { 839 name string 840 config *InputConfig 841 } 842 843 func BenchmarkFileInput(b *testing.B) { 844 cases := []fileInputBenchmark{ 845 { 846 "Default", 847 NewInputConfig("test_id"), 848 }, 849 { 850 "NoFileName", 851 func() *InputConfig { 852 cfg := NewInputConfig("test_id") 853 cfg.IncludeFileName = false 854 return cfg 855 }(), 856 }, 857 } 858 859 for _, tc := range cases { 860 b.Run(tc.name, func(b *testing.B) { 861 tempDir := testutil.NewTempDir(b) 862 path := filepath.Join(tempDir, "in.log") 863 864 cfg := tc.config 865 cfg.OutputIDs = []string{"fake"} 866 cfg.Include = []string{path} 867 cfg.StartAt = "beginning" 868 869 fileOperator, err := cfg.Build(testutil.NewBuildContext(b)) 870 require.NoError(b, err) 871 872 fakeOutput := testutil.NewFakeOutput(b) 873 err = fileOperator.SetOutputs([]operator.Operator{fakeOutput}) 874 require.NoError(b, err) 875 876 err = fileOperator.Start() 877 require.NoError(b, err) 878 879 file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) 880 require.NoError(b, err) 881 882 for i := 0; i < b.N; i++ { 883 file.WriteString("testlog\n") 884 } 885 886 b.ResetTimer() 887 for i := 0; i < b.N; i++ { 888 <-fakeOutput.Received 889 } 890 }) 891 } 892 }