github.com/observiq/carbon@v0.9.11-0.20200820160507-1b872e368a5e/operator/builtin/input/file/file_test.go (about)

     1  package file
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"io/ioutil"
     7  	"math/rand"
     8  	"os"
     9  	"path/filepath"
    10  	"runtime"
    11  	"strconv"
    12  	"testing"
    13  	"time"
    14  
    15  	"github.com/observiq/carbon/entry"
    16  	"github.com/observiq/carbon/operator"
    17  	"github.com/observiq/carbon/testutil"
    18  	"github.com/stretchr/testify/mock"
    19  	"github.com/stretchr/testify/require"
    20  )
    21  
    22  func newTestFileSource(t *testing.T) (*InputOperator, chan *entry.Entry) {
    23  	mockOutput := testutil.NewMockOperator("output")
    24  	receivedEntries := make(chan *entry.Entry, 1000)
    25  	mockOutput.On("Process", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) {
    26  		receivedEntries <- args.Get(1).(*entry.Entry)
    27  	})
    28  
    29  	cfg := NewInputConfig("testfile")
    30  	cfg.PollInterval = operator.Duration{Duration: 50 * time.Millisecond}
    31  	cfg.StartAt = "beginning"
    32  	cfg.Include = []string{"should-be-overwritten"}
    33  
    34  	pg, err := cfg.Build(testutil.NewBuildContext(t))
    35  	if err != nil {
    36  		t.Fatalf("Error building operator: %s", err)
    37  	}
    38  	source := pg.(*InputOperator)
    39  	source.OutputOperators = []operator.Operator{mockOutput}
    40  
    41  	return source, receivedEntries
    42  }
    43  
    44  func TestFileSource_Build(t *testing.T) {
    45  	t.Parallel()
    46  	mockOutput := testutil.NewMockOperator("mock")
    47  
    48  	basicConfig := func() *InputConfig {
    49  		cfg := NewInputConfig("testfile")
    50  		cfg.OutputIDs = []string{"mock"}
    51  		cfg.Include = []string{"/var/log/testpath.*"}
    52  		cfg.Exclude = []string{"/var/log/testpath.ex*"}
    53  		cfg.PollInterval = operator.Duration{Duration: 10 * time.Millisecond}
    54  		return cfg
    55  	}
    56  
    57  	cases := []struct {
    58  		name             string
    59  		modifyBaseConfig func(*InputConfig)
    60  		errorRequirement require.ErrorAssertionFunc
    61  		validate         func(*testing.T, *InputOperator)
    62  	}{
    63  		{
    64  			"Basic",
    65  			func(f *InputConfig) { return },
    66  			require.NoError,
    67  			func(t *testing.T, f *InputOperator) {
    68  				require.Equal(t, f.OutputOperators[0], mockOutput)
    69  				require.Equal(t, f.Include, []string{"/var/log/testpath.*"})
    70  				require.Equal(t, f.FilePathField, entry.NewNilField())
    71  				require.Equal(t, f.FileNameField, entry.NewLabelField("file_name"))
    72  				require.Equal(t, f.PollInterval, 10*time.Millisecond)
    73  			},
    74  		},
    75  		{
    76  			"BadIncludeGlob",
    77  			func(f *InputConfig) {
    78  				f.Include = []string{"["}
    79  			},
    80  			require.Error,
    81  			nil,
    82  		},
    83  		{
    84  			"BadExcludeGlob",
    85  			func(f *InputConfig) {
    86  				f.Include = []string{"["}
    87  			},
    88  			require.Error,
    89  			nil,
    90  		},
    91  		{
    92  			"MultilineConfiguredStartAndEndPatterns",
    93  			func(f *InputConfig) {
    94  				f.Multiline = &MultilineConfig{
    95  					LineEndPattern:   "Exists",
    96  					LineStartPattern: "Exists",
    97  				}
    98  			},
    99  			require.Error,
   100  			nil,
   101  		},
   102  		{
   103  			"MultilineConfiguredStartPattern",
   104  			func(f *InputConfig) {
   105  				f.Multiline = &MultilineConfig{
   106  					LineStartPattern: "START.*",
   107  				}
   108  			},
   109  			require.NoError,
   110  			func(t *testing.T, f *InputOperator) {},
   111  		},
   112  		{
   113  			"MultilineConfiguredEndPattern",
   114  			func(f *InputConfig) {
   115  				f.Multiline = &MultilineConfig{
   116  					LineEndPattern: "END.*",
   117  				}
   118  			},
   119  			require.NoError,
   120  			func(t *testing.T, f *InputOperator) {},
   121  		},
   122  	}
   123  
   124  	for _, tc := range cases {
   125  		t.Run(tc.name, func(t *testing.T) {
   126  			cfg := basicConfig()
   127  			tc.modifyBaseConfig(cfg)
   128  
   129  			plg, err := cfg.Build(testutil.NewBuildContext(t))
   130  			tc.errorRequirement(t, err)
   131  			if err != nil {
   132  				return
   133  			}
   134  
   135  			err = plg.SetOutputs([]operator.Operator{mockOutput})
   136  			require.NoError(t, err)
   137  
   138  			fileInput := plg.(*InputOperator)
   139  			tc.validate(t, fileInput)
   140  		})
   141  	}
   142  }
   143  
   144  func TestFileSource_CleanStop(t *testing.T) {
   145  	t.Parallel()
   146  	t.Skip(`Skipping due to goroutine leak in opencensus.
   147  See this issue for details: https://github.com/census-instrumentation/opencensus-go/issues/1191#issuecomment-610440163`)
   148  	// defer goleak.VerifyNone(t)
   149  
   150  	source, _ := newTestFileSource(t)
   151  
   152  	tempDir := testutil.NewTempDir(t)
   153  
   154  	tempFile, err := ioutil.TempFile(tempDir, "")
   155  	require.NoError(t, err)
   156  
   157  	source.Include = []string{tempFile.Name()}
   158  
   159  	err = source.Start()
   160  	require.NoError(t, err)
   161  	source.Stop()
   162  }
   163  
   164  func TestFileSource_AddFields(t *testing.T) {
   165  	t.Parallel()
   166  	source, logReceived := newTestFileSource(t)
   167  	tempDir := testutil.NewTempDir(t)
   168  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   169  	source.FilePathField = entry.NewLabelField("path")
   170  	source.FileNameField = entry.NewLabelField("file_name")
   171  
   172  	// Create a file, then start
   173  	temp, err := ioutil.TempFile(tempDir, "")
   174  	require.NoError(t, err)
   175  	defer temp.Close()
   176  
   177  	_, err = temp.WriteString("testlog\n")
   178  	require.NoError(t, err)
   179  
   180  	err = source.Start()
   181  	require.NoError(t, err)
   182  	defer source.Stop()
   183  
   184  	select {
   185  	case e := <-logReceived:
   186  		require.Equal(t, filepath.Base(temp.Name()), e.Labels["file_name"])
   187  		require.Equal(t, temp.Name(), e.Labels["path"])
   188  	case <-time.After(time.Second):
   189  		require.FailNow(t, "Timed out waiting for message")
   190  	}
   191  }
   192  
   193  func TestFileSource_ReadExistingLogs(t *testing.T) {
   194  	t.Parallel()
   195  	source, logReceived := newTestFileSource(t)
   196  	tempDir := testutil.NewTempDir(t)
   197  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   198  
   199  	// Create a file, then start
   200  	temp, err := ioutil.TempFile(tempDir, "")
   201  	require.NoError(t, err)
   202  	defer temp.Close()
   203  
   204  	_, err = temp.WriteString("testlog\n")
   205  	require.NoError(t, err)
   206  
   207  	err = source.Start()
   208  	require.NoError(t, err)
   209  	defer source.Stop()
   210  
   211  	waitForMessage(t, logReceived, "testlog")
   212  }
   213  
   214  func TestFileSource_IncludeFile(t *testing.T) {
   215  	t.Parallel()
   216  	source, logReceived := newTestFileSource(t)
   217  	tempDir := testutil.NewTempDir(t)
   218  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   219  	source.FileNameField = entry.NewLabelField("file_name")
   220  	source.FilePathField = entry.NewLabelField("file_path")
   221  
   222  	// Create a file, then start
   223  	temp, err := ioutil.TempFile(tempDir, "")
   224  	require.NoError(t, err)
   225  	defer temp.Close()
   226  
   227  	_, err = temp.WriteString("testlog\n")
   228  	require.NoError(t, err)
   229  
   230  	err = source.Start()
   231  	require.NoError(t, err)
   232  	defer source.Stop()
   233  
   234  	select {
   235  	case e := <-logReceived:
   236  		require.Equal(t, e.Labels["file_name"], filepath.Base(temp.Name()))
   237  		require.Equal(t, e.Labels["file_path"], temp.Name())
   238  	case <-time.After(time.Second):
   239  		require.FailNow(t, "Timed out waiting for message")
   240  	}
   241  }
   242  
   243  func TestFileSource_ReadNewLogs(t *testing.T) {
   244  	t.Parallel()
   245  	source, logReceived := newTestFileSource(t)
   246  	tempDir := testutil.NewTempDir(t)
   247  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   248  
   249  	// Start first, then create a new file
   250  	err := source.Start()
   251  	require.NoError(t, err)
   252  	defer source.Stop()
   253  
   254  	temp, err := ioutil.TempFile(tempDir, "")
   255  	require.NoError(t, err)
   256  	defer temp.Close()
   257  
   258  	_, err = temp.WriteString("testlog\n")
   259  	require.NoError(t, err)
   260  
   261  	waitForMessage(t, logReceived, "testlog")
   262  }
   263  
   264  func TestFileSource_ReadExistingAndNewLogs(t *testing.T) {
   265  	t.Parallel()
   266  	source, logReceived := newTestFileSource(t)
   267  	tempDir := testutil.NewTempDir(t)
   268  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   269  
   270  	temp, err := ioutil.TempFile(tempDir, "")
   271  	require.NoError(t, err)
   272  	defer temp.Close()
   273  
   274  	_, err = temp.WriteString("testlog1\n")
   275  	require.NoError(t, err)
   276  
   277  	err = source.Start()
   278  	require.NoError(t, err)
   279  	defer source.Stop()
   280  
   281  	_, err = temp.WriteString("testlog2\n")
   282  	require.NoError(t, err)
   283  
   284  	waitForMessage(t, logReceived, "testlog1")
   285  	waitForMessage(t, logReceived, "testlog2")
   286  }
   287  
   288  func TestFileSource_StartAtEnd(t *testing.T) {
   289  	t.Parallel()
   290  	source, logReceived := newTestFileSource(t)
   291  	tempDir := testutil.NewTempDir(t)
   292  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   293  	source.startAtBeginning = false
   294  
   295  	temp, err := ioutil.TempFile(tempDir, "")
   296  	require.NoError(t, err)
   297  	defer temp.Close()
   298  
   299  	_, err = temp.WriteString("testlog1\n")
   300  	require.NoError(t, err)
   301  
   302  	err = source.Start()
   303  	require.NoError(t, err)
   304  	defer source.Stop()
   305  
   306  	// Wait until file has been read the first time
   307  	time.Sleep(200 * time.Millisecond)
   308  
   309  	_, err = temp.WriteString("testlog2\n")
   310  	require.NoError(t, err)
   311  	temp.Close()
   312  
   313  	waitForMessage(t, logReceived, "testlog2")
   314  }
   315  
   316  func TestFileSource_StartAtEndNewFile(t *testing.T) {
   317  	t.Parallel()
   318  	source, logReceived := newTestFileSource(t)
   319  	tempDir := testutil.NewTempDir(t)
   320  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   321  	source.startAtBeginning = false
   322  
   323  	err := source.Start()
   324  	require.NoError(t, err)
   325  	defer source.Stop()
   326  
   327  	// Wait for the first check to complete
   328  	time.Sleep(200 * time.Millisecond)
   329  
   330  	temp, err := ioutil.TempFile(tempDir, "")
   331  	require.NoError(t, err)
   332  	defer temp.Close()
   333  
   334  	_, err = temp.WriteString("testlog1\ntestlog2\n")
   335  	require.NoError(t, err)
   336  
   337  	waitForMessage(t, logReceived, "testlog1")
   338  	waitForMessage(t, logReceived, "testlog2")
   339  }
   340  
   341  func TestFileSource_MultiFileSimple(t *testing.T) {
   342  	t.Parallel()
   343  	source, logReceived := newTestFileSource(t)
   344  	tempDir := testutil.NewTempDir(t)
   345  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   346  
   347  	temp1, err := ioutil.TempFile(tempDir, "")
   348  	require.NoError(t, err)
   349  	temp2, err := ioutil.TempFile(tempDir, "")
   350  	require.NoError(t, err)
   351  
   352  	_, err = temp1.WriteString("testlog1\n")
   353  	require.NoError(t, err)
   354  	_, err = temp2.WriteString("testlog2\n")
   355  	require.NoError(t, err)
   356  
   357  	err = source.Start()
   358  	require.NoError(t, err)
   359  	defer source.Stop()
   360  
   361  	waitForMessages(t, logReceived, []string{"testlog1", "testlog2"})
   362  }
   363  
   364  func TestFileSource_MoveFile(t *testing.T) {
   365  	if runtime.GOOS == "windows" {
   366  		t.Skip("Moving files while open is unsupported on Windows")
   367  	}
   368  	t.Parallel()
   369  	source, logReceived := newTestFileSource(t)
   370  	tempDir := testutil.NewTempDir(t)
   371  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   372  
   373  	temp1, err := ioutil.TempFile(tempDir, "")
   374  	require.NoError(t, err)
   375  
   376  	_, err = temp1.WriteString("testlog1\n")
   377  	require.NoError(t, err)
   378  	temp1.Close()
   379  
   380  	err = source.Start()
   381  	require.NoError(t, err)
   382  	defer source.Stop()
   383  
   384  	waitForMessage(t, logReceived, "testlog1")
   385  	time.Sleep(200 * time.Millisecond)
   386  
   387  	i := 0
   388  	for {
   389  		err = os.Rename(temp1.Name(), fmt.Sprintf("%s.2", temp1.Name()))
   390  		if err != nil {
   391  			if i < 3 {
   392  				t.Error(err)
   393  				i++
   394  				time.Sleep(10 * time.Millisecond)
   395  				continue
   396  			} else {
   397  				require.NoError(t, err)
   398  			}
   399  		}
   400  
   401  		break
   402  	}
   403  
   404  	expectNoMessages(t, logReceived)
   405  }
   406  
   407  func TestFileSource_TruncateThenWrite(t *testing.T) {
   408  	t.Parallel()
   409  	source, logReceived := newTestFileSource(t)
   410  	tempDir := testutil.NewTempDir(t)
   411  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   412  
   413  	temp1, err := ioutil.TempFile(tempDir, "")
   414  	require.NoError(t, err)
   415  
   416  	_, err = temp1.WriteString("testlog1\n")
   417  	require.NoError(t, err)
   418  	_, err = temp1.WriteString("testlog2\n")
   419  	require.NoError(t, err)
   420  
   421  	err = source.Start()
   422  	require.NoError(t, err)
   423  	defer source.Stop()
   424  
   425  	waitForMessage(t, logReceived, "testlog1")
   426  	waitForMessage(t, logReceived, "testlog2")
   427  
   428  	err = temp1.Truncate(0)
   429  	require.NoError(t, err)
   430  	temp1.Seek(0, 0)
   431  
   432  	_, err = temp1.WriteString("testlog3\n")
   433  	require.NoError(t, err)
   434  
   435  	waitForMessage(t, logReceived, "testlog3")
   436  	expectNoMessages(t, logReceived)
   437  }
   438  
   439  func TestFileSource_CopyTruncateWriteBoth(t *testing.T) {
   440  	t.Parallel()
   441  	source, logReceived := newTestFileSource(t)
   442  	tempDir := testutil.NewTempDir(t)
   443  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   444  
   445  	temp1, err := ioutil.TempFile(tempDir, "")
   446  	require.NoError(t, err)
   447  	defer temp1.Close()
   448  
   449  	_, err = temp1.WriteString("testlog1\n")
   450  	require.NoError(t, err)
   451  	_, err = temp1.WriteString("testlog2\n")
   452  	require.NoError(t, err)
   453  
   454  	err = source.Start()
   455  	require.NoError(t, err)
   456  	defer source.Stop()
   457  
   458  	waitForMessage(t, logReceived, "testlog1")
   459  	waitForMessage(t, logReceived, "testlog2")
   460  
   461  	time.Sleep(50 * time.Millisecond)
   462  
   463  	temp2, err := ioutil.TempFile(tempDir, "")
   464  	require.NoError(t, err)
   465  	defer temp2.Close()
   466  
   467  	_, err = io.Copy(temp1, temp2)
   468  	require.NoError(t, err)
   469  
   470  	// Truncate original file
   471  	err = temp1.Truncate(0)
   472  	temp1.Seek(0, 0)
   473  	require.NoError(t, err)
   474  
   475  	// Write to original and new file
   476  	_, err = temp1.WriteString("testlog3\n")
   477  	require.NoError(t, err)
   478  
   479  	waitForMessage(t, logReceived, "testlog3")
   480  
   481  	_, err = temp2.WriteString("testlog4\n")
   482  	require.NoError(t, err)
   483  
   484  	waitForMessage(t, logReceived, "testlog4")
   485  }
   486  
   487  func TestFileSource_OffsetsAfterRestart(t *testing.T) {
   488  	t.Parallel()
   489  	source, logReceived := newTestFileSource(t)
   490  	tempDir := testutil.NewTempDir(t)
   491  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   492  
   493  	temp1, err := ioutil.TempFile(tempDir, "")
   494  	require.NoError(t, err)
   495  
   496  	// Write to a file
   497  	_, err = temp1.WriteString("testlog1\n")
   498  	require.NoError(t, err)
   499  
   500  	// Start the source
   501  	err = source.Start()
   502  	require.NoError(t, err)
   503  	defer source.Stop()
   504  
   505  	waitForMessage(t, logReceived, "testlog1")
   506  
   507  	// Restart the source
   508  	err = source.Stop()
   509  	require.NoError(t, err)
   510  	err = source.Start()
   511  	require.NoError(t, err)
   512  
   513  	// Write a new log
   514  	_, err = temp1.WriteString("testlog2\n")
   515  	require.NoError(t, err)
   516  
   517  	waitForMessage(t, logReceived, "testlog2")
   518  }
   519  
   520  func TestFileSource_OffsetsAfterRestart_BigFiles(t *testing.T) {
   521  	t.Parallel()
   522  	source, logReceived := newTestFileSource(t)
   523  	tempDir := testutil.NewTempDir(t)
   524  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   525  
   526  	log1 := stringWithLength(1000)
   527  	log2 := stringWithLength(1000)
   528  
   529  	temp1, err := ioutil.TempFile(tempDir, "")
   530  	require.NoError(t, err)
   531  
   532  	// Write to a file
   533  	_, err = temp1.WriteString(log1 + "\n")
   534  	require.NoError(t, err)
   535  
   536  	// Start the source
   537  	err = source.Start()
   538  	require.NoError(t, err)
   539  
   540  	waitForMessage(t, logReceived, log1)
   541  
   542  	// Restart the source
   543  	err = source.Stop()
   544  	require.NoError(t, err)
   545  	err = source.Start()
   546  	require.NoError(t, err)
   547  	defer source.Stop()
   548  
   549  	_, err = temp1.WriteString(log2 + "\n")
   550  	require.NoError(t, err)
   551  
   552  	waitForMessage(t, logReceived, log2)
   553  }
   554  
   555  func TestFileSource_OffsetsAfterRestart_BigFilesWrittenWhileOff(t *testing.T) {
   556  	t.Parallel()
   557  	source, logReceived := newTestFileSource(t)
   558  	tempDir := testutil.NewTempDir(t)
   559  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   560  
   561  	log1 := stringWithLength(1000)
   562  	log2 := stringWithLength(1000)
   563  
   564  	temp1, err := ioutil.TempFile(tempDir, "")
   565  	require.NoError(t, err)
   566  
   567  	// Write to a file
   568  	_, err = temp1.WriteString(log1 + "\n")
   569  	require.NoError(t, err)
   570  
   571  	// Start the source
   572  	err = source.Start()
   573  	require.NoError(t, err)
   574  
   575  	waitForMessage(t, logReceived, log1)
   576  
   577  	// Restart the source
   578  	err = source.Stop()
   579  	require.NoError(t, err)
   580  
   581  	_, err = temp1.WriteString(log2 + "\n")
   582  	require.NoError(t, err)
   583  
   584  	err = source.Start()
   585  	require.NoError(t, err)
   586  	defer source.Stop()
   587  
   588  	waitForMessage(t, logReceived, log2)
   589  }
   590  
   591  func TestFileSource_FileMovedWhileOff_BigFiles(t *testing.T) {
   592  	t.Parallel()
   593  	source, logReceived := newTestFileSource(t)
   594  	tempDir := testutil.NewTempDir(t)
   595  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   596  
   597  	log1 := stringWithLength(1000)
   598  	log2 := stringWithLength(1000)
   599  
   600  	temp1, err := ioutil.TempFile(tempDir, "")
   601  	require.NoError(t, err)
   602  
   603  	// Write to a file
   604  	_, err = temp1.WriteString(log1 + "\n")
   605  	require.NoError(t, err)
   606  
   607  	// Start the source
   608  	err = source.Start()
   609  	require.NoError(t, err)
   610  
   611  	waitForMessage(t, logReceived, log1)
   612  
   613  	// Stop the source, then rename and write a new log
   614  	err = source.Stop()
   615  	require.NoError(t, err)
   616  
   617  	_, err = temp1.WriteString(log2 + "\n")
   618  	require.NoError(t, err)
   619  	temp1.Close()
   620  
   621  	err = os.Rename(temp1.Name(), fmt.Sprintf("%s2", temp1.Name()))
   622  	require.NoError(t, err)
   623  
   624  	err = source.Start()
   625  	require.NoError(t, err)
   626  	defer source.Stop()
   627  
   628  	waitForMessage(t, logReceived, log2)
   629  }
   630  
   631  func TestFileSource_FileMovedWhileOff_SmallFiles(t *testing.T) {
   632  	t.Parallel()
   633  	source, logReceived := newTestFileSource(t)
   634  	tempDir := testutil.NewTempDir(t)
   635  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   636  
   637  	log1 := stringWithLength(10)
   638  	log2 := stringWithLength(10)
   639  
   640  	temp1, err := ioutil.TempFile(tempDir, "")
   641  	require.NoError(t, err)
   642  
   643  	// Write to a file
   644  	_, err = temp1.WriteString(log1 + "\n")
   645  	require.NoError(t, err)
   646  
   647  	// Start the source
   648  	err = source.Start()
   649  	require.NoError(t, err)
   650  
   651  	waitForMessage(t, logReceived, log1)
   652  	time.Sleep(50 * time.Millisecond)
   653  
   654  	// Restart the source
   655  	err = source.Stop()
   656  	require.NoError(t, err)
   657  
   658  	_, err = temp1.WriteString(log2 + "\n")
   659  	require.NoError(t, err)
   660  	temp1.Close()
   661  
   662  	err = os.Rename(temp1.Name(), fmt.Sprintf("%s2", temp1.Name()))
   663  	require.NoError(t, err)
   664  
   665  	err = source.Start()
   666  	require.NoError(t, err)
   667  	defer source.Stop()
   668  
   669  	waitForMessage(t, logReceived, log2)
   670  }
   671  
   672  func TestFileSource_ManyLogsDelivered(t *testing.T) {
   673  	t.Parallel()
   674  	source, logReceived := newTestFileSource(t)
   675  	tempDir := testutil.NewTempDir(t)
   676  	source.Include = []string{fmt.Sprintf("%s/*", tempDir)}
   677  
   678  	temp1, err := ioutil.TempFile(tempDir, "")
   679  	require.NoError(t, err)
   680  
   681  	count := 1000
   682  	expectedMessages := make([]string, 0, count)
   683  	for i := 0; i < count; i++ {
   684  		expectedMessages = append(expectedMessages, strconv.Itoa(i))
   685  	}
   686  
   687  	// Start the source
   688  	err = source.Start()
   689  	require.NoError(t, err)
   690  	defer source.Stop()
   691  
   692  	// Write lots of logs
   693  	for _, message := range expectedMessages {
   694  		temp1.WriteString(message + "\n")
   695  	}
   696  
   697  	// Expect each of them to come through
   698  	for _, message := range expectedMessages {
   699  		waitForMessage(t, logReceived, message)
   700  	}
   701  
   702  	expectNoMessages(t, logReceived)
   703  }
   704  
   705  func stringWithLength(length int) string {
   706  	charset := "abcdefghijklmnopqrstuvwxyz"
   707  	b := make([]byte, length)
   708  	for i := range b {
   709  		b[i] = charset[rand.Intn(len(charset))]
   710  	}
   711  	return string(b)
   712  }
   713  
   714  func waitForMessage(t *testing.T, c chan *entry.Entry, expected string) {
   715  	select {
   716  	case e := <-c:
   717  		require.Equal(t, expected, e.Record.(string))
   718  	case <-time.After(time.Second):
   719  		require.FailNow(t, "Timed out waiting for message")
   720  	}
   721  }
   722  
   723  func waitForMessages(t *testing.T, c chan *entry.Entry, expected []string) {
   724  	receivedMessages := make([]string, 0, 100)
   725  LOOP:
   726  	for {
   727  		select {
   728  		case e := <-c:
   729  			receivedMessages = append(receivedMessages, e.Record.(string))
   730  			if len(receivedMessages) == len(expected) {
   731  				break LOOP
   732  			}
   733  		case <-time.After(time.Second):
   734  			require.FailNow(t, "Timed out waiting for expected messages")
   735  		}
   736  	}
   737  
   738  	require.ElementsMatch(t, expected, receivedMessages)
   739  }
   740  
   741  func expectNoMessages(t *testing.T, c chan *entry.Entry) {
   742  	select {
   743  	case e := <-c:
   744  		require.FailNow(t, "Received unexpected message", "Message: %s", e.Record.(string))
   745  	case <-time.After(200 * time.Millisecond):
   746  	}
   747  }
   748  
   749  func TestEncodings(t *testing.T) {
   750  	t.Parallel()
   751  	cases := []struct {
   752  		name     string
   753  		contents []byte
   754  		encoding string
   755  		expected [][]byte
   756  	}{
   757  		{
   758  			"Nop",
   759  			[]byte{0xc5, '\n'},
   760  			"",
   761  			[][]byte{{0xc5}},
   762  		},
   763  		{
   764  			"InvalidUTFReplacement",
   765  			[]byte{0xc5, '\n'},
   766  			"utf8",
   767  			[][]byte{{0xef, 0xbf, 0xbd}},
   768  		},
   769  		{
   770  			"ValidUTF8",
   771  			[]byte("foo\n"),
   772  			"utf8",
   773  			[][]byte{[]byte("foo")},
   774  		},
   775  		{
   776  			"ChineseCharacter",
   777  			[]byte{230, 138, 152, '\n'}, // 折\n
   778  			"utf8",
   779  			[][]byte{{230, 138, 152}},
   780  		},
   781  		{
   782  			"SmileyFaceUTF16",
   783  			[]byte{216, 61, 222, 0, 0, 10}, // 😀\n
   784  			"utf-16be",
   785  			[][]byte{{240, 159, 152, 128}},
   786  		},
   787  		{
   788  			"SmileyFaceNewlineUTF16",
   789  			[]byte{216, 61, 222, 0, 0, 10, 0, 102, 0, 111, 0, 111}, // 😀\nfoo
   790  			"utf-16be",
   791  			[][]byte{{240, 159, 152, 128}, {102, 111, 111}},
   792  		},
   793  		{
   794  			"SmileyFaceNewlineUTF16LE",
   795  			[]byte{61, 216, 0, 222, 10, 0, 102, 0, 111, 0, 111, 0}, // 😀\nfoo
   796  			"utf-16le",
   797  			[][]byte{{240, 159, 152, 128}, {102, 111, 111}},
   798  		},
   799  		{
   800  			"ChineseCharacterBig5",
   801  			[]byte{167, 233, 10}, // 折\n
   802  			"big5",
   803  			[][]byte{{230, 138, 152}},
   804  		},
   805  	}
   806  
   807  	for _, tc := range cases {
   808  		t.Run(tc.name, func(t *testing.T) {
   809  			tempDir := testutil.NewTempDir(t)
   810  			path := filepath.Join(tempDir, "in.log")
   811  			err := ioutil.WriteFile(path, tc.contents, 0777)
   812  			require.NoError(t, err)
   813  
   814  			source, receivedEntries := newTestFileSource(t)
   815  			source.Include = []string{path}
   816  			source.encoding, err = lookupEncoding(tc.encoding)
   817  			require.NoError(t, err)
   818  			source.SplitFunc, err = NewNewlineSplitFunc(source.encoding)
   819  			require.NoError(t, err)
   820  			require.NotNil(t, source.encoding)
   821  
   822  			err = source.Start()
   823  			require.NoError(t, err)
   824  			defer source.Stop()
   825  
   826  			for _, expected := range tc.expected {
   827  				select {
   828  				case entry := <-receivedEntries:
   829  					require.Equal(t, expected, []byte(entry.Record.(string)))
   830  				case <-time.After(time.Second):
   831  					require.FailNow(t, "Timed out waiting for entry to be read")
   832  				}
   833  			}
   834  		})
   835  	}
   836  }
   837  
   838  type fileInputBenchmark struct {
   839  	name   string
   840  	config *InputConfig
   841  }
   842  
   843  func BenchmarkFileInput(b *testing.B) {
   844  	cases := []fileInputBenchmark{
   845  		{
   846  			"Default",
   847  			NewInputConfig("test_id"),
   848  		},
   849  		{
   850  			"NoFileName",
   851  			func() *InputConfig {
   852  				cfg := NewInputConfig("test_id")
   853  				cfg.IncludeFileName = false
   854  				return cfg
   855  			}(),
   856  		},
   857  	}
   858  
   859  	for _, tc := range cases {
   860  		b.Run(tc.name, func(b *testing.B) {
   861  			tempDir := testutil.NewTempDir(b)
   862  			path := filepath.Join(tempDir, "in.log")
   863  
   864  			cfg := tc.config
   865  			cfg.OutputIDs = []string{"fake"}
   866  			cfg.Include = []string{path}
   867  			cfg.StartAt = "beginning"
   868  
   869  			fileOperator, err := cfg.Build(testutil.NewBuildContext(b))
   870  			require.NoError(b, err)
   871  
   872  			fakeOutput := testutil.NewFakeOutput(b)
   873  			err = fileOperator.SetOutputs([]operator.Operator{fakeOutput})
   874  			require.NoError(b, err)
   875  
   876  			err = fileOperator.Start()
   877  			require.NoError(b, err)
   878  
   879  			file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666)
   880  			require.NoError(b, err)
   881  
   882  			for i := 0; i < b.N; i++ {
   883  				file.WriteString("testlog\n")
   884  			}
   885  
   886  			b.ResetTimer()
   887  			for i := 0; i < b.N; i++ {
   888  				<-fakeOutput.Received
   889  			}
   890  		})
   891  	}
   892  }