github.com/Jeffail/benthos/v3@v3.65.0/lib/input/reader/lines_test.go (about)

     1  package reader
     2  
     3  import (
     4  	"bytes"
     5  	"io"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/Jeffail/benthos/v3/lib/types"
    10  )
    11  
    12  func TestReaderSinglePart(t *testing.T) {
    13  	messages := []string{
    14  		"first message",
    15  		"second message",
    16  		"third message",
    17  	}
    18  
    19  	var handle bytes.Buffer
    20  
    21  	for _, msg := range messages {
    22  		handle.Write([]byte(msg))
    23  		handle.Write([]byte("\n"))
    24  		handle.Write([]byte("\n")) // Try some empty messages
    25  	}
    26  
    27  	ctored := false
    28  	f, err := NewLines(
    29  		func() (io.Reader, error) {
    30  			if ctored {
    31  				return nil, io.EOF
    32  			}
    33  			ctored = true
    34  			return &handle, nil
    35  		},
    36  		func() {},
    37  	)
    38  	if err != nil {
    39  		t.Fatal(err)
    40  	}
    41  
    42  	defer func() {
    43  		f.CloseAsync()
    44  		if err := f.WaitForClose(time.Second); err != nil {
    45  			t.Error(err)
    46  		}
    47  	}()
    48  
    49  	if err = f.Connect(); err != nil {
    50  		t.Fatal(err)
    51  	}
    52  
    53  	for _, msg := range messages {
    54  		var resMsg types.Message
    55  		if resMsg, err = f.Read(); err != nil {
    56  			t.Fatal(err)
    57  		}
    58  		if res := string(resMsg.Get(0).Get()); res != msg {
    59  			t.Errorf("Wrong result, %v != %v", res, msg)
    60  		}
    61  		if err = f.Acknowledge(nil); err != nil {
    62  			t.Error(err)
    63  		}
    64  	}
    65  
    66  	if _, err = f.Read(); err != types.ErrNotConnected {
    67  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrNotConnected)
    68  	}
    69  
    70  	if err = f.Connect(); err != types.ErrTypeClosed {
    71  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrTypeClosed)
    72  	}
    73  }
    74  
    75  func TestReaderSinglePartAppended(t *testing.T) {
    76  	messages := []string{
    77  		"first message",
    78  		"second message",
    79  		"third message",
    80  	}
    81  
    82  	var handle bytes.Buffer
    83  
    84  	for _, msg := range messages {
    85  		handle.Write([]byte(msg))
    86  		handle.Write([]byte("\n"))
    87  		handle.Write([]byte("\n")) // Try some empty messages
    88  	}
    89  
    90  	ctored := false
    91  	f, err := NewLines(
    92  		func() (io.Reader, error) {
    93  			if ctored {
    94  				return nil, io.EOF
    95  			}
    96  			ctored = true
    97  			return &handle, nil
    98  		},
    99  		func() {},
   100  	)
   101  	if err != nil {
   102  		t.Fatal(err)
   103  	}
   104  
   105  	defer func() {
   106  		f.CloseAsync()
   107  		if err := f.WaitForClose(time.Second); err != nil {
   108  			t.Error(err)
   109  		}
   110  	}()
   111  
   112  	if err = f.Connect(); err != nil {
   113  		t.Fatal(err)
   114  	}
   115  
   116  	parts := [][]byte{}
   117  	for _, msg := range messages {
   118  		var resMsg types.Message
   119  		if resMsg, err = f.Read(); err != nil {
   120  			t.Fatal(err)
   121  		}
   122  		rBytes := resMsg.Get(0).Get()
   123  		if res := string(rBytes); res != msg {
   124  			t.Errorf("Wrong result, %v != %v", res, msg)
   125  		}
   126  		parts = append(parts, rBytes)
   127  	}
   128  	if err = f.Acknowledge(nil); err != nil {
   129  		t.Error(err)
   130  	}
   131  
   132  	for i, msg := range messages {
   133  		parts[i] = append(parts[i], []byte(" foo")...)
   134  		if exp, act := msg+" foo", string(parts[i]); act != exp {
   135  			t.Errorf("Wrong appended result, %v != %v", act, exp)
   136  		}
   137  	}
   138  
   139  	if _, err = f.Read(); err != types.ErrNotConnected {
   140  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrNotConnected)
   141  	}
   142  
   143  	if err = f.Connect(); err != types.ErrTypeClosed {
   144  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrTypeClosed)
   145  	}
   146  }
   147  
   148  func TestReaderSinglePartMultiReaders(t *testing.T) {
   149  	messages := []string{
   150  		"first message",
   151  		"second message",
   152  		"third message",
   153  	}
   154  
   155  	var handle1, handle2 bytes.Buffer
   156  
   157  	for _, msg := range messages {
   158  		handle1.Write([]byte(msg))
   159  		handle1.Write([]byte("\n"))
   160  		handle2.Write([]byte(msg))
   161  		handle2.Write([]byte("\n"))
   162  	}
   163  
   164  	ctored1, ctored2 := false, false
   165  	f, err := NewLines(
   166  		func() (io.Reader, error) {
   167  			if ctored2 {
   168  				return nil, io.EOF
   169  			}
   170  			if ctored1 {
   171  				ctored2 = true
   172  				return &handle2, nil
   173  			}
   174  			ctored1 = true
   175  			return &handle1, nil
   176  		},
   177  		func() {},
   178  	)
   179  	if err != nil {
   180  		t.Fatal(err)
   181  	}
   182  
   183  	defer func() {
   184  		f.CloseAsync()
   185  		if err := f.WaitForClose(time.Second); err != nil {
   186  			t.Error(err)
   187  		}
   188  	}()
   189  
   190  	if err = f.Connect(); err != nil {
   191  		t.Fatal(err)
   192  	}
   193  
   194  	for _, msg := range messages {
   195  		var resMsg types.Message
   196  		if resMsg, err = f.Read(); err != nil {
   197  			t.Error(err)
   198  		} else if res := string(resMsg.Get(0).Get()); res != msg {
   199  			t.Errorf("Wrong result, %v != %v", res, msg)
   200  		}
   201  		if err = f.Acknowledge(nil); err != nil {
   202  			t.Error(err)
   203  		}
   204  	}
   205  
   206  	if _, err = f.Read(); err != types.ErrNotConnected {
   207  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrNotConnected)
   208  	}
   209  
   210  	if err = f.Connect(); err != nil {
   211  		t.Error(err)
   212  	}
   213  
   214  	for _, msg := range messages {
   215  		var resMsg types.Message
   216  		if resMsg, err = f.Read(); err != nil {
   217  			t.Error(err)
   218  		} else if res := string(resMsg.Get(0).Get()); res != msg {
   219  			t.Errorf("Wrong result, %v != %v", res, msg)
   220  		}
   221  		if err = f.Acknowledge(nil); err != nil {
   222  			t.Error(err)
   223  		}
   224  	}
   225  
   226  	if _, err = f.Read(); err != types.ErrNotConnected {
   227  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrNotConnected)
   228  	}
   229  
   230  	if err = f.Connect(); err != types.ErrTypeClosed {
   231  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrTypeClosed)
   232  	}
   233  }
   234  
   235  func TestReaderSinglePartCustomDelim(t *testing.T) {
   236  	messages := []string{
   237  		"first message",
   238  		"second message",
   239  		"third message",
   240  	}
   241  
   242  	var handle bytes.Buffer
   243  
   244  	for _, msg := range messages {
   245  		handle.Write([]byte(msg))
   246  		handle.Write([]byte("<FOO>"))
   247  		handle.Write([]byte("<FOO>")) // Try some empty messages
   248  	}
   249  
   250  	ctored := false
   251  	f, err := NewLines(
   252  		func() (io.Reader, error) {
   253  			if ctored {
   254  				return nil, io.EOF
   255  			}
   256  			ctored = true
   257  			return &handle, nil
   258  		},
   259  		func() {},
   260  		OptLinesSetDelimiter("<FOO>"),
   261  	)
   262  	if err != nil {
   263  		t.Fatal(err)
   264  	}
   265  
   266  	defer func() {
   267  		f.CloseAsync()
   268  		if err := f.WaitForClose(time.Second); err != nil {
   269  			t.Error(err)
   270  		}
   271  	}()
   272  
   273  	if err = f.Connect(); err != nil {
   274  		t.Fatal(err)
   275  	}
   276  
   277  	for _, msg := range messages {
   278  		var resMsg types.Message
   279  		if resMsg, err = f.Read(); err != nil {
   280  			t.Error(err)
   281  		} else if res := string(resMsg.Get(0).Get()); res != msg {
   282  			t.Errorf("Wrong result, %v != %v", res, msg)
   283  		}
   284  		if err = f.Acknowledge(nil); err != nil {
   285  			t.Error(err)
   286  		}
   287  	}
   288  
   289  	if _, err = f.Read(); err != types.ErrNotConnected {
   290  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrNotConnected)
   291  	}
   292  
   293  	if err = f.Connect(); err != types.ErrTypeClosed {
   294  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrTypeClosed)
   295  	}
   296  }
   297  
   298  func TestReaderMultiPart(t *testing.T) {
   299  	var handle bytes.Buffer
   300  
   301  	messages := [][]string{
   302  		{
   303  			"first message",
   304  			"1",
   305  			"2",
   306  		},
   307  		{
   308  			"second message",
   309  			"3",
   310  			"4",
   311  		},
   312  		{
   313  			"third message",
   314  			"5",
   315  			"6",
   316  		},
   317  	}
   318  
   319  	for _, msg := range messages {
   320  		for _, part := range msg {
   321  			handle.Write([]byte(part))
   322  			handle.Write([]byte("\n"))
   323  		}
   324  		handle.Write([]byte("\n"))
   325  	}
   326  
   327  	ctored := false
   328  	f, err := NewLines(
   329  		func() (io.Reader, error) {
   330  			if ctored {
   331  				return nil, io.EOF
   332  			}
   333  			ctored = true
   334  			return &handle, nil
   335  		},
   336  		func() {},
   337  		OptLinesSetMultipart(true),
   338  	)
   339  	if err != nil {
   340  		t.Fatal(err)
   341  	}
   342  
   343  	defer func() {
   344  		f.CloseAsync()
   345  		if err := f.WaitForClose(time.Second); err != nil {
   346  			t.Error(err)
   347  		}
   348  	}()
   349  
   350  	if err = f.Connect(); err != nil {
   351  		t.Fatal(err)
   352  	}
   353  
   354  	for _, msg := range messages {
   355  		var resMsg types.Message
   356  		if resMsg, err = f.Read(); err != nil {
   357  			t.Error(err)
   358  		} else {
   359  			for i, part := range msg {
   360  				if res := string(resMsg.Get(i).Get()); res != part {
   361  					t.Errorf("Wrong result, %v != %v", res, part)
   362  				}
   363  			}
   364  		}
   365  		if err = f.Acknowledge(nil); err != nil {
   366  			t.Error(err)
   367  		}
   368  	}
   369  
   370  	if _, err = f.Read(); err != types.ErrNotConnected {
   371  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrNotConnected)
   372  	}
   373  
   374  	if err = f.Connect(); err != types.ErrTypeClosed {
   375  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrTypeClosed)
   376  	}
   377  }
   378  
   379  func TestReaderMultiPartNoAck(t *testing.T) {
   380  	var handle bytes.Buffer
   381  
   382  	messages := [][]string{
   383  		{
   384  			"first message",
   385  			"1",
   386  			"2",
   387  		},
   388  		{
   389  			"second message",
   390  			"3",
   391  			"4",
   392  		},
   393  		{
   394  			"third message",
   395  			"5",
   396  			"6",
   397  		},
   398  	}
   399  
   400  	for _, msg := range messages {
   401  		for _, part := range msg {
   402  			handle.Write([]byte(part))
   403  			handle.Write([]byte("\n"))
   404  		}
   405  		handle.Write([]byte("\n"))
   406  	}
   407  
   408  	ctored := false
   409  	f, err := NewLines(
   410  		func() (io.Reader, error) {
   411  			if ctored {
   412  				return nil, io.EOF
   413  			}
   414  			ctored = true
   415  			return &handle, nil
   416  		},
   417  		func() {},
   418  		OptLinesSetMultipart(true),
   419  	)
   420  	if err != nil {
   421  		t.Fatal(err)
   422  	}
   423  
   424  	defer func() {
   425  		f.CloseAsync()
   426  		if err := f.WaitForClose(time.Second); err != nil {
   427  			t.Error(err)
   428  		}
   429  	}()
   430  
   431  	if err = f.Connect(); err != nil {
   432  		t.Fatal(err)
   433  	}
   434  
   435  	for _, msg := range messages {
   436  		var resMsg types.Message
   437  		if resMsg, err = f.Read(); err != nil {
   438  			t.Error(err)
   439  		} else {
   440  			for i, part := range msg {
   441  				if res := string(resMsg.Get(i).Get()); res != part {
   442  					t.Errorf("Wrong result, %v != %v", res, part)
   443  				}
   444  			}
   445  		}
   446  	}
   447  	if err = f.Acknowledge(nil); err != nil {
   448  		t.Error(err)
   449  	}
   450  
   451  	if _, err = f.Read(); err != types.ErrNotConnected {
   452  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrNotConnected)
   453  	}
   454  
   455  	if err = f.Connect(); err != types.ErrTypeClosed {
   456  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrTypeClosed)
   457  	}
   458  }
   459  
   460  func TestReaderMultiPartCustomDelim(t *testing.T) {
   461  	var handle bytes.Buffer
   462  
   463  	messages := [][]string{
   464  		{
   465  			"first message",
   466  			"1",
   467  			"2",
   468  		},
   469  		{
   470  			"second message",
   471  			"3",
   472  			"4",
   473  		},
   474  		{
   475  			"third message",
   476  			"5",
   477  			"6",
   478  		},
   479  	}
   480  
   481  	for _, msg := range messages {
   482  		for _, part := range msg {
   483  			handle.Write([]byte(part))
   484  			handle.Write([]byte("<FOO>"))
   485  		}
   486  		handle.Write([]byte("<FOO>"))
   487  	}
   488  
   489  	ctored := false
   490  	f, err := NewLines(
   491  		func() (io.Reader, error) {
   492  			if ctored {
   493  				return nil, io.EOF
   494  			}
   495  			ctored = true
   496  			return &handle, nil
   497  		},
   498  		func() {},
   499  		OptLinesSetMultipart(true),
   500  		OptLinesSetDelimiter("<FOO>"),
   501  	)
   502  	if err != nil {
   503  		t.Fatal(err)
   504  	}
   505  
   506  	defer func() {
   507  		f.CloseAsync()
   508  		if err := f.WaitForClose(time.Second); err != nil {
   509  			t.Error(err)
   510  		}
   511  	}()
   512  
   513  	if err = f.Connect(); err != nil {
   514  		t.Fatal(err)
   515  	}
   516  
   517  	for _, msg := range messages {
   518  		var resMsg types.Message
   519  		if resMsg, err = f.Read(); err != nil {
   520  			t.Error(err)
   521  		} else {
   522  			for i, part := range msg {
   523  				if res := string(resMsg.Get(i).Get()); res != part {
   524  					t.Errorf("Wrong result, %v != %v", res, part)
   525  				}
   526  			}
   527  		}
   528  		if err = f.Acknowledge(nil); err != nil {
   529  			t.Error(err)
   530  		}
   531  	}
   532  
   533  	if _, err = f.Read(); err != types.ErrNotConnected {
   534  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrNotConnected)
   535  	}
   536  
   537  	if err = f.Connect(); err != types.ErrTypeClosed {
   538  		t.Errorf("Wrong error returned: %v != %v", err, types.ErrTypeClosed)
   539  	}
   540  }