github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/group_by_test.go (about)

     1  package processor
     2  
     3  import (
     4  	"reflect"
     5  	"testing"
     6  
     7  	"github.com/Jeffail/benthos/v3/lib/condition"
     8  	"github.com/Jeffail/benthos/v3/lib/log"
     9  	"github.com/Jeffail/benthos/v3/lib/message"
    10  	"github.com/Jeffail/benthos/v3/lib/metrics"
    11  	"github.com/stretchr/testify/assert"
    12  	"github.com/stretchr/testify/require"
    13  )
    14  
    15  //------------------------------------------------------------------------------
    16  
    17  func TestGroupBy(t *testing.T) {
    18  	conf := NewConfig()
    19  	conf.Type = TypeGroupBy
    20  
    21  	procConf := NewConfig()
    22  	procConf.Type = TypeArchive
    23  	procConf.Archive.Format = "lines"
    24  
    25  	conf.GroupBy = append(conf.GroupBy, GroupByElement{
    26  		Check: `content().contains("foo")`,
    27  		Processors: []Config{
    28  			procConf,
    29  		},
    30  	})
    31  
    32  	procConf = NewConfig()
    33  	procConf.Type = TypeBloblang
    34  	procConf.Bloblang = `root = content().uppercase()`
    35  
    36  	procConf2 := NewConfig()
    37  	procConf2.Type = TypeBloblang
    38  	procConf2.Bloblang = `root = content().trim()`
    39  
    40  	conf.GroupBy = append(conf.GroupBy, GroupByElement{
    41  		Check: `content().contains("bar")`,
    42  		Processors: []Config{
    43  			procConf,
    44  			procConf2,
    45  		},
    46  	})
    47  
    48  	proc, err := New(conf, nil, log.Noop(), metrics.Noop())
    49  	require.NoError(t, err)
    50  
    51  	exp := [][][]byte{
    52  		{
    53  			[]byte(` hello foo world 1 
    54   hello foo bar world 1 
    55   hello foo world 2 
    56   hello foo bar world 2 `),
    57  		},
    58  		{
    59  			[]byte(`HELLO BAR WORLD 1`),
    60  			[]byte(`HELLO BAR WORLD 2`),
    61  		},
    62  		{
    63  			[]byte(` hello world 1 `),
    64  			[]byte(` hello world 2 `),
    65  		},
    66  	}
    67  	act := [][][]byte{}
    68  
    69  	input := message.New([][]byte{
    70  		[]byte(` hello foo world 1 `),
    71  		[]byte(` hello world 1 `),
    72  		[]byte(` hello foo bar world 1 `),
    73  		[]byte(` hello bar world 1 `),
    74  		[]byte(` hello foo world 2 `),
    75  		[]byte(` hello world 2 `),
    76  		[]byte(` hello foo bar world 2 `),
    77  		[]byte(` hello bar world 2 `),
    78  	})
    79  	msgs, res := proc.ProcessMessage(input)
    80  	require.Nil(t, res)
    81  
    82  	for _, msg := range msgs {
    83  		act = append(act, message.GetAllBytes(msg))
    84  	}
    85  	assert.Equal(t, exp, act)
    86  }
    87  
    88  func TestGroupByErrs(t *testing.T) {
    89  	conf := NewConfig()
    90  	conf.Type = TypeGroupBy
    91  
    92  	procConf := NewConfig()
    93  	procConf.Type = TypeArchive
    94  	procConf.Archive.Format = "lines"
    95  
    96  	conf.GroupBy = append(conf.GroupBy, GroupByElement{
    97  		Processors: []Config{
    98  			procConf,
    99  		},
   100  	})
   101  
   102  	_, err := New(conf, nil, log.Noop(), metrics.Noop())
   103  	require.EqualError(t, err, "a group definition must have a check query")
   104  
   105  	cond := condition.NewConfig()
   106  	cond.Text.Arg = "foobar"
   107  
   108  	conf.GroupBy[0] = GroupByElement{
   109  		Condition: cond,
   110  		Check:     "foo.bar",
   111  		Processors: []Config{
   112  			procConf,
   113  		},
   114  	}
   115  
   116  	_, err = New(conf, nil, log.Noop(), metrics.Noop())
   117  	require.EqualError(t, err, "cannot specify both a condition and a check in a group")
   118  }
   119  
   120  func TestGroupByDeprecated(t *testing.T) {
   121  	conf := NewConfig()
   122  	conf.Type = TypeGroupBy
   123  
   124  	condConf := condition.NewConfig()
   125  	condConf.Type = condition.TypeText
   126  	condConf.Text.Arg = "foo"
   127  	condConf.Text.Operator = "contains"
   128  
   129  	procConf := NewConfig()
   130  	procConf.Type = TypeArchive
   131  	procConf.Archive.Format = "lines"
   132  
   133  	conf.GroupBy = append(conf.GroupBy, GroupByElement{
   134  		Condition: condConf,
   135  		Processors: []Config{
   136  			procConf,
   137  		},
   138  	})
   139  
   140  	condConf = condition.NewConfig()
   141  	condConf.Type = condition.TypeText
   142  	condConf.Text.Arg = "bar"
   143  	condConf.Text.Operator = "contains"
   144  
   145  	procConf = NewConfig()
   146  	procConf.Type = TypeText
   147  	procConf.Text.Operator = "to_upper"
   148  
   149  	procConf2 := NewConfig()
   150  	procConf2.Type = TypeText
   151  	procConf2.Text.Operator = "trim_space"
   152  
   153  	conf.GroupBy = append(conf.GroupBy, GroupByElement{
   154  		Condition: condConf,
   155  		Processors: []Config{
   156  			procConf,
   157  			procConf2,
   158  		},
   159  	})
   160  
   161  	proc, err := New(conf, nil, log.Noop(), metrics.Noop())
   162  	if err != nil {
   163  		t.Fatal(err)
   164  	}
   165  
   166  	exp := [][][]byte{
   167  		{
   168  			[]byte(` hello foo world 1 
   169   hello foo bar world 1 
   170   hello foo world 2 
   171   hello foo bar world 2 `),
   172  		},
   173  		{
   174  			[]byte(`HELLO BAR WORLD 1`),
   175  			[]byte(`HELLO BAR WORLD 2`),
   176  		},
   177  		{
   178  			[]byte(` hello world 1 `),
   179  			[]byte(` hello world 2 `),
   180  		},
   181  	}
   182  	act := [][][]byte{}
   183  
   184  	input := message.New([][]byte{
   185  		[]byte(` hello foo world 1 `),
   186  		[]byte(` hello world 1 `),
   187  		[]byte(` hello foo bar world 1 `),
   188  		[]byte(` hello bar world 1 `),
   189  		[]byte(` hello foo world 2 `),
   190  		[]byte(` hello world 2 `),
   191  		[]byte(` hello foo bar world 2 `),
   192  		[]byte(` hello bar world 2 `),
   193  	})
   194  	msgs, res := proc.ProcessMessage(input)
   195  	if res != nil {
   196  		t.Fatal(res.Error())
   197  	}
   198  
   199  	for _, msg := range msgs {
   200  		act = append(act, message.GetAllBytes(msg))
   201  	}
   202  	if !reflect.DeepEqual(exp, act) {
   203  		t.Errorf("Wrong result: %s != %s", act, exp)
   204  	}
   205  }
   206  
   207  //------------------------------------------------------------------------------