github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/grok_test.go (about)

     1  package processor
     2  
     3  import (
     4  	"os"
     5  	"path/filepath"
     6  	"reflect"
     7  	"testing"
     8  
     9  	"github.com/Jeffail/benthos/v3/lib/log"
    10  	"github.com/Jeffail/benthos/v3/lib/message"
    11  	"github.com/Jeffail/benthos/v3/lib/metrics"
    12  	"github.com/stretchr/testify/assert"
    13  	"github.com/stretchr/testify/require"
    14  )
    15  
    16  func TestGrokAllParts(t *testing.T) {
    17  	conf := NewConfig()
    18  	conf.Grok.Parts = []int{}
    19  	conf.Grok.Patterns = []string{
    20  		"%{WORD:first},%{INT:second:int}",
    21  	}
    22  
    23  	testLog := log.Noop()
    24  
    25  	gSet, err := NewGrok(conf, nil, testLog, metrics.Noop())
    26  	if err != nil {
    27  		t.Fatal(err)
    28  	}
    29  
    30  	msgIn := message.New([][]byte{
    31  		[]byte(`foo,0`),
    32  		[]byte(`foo,1`),
    33  		[]byte(`foo,2`),
    34  	})
    35  	msgs, res := gSet.ProcessMessage(msgIn)
    36  	if len(msgs) != 1 {
    37  		t.Fatal("Wrong count of messages")
    38  	}
    39  	if res != nil {
    40  		t.Fatal("Non-nil result")
    41  	}
    42  
    43  	exp := [][]byte{
    44  		[]byte(`{"first":"foo","second":0}`),
    45  		[]byte(`{"first":"foo","second":1}`),
    46  		[]byte(`{"first":"foo","second":2}`),
    47  	}
    48  	act := message.GetAllBytes(msgs[0])
    49  	if !reflect.DeepEqual(act, exp) {
    50  		t.Errorf("Wrong output from grok: %s != %s", act, exp)
    51  	}
    52  }
    53  
    54  func TestGrok(t *testing.T) {
    55  	tLog := log.Noop()
    56  	tStats := metrics.Noop()
    57  
    58  	type gTest struct {
    59  		name        string
    60  		pattern     string
    61  		input       string
    62  		output      string
    63  		definitions map[string]string
    64  	}
    65  
    66  	tests := []gTest{
    67  		{
    68  			name:    "Common apache parsing",
    69  			pattern: "%{COMMONAPACHELOG}",
    70  			input:   `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`,
    71  			output:  `{"auth":"-","bytes":"207","clientip":"127.0.0.1","httpversion":"1.1","ident":"-","request":"/index.php","response":"404","timestamp":"23/Apr/2014:22:58:32 +0200","verb":"GET"}`,
    72  		},
    73  		{
    74  			name: "Test pattern definitions",
    75  			definitions: map[string]string{
    76  				"ACTION": "(pass|deny)",
    77  			},
    78  			input:   `pass connection from 127.0.0.1`,
    79  			pattern: "%{ACTION:action} connection from %{IPV4:ipv4}",
    80  			output:  `{"action":"pass","ipv4":"127.0.0.1"}`,
    81  		},
    82  		{
    83  			name:    "Test dot path in name definition",
    84  			input:   `foo 5 bazes from 192.0.1.11`,
    85  			pattern: "%{WORD:nested.name} %{INT:nested.value:int} bazes from %{IPV4:nested.ipv4}",
    86  			output:  `{"nested":{"ipv4":"192.0.1.11","name":"foo","value":5}}`,
    87  		},
    88  	}
    89  
    90  	for _, test := range tests {
    91  		t.Run(test.name, func(t *testing.T) {
    92  			conf := NewConfig()
    93  			conf.Grok.Parts = []int{0}
    94  			conf.Grok.Patterns = []string{test.pattern}
    95  			conf.Grok.PatternDefinitions = test.definitions
    96  
    97  			gSet, err := NewGrok(conf, nil, tLog, tStats)
    98  			require.NoError(t, err)
    99  
   100  			inMsg := message.New([][]byte{[]byte(test.input)})
   101  			msgs, _ := gSet.ProcessMessage(inMsg)
   102  			require.Len(t, msgs, 1)
   103  
   104  			assert.Equal(t, test.output, string(msgs[0].Get(0).Get()))
   105  		})
   106  	}
   107  
   108  	for _, test := range tests {
   109  		t.Run(test.name, func(t *testing.T) {
   110  			conf := NewConfig()
   111  			conf.Grok.Parts = []int{0}
   112  			conf.Grok.Expressions = []string{test.pattern}
   113  			conf.Grok.PatternDefinitions = test.definitions
   114  
   115  			gSet, err := NewGrok(conf, nil, tLog, tStats)
   116  			require.NoError(t, err)
   117  
   118  			inMsg := message.New([][]byte{[]byte(test.input)})
   119  			msgs, _ := gSet.ProcessMessage(inMsg)
   120  			require.Len(t, msgs, 1)
   121  
   122  			assert.Equal(t, test.output, string(msgs[0].Get(0).Get()))
   123  		})
   124  	}
   125  }
   126  
   127  func TestGrokFileImports(t *testing.T) {
   128  	tmpDir := t.TempDir()
   129  
   130  	err := os.WriteFile(filepath.Join(tmpDir, "foos"), []byte(`
   131  FOOFLAT %{WORD:first} %{WORD:second} %{WORD:third}
   132  FOONESTED %{INT:nested.first:int} %{WORD:nested.second} %{WORD:nested.third}
   133  `), 0o777)
   134  	require.NoError(t, err)
   135  
   136  	conf := NewConfig()
   137  	conf.Grok.Parts = []int{0}
   138  	conf.Grok.Expressions = []string{`%{FOONESTED}`, `%{FOOFLAT}`}
   139  	conf.Grok.PatternPaths = []string{tmpDir}
   140  
   141  	gSet, err := NewGrok(conf, nil, log.Noop(), metrics.Noop())
   142  	require.NoError(t, err)
   143  
   144  	inMsg := message.New([][]byte{[]byte(`hello foo bar`)})
   145  	msgs, _ := gSet.ProcessMessage(inMsg)
   146  	require.Len(t, msgs, 1)
   147  	assert.Equal(t, `{"first":"hello","second":"foo","third":"bar"}`, string(msgs[0].Get(0).Get()))
   148  
   149  	inMsg = message.New([][]byte{[]byte(`10 foo bar`)})
   150  	msgs, _ = gSet.ProcessMessage(inMsg)
   151  	require.Len(t, msgs, 1)
   152  	assert.Equal(t, `{"nested":{"first":10,"second":"foo","third":"bar"}}`, string(msgs[0].Get(0).Get()))
   153  }