github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/grok_test.go (about) 1 package processor 2 3 import ( 4 "os" 5 "path/filepath" 6 "reflect" 7 "testing" 8 9 "github.com/Jeffail/benthos/v3/lib/log" 10 "github.com/Jeffail/benthos/v3/lib/message" 11 "github.com/Jeffail/benthos/v3/lib/metrics" 12 "github.com/stretchr/testify/assert" 13 "github.com/stretchr/testify/require" 14 ) 15 16 func TestGrokAllParts(t *testing.T) { 17 conf := NewConfig() 18 conf.Grok.Parts = []int{} 19 conf.Grok.Patterns = []string{ 20 "%{WORD:first},%{INT:second:int}", 21 } 22 23 testLog := log.Noop() 24 25 gSet, err := NewGrok(conf, nil, testLog, metrics.Noop()) 26 if err != nil { 27 t.Fatal(err) 28 } 29 30 msgIn := message.New([][]byte{ 31 []byte(`foo,0`), 32 []byte(`foo,1`), 33 []byte(`foo,2`), 34 }) 35 msgs, res := gSet.ProcessMessage(msgIn) 36 if len(msgs) != 1 { 37 t.Fatal("Wrong count of messages") 38 } 39 if res != nil { 40 t.Fatal("Non-nil result") 41 } 42 43 exp := [][]byte{ 44 []byte(`{"first":"foo","second":0}`), 45 []byte(`{"first":"foo","second":1}`), 46 []byte(`{"first":"foo","second":2}`), 47 } 48 act := message.GetAllBytes(msgs[0]) 49 if !reflect.DeepEqual(act, exp) { 50 t.Errorf("Wrong output from grok: %s != %s", act, exp) 51 } 52 } 53 54 func TestGrok(t *testing.T) { 55 tLog := log.Noop() 56 tStats := metrics.Noop() 57 58 type gTest struct { 59 name string 60 pattern string 61 input string 62 output string 63 definitions map[string]string 64 } 65 66 tests := []gTest{ 67 { 68 name: "Common apache parsing", 69 pattern: "%{COMMONAPACHELOG}", 70 input: `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`, 71 output: `{"auth":"-","bytes":"207","clientip":"127.0.0.1","httpversion":"1.1","ident":"-","request":"/index.php","response":"404","timestamp":"23/Apr/2014:22:58:32 +0200","verb":"GET"}`, 72 }, 73 { 74 name: "Test pattern definitions", 75 definitions: map[string]string{ 76 "ACTION": "(pass|deny)", 77 }, 78 input: `pass connection from 127.0.0.1`, 79 pattern: "%{ACTION:action} connection from %{IPV4:ipv4}", 80 output: `{"action":"pass","ipv4":"127.0.0.1"}`, 81 }, 82 { 83 name: "Test dot path in name definition", 84 input: `foo 5 bazes from 192.0.1.11`, 85 pattern: "%{WORD:nested.name} %{INT:nested.value:int} bazes from %{IPV4:nested.ipv4}", 86 output: `{"nested":{"ipv4":"192.0.1.11","name":"foo","value":5}}`, 87 }, 88 } 89 90 for _, test := range tests { 91 t.Run(test.name, func(t *testing.T) { 92 conf := NewConfig() 93 conf.Grok.Parts = []int{0} 94 conf.Grok.Patterns = []string{test.pattern} 95 conf.Grok.PatternDefinitions = test.definitions 96 97 gSet, err := NewGrok(conf, nil, tLog, tStats) 98 require.NoError(t, err) 99 100 inMsg := message.New([][]byte{[]byte(test.input)}) 101 msgs, _ := gSet.ProcessMessage(inMsg) 102 require.Len(t, msgs, 1) 103 104 assert.Equal(t, test.output, string(msgs[0].Get(0).Get())) 105 }) 106 } 107 108 for _, test := range tests { 109 t.Run(test.name, func(t *testing.T) { 110 conf := NewConfig() 111 conf.Grok.Parts = []int{0} 112 conf.Grok.Expressions = []string{test.pattern} 113 conf.Grok.PatternDefinitions = test.definitions 114 115 gSet, err := NewGrok(conf, nil, tLog, tStats) 116 require.NoError(t, err) 117 118 inMsg := message.New([][]byte{[]byte(test.input)}) 119 msgs, _ := gSet.ProcessMessage(inMsg) 120 require.Len(t, msgs, 1) 121 122 assert.Equal(t, test.output, string(msgs[0].Get(0).Get())) 123 }) 124 } 125 } 126 127 func TestGrokFileImports(t *testing.T) { 128 tmpDir := t.TempDir() 129 130 err := os.WriteFile(filepath.Join(tmpDir, "foos"), []byte(` 131 FOOFLAT %{WORD:first} %{WORD:second} %{WORD:third} 132 FOONESTED %{INT:nested.first:int} %{WORD:nested.second} %{WORD:nested.third} 133 `), 0o777) 134 require.NoError(t, err) 135 136 conf := NewConfig() 137 conf.Grok.Parts = []int{0} 138 conf.Grok.Expressions = []string{`%{FOONESTED}`, `%{FOOFLAT}`} 139 conf.Grok.PatternPaths = []string{tmpDir} 140 141 gSet, err := NewGrok(conf, nil, log.Noop(), metrics.Noop()) 142 require.NoError(t, err) 143 144 inMsg := message.New([][]byte{[]byte(`hello foo bar`)}) 145 msgs, _ := gSet.ProcessMessage(inMsg) 146 require.Len(t, msgs, 1) 147 assert.Equal(t, `{"first":"hello","second":"foo","third":"bar"}`, string(msgs[0].Get(0).Get())) 148 149 inMsg = message.New([][]byte{[]byte(`10 foo bar`)}) 150 msgs, _ = gSet.ProcessMessage(inMsg) 151 require.Len(t, msgs, 1) 152 assert.Equal(t, `{"nested":{"first":10,"second":"foo","third":"bar"}}`, string(msgs[0].Get(0).Get())) 153 }