bitbucket.org/Aishee/synsec@v0.0.0-20210414005726-236fc01a153d/pkg/parser/parsing_test.go (about) 1 package parser 2 3 import ( 4 "bytes" 5 "fmt" 6 "html/template" 7 "io" 8 "io/ioutil" 9 "os" 10 "sort" 11 "strings" 12 "testing" 13 14 "bitbucket.org/Aishee/synsec/pkg/exprhelpers" 15 "bitbucket.org/Aishee/synsec/pkg/types" 16 "github.com/davecgh/go-spew/spew" 17 log "github.com/sirupsen/logrus" 18 "gopkg.in/yaml.v2" 19 ) 20 21 type TestFile struct { 22 Lines []types.Event `yaml:"lines,omitempty"` 23 Results []types.Event `yaml:"results,omitempty"` 24 } 25 26 var debug bool = false 27 28 func TestParser(t *testing.T) { 29 debug = true 30 log.SetLevel(log.InfoLevel) 31 var envSetting = os.Getenv("TEST_ONLY") 32 pctx, ectx, err := prepTests() 33 if err != nil { 34 t.Fatalf("failed to load env : %s", err) 35 } 36 //Init the enricher 37 if envSetting != "" { 38 if err := testOneParser(pctx, ectx, envSetting, nil); err != nil { 39 t.Fatalf("Test '%s' failed : %s", envSetting, err) 40 } 41 } else { 42 fds, err := ioutil.ReadDir("./tests/") 43 if err != nil { 44 t.Fatalf("Unable to read test directory : %s", err) 45 } 46 for _, fd := range fds { 47 if !fd.IsDir() { 48 continue 49 } 50 fname := "./tests/" + fd.Name() 51 log.Infof("Running test on %s", fname) 52 if err := testOneParser(pctx, ectx, fname, nil); err != nil { 53 t.Fatalf("Test '%s' failed : %s", fname, err) 54 } 55 } 56 } 57 58 } 59 60 func BenchmarkParser(t *testing.B) { 61 log.Printf("start bench !!!!") 62 debug = false 63 log.SetLevel(log.ErrorLevel) 64 pctx, ectx, err := prepTests() 65 if err != nil { 66 t.Fatalf("failed to load env : %s", err) 67 } 68 var envSetting = os.Getenv("TEST_ONLY") 69 70 if envSetting != "" { 71 if err := testOneParser(pctx, ectx, envSetting, t); err != nil { 72 t.Fatalf("Test '%s' failed : %s", envSetting, err) 73 } 74 } else { 75 fds, err := ioutil.ReadDir("./tests/") 76 if err != nil { 77 t.Fatalf("Unable to read test directory : %s", err) 78 } 79 for _, fd := range fds { 80 if !fd.IsDir() { 81 continue 82 } 83 fname := "./tests/" + fd.Name() 84 log.Infof("Running test on %s", fname) 85 if err := testOneParser(pctx, ectx, fname, t); err != nil { 86 t.Fatalf("Test '%s' failed : %s", fname, err) 87 } 88 } 89 } 90 } 91 92 func testOneParser(pctx *UnixParserCtx, ectx []EnricherCtx, dir string, b *testing.B) error { 93 94 var ( 95 err error 96 pnodes []Node 97 98 parser_configs []Stagefile 99 ) 100 log.Warningf("testing %s", dir) 101 parser_cfg_file := fmt.Sprintf("%s/parsers.yaml", dir) 102 cfg, err := ioutil.ReadFile(parser_cfg_file) 103 if err != nil { 104 return fmt.Errorf("failed opening %s : %s", parser_cfg_file, err) 105 } 106 tmpl, err := template.New("test").Parse(string(cfg)) 107 if err != nil { 108 return fmt.Errorf("failed to parse template %s : %s", cfg, err) 109 } 110 var out bytes.Buffer 111 err = tmpl.Execute(&out, map[string]string{"TestDirectory": dir}) 112 if err != nil { 113 panic(err) 114 } 115 if err := yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { 116 return fmt.Errorf("failed unmarshaling %s : %s", parser_cfg_file, err) 117 } 118 119 pnodes, err = LoadStages(parser_configs, pctx, ectx) 120 if err != nil { 121 return fmt.Errorf("unable to load parser config : %s", err) 122 } 123 124 //TBD: Load post overflows 125 //func testFile(t *testing.T, file string, pctx UnixParserCtx, nodes []Node) bool { 126 parser_test_file := fmt.Sprintf("%s/test.yaml", dir) 127 tests := loadTestFile(parser_test_file) 128 count := 1 129 if b != nil { 130 count = b.N 131 b.ResetTimer() 132 } 133 for n := 0; n < count; n++ { 134 if testFile(tests, *pctx, pnodes) != true { 135 return fmt.Errorf("test failed !") 136 } 137 } 138 return nil 139 } 140 141 //prepTests is going to do the initialisation of parser : it's going to load enrichment plugins and load the patterns. This is done here so that we don't redo it for each test 142 func prepTests() (*UnixParserCtx, []EnricherCtx, error) { 143 var ( 144 err error 145 pctx *UnixParserCtx 146 ectx []EnricherCtx 147 ) 148 149 err = exprhelpers.Init() 150 if err != nil { 151 log.Fatalf("exprhelpers init failed: %s", err) 152 } 153 154 //Load enrichment 155 datadir := "./test_data/" 156 ectx, err = Loadplugin(datadir) 157 if err != nil { 158 log.Fatalf("failed to load plugin geoip : %v", err) 159 } 160 log.Printf("Loaded -> %+v", ectx) 161 162 //Load the parser patterns 163 cfgdir := "../../config/" 164 165 /* this should be refactored to 2 lines :p */ 166 // Init the parser 167 pctx, err = Init(map[string]interface{}{"patterns": cfgdir + string("/patterns/"), "data": "./tests/"}) 168 if err != nil { 169 return nil, nil, fmt.Errorf("failed to initialize parser : %v", err) 170 } 171 return pctx, ectx, nil 172 } 173 174 func loadTestFile(file string) []TestFile { 175 yamlFile, err := os.Open(file) 176 if err != nil { 177 log.Fatalf("yamlFile.Get err #%v ", err) 178 } 179 dec := yaml.NewDecoder(yamlFile) 180 dec.SetStrict(true) 181 var testSet []TestFile 182 for { 183 tf := TestFile{} 184 err := dec.Decode(&tf) 185 if err != nil { 186 if err == io.EOF { 187 break 188 } 189 log.Fatalf("Failed to load testfile '%s' yaml error : %v", file, err) 190 return nil 191 } 192 testSet = append(testSet, tf) 193 } 194 return testSet 195 } 196 197 func matchEvent(expected types.Event, out types.Event, debug bool) ([]string, bool) { 198 var retInfo []string 199 var valid bool = false 200 expectMaps := []map[string]string{expected.Parsed, expected.Meta, expected.Enriched} 201 outMaps := []map[string]string{out.Parsed, out.Meta, out.Enriched} 202 outLabels := []string{"Parsed", "Meta", "Enriched"} 203 204 //allow to check as well for stage and processed flags 205 if expected.Stage != "" { 206 if expected.Stage != out.Stage { 207 if debug { 208 retInfo = append(retInfo, fmt.Sprintf("mismatch stage %s != %s", expected.Stage, out.Stage)) 209 } 210 goto checkFinished 211 } else { 212 valid = true 213 if debug { 214 retInfo = append(retInfo, fmt.Sprintf("ok stage %s == %s", expected.Stage, out.Stage)) 215 } 216 } 217 } 218 219 if expected.Process != out.Process { 220 if debug { 221 retInfo = append(retInfo, fmt.Sprintf("mismatch process %t != %t", expected.Process, out.Process)) 222 } 223 goto checkFinished 224 } else { 225 valid = true 226 if debug { 227 retInfo = append(retInfo, fmt.Sprintf("ok process %t == %t", expected.Process, out.Process)) 228 } 229 } 230 231 if expected.Whitelisted != out.Whitelisted { 232 if debug { 233 retInfo = append(retInfo, fmt.Sprintf("mismatch whitelist %t != %t", expected.Whitelisted, out.Whitelisted)) 234 } 235 goto checkFinished 236 } else { 237 if debug { 238 retInfo = append(retInfo, fmt.Sprintf("ok whitelist %t == %t", expected.Whitelisted, out.Whitelisted)) 239 } 240 valid = true 241 } 242 243 for mapIdx := 0; mapIdx < len(expectMaps); mapIdx++ { 244 for expKey, expVal := range expectMaps[mapIdx] { 245 if outVal, ok := outMaps[mapIdx][expKey]; ok { 246 if outVal == expVal { //ok entry 247 if debug { 248 retInfo = append(retInfo, fmt.Sprintf("ok %s[%s] %s == %s", outLabels[mapIdx], expKey, expVal, outVal)) 249 } 250 valid = true 251 } else { //mismatch entry 252 if debug { 253 retInfo = append(retInfo, fmt.Sprintf("mismatch %s[%s] %s != %s", outLabels[mapIdx], expKey, expVal, outVal)) 254 } 255 goto checkFinished 256 } 257 } else { //missing entry 258 if debug { 259 retInfo = append(retInfo, fmt.Sprintf("missing entry %s[%s]", outLabels[mapIdx], expKey)) 260 } 261 valid = false 262 goto checkFinished 263 } 264 } 265 } 266 checkFinished: 267 if valid { 268 if debug { 269 retInfo = append(retInfo, fmt.Sprintf("OK ! %s", strings.Join(retInfo, "/"))) 270 } 271 } else { 272 if debug { 273 retInfo = append(retInfo, fmt.Sprintf("KO ! %s", strings.Join(retInfo, "/"))) 274 } 275 } 276 return retInfo, valid 277 } 278 279 func testSubSet(testSet TestFile, pctx UnixParserCtx, nodes []Node) (bool, error) { 280 var results []types.Event 281 282 for _, in := range testSet.Lines { 283 out, err := Parse(pctx, in, nodes) 284 if err != nil { 285 log.Errorf("Failed to process %s : %v", spew.Sdump(in), err) 286 } 287 //log.Infof("Parser output : %s", spew.Sdump(out)) 288 results = append(results, out) 289 } 290 log.Infof("parsed %d lines", len(testSet.Lines)) 291 log.Infof("got %d results", len(results)) 292 293 /* 294 check the results we got against the expected ones 295 only the keys of the expected part are checked against result 296 */ 297 if len(testSet.Results) == 0 && len(results) == 0 { 298 log.Fatalf("No results, no tests, abort.") 299 return false, fmt.Errorf("no tests, no results") 300 } 301 302 reCheck: 303 failinfo := []string{} 304 for ridx, result := range results { 305 for eidx, expected := range testSet.Results { 306 explain, match := matchEvent(expected, result, debug) 307 if match == true { 308 log.Infof("expected %d/%d matches result %d/%d", eidx, len(testSet.Results), ridx, len(results)) 309 if len(explain) > 0 { 310 log.Printf("-> %s", explain[len(explain)-1]) 311 } 312 //don't do this at home : delete current element from list and redo 313 results[len(results)-1], results[ridx] = results[ridx], results[len(results)-1] 314 results = results[:len(results)-1] 315 316 testSet.Results[len(testSet.Results)-1], testSet.Results[eidx] = testSet.Results[eidx], testSet.Results[len(testSet.Results)-1] 317 testSet.Results = testSet.Results[:len(testSet.Results)-1] 318 319 goto reCheck 320 } else { 321 failinfo = append(failinfo, explain...) 322 } 323 } 324 } 325 if len(results) > 0 { 326 log.Printf("Errors : %s", strings.Join(failinfo, " / ")) 327 return false, fmt.Errorf("leftover results : %+v", results) 328 } 329 if len(testSet.Results) > 0 { 330 log.Printf("Errors : %s", strings.Join(failinfo, " / ")) 331 return false, fmt.Errorf("leftover expected results : %+v", testSet.Results) 332 } 333 return true, nil 334 } 335 336 func testFile(testSet []TestFile, pctx UnixParserCtx, nodes []Node) bool { 337 log.Warningf("Going to process one test set") 338 for _, tf := range testSet { 339 //func testSubSet(testSet TestFile, pctx UnixParserCtx, nodes []Node) (bool, error) { 340 testOk, err := testSubSet(tf, pctx, nodes) 341 if err != nil { 342 log.Fatalf("test failed : %s", err) 343 } 344 if !testOk { 345 log.Fatalf("failed test : %+v", tf) 346 } 347 } 348 return true 349 } 350 351 /*THIS IS ONLY PRESENT TO BE ABLE TO GENERATE DOCUMENTATION OF EXISTING PATTERNS*/ 352 type Pair struct { 353 Key string 354 Value string 355 } 356 357 type PairList []Pair 358 359 func (p PairList) Len() int { return len(p) } 360 func (p PairList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 361 func (p PairList) Less(i, j int) bool { return len(p[i].Value) < len(p[j].Value) } 362 363 func TestGeneratePatternsDoc(t *testing.T) { 364 if os.Getenv("GO_WANT_TEST_DOC") != "1" { 365 return 366 } 367 368 pctx, err := Init(map[string]interface{}{"patterns": "../../config/patterns/", "data": "./tests/"}) 369 if err != nil { 370 t.Fatalf("unable to load patterns : %s", err) 371 } 372 log.Infof("-> %s", spew.Sdump(pctx)) 373 /*don't judge me, we do it for the users*/ 374 p := make(PairList, len(pctx.Grok)) 375 376 i := 0 377 for key, val := range pctx.Grok { 378 p[i] = Pair{key, val} 379 p[i].Value = strings.Replace(p[i].Value, "{%{", "\\{\\%\\{", -1) 380 i++ 381 } 382 sort.Sort(p) 383 384 f, err := os.OpenFile("./patterns-documentation.md", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) 385 if err != nil { 386 t.Fatalf("failed to open : %s", err) 387 } 388 if _, err := f.WriteString("# Patterns documentation\n\n"); err != nil { 389 t.Fatal("failed to write to file") 390 } 391 if _, err := f.WriteString("You will find here a generated documentation of all the patterns loaded by synsec.\n"); err != nil { 392 t.Fatal("failed to write to file") 393 } 394 if _, err := f.WriteString("They are sorted by pattern length, and are meant to be used in parsers, in the form %{PATTERN_NAME}.\n"); err != nil { 395 t.Fatal("failed to write to file") 396 } 397 if _, err := f.WriteString("\n\n"); err != nil { 398 t.Fatal("failed to write to file") 399 } 400 for _, k := range p { 401 if _, err := f.WriteString(fmt.Sprintf("## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value)); err != nil { 402 t.Fatal("failed to write to file") 403 } 404 fmt.Printf("%v\t%v\n", k.Key, k.Value) 405 } 406 if _, err := f.WriteString("\n"); err != nil { 407 t.Fatal("failed to write to file") 408 } 409 if _, err := f.WriteString("# Documentation generation\n"); err != nil { 410 t.Fatal("failed to write to file") 411 } 412 if _, err := f.WriteString("This documentation is generated by `pkg/parser` : `GO_WANT_TEST_DOC=1 go test -run TestGeneratePatternsDoc`\n"); err != nil { 413 t.Fatal("failed to write to file") 414 } 415 f.Close() 416 417 }