bitbucket.org/Aishee/synsec@v0.0.0-20210414005726-236fc01a153d/pkg/parser/stage.go (about)

     1  package parser
     2  
     3  /*
     4   This file contains
     5   - the runtime definition of parser
     6   - the compilation/parsing routines of parser configuration
     7  */
     8  
     9  import (
    10  	//"fmt"
    11  
    12  	"fmt"
    13  	"io"
    14  	_ "net/http/pprof"
    15  	"os"
    16  	"sort"
    17  	"strings"
    18  	"time"
    19  
    20  	"bitbucket.org/Aishee/synsec/pkg/cwversion"
    21  	"bitbucket.org/Aishee/synsec/pkg/exprhelpers"
    22  
    23  	log "github.com/sirupsen/logrus"
    24  
    25  	"github.com/goombaio/namegenerator"
    26  	yaml "gopkg.in/yaml.v2"
    27  )
    28  
    29  var seed namegenerator.Generator = namegenerator.NewNameGenerator(time.Now().UTC().UnixNano())
    30  
    31  /*
    32   identify generic component to alter maps, smartfilters ? (static, conditional static etc.)
    33  */
    34  
    35  type Stagefile struct {
    36  	Filename string `yaml:"filename"`
    37  	Stage    string `yaml:"stage"`
    38  }
    39  
    40  func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx []EnricherCtx) ([]Node, error) {
    41  	var nodes []Node
    42  	tmpstages := make(map[string]bool)
    43  	pctx.Stages = []string{}
    44  
    45  	for _, stageFile := range stageFiles {
    46  		if !strings.HasSuffix(stageFile.Filename, ".yaml") {
    47  			log.Warningf("skip non yaml : %s", stageFile.Filename)
    48  			continue
    49  		}
    50  		log.Debugf("loading parser file '%s'", stageFile)
    51  		st, err := os.Stat(stageFile.Filename)
    52  		if err != nil {
    53  			return nil, fmt.Errorf("failed to stat %s : %v", stageFile, err)
    54  		}
    55  		if st.IsDir() {
    56  			continue
    57  		}
    58  		yamlFile, err := os.Open(stageFile.Filename)
    59  		if err != nil {
    60  			return nil, fmt.Errorf("can't access parsing configuration file %s : %s", stageFile.Filename, err)
    61  		}
    62  		//process the yaml
    63  		dec := yaml.NewDecoder(yamlFile)
    64  		dec.SetStrict(true)
    65  		nodesCount := 0
    66  		for {
    67  			node := Node{}
    68  			node.OnSuccess = "continue" //default behaviour is to continue
    69  			err = dec.Decode(&node)
    70  			if err != nil {
    71  				if err == io.EOF {
    72  					log.Tracef("End of yaml file")
    73  					break
    74  				}
    75  				log.Fatalf("Error decoding parsing configuration file '%s': %v", stageFile.Filename, err)
    76  			}
    77  
    78  			//check for empty bucket
    79  			if node.Name == "" && node.Description == "" && node.Author == "" {
    80  				log.Infof("Node in %s has no name,author or description. Skipping.", stageFile.Filename)
    81  				continue
    82  			}
    83  			//check compat
    84  			if node.FormatVersion == "" {
    85  				log.Tracef("no version in %s, assuming '1.0'", node.Name)
    86  				node.FormatVersion = "1.0"
    87  			}
    88  			ok, err := cwversion.Statisfies(node.FormatVersion, cwversion.Constraint_parser)
    89  			if err != nil {
    90  				log.Fatalf("Failed to check version : %s", err)
    91  			}
    92  			if !ok {
    93  				log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, cwversion.Constraint_parser)
    94  				continue
    95  			}
    96  
    97  			node.Stage = stageFile.Stage
    98  			if _, ok := tmpstages[stageFile.Stage]; !ok {
    99  				tmpstages[stageFile.Stage] = true
   100  			}
   101  			//compile the node : grok pattern and expression
   102  			err = node.compile(pctx, ectx)
   103  			if err != nil {
   104  				if node.Name != "" {
   105  					return nil, fmt.Errorf("failed to compile node '%s' in '%s' : %s", node.Name, stageFile.Filename, err.Error())
   106  				}
   107  				return nil, fmt.Errorf("failed to compile node in '%s' : %s", stageFile.Filename, err.Error())
   108  			}
   109  			/* if the stage is empty, the node is empty, it's a trailing entry in users yaml file */
   110  			if node.Stage == "" {
   111  				continue
   112  			}
   113  
   114  			if len(node.Data) > 0 {
   115  				for _, data := range node.Data {
   116  					err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type)
   117  					if err != nil {
   118  						log.Errorf(err.Error())
   119  					}
   120  				}
   121  			}
   122  			nodes = append(nodes, node)
   123  			nodesCount++
   124  		}
   125  		log.WithFields(log.Fields{"file": stageFile.Filename}).Infof("Loaded %d parser nodes", nodesCount)
   126  	}
   127  
   128  	for k := range tmpstages {
   129  		pctx.Stages = append(pctx.Stages, k)
   130  	}
   131  	sort.Strings(pctx.Stages)
   132  	log.Infof("Loaded %d nodes, %d stages", len(nodes), len(pctx.Stages))
   133  
   134  	return nodes, nil
   135  }