github.com/crowdsecurity/crowdsec@v1.6.1/pkg/parser/stage.go (about)

     1  package parser
     2  
     3  /*
     4   This file contains
     5   - the runtime definition of parser
     6   - the compilation/parsing routines of parser configuration
     7  */
     8  
     9  import (
    10  	"errors"
    11  	"fmt"
    12  	"io"
    13  	_ "net/http/pprof"
    14  	"os"
    15  	"sort"
    16  	"strings"
    17  	"time"
    18  
    19  	"github.com/goombaio/namegenerator"
    20  	log "github.com/sirupsen/logrus"
    21  	yaml "gopkg.in/yaml.v2"
    22  
    23  	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
    24  	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
    25  )
    26  
    27  var seed namegenerator.Generator = namegenerator.NewNameGenerator(time.Now().UTC().UnixNano())
    28  
    29  /*
    30   identify generic component to alter maps, smartfilters ? (static, conditional static etc.)
    31  */
    32  
    33  type Stagefile struct {
    34  	Filename string `yaml:"filename"`
    35  	Stage    string `yaml:"stage"`
    36  }
    37  
    38  func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ([]Node, error) {
    39  	var nodes []Node
    40  	tmpstages := make(map[string]bool)
    41  	pctx.Stages = []string{}
    42  
    43  	for _, stageFile := range stageFiles {
    44  		if !strings.HasSuffix(stageFile.Filename, ".yaml") && !strings.HasSuffix(stageFile.Filename, ".yml") {
    45  			log.Warningf("skip non yaml : %s", stageFile.Filename)
    46  			continue
    47  		}
    48  		log.Debugf("loading parser file '%s'", stageFile)
    49  		st, err := os.Stat(stageFile.Filename)
    50  		if err != nil {
    51  			return nil, fmt.Errorf("failed to stat %s : %v", stageFile, err)
    52  		}
    53  		if st.IsDir() {
    54  			continue
    55  		}
    56  		yamlFile, err := os.Open(stageFile.Filename)
    57  		if err != nil {
    58  			return nil, fmt.Errorf("can't access parsing configuration file %s : %s", stageFile.Filename, err)
    59  		}
    60  		defer yamlFile.Close()
    61  		//process the yaml
    62  		dec := yaml.NewDecoder(yamlFile)
    63  		dec.SetStrict(true)
    64  		nodesCount := 0
    65  		for {
    66  			node := Node{}
    67  			node.OnSuccess = "continue" //default behavior is to continue
    68  			err = dec.Decode(&node)
    69  			if err != nil {
    70  				if errors.Is(err, io.EOF) {
    71  					log.Tracef("End of yaml file")
    72  					break
    73  				}
    74  				return nil, fmt.Errorf("error decoding parsing configuration file '%s': %v", stageFile.Filename, err)
    75  			}
    76  
    77  			//check for empty bucket
    78  			if node.Name == "" && node.Description == "" && node.Author == "" {
    79  				log.Infof("Node in %s has no name, author or description. Skipping.", stageFile.Filename)
    80  				continue
    81  			}
    82  			//check compat
    83  			if node.FormatVersion == "" {
    84  				log.Tracef("no version in %s, assuming '1.0'", node.Name)
    85  				node.FormatVersion = "1.0"
    86  			}
    87  			ok, err := cwversion.Satisfies(node.FormatVersion, cwversion.Constraint_parser)
    88  			if err != nil {
    89  				return nil, fmt.Errorf("failed to check version : %s", err)
    90  			}
    91  			if !ok {
    92  				log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, cwversion.Constraint_parser)
    93  				continue
    94  			}
    95  
    96  			node.Stage = stageFile.Stage
    97  			if _, ok := tmpstages[stageFile.Stage]; !ok {
    98  				tmpstages[stageFile.Stage] = true
    99  			}
   100  			//compile the node : grok pattern and expression
   101  			err = node.compile(pctx, ectx)
   102  			if err != nil {
   103  				if node.Name != "" {
   104  					return nil, fmt.Errorf("failed to compile node '%s' in '%s' : %s", node.Name, stageFile.Filename, err)
   105  				}
   106  				return nil, fmt.Errorf("failed to compile node in '%s' : %s", stageFile.Filename, err)
   107  			}
   108  			/* if the stage is empty, the node is empty, it's a trailing entry in users yaml file */
   109  			if node.Stage == "" {
   110  				continue
   111  			}
   112  
   113  			for _, data := range node.Data {
   114  				err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type)
   115  				if err != nil {
   116  					log.Error(err)
   117  				}
   118  				if data.Type == "regexp" { //cache only makes sense for regexp
   119  					exprhelpers.RegexpCacheInit(data.DestPath, *data)
   120  				}
   121  			}
   122  
   123  			nodes = append(nodes, node)
   124  			nodesCount++
   125  		}
   126  		log.WithFields(log.Fields{"file": stageFile.Filename, "stage": stageFile.Stage}).Infof("Loaded %d parser nodes", nodesCount)
   127  	}
   128  
   129  	for k := range tmpstages {
   130  		pctx.Stages = append(pctx.Stages, k)
   131  	}
   132  	sort.Strings(pctx.Stages)
   133  	log.Infof("Loaded %d nodes from %d stages", len(nodes), len(pctx.Stages))
   134  
   135  	return nodes, nil
   136  }