github.com/crowdsecurity/crowdsec@v1.6.1/pkg/parser/unix_parser.go (about)

     1  package parser
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  	"path/filepath"
     7  	"sort"
     8  	"strings"
     9  
    10  	log "github.com/sirupsen/logrus"
    11  
    12  	"github.com/crowdsecurity/grokky"
    13  
    14  	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
    15  	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
    16  	"github.com/crowdsecurity/crowdsec/pkg/fflag"
    17  )
    18  
    19  type UnixParserCtx struct {
    20  	Grok       grokky.Host
    21  	Stages     []string
    22  	Profiling  bool
    23  	DataFolder string
    24  }
    25  
    26  type Parsers struct {
    27  	Ctx             *UnixParserCtx
    28  	Povfwctx        *UnixParserCtx
    29  	StageFiles      []Stagefile
    30  	PovfwStageFiles []Stagefile
    31  	Nodes           []Node
    32  	Povfwnodes      []Node
    33  	EnricherCtx     EnricherCtx
    34  }
    35  
    36  func Init(c map[string]interface{}) (*UnixParserCtx, error) {
    37  	r := UnixParserCtx{}
    38  	r.Grok = grokky.NewBase()
    39  	r.Grok.UseRe2 = fflag.Re2GrokSupport.IsEnabled()
    40  	files, err := os.ReadDir(c["patterns"].(string))
    41  	if err != nil {
    42  		return nil, err
    43  	}
    44  	r.DataFolder = c["data"].(string)
    45  	for _, f := range files {
    46  		if strings.Contains(f.Name(), ".") {
    47  			continue
    48  		}
    49  		if err := r.Grok.AddFromFile(filepath.Join(c["patterns"].(string), f.Name())); err != nil {
    50  			log.Errorf("failed to load pattern %s : %v", f.Name(), err)
    51  			return nil, err
    52  		}
    53  	}
    54  	log.Debugf("Loaded %d pattern files", len(files))
    55  	return &r, nil
    56  }
    57  
    58  // Return new parsers
    59  // nodes and povfwnodes are already initialized in parser.LoadStages
    60  func NewParsers(hub *cwhub.Hub) *Parsers {
    61  	parsers := &Parsers{
    62  		Ctx:             &UnixParserCtx{},
    63  		Povfwctx:        &UnixParserCtx{},
    64  		StageFiles:      make([]Stagefile, 0),
    65  		PovfwStageFiles: make([]Stagefile, 0),
    66  	}
    67  
    68  	for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
    69  		for _, hubParserItem := range hub.GetItemMap(itemType) {
    70  			if hubParserItem.State.Installed {
    71  				stagefile := Stagefile{
    72  					Filename: hubParserItem.State.LocalPath,
    73  					Stage:    hubParserItem.Stage,
    74  				}
    75  				if itemType == cwhub.PARSERS {
    76  					parsers.StageFiles = append(parsers.StageFiles, stagefile)
    77  				}
    78  				if itemType == cwhub.POSTOVERFLOWS {
    79  					parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
    80  				}
    81  			}
    82  		}
    83  	}
    84  	if parsers.StageFiles != nil {
    85  		sort.Slice(parsers.StageFiles, func(i, j int) bool {
    86  			return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
    87  		})
    88  	}
    89  	if parsers.PovfwStageFiles != nil {
    90  		sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
    91  			return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
    92  		})
    93  	}
    94  
    95  	return parsers
    96  }
    97  
    98  func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
    99  	var err error
   100  
   101  	patternsDir := cConfig.ConfigPaths.PatternDir
   102  	log.Infof("Loading grok library %s", patternsDir)
   103  	/* load base regexps for two grok parsers */
   104  	parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
   105  		"data": cConfig.ConfigPaths.DataDir})
   106  	if err != nil {
   107  		return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
   108  	}
   109  	parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
   110  		"data": cConfig.ConfigPaths.DataDir})
   111  	if err != nil {
   112  		return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
   113  	}
   114  
   115  	/*
   116  		Load enrichers
   117  	*/
   118  	log.Infof("Loading enrich plugins")
   119  
   120  	parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
   121  	if err != nil {
   122  		return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
   123  	}
   124  
   125  	/*
   126  	 Load the actual parsers
   127  	*/
   128  
   129  	log.Infof("Loading parsers from %d files", len(parsers.StageFiles))
   130  
   131  	parsers.Nodes, err = LoadStages(parsers.StageFiles, parsers.Ctx, parsers.EnricherCtx)
   132  	if err != nil {
   133  		return parsers, fmt.Errorf("failed to load parser config : %v", err)
   134  	}
   135  
   136  	if len(parsers.PovfwStageFiles) > 0 {
   137  		log.Infof("Loading postoverflow parsers")
   138  		parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx)
   139  	} else {
   140  		log.Infof("No postoverflow parsers to load")
   141  		parsers.Povfwnodes = []Node{}
   142  	}
   143  
   144  	if err != nil {
   145  		return parsers, fmt.Errorf("failed to load postoverflow config : %v", err)
   146  	}
   147  
   148  	if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
   149  		parsers.Ctx.Profiling = true
   150  		parsers.Povfwctx.Profiling = true
   151  	}
   152  	/*
   153  		Reset CTX grok to reduce memory footprint after we compile all the patterns
   154  	*/
   155  	parsers.Ctx.Grok = grokky.Host{}
   156  	parsers.Povfwctx.Grok = grokky.Host{}
   157  	parsers.StageFiles = []Stagefile{}
   158  	parsers.PovfwStageFiles = []Stagefile{}
   159  	return parsers, nil
   160  }