github.com/crowdsecurity/crowdsec@v1.6.1/cmd/crowdsec/output.go (about)

     1  package main
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"sync"
     7  	"time"
     8  
     9  	log "github.com/sirupsen/logrus"
    10  
    11  	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
    12  	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
    13  	"github.com/crowdsecurity/crowdsec/pkg/models"
    14  	"github.com/crowdsecurity/crowdsec/pkg/parser"
    15  	"github.com/crowdsecurity/crowdsec/pkg/types"
    16  )
    17  
    18  func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
    19  	var dedupCache []*models.Alert
    20  
    21  	for idx, alert := range alerts {
    22  		log.Tracef("alert %d/%d", idx, len(alerts))
    23  		/*if we have more than one source, we need to dedup */
    24  		if len(alert.Sources) == 0 || len(alert.Sources) == 1 {
    25  			dedupCache = append(dedupCache, alert.Alert)
    26  			continue
    27  		}
    28  
    29  		for k, src := range alert.Sources {
    30  			refsrc := *alert.Alert // copy
    31  
    32  			log.Tracef("source[%s]", k)
    33  
    34  			refsrc.Source = &src
    35  			dedupCache = append(dedupCache, &refsrc)
    36  		}
    37  	}
    38  
    39  	if len(dedupCache) != len(alerts) {
    40  		log.Tracef("went from %d to %d alerts", len(alerts), len(dedupCache))
    41  	}
    42  
    43  	return dedupCache, nil
    44  }
    45  
    46  func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error {
    47  	ctx := context.Background()
    48  	alertsToPush, err := dedupAlerts(alerts)
    49  
    50  	if err != nil {
    51  		return fmt.Errorf("failed to transform alerts for api: %w", err)
    52  	}
    53  
    54  	_, _, err = client.Alerts.Add(ctx, alertsToPush)
    55  	if err != nil {
    56  		return fmt.Errorf("failed sending alert to LAPI: %w", err)
    57  	}
    58  
    59  	return nil
    60  }
    61  
    62  var bucketOverflows []types.Event
    63  
    64  func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets, postOverflowCTX parser.UnixParserCtx,
    65  	postOverflowNodes []parser.Node, client *apiclient.ApiClient) error {
    66  	var (
    67  		cache      []types.RuntimeAlert
    68  		cacheMutex sync.Mutex
    69  	)
    70  
    71  	ticker := time.NewTicker(1 * time.Second)
    72  LOOP:
    73  	for {
    74  		select {
    75  		case <-ticker.C:
    76  			if len(cache) > 0 {
    77  				cacheMutex.Lock()
    78  				cachecopy := cache
    79  				newcache := make([]types.RuntimeAlert, 0)
    80  				cache = newcache
    81  				cacheMutex.Unlock()
    82  				if err := PushAlerts(cachecopy, client); err != nil {
    83  					log.Errorf("while pushing to api : %s", err)
    84  					// just push back the events to the queue
    85  					cacheMutex.Lock()
    86  					cache = append(cache, cachecopy...)
    87  					cacheMutex.Unlock()
    88  				}
    89  			}
    90  		case <-outputsTomb.Dying():
    91  			if len(cache) > 0 {
    92  				cacheMutex.Lock()
    93  				cachecopy := cache
    94  				cacheMutex.Unlock()
    95  				if err := PushAlerts(cachecopy, client); err != nil {
    96  					log.Errorf("while pushing leftovers to api : %s", err)
    97  				}
    98  			}
    99  
   100  			break LOOP
   101  		case event := <-overflow:
   102  			/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
   103  			if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
   104  				buckets.Bucket_map.Delete(event.Overflow.Mapkey)
   105  				break
   106  			}
   107  			/* process post overflow parser nodes */
   108  			event, err := parser.Parse(postOverflowCTX, event, postOverflowNodes)
   109  			if err != nil {
   110  				return fmt.Errorf("postoverflow failed: %w", err)
   111  			}
   112  			log.Printf("%s", *event.Overflow.Alert.Message)
   113  			// if the Alert is nil, it's to signal bucket is ready for GC, don't track this
   114  			// dump after postoveflow processing to avoid missing whitelist info
   115  			if dumpStates && event.Overflow.Alert != nil {
   116  				if bucketOverflows == nil {
   117  					bucketOverflows = make([]types.Event, 0)
   118  				}
   119  				bucketOverflows = append(bucketOverflows, event)
   120  			}
   121  			if event.Overflow.Whitelisted {
   122  				log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
   123  				continue
   124  			}
   125  			if event.Overflow.Reprocess {
   126  				log.Debugf("Overflow being reprocessed.")
   127  				input <- event
   128  			}
   129  			if dumpStates {
   130  				continue
   131  			}
   132  
   133  			cacheMutex.Lock()
   134  			cache = append(cache, event.Overflow)
   135  			cacheMutex.Unlock()
   136  		}
   137  	}
   138  
   139  	ticker.Stop()
   140  
   141  	return nil
   142  }