github.com/Jeffail/benthos/v3@v3.65.0/internal/metadata/include_filter.go (about)

     1  package metadata
     2  
     3  import (
     4  	"fmt"
     5  	"regexp"
     6  	"strings"
     7  
     8  	"github.com/Jeffail/benthos/v3/internal/docs"
     9  	"github.com/Jeffail/benthos/v3/lib/types"
    10  )
    11  
    12  // IncludeFilterDocs returns a docs spec for a metadata filter where keys are
    13  // ignored by default and must be explicitly included.
    14  func IncludeFilterDocs() docs.FieldSpecs {
    15  	return docs.FieldSpecs{
    16  		docs.FieldString(
    17  			"include_prefixes", "Provide a list of explicit metadata key prefixes to match against.",
    18  			[]string{"foo_", "bar_"},
    19  			[]string{"kafka_"},
    20  			[]string{"content-"},
    21  		).Array(),
    22  		docs.FieldString(
    23  			"include_patterns", "Provide a list of explicit metadata key regular expression (re2) patterns to match against.",
    24  			[]string{".*"},
    25  			[]string{"_timestamp_unix$"},
    26  		).Array(),
    27  	}
    28  }
    29  
    30  // IncludeFilterConfig contains configuration fields for a metadata filter where
    31  // keys are ignored by default and must be explicitly included.
    32  type IncludeFilterConfig struct {
    33  	IncludePrefixes []string `json:"include_prefixes" yaml:"include_prefixes"`
    34  	IncludePatterns []string `json:"include_patterns" yaml:"include_patterns"`
    35  }
    36  
    37  // NewIncludeFilterConfig returns an IncludeFilterConfig struct with default
    38  // values.
    39  func NewIncludeFilterConfig() IncludeFilterConfig {
    40  	return IncludeFilterConfig{
    41  		IncludePrefixes: []string{},
    42  		IncludePatterns: []string{},
    43  	}
    44  }
    45  
    46  // CreateFilter attempts to construct a filter object.
    47  func (c IncludeFilterConfig) CreateFilter() (*IncludeFilter, error) {
    48  	var includePatterns []*regexp.Regexp
    49  	for _, pattern := range c.IncludePatterns {
    50  		compiledPattern, err := regexp.Compile(pattern)
    51  		if err != nil {
    52  			return nil, fmt.Errorf("failed to compile regexp %q: %s", pattern, err)
    53  		}
    54  		includePatterns = append(includePatterns, compiledPattern)
    55  	}
    56  	return &IncludeFilter{
    57  		includePrefixes: c.IncludePrefixes,
    58  		includePatterns: includePatterns,
    59  	}, nil
    60  }
    61  
    62  // IncludeFilter provides a way to filter keys based on a Config.
    63  type IncludeFilter struct {
    64  	includePrefixes []string
    65  	includePatterns []*regexp.Regexp
    66  }
    67  
    68  // IsSet returns true if there are any rules configured for matching keys.
    69  func (f *IncludeFilter) IsSet() bool {
    70  	return len(f.includePrefixes) > 0 || len(f.includePatterns) > 0
    71  }
    72  
    73  // Match returns true if the provided string matches the configured filters and
    74  // false otherwise. It also returns false if no filters are configured.
    75  func (f *IncludeFilter) Match(str string) bool {
    76  	for _, prefix := range f.includePrefixes {
    77  		if strings.HasPrefix(str, prefix) {
    78  			return true
    79  		}
    80  	}
    81  	for _, pattern := range f.includePatterns {
    82  		if matched := pattern.MatchString(str); matched {
    83  			return true
    84  		}
    85  	}
    86  	return false
    87  }
    88  
    89  // Iter applies a function to each metadata key value pair that passes the
    90  // filter.
    91  func (f *IncludeFilter) Iter(m types.Metadata, fn func(k, v string) error) error {
    92  	return m.Iter(func(k, v string) error {
    93  		if !f.Match(k) {
    94  			return nil
    95  		}
    96  		return fn(k, v)
    97  	})
    98  }