github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/blocksconvert/scanner/scanner_processor.go (about)

     1  package scanner
     2  
     3  import (
     4  	"path/filepath"
     5  	"regexp"
     6  	"strconv"
     7  
     8  	"github.com/pkg/errors"
     9  	"github.com/prometheus/client_golang/prometheus"
    10  
    11  	"github.com/cortexproject/cortex/pkg/chunk"
    12  	"github.com/cortexproject/cortex/tools/blocksconvert"
    13  )
    14  
    15  // Results from processor are passed to this function.
    16  type planEntryFn func(dir string, file string, entry blocksconvert.PlanEntry, header func() blocksconvert.PlanEntry) error
    17  
    18  // Processor implements IndexEntryProcessor. It caches chunks for single series until it finds
    19  // that another series has arrived, at which point it writes it to the file.
    20  // IndexReader guarantees correct order of entries.
    21  type processor struct {
    22  	dir      string
    23  	resultFn planEntryFn
    24  
    25  	series  prometheus.Counter
    26  	scanned *prometheus.CounterVec
    27  
    28  	allowedUsers      blocksconvert.AllowedUsers
    29  	ignoredUsersRegex *regexp.Regexp
    30  	ignoredUsers      map[string]struct{}
    31  	ignoredEntries    prometheus.Counter
    32  
    33  	lastKey key
    34  	chunks  []string
    35  }
    36  
    37  // Key is full series ID, used by processor to find out whether subsequent index entries belong to the same series
    38  // or not.
    39  type key struct {
    40  	user     string
    41  	dayIndex int
    42  	seriesID string
    43  }
    44  
    45  func newProcessor(dir string, resultFn planEntryFn, allowed blocksconvert.AllowedUsers, ignoredUsers *regexp.Regexp, series prometheus.Counter, scannedEntries *prometheus.CounterVec, ignoredEntries prometheus.Counter) *processor {
    46  	w := &processor{
    47  		dir:      dir,
    48  		resultFn: resultFn,
    49  		series:   series,
    50  		scanned:  scannedEntries,
    51  
    52  		allowedUsers:      allowed,
    53  		ignoredUsersRegex: ignoredUsers,
    54  		ignoredUsers:      map[string]struct{}{},
    55  		ignoredEntries:    ignoredEntries,
    56  	}
    57  
    58  	return w
    59  }
    60  
    61  func (w *processor) ProcessIndexEntry(indexEntry chunk.IndexEntry) error {
    62  	switch {
    63  	case IsMetricToSeriesMapping(indexEntry.RangeValue):
    64  		w.scanned.WithLabelValues("metric-to-series").Inc()
    65  		return nil
    66  
    67  	case IsMetricLabelToLabelValueMapping(indexEntry.RangeValue):
    68  		w.scanned.WithLabelValues("metric-label-to-label-value").Inc()
    69  		return nil
    70  
    71  	case IsSeriesToLabelValues(indexEntry.RangeValue):
    72  		w.scanned.WithLabelValues("series-to-label-values").Inc()
    73  		return nil
    74  
    75  	case IsSeriesToChunkMapping(indexEntry.RangeValue):
    76  		w.scanned.WithLabelValues("series-to-chunk").Inc()
    77  		// We will process these, don't return yet.
    78  
    79  	default:
    80  		// Should not happen.
    81  		w.scanned.WithLabelValues("unknown-" + UnknownIndexEntryType(indexEntry.RangeValue)).Inc()
    82  		return nil
    83  	}
    84  
    85  	user, index, seriesID, chunkID, err := GetSeriesToChunkMapping(indexEntry.HashValue, indexEntry.RangeValue)
    86  	if err != nil {
    87  		return err
    88  	}
    89  
    90  	if !w.AcceptUser(user) {
    91  		return nil
    92  	}
    93  
    94  	k := key{
    95  		user:     user,
    96  		dayIndex: index,
    97  		seriesID: seriesID,
    98  	}
    99  
   100  	if w.lastKey != k && len(w.chunks) > 0 {
   101  		err := w.Flush()
   102  		if err != nil {
   103  			return errors.Wrap(err, "failed to flush chunks")
   104  		}
   105  	}
   106  
   107  	w.lastKey = k
   108  	w.chunks = append(w.chunks, chunkID)
   109  	return nil
   110  }
   111  
   112  func (w *processor) AcceptUser(user string) bool {
   113  	if _, found := w.ignoredUsers[user]; found {
   114  		w.ignoredEntries.Inc()
   115  		return false
   116  	}
   117  	if !w.allowedUsers.IsAllowed(user) || (w.ignoredUsersRegex != nil && w.ignoredUsersRegex.MatchString(user)) {
   118  		w.ignoredEntries.Inc()
   119  		w.ignoredUsers[user] = struct{}{}
   120  		return false
   121  	}
   122  	return true
   123  }
   124  
   125  func (w *processor) Flush() error {
   126  	if len(w.chunks) == 0 {
   127  		return nil
   128  	}
   129  
   130  	k := w.lastKey
   131  
   132  	err := w.resultFn(filepath.Join(w.dir, k.user), strconv.Itoa(k.dayIndex)+".plan", blocksconvert.PlanEntry{
   133  		SeriesID: w.lastKey.seriesID,
   134  		Chunks:   w.chunks,
   135  	}, func() blocksconvert.PlanEntry {
   136  		return blocksconvert.PlanEntry{
   137  			User:     k.user,
   138  			DayIndex: k.dayIndex,
   139  		}
   140  	})
   141  
   142  	if err != nil {
   143  		return err
   144  	}
   145  
   146  	w.series.Inc()
   147  	w.chunks = nil
   148  	return nil
   149  }