github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/blocksconvert/scanner/scanner_processor_test.go (about)

     1  package scanner
     2  
     3  import (
     4  	"fmt"
     5  	"path"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/prometheus/client_golang/prometheus"
    10  	"github.com/prometheus/common/model"
    11  	"github.com/prometheus/prometheus/pkg/labels"
    12  	"github.com/stretchr/testify/require"
    13  
    14  	"github.com/cortexproject/cortex/pkg/chunk"
    15  	"github.com/cortexproject/cortex/tools/blocksconvert"
    16  )
    17  
    18  func TestProcessorError(t *testing.T) {
    19  	entries := map[string][]blocksconvert.PlanEntry{}
    20  	resultFn := func(dir string, file string, entry blocksconvert.PlanEntry, header func() blocksconvert.PlanEntry) error {
    21  		full := path.Join(dir, file)
    22  
    23  		entries[full] = append(entries[full], entry)
    24  		return nil
    25  	}
    26  
    27  	p := newProcessor("output", resultFn, nil, nil, prometheus.NewCounter(prometheus.CounterOpts{}), prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"type"}), prometheus.NewCounter(prometheus.CounterOpts{}))
    28  
    29  	// Day 18500
    30  	now := time.Unix(1598456178, 0)
    31  	startOfDay := model.TimeFromUnixNano(now.Truncate(24 * time.Hour).UnixNano())
    32  
    33  	pc := chunk.PeriodConfig{
    34  		From:       chunk.DayTime{Time: startOfDay},
    35  		IndexType:  "bigtable",
    36  		ObjectType: "gcs",
    37  		Schema:     "v9",
    38  		IndexTables: chunk.PeriodicTableConfig{
    39  			Prefix: "index_",
    40  			Period: 7 * 24 * time.Hour,
    41  		},
    42  		ChunkTables: chunk.PeriodicTableConfig{
    43  			Prefix: "chunks_",
    44  			Period: 7 * 24 * time.Hour,
    45  		},
    46  	}
    47  	schema, err := pc.CreateSchema()
    48  	require.NoError(t, err)
    49  
    50  	sschema := schema.(chunk.SeriesStoreSchema)
    51  
    52  	// Label write entries are ignored by the processor
    53  	{
    54  		_, ies, err := sschema.GetCacheKeysAndLabelWriteEntries(startOfDay.Add(1*time.Hour), startOfDay.Add(2*time.Hour), "testUser", "test_metric", labels.Labels{
    55  			{Name: "__name__", Value: "test_metric"},
    56  		}, "chunkID")
    57  		require.NoError(t, err)
    58  		for _, es := range ies {
    59  			passEntriesToProcessor(t, p, es)
    60  		}
    61  	}
    62  
    63  	// Processor expects all entries for same series to arrive in sequence, before receiving different series. BigTable reader guarantees that.
    64  	{
    65  		es, err := sschema.GetChunkWriteEntries(startOfDay.Add(2*time.Hour), startOfDay.Add(3*time.Hour), "testUser", "test_metric", labels.Labels{
    66  			{Name: "__name__", Value: "test_metric"},
    67  		}, "chunkID_1")
    68  		require.NoError(t, err)
    69  		passEntriesToProcessor(t, p, es)
    70  	}
    71  
    72  	{
    73  		es, err := sschema.GetChunkWriteEntries(startOfDay.Add(23*time.Hour), startOfDay.Add(25*time.Hour), "testUser", "test_metric", labels.Labels{
    74  			{Name: "__name__", Value: "test_metric"},
    75  		}, "chunkID_2")
    76  		require.NoError(t, err)
    77  		passEntriesToProcessor(t, p, es)
    78  	}
    79  
    80  	{
    81  		es, err := sschema.GetChunkWriteEntries(startOfDay.Add(5*time.Hour), startOfDay.Add(6*time.Hour), "testUser", "different_metric", labels.Labels{
    82  			{Name: "__name__", Value: "different_metric"},
    83  		}, "chunkID_5")
    84  		require.NoError(t, err)
    85  		passEntriesToProcessor(t, p, es)
    86  	}
    87  
    88  	require.NoError(t, p.Flush())
    89  
    90  	// Now let's compare what we have received.
    91  	require.Equal(t, map[string][]blocksconvert.PlanEntry{
    92  		"output/testUser/18500.plan": {
    93  			{
    94  				SeriesID: "eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
    95  				Chunks:   []string{"chunkID_1", "chunkID_2"},
    96  			},
    97  			{
    98  				SeriesID: "+afMmul/w5PDAAWGPd7y8+xyBq5IN+Q2/ZnPnrMEI+k",
    99  				Chunks:   []string{"chunkID_5"},
   100  			},
   101  		},
   102  
   103  		"output/testUser/18501.plan": {
   104  			{
   105  				SeriesID: "eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M",
   106  				Chunks:   []string{"chunkID_2"},
   107  			},
   108  		},
   109  	}, entries)
   110  }
   111  
   112  func passEntriesToProcessor(t *testing.T, p *processor, es []chunk.IndexEntry) {
   113  	for _, ie := range es {
   114  		fmt.Printf("%q %q %q\n", ie.HashValue, ie.RangeValue, ie.Value)
   115  
   116  		require.NoError(t, p.ProcessIndexEntry(ie))
   117  	}
   118  }