github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/blocksconvert/scanner/scanner_test.go (about)

     1  package scanner
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"os"
     8  	"path/filepath"
     9  	"strings"
    10  	"testing"
    11  
    12  	"github.com/go-kit/log"
    13  	"github.com/prometheus/client_golang/prometheus"
    14  	"github.com/stretchr/testify/require"
    15  	"github.com/thanos-io/thanos/pkg/objstore"
    16  
    17  	util_log "github.com/cortexproject/cortex/pkg/util/log"
    18  	"github.com/cortexproject/cortex/tools/blocksconvert"
    19  )
    20  
    21  func TestVerifyPlanFile(t *testing.T) {
    22  	testCases := map[string]struct {
    23  		content  string
    24  		errorMsg string
    25  	}{
    26  		"Minimum valid plan file, with no series": {
    27  			content:  `{"user": "test", "day_index": 12345}{"complete": true}`,
    28  			errorMsg: "",
    29  		},
    30  		"no header": {
    31  			content:  `{"complete": true}`,
    32  			errorMsg: "failed to read plan file header: no user or day index found",
    33  		},
    34  		"no footer": {
    35  			content:  `{"user": "test", "day_index": 12345}`,
    36  			errorMsg: "no footer found in the plan",
    37  		},
    38  		"data after footer": {
    39  			content:  `{"user": "test", "day_index": 12345}{"complete": true}{"sid": "some seriesID", "cs": ["chunk1", "chunk2"]}`,
    40  			errorMsg: "plan entries found after plan footer",
    41  		},
    42  		"valid plan with single series": {
    43  			content: `
    44  				{"user": "test", "day_index": 12345}
    45  				{"sid": "some seriesID", "cs": ["chunk1", "chunk2"]}
    46  				{"complete": true}`,
    47  			errorMsg: "",
    48  		},
    49  		"series with no chunks": {
    50  			content: `
    51  				{"user": "test", "day_index": 12345}
    52  				{"sid": "AAAAAA"}
    53  				{"complete": true}`,
    54  			errorMsg: fmt.Sprintf("entry for seriesID %s has no chunks", "AAAAAA"),
    55  		},
    56  		"multiple series entries": {
    57  			content: `
    58  				{"user": "test", "day_index": 12345}
    59  				{"sid": "AAA", "cs": ["chunk1", "chunk2"]}
    60  				{"sid": "AAA", "cs": ["chunk3", "chunk4"]}
    61  				{"complete": true}`,
    62  			errorMsg: "multiple entries for series AAA found in plan",
    63  		},
    64  	}
    65  
    66  	for name, tc := range testCases {
    67  		if tc.errorMsg == "" {
    68  			require.NoError(t, verifyPlanFile(strings.NewReader(tc.content)), name)
    69  		} else {
    70  			require.EqualError(t, verifyPlanFile(strings.NewReader(tc.content)), tc.errorMsg, name)
    71  		}
    72  	}
    73  }
    74  
    75  func TestVerifyPlansDir(t *testing.T) {
    76  	dir, err := ioutil.TempDir("", "plans")
    77  	require.NoError(t, err)
    78  	t.Cleanup(func() {
    79  		_ = os.RemoveAll(dir)
    80  	})
    81  
    82  	of := newOpenFiles(prometheus.NewGauge(prometheus.GaugeOpts{}))
    83  	// This file is checked first, and no error is reported for it.
    84  	require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{User: "user1", DayIndex: 123}, nil))
    85  	require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c1, c2"}}, nil))
    86  	require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{Complete: true}, nil))
    87  
    88  	require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{User: "user2", DayIndex: 456}, nil))
    89  	require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c1, c2"}}, nil))
    90  	require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c3, c4"}}, nil))
    91  
    92  	require.NoError(t, of.closeAllFiles(nil))
    93  
    94  	err = verifyPlanFiles(context.Background(), dir, util_log.Logger)
    95  	require.Error(t, err)
    96  	require.True(t, strings.Contains(err.Error(), "456.plan"))
    97  	require.True(t, strings.Contains(err.Error(), "multiple entries for series s1 found in plan"))
    98  }
    99  
   100  func TestUploadPlans(t *testing.T) {
   101  	dir, err := ioutil.TempDir("", "upload")
   102  	require.NoError(t, err)
   103  	t.Cleanup(func() {
   104  		_ = os.RemoveAll(dir)
   105  	})
   106  
   107  	require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1"), 0700))
   108  	require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2"), 0700))
   109  	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user1", "plan1"), []byte("plan1"), 0600))
   110  	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user1", "plan2"), []byte("plan2"), 0600))
   111  	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "plan3"), []byte("plan3"), 0600))
   112  
   113  	inmem := objstore.NewInMemBucket()
   114  
   115  	require.NoError(t, uploadPlansConcurrently(context.Background(), log.NewNopLogger(), dir, inmem, "bucket-prefix", 5))
   116  
   117  	objs := inmem.Objects()
   118  	require.Equal(t, objs, map[string][]byte{
   119  		"bucket-prefix/user1/plan1": []byte("plan1"),
   120  		"bucket-prefix/user1/plan2": []byte("plan2"),
   121  		"bucket-prefix/user2/plan3": []byte("plan3"),
   122  	})
   123  }