github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/tools/blocksconvert/scanner/scanner_test.go (about) 1 package scanner 2 3 import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "strings" 10 "testing" 11 12 "github.com/go-kit/log" 13 "github.com/prometheus/client_golang/prometheus" 14 "github.com/stretchr/testify/require" 15 "github.com/thanos-io/thanos/pkg/objstore" 16 17 util_log "github.com/cortexproject/cortex/pkg/util/log" 18 "github.com/cortexproject/cortex/tools/blocksconvert" 19 ) 20 21 func TestVerifyPlanFile(t *testing.T) { 22 testCases := map[string]struct { 23 content string 24 errorMsg string 25 }{ 26 "Minimum valid plan file, with no series": { 27 content: `{"user": "test", "day_index": 12345}{"complete": true}`, 28 errorMsg: "", 29 }, 30 "no header": { 31 content: `{"complete": true}`, 32 errorMsg: "failed to read plan file header: no user or day index found", 33 }, 34 "no footer": { 35 content: `{"user": "test", "day_index": 12345}`, 36 errorMsg: "no footer found in the plan", 37 }, 38 "data after footer": { 39 content: `{"user": "test", "day_index": 12345}{"complete": true}{"sid": "some seriesID", "cs": ["chunk1", "chunk2"]}`, 40 errorMsg: "plan entries found after plan footer", 41 }, 42 "valid plan with single series": { 43 content: ` 44 {"user": "test", "day_index": 12345} 45 {"sid": "some seriesID", "cs": ["chunk1", "chunk2"]} 46 {"complete": true}`, 47 errorMsg: "", 48 }, 49 "series with no chunks": { 50 content: ` 51 {"user": "test", "day_index": 12345} 52 {"sid": "AAAAAA"} 53 {"complete": true}`, 54 errorMsg: fmt.Sprintf("entry for seriesID %s has no chunks", "AAAAAA"), 55 }, 56 "multiple series entries": { 57 content: ` 58 {"user": "test", "day_index": 12345} 59 {"sid": "AAA", "cs": ["chunk1", "chunk2"]} 60 {"sid": "AAA", "cs": ["chunk3", "chunk4"]} 61 {"complete": true}`, 62 errorMsg: "multiple entries for series AAA found in plan", 63 }, 64 } 65 66 for name, tc := range testCases { 67 if tc.errorMsg == "" { 68 require.NoError(t, verifyPlanFile(strings.NewReader(tc.content)), name) 69 } else { 70 require.EqualError(t, verifyPlanFile(strings.NewReader(tc.content)), tc.errorMsg, name) 71 } 72 } 73 } 74 75 func TestVerifyPlansDir(t *testing.T) { 76 dir, err := ioutil.TempDir("", "plans") 77 require.NoError(t, err) 78 t.Cleanup(func() { 79 _ = os.RemoveAll(dir) 80 }) 81 82 of := newOpenFiles(prometheus.NewGauge(prometheus.GaugeOpts{})) 83 // This file is checked first, and no error is reported for it. 84 require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{User: "user1", DayIndex: 123}, nil)) 85 require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c1, c2"}}, nil)) 86 require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{Complete: true}, nil)) 87 88 require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{User: "user2", DayIndex: 456}, nil)) 89 require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c1, c2"}}, nil)) 90 require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c3, c4"}}, nil)) 91 92 require.NoError(t, of.closeAllFiles(nil)) 93 94 err = verifyPlanFiles(context.Background(), dir, util_log.Logger) 95 require.Error(t, err) 96 require.True(t, strings.Contains(err.Error(), "456.plan")) 97 require.True(t, strings.Contains(err.Error(), "multiple entries for series s1 found in plan")) 98 } 99 100 func TestUploadPlans(t *testing.T) { 101 dir, err := ioutil.TempDir("", "upload") 102 require.NoError(t, err) 103 t.Cleanup(func() { 104 _ = os.RemoveAll(dir) 105 }) 106 107 require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1"), 0700)) 108 require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2"), 0700)) 109 require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user1", "plan1"), []byte("plan1"), 0600)) 110 require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user1", "plan2"), []byte("plan2"), 0600)) 111 require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "plan3"), []byte("plan3"), 0600)) 112 113 inmem := objstore.NewInMemBucket() 114 115 require.NoError(t, uploadPlansConcurrently(context.Background(), log.NewNopLogger(), dir, inmem, "bucket-prefix", 5)) 116 117 objs := inmem.Objects() 118 require.Equal(t, objs, map[string][]byte{ 119 "bucket-prefix/user1/plan1": []byte("plan1"), 120 "bucket-prefix/user1/plan2": []byte("plan2"), 121 "bucket-prefix/user2/plan3": []byte("plan3"), 122 }) 123 }