github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/pacer_test.go (about) 1 // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package pebble 6 7 import ( 8 "fmt" 9 "math" 10 "math/rand" 11 "sort" 12 "testing" 13 "time" 14 15 "github.com/stretchr/testify/require" 16 ) 17 18 func TestDeletionPacer(t *testing.T) { 19 const MB = 1 << 20 20 const GB = 1 << 30 21 testCases := []struct { 22 freeBytes uint64 23 obsoleteBytes uint64 24 liveBytes uint64 25 // history of deletion reporting; first value in the pair is the time, 26 // second value is the deleted bytes. The time of pacing is the same as the 27 // last time in the history. 28 history [][2]int64 29 // expected pacing rate in MB/s. 30 expected float64 31 }{ 32 { 33 freeBytes: 160 * GB, 34 obsoleteBytes: 1 * MB, 35 liveBytes: 160 * MB, 36 expected: 100.0, 37 }, 38 // As freeBytes is 2GB below the free space threshold, rate should be 39 // increased by 204.8MB/s. 40 { 41 freeBytes: 14 * GB, 42 obsoleteBytes: 1 * MB, 43 liveBytes: 100 * MB, 44 expected: 304.8, 45 }, 46 // As freeBytes is 10GB below the free space threshold, rate should be 47 // increased to by 1GB/s. 48 { 49 freeBytes: 6 * GB, 50 obsoleteBytes: 1 * MB, 51 liveBytes: 100 * MB, 52 expected: 1124.0, 53 }, 54 // obsoleteBytesRatio is 50%. We need to delete 30GB within 5 minutes. 55 { 56 freeBytes: 500 * GB, 57 obsoleteBytes: 50 * GB, 58 liveBytes: 100 * GB, 59 expected: 202.4, 60 }, 61 // When obsolete ratio unknown, there should be no throttling. 62 { 63 freeBytes: 500 * GB, 64 obsoleteBytes: 0, 65 liveBytes: 0, 66 expected: math.Inf(1), 67 }, 68 // History shows 200MB/sec deletions on average over last 5 minutes. 69 { 70 freeBytes: 160 * GB, 71 obsoleteBytes: 1 * MB, 72 liveBytes: 160 * MB, 73 history: [][2]int64{{0, 5 * 60 * 200 * MB}}, 74 expected: 200.0, 75 }, 76 // History shows 200MB/sec deletions on average over last 5 minutes and 77 // freeBytes is 10GB below the threshold. 78 { 79 freeBytes: 6 * GB, 80 obsoleteBytes: 1 * MB, 81 liveBytes: 160 * MB, 82 history: [][2]int64{{0, 5 * 60 * 200 * MB}}, 83 expected: 1224.0, 84 }, 85 // History shows 200MB/sec deletions on average over last 5 minutes and 86 // obsoleteBytesRatio is 50%. 87 { 88 freeBytes: 500 * GB, 89 obsoleteBytes: 50 * GB, 90 liveBytes: 100 * GB, 91 history: [][2]int64{{0, 5 * 60 * 200 * MB}}, 92 expected: 302.4, 93 }, 94 // History shows 1000MB/sec deletions on average over last 5 minutes. 95 { 96 freeBytes: 160 * GB, 97 obsoleteBytes: 1 * MB, 98 liveBytes: 160 * MB, 99 history: [][2]int64{{0, 60 * 1000 * MB}, {3 * 60, 60 * 4 * 1000 * MB}, {4 * 60, 0}}, 100 expected: 1000.0, 101 }, 102 // First entry in history is too old, it should be discarded. 103 { 104 freeBytes: 160 * GB, 105 obsoleteBytes: 1 * MB, 106 liveBytes: 160 * MB, 107 history: [][2]int64{{0, 10 * 60 * 10000 * MB}, {3 * 60, 4 * 60 * 200 * MB}, {7 * 60, 1 * 60 * 200 * MB}}, 108 expected: 200.0, 109 }, 110 } 111 for tcIdx, tc := range testCases { 112 t.Run(fmt.Sprintf("%d", tcIdx), func(t *testing.T) { 113 getInfo := func() deletionPacerInfo { 114 return deletionPacerInfo{ 115 freeBytes: tc.freeBytes, 116 liveBytes: tc.liveBytes, 117 obsoleteBytes: tc.obsoleteBytes, 118 } 119 } 120 start := time.Now() 121 last := start 122 pacer := newDeletionPacer(start, 100*MB, getInfo) 123 for _, h := range tc.history { 124 last = start.Add(time.Second * time.Duration(h[0])) 125 pacer.ReportDeletion(last, uint64(h[1])) 126 } 127 result := 1.0 / pacer.PacingDelay(last, 1*MB) 128 require.InDelta(t, tc.expected, result, 1e-7) 129 }) 130 } 131 } 132 133 // TestDeletionPacerHistory tests the history helper by crosschecking Sum() 134 // against a naive implementation. 135 func TestDeletionPacerHistory(t *testing.T) { 136 type event struct { 137 time time.Time 138 // If report is 0, this event is a Sum(). Otherwise it is an Add(). 139 report int64 140 } 141 numEvents := 1 + rand.Intn(200) 142 timeframe := time.Duration(1+rand.Intn(60*100)) * time.Second 143 events := make([]event, numEvents) 144 startTime := time.Now() 145 for i := range events { 146 events[i].time = startTime.Add(time.Duration(rand.Int63n(int64(timeframe)))) 147 if rand.Intn(3) == 0 { 148 events[i].report = 0 149 } else { 150 events[i].report = int64(rand.Intn(100000)) 151 } 152 } 153 sort.Slice(events, func(i, j int) bool { 154 return events[i].time.Before(events[j].time) 155 }) 156 157 var h history 158 h.Init(startTime, timeframe) 159 160 // partialSums[i] := SUM_j<i events[j].report 161 partialSums := make([]int64, len(events)+1) 162 for i := range events { 163 partialSums[i+1] = partialSums[i] + events[i].report 164 } 165 166 for i, e := range events { 167 if e.report != 0 { 168 h.Add(e.time, e.report) 169 continue 170 } 171 172 result := h.Sum(e.time) 173 174 // getIdx returns the largest event index <= i that is before the cutoff 175 // time. 176 getIdx := func(cutoff time.Time) int { 177 for j := i; j >= 0; j-- { 178 if events[j].time.Before(cutoff) { 179 return j 180 } 181 } 182 return -1 183 } 184 185 // Sum all report values in the last timeframe, and see if recent events 186 // (allowing 1% error in the cutoff time) match the result. 187 a := getIdx(e.time.Add(-timeframe * (historyEpochs + 1) / historyEpochs)) 188 b := getIdx(e.time.Add(-timeframe * (historyEpochs - 1) / historyEpochs)) 189 found := false 190 for j := a; j <= b; j++ { 191 if partialSums[i+1]-partialSums[j+1] == result { 192 found = true 193 break 194 } 195 } 196 if !found { 197 t.Fatalf("incorrect Sum() result %d; %v", result, events[a+1:i+1]) 198 } 199 } 200 }