vitess.io/vitess@v0.16.2/go/vt/throttler/memory.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package throttler 18 19 import ( 20 "fmt" 21 "sort" 22 "time" 23 ) 24 25 // memory tracks "good" and "bad" throttler rates where good rates are below 26 // the system capacity and bad are above it. 27 // 28 // It is based on the fact that the MySQL performance degrades with an 29 // increasing number of rows. Therefore, the implementation stores only one 30 // bad rate which will get lower over time and turn known good rates into 31 // bad ones. 32 // 33 // To protect against temporary performance degradations, a stable bad rate, 34 // which hasn't changed for a certain time, "ages out" and will be increased 35 // again. This ensures that rates above past bad rates will be tested again in 36 // the future. 37 // 38 // To simplify tracking all possible rates, they are slotted into buckets 39 // e.g. of the size 5. 40 type memory struct { 41 bucketSize int 42 43 good []int64 44 bad int64 45 nextBadRateAging time.Time 46 47 ageBadRateAfter time.Duration 48 badRateIncrease float64 49 } 50 51 func newMemory(bucketSize int, ageBadRateAfter time.Duration, badRateIncrease float64) *memory { 52 if bucketSize == 0 { 53 bucketSize = 1 54 } 55 return &memory{ 56 bucketSize: bucketSize, 57 good: make([]int64, 0), 58 ageBadRateAfter: ageBadRateAfter, 59 badRateIncrease: badRateIncrease, 60 } 61 } 62 63 func (m *memory) updateAgingConfiguration(ageBadRateAfter time.Duration, badRateIncrease float64) { 64 if !m.nextBadRateAging.IsZero() { 65 // Adjust the current age timer immediately. 66 m.nextBadRateAging = m.nextBadRateAging.Add(ageBadRateAfter).Add(-m.ageBadRateAfter) 67 } 68 m.ageBadRateAfter = ageBadRateAfter 69 m.badRateIncrease = badRateIncrease 70 } 71 72 // int64Slice is used to sort int64 slices. 73 type int64Slice []int64 74 75 func (a int64Slice) Len() int { return len(a) } 76 func (a int64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 77 func (a int64Slice) Less(i, j int) bool { return a[i] < a[j] } 78 79 func searchInt64s(a []int64, x int64) int { 80 return sort.Search(len(a), func(i int) bool { return a[i] >= x }) 81 } 82 83 func (m *memory) markGood(rate int64) error { 84 rate = m.roundDown(rate) 85 86 if lowestBad := m.lowestBad(); lowestBad != 0 && rate > lowestBad { 87 return fmt.Errorf("ignoring higher good rate of %v because we assume that the known maximum capacity (currently at %v) can only degrade", rate, lowestBad) 88 } 89 90 // Skip rates which already exist. 91 i := searchInt64s(m.good, rate) 92 if i < len(m.good) && m.good[i] == rate { 93 return nil 94 } 95 96 m.good = append(m.good, rate) 97 sort.Sort(int64Slice(m.good)) 98 return nil 99 } 100 101 func (m *memory) markBad(rate int64, now time.Time) error { 102 // Bad rates are rounded up instead of down to not be too extreme on the 103 // reduction and account for some margin of error. 104 rate = m.roundUp(rate) 105 106 // Ignore higher bad rates than the current one. 107 if m.bad != 0 && rate >= m.bad { 108 return nil 109 } 110 111 // Ignore bad rates which are too drastic. This prevents that temporary 112 // hiccups e.g. during a reparent, are stored in the memory. 113 // TODO(mberlin): Remove this once we let bad values expire over time. 114 highestGood := m.highestGood() 115 if rate < highestGood { 116 decrease := float64(highestGood) - float64(rate) 117 degradation := decrease / float64(highestGood) 118 if degradation > 0.1 { 119 return fmt.Errorf("ignoring lower bad rate of %v because such a high degradation (%.1f%%) is unlikely (current highest good: %v)", rate, degradation*100, highestGood) 120 } 121 } 122 123 // Delete all good values which turned bad. 124 goodLength := len(m.good) 125 for i := goodLength - 1; i >= 0; i-- { 126 goodRate := m.good[i] 127 if goodRate >= rate { 128 goodLength = i 129 } else { 130 break 131 } 132 } 133 m.good = m.good[:goodLength] 134 135 m.bad = rate 136 m.touchBadRateAge(now) 137 return nil 138 } 139 140 // touchBadRateAge records that the bad rate was changed and the aging should be 141 // further delayed. 142 func (m *memory) touchBadRateAge(now time.Time) { 143 m.nextBadRateAging = now.Add(m.ageBadRateAfter) 144 } 145 146 func (m *memory) ageBadRate(now time.Time) { 147 if m.badRateIncrease == 0 { 148 return 149 } 150 if m.bad == 0 { 151 return 152 } 153 if m.nextBadRateAging.IsZero() { 154 return 155 } 156 if now.Before(m.nextBadRateAging) { 157 return 158 } 159 160 newBad := float64(m.bad) * (1 + m.badRateIncrease) 161 if int64(newBad) == m.bad { 162 // Increase had no effect. Increase it at least by the granularity. 163 newBad += memoryGranularity 164 } 165 m.bad = int64(newBad) 166 m.touchBadRateAge(now) 167 } 168 169 func (m *memory) highestGood() int64 { 170 if len(m.good) == 0 { 171 return 0 172 } 173 174 return m.good[len(m.good)-1] 175 } 176 177 func (m *memory) lowestBad() int64 { 178 return m.bad 179 } 180 181 func (m *memory) roundDown(rate int64) int64 { 182 return rate / int64(m.bucketSize) * int64(m.bucketSize) 183 } 184 185 func (m *memory) roundUp(rate int64) int64 { 186 ceil := rate / int64(m.bucketSize) * int64(m.bucketSize) 187 if rate%int64(m.bucketSize) != 0 { 188 ceil += int64(m.bucketSize) 189 } 190 return ceil 191 }