vitess.io/vitess@v0.16.2/go/vt/throttler/memory.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package throttler
    18  
    19  import (
    20  	"fmt"
    21  	"sort"
    22  	"time"
    23  )
    24  
    25  // memory tracks "good" and "bad" throttler rates where good rates are below
    26  // the system capacity and bad are above it.
    27  //
    28  // It is based on the fact that the MySQL performance degrades with an
    29  // increasing number of rows. Therefore, the implementation stores only one
    30  // bad rate which will get lower over time and turn known good rates into
    31  // bad ones.
    32  //
    33  // To protect against temporary performance degradations, a stable bad rate,
    34  // which hasn't changed for a certain time, "ages out" and will be increased
    35  // again. This ensures that rates above past bad rates will be tested again in
    36  // the future.
    37  //
    38  // To simplify tracking all possible rates, they are slotted into buckets
    39  // e.g. of the size 5.
    40  type memory struct {
    41  	bucketSize int
    42  
    43  	good             []int64
    44  	bad              int64
    45  	nextBadRateAging time.Time
    46  
    47  	ageBadRateAfter time.Duration
    48  	badRateIncrease float64
    49  }
    50  
    51  func newMemory(bucketSize int, ageBadRateAfter time.Duration, badRateIncrease float64) *memory {
    52  	if bucketSize == 0 {
    53  		bucketSize = 1
    54  	}
    55  	return &memory{
    56  		bucketSize:      bucketSize,
    57  		good:            make([]int64, 0),
    58  		ageBadRateAfter: ageBadRateAfter,
    59  		badRateIncrease: badRateIncrease,
    60  	}
    61  }
    62  
    63  func (m *memory) updateAgingConfiguration(ageBadRateAfter time.Duration, badRateIncrease float64) {
    64  	if !m.nextBadRateAging.IsZero() {
    65  		// Adjust the current age timer immediately.
    66  		m.nextBadRateAging = m.nextBadRateAging.Add(ageBadRateAfter).Add(-m.ageBadRateAfter)
    67  	}
    68  	m.ageBadRateAfter = ageBadRateAfter
    69  	m.badRateIncrease = badRateIncrease
    70  }
    71  
    72  // int64Slice is used to sort int64 slices.
    73  type int64Slice []int64
    74  
    75  func (a int64Slice) Len() int           { return len(a) }
    76  func (a int64Slice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
    77  func (a int64Slice) Less(i, j int) bool { return a[i] < a[j] }
    78  
    79  func searchInt64s(a []int64, x int64) int {
    80  	return sort.Search(len(a), func(i int) bool { return a[i] >= x })
    81  }
    82  
    83  func (m *memory) markGood(rate int64) error {
    84  	rate = m.roundDown(rate)
    85  
    86  	if lowestBad := m.lowestBad(); lowestBad != 0 && rate > lowestBad {
    87  		return fmt.Errorf("ignoring higher good rate of %v because we assume that the known maximum capacity (currently at %v) can only degrade", rate, lowestBad)
    88  	}
    89  
    90  	// Skip rates which already exist.
    91  	i := searchInt64s(m.good, rate)
    92  	if i < len(m.good) && m.good[i] == rate {
    93  		return nil
    94  	}
    95  
    96  	m.good = append(m.good, rate)
    97  	sort.Sort(int64Slice(m.good))
    98  	return nil
    99  }
   100  
   101  func (m *memory) markBad(rate int64, now time.Time) error {
   102  	// Bad rates are rounded up instead of down to not be too extreme on the
   103  	// reduction and account for some margin of error.
   104  	rate = m.roundUp(rate)
   105  
   106  	// Ignore higher bad rates than the current one.
   107  	if m.bad != 0 && rate >= m.bad {
   108  		return nil
   109  	}
   110  
   111  	// Ignore bad rates which are too drastic. This prevents that temporary
   112  	// hiccups e.g. during a reparent, are stored in the memory.
   113  	// TODO(mberlin): Remove this once we let bad values expire over time.
   114  	highestGood := m.highestGood()
   115  	if rate < highestGood {
   116  		decrease := float64(highestGood) - float64(rate)
   117  		degradation := decrease / float64(highestGood)
   118  		if degradation > 0.1 {
   119  			return fmt.Errorf("ignoring lower bad rate of %v because such a high degradation (%.1f%%) is unlikely (current highest good: %v)", rate, degradation*100, highestGood)
   120  		}
   121  	}
   122  
   123  	// Delete all good values which turned bad.
   124  	goodLength := len(m.good)
   125  	for i := goodLength - 1; i >= 0; i-- {
   126  		goodRate := m.good[i]
   127  		if goodRate >= rate {
   128  			goodLength = i
   129  		} else {
   130  			break
   131  		}
   132  	}
   133  	m.good = m.good[:goodLength]
   134  
   135  	m.bad = rate
   136  	m.touchBadRateAge(now)
   137  	return nil
   138  }
   139  
   140  // touchBadRateAge records that the bad rate was changed and the aging should be
   141  // further delayed.
   142  func (m *memory) touchBadRateAge(now time.Time) {
   143  	m.nextBadRateAging = now.Add(m.ageBadRateAfter)
   144  }
   145  
   146  func (m *memory) ageBadRate(now time.Time) {
   147  	if m.badRateIncrease == 0 {
   148  		return
   149  	}
   150  	if m.bad == 0 {
   151  		return
   152  	}
   153  	if m.nextBadRateAging.IsZero() {
   154  		return
   155  	}
   156  	if now.Before(m.nextBadRateAging) {
   157  		return
   158  	}
   159  
   160  	newBad := float64(m.bad) * (1 + m.badRateIncrease)
   161  	if int64(newBad) == m.bad {
   162  		// Increase had no effect. Increase it at least by the granularity.
   163  		newBad += memoryGranularity
   164  	}
   165  	m.bad = int64(newBad)
   166  	m.touchBadRateAge(now)
   167  }
   168  
   169  func (m *memory) highestGood() int64 {
   170  	if len(m.good) == 0 {
   171  		return 0
   172  	}
   173  
   174  	return m.good[len(m.good)-1]
   175  }
   176  
   177  func (m *memory) lowestBad() int64 {
   178  	return m.bad
   179  }
   180  
   181  func (m *memory) roundDown(rate int64) int64 {
   182  	return rate / int64(m.bucketSize) * int64(m.bucketSize)
   183  }
   184  
   185  func (m *memory) roundUp(rate int64) int64 {
   186  	ceil := rate / int64(m.bucketSize) * int64(m.bucketSize)
   187  	if rate%int64(m.bucketSize) != 0 {
   188  		ceil += int64(m.bucketSize)
   189  	}
   190  	return ceil
   191  }