github.com/m3db/m3@v1.5.0/src/dbnode/storage/forward_index_dice.go (about)

     1  // Copyright (c) 2019 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package storage
    22  
    23  import (
    24  	"fmt"
    25  	"time"
    26  
    27  	xtime "github.com/m3db/m3/src/x/time"
    28  )
    29  
    30  // forwardIndexDice is a die roll that adds a chance for incoming index writes
    31  // arriving near a block boundary to be duplicated and written to the next block
    32  // index, adding jitter and smoothing index load so that block boundaries do not
    33  // cause a huge influx of new documents that all need to be indexed at once.
    34  type forwardIndexDice struct {
    35  	enabled   bool
    36  	blockSize time.Duration
    37  
    38  	forwardIndexThreshold time.Duration
    39  	forwardIndexDice      dice
    40  }
    41  
    42  func newForwardIndexDice(
    43  	opts Options,
    44  ) (forwardIndexDice, error) {
    45  	var (
    46  		indexOpts  = opts.IndexOptions()
    47  		seriesOpts = opts.SeriesOptions()
    48  
    49  		probability = indexOpts.ForwardIndexProbability()
    50  	)
    51  
    52  	// NB: if not enabled, return a no-op forward index dice.
    53  	if probability == 0 {
    54  		return forwardIndexDice{}, nil
    55  	}
    56  
    57  	var (
    58  		threshold = indexOpts.ForwardIndexThreshold()
    59  
    60  		retention    = seriesOpts.RetentionOptions()
    61  		bufferFuture = retention.BufferFuture()
    62  		blockSize    = retention.BlockSize()
    63  
    64  		forwardIndexThreshold time.Duration
    65  	)
    66  
    67  	if threshold < 0 || threshold > 1 {
    68  		return forwardIndexDice{},
    69  			fmt.Errorf("invalid forward write threshold %f", threshold)
    70  	}
    71  
    72  	bufferFragment := float64(bufferFuture) * threshold
    73  	forwardIndexThreshold = blockSize - time.Duration(bufferFragment)
    74  
    75  	dice, err := newDice(probability)
    76  	if err != nil {
    77  		return forwardIndexDice{},
    78  			fmt.Errorf("cannot create forward write dice: %s", err)
    79  	}
    80  
    81  	return forwardIndexDice{
    82  		enabled:   true,
    83  		blockSize: blockSize,
    84  
    85  		forwardIndexThreshold: forwardIndexThreshold,
    86  		forwardIndexDice:      dice,
    87  	}, nil
    88  }
    89  
    90  // roll decides if a timestamp is eligible for forward index writes.
    91  func (o *forwardIndexDice) roll(timestamp xtime.UnixNano) bool {
    92  	if !o.enabled {
    93  		return false
    94  	}
    95  
    96  	threshold := timestamp.Truncate(o.blockSize).Add(o.forwardIndexThreshold)
    97  	if !timestamp.Before(threshold) {
    98  		return o.forwardIndexDice.Roll()
    99  	}
   100  
   101  	return false
   102  }