github.com/m3db/m3@v1.5.0/src/query/storage/m3/consolidators/step_consolidator.go (about)

     1  // Copyright (c) 2018 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package consolidators
    22  
    23  import (
    24  	"time"
    25  
    26  	"github.com/m3db/m3/src/dbnode/ts"
    27  	xtime "github.com/m3db/m3/src/x/time"
    28  )
    29  
    30  func removeStale(
    31  	earliestLookback xtime.UnixNano,
    32  	dps []ts.Datapoint,
    33  ) []ts.Datapoint {
    34  	for i, dp := range dps {
    35  		if !dp.TimestampNanos.Before(earliestLookback) {
    36  			return dps[i:]
    37  		}
    38  	}
    39  
    40  	return dps[:0]
    41  }
    42  
    43  // StepLookbackConsolidator is a helper for consolidating series in a step-wise
    44  // fashion. It takes a 'step' of values, which represents a vertical
    45  // slice of time across a list of series, and consolidates when a
    46  // valid step has been reached.
    47  type StepLookbackConsolidator struct {
    48  	stepSize         time.Duration
    49  	earliestLookback xtime.UnixNano
    50  	datapoints       []ts.Datapoint
    51  	buffer           []float64
    52  	unconsumed       []float64
    53  	fn               ConsolidationFunc
    54  }
    55  
    56  // Ensure StepLookbackConsolidator satisfies StepCollector.
    57  var _ StepCollector = (*StepLookbackConsolidator)(nil)
    58  
    59  // NewStepLookbackConsolidator creates a multivalue consolidator used for
    60  // step iteration across a series list with a given lookback.
    61  func NewStepLookbackConsolidator(
    62  	lookbackDuration, stepSize time.Duration,
    63  	startTime xtime.UnixNano,
    64  	fn ConsolidationFunc,
    65  ) *StepLookbackConsolidator {
    66  	datapoints := make([]ts.Datapoint, 0, initLength)
    67  	buffer := make([]float64, BufferSteps)
    68  	return &StepLookbackConsolidator{
    69  		stepSize:         stepSize,
    70  		earliestLookback: startTime.Add(-1 * lookbackDuration),
    71  		datapoints:       datapoints,
    72  		buffer:           buffer,
    73  		unconsumed:       buffer[:0],
    74  		fn:               fn,
    75  	}
    76  }
    77  
    78  // AddPoint adds a datapoint to a given step if it's within the valid
    79  // time period; otherwise drops it silently, which is fine for consolidation.
    80  func (c *StepLookbackConsolidator) AddPoint(dp ts.Datapoint) {
    81  	if dp.TimestampNanos.Before(c.earliestLookback) {
    82  		// this datapoint is too far in the past, it can be dropped.
    83  		return
    84  	}
    85  
    86  	c.datapoints = append(c.datapoints, dp)
    87  }
    88  
    89  // BufferStep adds viable points to the next unconsumed buffer step.
    90  func (c *StepLookbackConsolidator) BufferStep() {
    91  	c.earliestLookback = c.earliestLookback.Add(c.stepSize)
    92  	val := c.fn(c.datapoints)
    93  
    94  	// Remove any datapoints not relevant to the next step now.
    95  	datapointsRelevant := removeStale(c.earliestLookback, c.datapoints)
    96  	if len(datapointsRelevant) > 0 {
    97  		// Move them back to the start of the slice to reuse the slice
    98  		// as best as possible.
    99  		c.datapoints = c.datapoints[:len(datapointsRelevant)]
   100  		copy(c.datapoints, datapointsRelevant)
   101  	} else {
   102  		// No relevant datapoints, repoint to the start of the buffer.
   103  		c.datapoints = c.datapoints[:0]
   104  	}
   105  
   106  	// Blindly append to unconsumed.
   107  	c.unconsumed = append(c.unconsumed, val)
   108  }
   109  
   110  // BufferStepCount indicates how many accumulated points are still unconsumed.
   111  func (c *StepLookbackConsolidator) BufferStepCount() int {
   112  	return len(c.unconsumed)
   113  }
   114  
   115  // ConsolidateAndMoveToNext consolidates the current values and moves the
   116  // consolidator to the next given value, purging stale values.
   117  func (c *StepLookbackConsolidator) ConsolidateAndMoveToNext() float64 {
   118  	if len(c.unconsumed) == 0 {
   119  		return c.fn(nil)
   120  	}
   121  
   122  	// Consume value.
   123  	val := c.unconsumed[0]
   124  	remaining := c.unconsumed[1:]
   125  
   126  	if len(remaining) > 0 {
   127  		// Move any unconsumed values to the front of unconsumed.
   128  		c.unconsumed = c.buffer[:len(remaining)]
   129  		copy(c.unconsumed, remaining)
   130  	} else {
   131  		// Otherwise just repoint to the start of the buffer.
   132  		c.unconsumed = c.buffer[:0]
   133  	}
   134  
   135  	return val
   136  }