github.com/m3db/m3@v1.5.0/src/cmd/services/m3comparator/main/parser/series_iterator_builder.go (about)

     1  // Copyright (c) 2019 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package parser
    22  
    23  import (
    24  	"time"
    25  
    26  	"github.com/m3db/m3/src/dbnode/encoding"
    27  	"github.com/m3db/m3/src/dbnode/encoding/m3tsz"
    28  	"github.com/m3db/m3/src/dbnode/ts"
    29  	"github.com/m3db/m3/src/dbnode/x/xio"
    30  	"github.com/m3db/m3/src/query/models"
    31  	"github.com/m3db/m3/src/x/ident"
    32  	xtime "github.com/m3db/m3/src/x/time"
    33  )
    34  
    35  // Data is a set of datapoints.
    36  type Data []ts.Datapoint
    37  
    38  // IngestSeries is a series that can be ingested by the parser.
    39  type IngestSeries struct {
    40  	Datapoints []Data
    41  	Tags       Tags
    42  }
    43  
    44  var iterAlloc = m3tsz.DefaultReaderIteratorAllocFn(encoding.NewOptions())
    45  
    46  func buildBlockReader(
    47  	block Data,
    48  	start xtime.UnixNano,
    49  	blockSize time.Duration,
    50  	opts Options,
    51  ) ([]xio.BlockReader, error) {
    52  	encoder := opts.EncoderPool.Get()
    53  	encoder.Reset(xtime.Now(), len(block), nil)
    54  	for _, dp := range block {
    55  		err := encoder.Encode(dp, xtime.Second, nil)
    56  		if err != nil {
    57  			encoder.Close()
    58  			return nil, err
    59  		}
    60  	}
    61  
    62  	segment := encoder.Discard()
    63  	return []xio.BlockReader{
    64  		{
    65  			SegmentReader: xio.NewSegmentReader(segment),
    66  			Start:         start,
    67  			BlockSize:     blockSize,
    68  		},
    69  	}, nil
    70  }
    71  
    72  func buildTagIteratorAndID(
    73  	parsedTags Tags,
    74  	opts models.TagOptions,
    75  ) (ident.TagIterator, ident.ID) {
    76  	var (
    77  		tags      = ident.Tags{}
    78  		modelTags = models.NewTags(len(parsedTags), opts)
    79  	)
    80  
    81  	for _, tag := range parsedTags {
    82  		name := tag.Name()
    83  		value := tag.Value()
    84  		modelTags = modelTags.AddOrUpdateTag(models.Tag{
    85  			Name:  []byte(name),
    86  			Value: []byte(value),
    87  		})
    88  
    89  		tags.Append(ident.StringTag(name, value))
    90  	}
    91  
    92  	id := string(modelTags.ID())
    93  	return ident.NewTagsIterator(tags), ident.StringID(id)
    94  }
    95  
    96  func buildSeriesIterator(
    97  	series IngestSeries,
    98  	start xtime.UnixNano,
    99  	blockSize time.Duration,
   100  	opts Options,
   101  ) (encoding.SeriesIterator, error) {
   102  	var (
   103  		points  = series.Datapoints
   104  		tags    = series.Tags
   105  		readers = make([][]xio.BlockReader, 0, len(points))
   106  	)
   107  
   108  	blockStart := start
   109  	for _, block := range points {
   110  		seriesBlock, err := buildBlockReader(block, blockStart, blockSize, opts)
   111  		if err != nil {
   112  			return nil, err
   113  		}
   114  
   115  		readers = append(readers, seriesBlock)
   116  		blockStart = blockStart.Add(blockSize)
   117  	}
   118  
   119  	multiReader := encoding.NewMultiReaderIterator(
   120  		iterAlloc,
   121  		opts.IteratorPools.MultiReaderIterator(),
   122  	)
   123  
   124  	sliceOfSlicesIter := xio.NewReaderSliceOfSlicesFromBlockReadersIterator(readers)
   125  	multiReader.ResetSliceOfSlices(sliceOfSlicesIter, nil)
   126  
   127  	end := start.Add(blockSize)
   128  	if len(points) > 0 {
   129  		lastBlock := points[len(points)-1]
   130  		end = lastBlock[len(lastBlock)-1].TimestampNanos
   131  	}
   132  
   133  	tagIter, id := buildTagIteratorAndID(tags, opts.TagOptions)
   134  	return encoding.NewSeriesIterator(
   135  		encoding.SeriesIteratorOptions{
   136  			ID:             id,
   137  			Namespace:      ident.StringID("ns"),
   138  			Tags:           tagIter,
   139  			StartInclusive: start,
   140  			EndExclusive:   end,
   141  			Replicas: []encoding.MultiReaderIterator{
   142  				multiReader,
   143  			},
   144  		}, nil), nil
   145  }
   146  
   147  // BuildSeriesIterators builds series iterators from parser data.
   148  func BuildSeriesIterators(
   149  	series []IngestSeries,
   150  	start xtime.UnixNano,
   151  	blockSize time.Duration,
   152  	opts Options,
   153  ) (encoding.SeriesIterators, error) {
   154  	iters := make([]encoding.SeriesIterator, 0, len(series))
   155  	for _, s := range series {
   156  		iter, err := buildSeriesIterator(s, start, blockSize, opts)
   157  		if err != nil {
   158  			return nil, err
   159  		}
   160  
   161  		iters = append(iters, iter)
   162  	}
   163  
   164  	return encoding.NewSeriesIterators(iters), nil
   165  }