github.com/m3db/m3@v1.5.0/src/dbnode/storage/fs_merge_with_mem.go (about)

     1  // Copyright (c) 2019 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package storage
    22  
    23  import (
    24  	"github.com/m3db/m3/src/dbnode/namespace"
    25  	"github.com/m3db/m3/src/dbnode/persist/fs"
    26  	"github.com/m3db/m3/src/dbnode/storage/block"
    27  	"github.com/m3db/m3/src/dbnode/storage/series"
    28  	"github.com/m3db/m3/src/dbnode/x/xio"
    29  	"github.com/m3db/m3/src/x/context"
    30  	"github.com/m3db/m3/src/x/ident"
    31  	xtime "github.com/m3db/m3/src/x/time"
    32  )
    33  
    34  // fsMergeWithMem implements fs.MergeWith, where the merge target is data in
    35  // memory. It relies on data structures being passed in that are created
    36  // within the shard informing fsMergeWithMem which series require merging.
    37  // These data structures enable efficient reading of data as well as keeping
    38  // track of which series were read so that the remaining series can be looped
    39  // through.
    40  type fsMergeWithMem struct {
    41  	shard              databaseShard
    42  	retriever          series.QueryableBlockRetriever
    43  	dirtySeries        *dirtySeriesMap
    44  	dirtySeriesToWrite map[xtime.UnixNano]*idList
    45  	reusableID         *ident.ReusableBytesID
    46  }
    47  
    48  func newFSMergeWithMem(
    49  	shard databaseShard,
    50  	retriever series.QueryableBlockRetriever,
    51  	dirtySeries *dirtySeriesMap,
    52  	dirtySeriesToWrite map[xtime.UnixNano]*idList,
    53  ) fs.MergeWith {
    54  	return &fsMergeWithMem{
    55  		shard:              shard,
    56  		retriever:          retriever,
    57  		dirtySeries:        dirtySeries,
    58  		dirtySeriesToWrite: dirtySeriesToWrite,
    59  		reusableID:         ident.NewReusableBytesID(),
    60  	}
    61  }
    62  
    63  func (m *fsMergeWithMem) Read(
    64  	ctx context.Context,
    65  	seriesID ident.ID,
    66  	blockStart xtime.UnixNano,
    67  	nsCtx namespace.Context,
    68  ) ([]xio.BlockReader, bool, error) {
    69  	// Check if this series is in memory (and thus requires merging).
    70  	element, exists := m.dirtySeries.Get(idAndBlockStart{
    71  		blockStart: blockStart,
    72  		id:         seriesID.Bytes(),
    73  	})
    74  	if !exists {
    75  		return nil, false, nil
    76  	}
    77  
    78  	// Series is in memory, so it will get merged with disk and
    79  	// written in this loop. Therefore, we need to remove it from
    80  	// the "to write" list so that the later loop does not rewrite
    81  	// it.
    82  	m.dirtySeriesToWrite[blockStart].Remove(element)
    83  
    84  	result, ok, err := m.fetchBlocks(ctx, seriesID, blockStart, nsCtx)
    85  	if err != nil {
    86  		return nil, false, err
    87  	}
    88  	return result.Blocks, ok, nil
    89  }
    90  
    91  func (m *fsMergeWithMem) fetchBlocks(
    92  	ctx context.Context,
    93  	id ident.ID,
    94  	blockStart xtime.UnixNano,
    95  	nsCtx namespace.Context,
    96  ) (block.FetchBlockResult, bool, error) {
    97  	currVersion, err := m.retriever.RetrievableBlockColdVersion(blockStart)
    98  	if err != nil {
    99  		return block.FetchBlockResult{}, false, err
   100  	}
   101  	nextVersion := currVersion + 1
   102  
   103  	result, err := m.shard.FetchBlocksForColdFlush(ctx, id, blockStart, nextVersion, nsCtx)
   104  	if err != nil {
   105  		return block.FetchBlockResult{}, false, err
   106  	}
   107  
   108  	if len(result.Blocks) > 0 {
   109  		return result, true, nil
   110  	}
   111  
   112  	return block.FetchBlockResult{}, false, nil
   113  }
   114  
   115  // The data passed to ForEachRemaining (through the fs.ForEachRemainingFn) is
   116  // basically a copy that will be finalized when the context is closed, but the
   117  // ID and tags are expected to live for as long as the caller of the MergeWith
   118  // requires them, so they should either be NoFinalize() or passed as copies.
   119  func (m *fsMergeWithMem) ForEachRemaining(
   120  	ctx context.Context,
   121  	blockStart xtime.UnixNano,
   122  	fn fs.ForEachRemainingFn,
   123  	nsCtx namespace.Context,
   124  ) error {
   125  	reusableID := m.reusableID
   126  	seriesList := m.dirtySeriesToWrite[blockStart]
   127  
   128  	for seriesElement := seriesList.Front(); seriesElement != nil; seriesElement = seriesElement.Next() {
   129  		seriesMetadata := seriesElement.Value
   130  		reusableID.Reset(seriesMetadata.ID)
   131  		mergeWithData, hasData, err := m.fetchBlocks(ctx, reusableID, blockStart, nsCtx)
   132  		if err != nil {
   133  			return err
   134  		}
   135  
   136  		if hasData {
   137  			err = fn(seriesMetadata, mergeWithData)
   138  			if err != nil {
   139  				return err
   140  			}
   141  		}
   142  	}
   143  
   144  	return nil
   145  }