github.com/m3db/m3@v1.5.0/src/dbnode/integration/commitlog_bootstrap_coldwrites_test.go (about)

     1  // +build integration
     2  
     3  // Copyright (c) 2019 Uber Technologies, Inc.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  package integration
    24  
    25  import (
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/m3db/m3/src/dbnode/integration/generate"
    30  	"github.com/m3db/m3/src/dbnode/namespace"
    31  	"github.com/m3db/m3/src/dbnode/retention"
    32  	xtime "github.com/m3db/m3/src/x/time"
    33  
    34  	"github.com/stretchr/testify/require"
    35  )
    36  
    37  func TestCommitLogBootstrapColdWrites(t *testing.T) {
    38  	testCommitLogBootstrapColdWrites(t, nil, nil)
    39  }
    40  
    41  func TestProtoCommitLogBootstrapColdWrites(t *testing.T) {
    42  	testCommitLogBootstrapColdWrites(t, setProtoTestOptions, setProtoTestInputConfig)
    43  }
    44  
    45  func testCommitLogBootstrapColdWrites(t *testing.T, setTestOpts setTestOptions, updateInputConfig generate.UpdateBlockConfig) {
    46  	if testing.Short() {
    47  		t.SkipNow() // Just skip if we're doing a short run
    48  	}
    49  
    50  	// Test setup
    51  	var (
    52  		ropts     = retention.NewOptions().SetRetentionPeriod(12 * time.Hour)
    53  		blockSize = ropts.BlockSize()
    54  	)
    55  	ns1, err := namespace.NewMetadata(testNamespaces[0], namespace.NewOptions().
    56  		SetRetentionOptions(ropts).
    57  		SetColdWritesEnabled(true))
    58  	require.NoError(t, err)
    59  	opts := NewTestOptions(t).
    60  		SetNamespaces([]namespace.Metadata{ns1})
    61  	if setTestOpts != nil {
    62  		opts = setTestOpts(t, opts)
    63  		ns1 = opts.Namespaces()[0]
    64  	}
    65  
    66  	setup, err := NewTestSetup(t, opts, nil)
    67  	require.NoError(t, err)
    68  	defer setup.Close()
    69  
    70  	commitLogOpts := setup.StorageOpts().CommitLogOptions().
    71  		SetFlushInterval(defaultIntegrationTestFlushInterval)
    72  	setup.SetStorageOpts(setup.StorageOpts().SetCommitLogOptions(commitLogOpts))
    73  
    74  	log := setup.StorageOpts().InstrumentOptions().Logger()
    75  	log.Info("commit log bootstrap test")
    76  
    77  	start := setup.NowFn()()
    78  
    79  	log.Info("writing data files")
    80  	dataFilesData := []generate.BlockConfig{
    81  		{IDs: []string{"foo", "bar"}, NumPoints: 100, Start: start.Add(-2 * blockSize)},
    82  		{IDs: []string{"foo", "baz"}, NumPoints: 50, Start: start.Add(-blockSize)},
    83  	}
    84  	if updateInputConfig != nil {
    85  		updateInputConfig(dataFilesData)
    86  	}
    87  	dataFilesSeriesMaps := generate.BlocksByStart(dataFilesData)
    88  	require.NoError(t, writeTestDataToDiskWithIndex(ns1, setup, dataFilesSeriesMaps))
    89  	log.Info("finished writing data files")
    90  
    91  	log.Info("writing commit logs")
    92  	commitLogData := []generate.BlockConfig{
    93  		{IDs: []string{"commitlog1", "commitlog2"}, NumPoints: 120, Start: start.Add(-2 * blockSize)},
    94  		{IDs: []string{"commitlog2", "commitlog3"}, NumPoints: 130, Start: start.Add(-blockSize)},
    95  	}
    96  	if updateInputConfig != nil {
    97  		updateInputConfig(commitLogData)
    98  	}
    99  	commitLogSeriesMaps := generate.BlocksByStart(commitLogData)
   100  	writeCommitLogData(t, setup, commitLogOpts, commitLogSeriesMaps, ns1, false)
   101  	log.Info("finished writing commit logs")
   102  
   103  	// Merge the two generated series maps together. We can only do this simply
   104  	// here because we know that they span the same block starts and they do
   105  	// not contains the same series.
   106  	allSeriesMaps := make(map[xtime.UnixNano]generate.SeriesBlock, len(dataFilesSeriesMaps))
   107  	for i := -2; i < 0; i++ {
   108  		unixNano := start.Add(time.Duration(i) * blockSize)
   109  		series := append(dataFilesSeriesMaps[unixNano], commitLogSeriesMaps[unixNano]...)
   110  		allSeriesMaps[unixNano] = series
   111  	}
   112  
   113  	// Setup bootstrapper after writing data so filesystem inspection can find it.
   114  	setupCommitLogBootstrapperWithFSInspection(t, setup, commitLogOpts)
   115  
   116  	setup.SetNowFn(start)
   117  	// Start the server with filesystem bootstrapper
   118  	require.NoError(t, setup.StartServer())
   119  	log.Debug("server is now up")
   120  
   121  	// Stop the server
   122  	defer func() {
   123  		require.NoError(t, setup.StopServer())
   124  		log.Debug("server is now down")
   125  	}()
   126  
   127  	// Verify in-memory data match what we expect - all writes from seriesMaps
   128  	// should be present
   129  	metadatasByShard := testSetupMetadatas(t, setup, testNamespaces[0], start.Add(-2*blockSize), start)
   130  	observedSeriesMaps := testSetupToSeriesMaps(t, setup, ns1, metadatasByShard)
   131  	verifySeriesMapsEqual(t, allSeriesMaps, observedSeriesMaps)
   132  }