github.com/m3db/m3@v1.5.0/src/dbnode/integration/commitlog_bootstrap_merge_test.go (about)

     1  // +build integration
     2  
     3  // Copyright (c) 2017 Uber Technologies, Inc.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  package integration
    24  
    25  import (
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/m3db/m3/src/dbnode/integration/generate"
    30  	"github.com/m3db/m3/src/dbnode/namespace"
    31  	"github.com/m3db/m3/src/dbnode/retention"
    32  
    33  	"github.com/stretchr/testify/require"
    34  )
    35  
    36  // Consider a database running with a single namespaces, and the following retention opts:
    37  //
    38  //           | BlockSize | Retention Period
    39  // ns1       |    2h     |      8h
    40  // commitLog |   15m     |      8h
    41  //
    42  // We have a block for each of the two at each marker in the diagram below.
    43  //
    44  // time (flowing left --> right):
    45  // time-label: t0  t1  t2  t3
    46  //  ns1        .   .          [blocksize . is 2h]
    47  //  commitlog      ,,,,,,,,,  [blocksize , is 30min]
    48  //
    49  // The test creates the blocks below, and verifies the bootstrappers are able to merge the data
    50  // successfully.
    51  // - ns1 block at t0 and t1
    52  // - commit log blocks from [t1, t3]
    53  func TestCommitLogAndFSMergeBootstrap(t *testing.T) {
    54  	if testing.Short() {
    55  		t.SkipNow()
    56  	}
    57  
    58  	// Test setup
    59  	var (
    60  		rOpts        = retention.NewOptions().SetRetentionPeriod(12 * time.Hour)
    61  		ns1BlockSize = 2 * time.Hour
    62  		ns1ROpts     = rOpts.SetBlockSize(ns1BlockSize)
    63  	)
    64  	ns1, err := namespace.NewMetadata(testNamespaces[0], namespace.NewOptions().SetRetentionOptions(ns1ROpts))
    65  	require.NoError(t, err)
    66  	opts := NewTestOptions(t).
    67  		SetNamespaces([]namespace.Metadata{ns1})
    68  
    69  	// Test setup
    70  	setup, err := NewTestSetup(t, opts, nil)
    71  	require.NoError(t, err)
    72  	defer setup.Close()
    73  
    74  	commitLogOpts := setup.StorageOpts().CommitLogOptions().
    75  		SetFlushInterval(defaultIntegrationTestFlushInterval)
    76  	setup.SetStorageOpts(setup.StorageOpts().SetCommitLogOptions(commitLogOpts))
    77  
    78  	log := setup.StorageOpts().InstrumentOptions().Logger()
    79  	log.Info("commit log + fs merge bootstrap test")
    80  
    81  	// generate and write test data
    82  	var (
    83  		t0 = setup.NowFn()()
    84  		t1 = t0.Add(ns1BlockSize)
    85  		t2 = t1.Add(ns1BlockSize)
    86  		t3 = t2.Add(ns1BlockSize)
    87  	)
    88  	blockConfigs := []generate.BlockConfig{
    89  		{IDs: []string{"foo", "bar"}, NumPoints: 20, Start: t0},
    90  		{IDs: []string{"nah", "baz"}, NumPoints: 50, Start: t1},
    91  		{IDs: []string{"hax", "ord"}, NumPoints: 30, Start: t2},
    92  	}
    93  	log.Info("generating data")
    94  	seriesMaps := generate.BlocksByStart(blockConfigs)
    95  	log.Info("generated data")
    96  
    97  	log.Info("writing filesets")
    98  	fsSeriesMaps := generate.SeriesBlocksByStart{
    99  		t0: seriesMaps[t0],
   100  		t1: seriesMaps[t1],
   101  	}
   102  	require.NoError(t, writeTestDataToDiskWithIndex(ns1, setup, fsSeriesMaps))
   103  
   104  	log.Info("writing commit logs")
   105  	commitlogSeriesMaps := generate.SeriesBlocksByStart{
   106  		t1: seriesMaps[t1],
   107  		t2: seriesMaps[t2],
   108  	}
   109  	writeCommitLogData(t, setup, commitLogOpts, commitlogSeriesMaps, ns1, false)
   110  
   111  	require.NoError(t, setup.InitializeBootstrappers(InitializeBootstrappersOptions{
   112  		CommitLogOptions: commitLogOpts,
   113  		WithCommitLog:    true,
   114  		WithFileSystem:   true,
   115  	}))
   116  
   117  	log.Info("moving time forward and starting server")
   118  	setup.SetNowFn(t3)
   119  	// Start the server
   120  	require.NoError(t, setup.StartServer())
   121  	log.Debug("server is now up")
   122  
   123  	// Stop the server
   124  	defer func() {
   125  		require.NoError(t, setup.StopServer())
   126  		log.Debug("server is now down")
   127  	}()
   128  
   129  	log.Info("validating bootstrapped data")
   130  	verifySeriesMaps(t, setup, ns1.ID(), seriesMaps)
   131  }