github.com/m3db/m3@v1.5.0/src/dbnode/integration/commitlog_bootstrap_multi_ns_test.go (about)

     1  // +build integration
     2  
     3  // Copyright (c) 2017 Uber Technologies, Inc.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  package integration
    24  
    25  import (
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/m3db/m3/src/dbnode/integration/generate"
    30  	"github.com/m3db/m3/src/dbnode/namespace"
    31  	"github.com/m3db/m3/src/dbnode/retention"
    32  
    33  	"github.com/stretchr/testify/require"
    34  )
    35  
    36  func TestCommitLogBootstrapMultipleNamespaces(t *testing.T) {
    37  	if testing.Short() {
    38  		t.SkipNow() // Just skip if we're doing a short run
    39  	}
    40  
    41  	// Test setup
    42  	var (
    43  		rOpts        = retention.NewOptions().SetRetentionPeriod(48 * time.Hour)
    44  		ns1BlockSize = time.Hour
    45  		ns2BlockSize = 30 * time.Minute
    46  		ns1ROpts     = rOpts.SetBlockSize(ns1BlockSize)
    47  		ns2ROpts     = rOpts.SetBlockSize(ns2BlockSize)
    48  	)
    49  
    50  	ns1, err := namespace.NewMetadata(testNamespaces[0], namespace.NewOptions().SetRetentionOptions(ns1ROpts))
    51  	require.NoError(t, err)
    52  	ns2, err := namespace.NewMetadata(testNamespaces[1], namespace.NewOptions().SetRetentionOptions(ns2ROpts))
    53  	require.NoError(t, err)
    54  
    55  	opts := NewTestOptions(t).
    56  		SetNamespaces([]namespace.Metadata{ns1, ns2})
    57  
    58  	// Test setup
    59  	setup, err := NewTestSetup(t, opts, nil)
    60  	require.NoError(t, err)
    61  	defer setup.Close()
    62  
    63  	commitLogOpts := setup.StorageOpts().CommitLogOptions().
    64  		SetFlushInterval(defaultIntegrationTestFlushInterval)
    65  	setup.SetStorageOpts(setup.StorageOpts().SetCommitLogOptions(commitLogOpts))
    66  
    67  	log := setup.StorageOpts().InstrumentOptions().Logger()
    68  
    69  	// Write test data for ns1
    70  	log.Info("generating data - ns1")
    71  	now := setup.NowFn()()
    72  	ns1SeriesMap := generate.BlocksByStart([]generate.BlockConfig{
    73  		{IDs: []string{"foo", "bar"}, NumPoints: 20, Start: now.Add(ns1BlockSize)},
    74  		{IDs: []string{"bar", "baz"}, NumPoints: 50, Start: now.Add(2 * ns1BlockSize)},
    75  		{IDs: []string{"and", "one"}, NumPoints: 40, Start: now.Add(3 * ns1BlockSize)},
    76  	})
    77  
    78  	setup.NamespaceMetadataOrFail(testNamespaces[0])
    79  	log.Info("writing data - ns1")
    80  	writeCommitLogData(t, setup, commitLogOpts, ns1SeriesMap, ns1, false)
    81  	log.Info("written data - ns1")
    82  
    83  	// Write test data for ns2
    84  	log.Info("generating data - ns2")
    85  	ns2SeriesMap := generate.BlocksByStart([]generate.BlockConfig{
    86  		{IDs: []string{"abc", "def"}, NumPoints: 20, Start: now.Add(ns2BlockSize)},
    87  		{IDs: []string{"xyz", "lmn"}, NumPoints: 50, Start: now.Add(2 * ns2BlockSize)},
    88  		{IDs: []string{"cat", "hax"}, NumPoints: 80, Start: now.Add(3 * ns2BlockSize)},
    89  		{IDs: []string{"why", "this"}, NumPoints: 40, Start: now.Add(4 * ns2BlockSize)},
    90  	})
    91  	setup.NamespaceMetadataOrFail(testNamespaces[1])
    92  	log.Info("writing data - ns2")
    93  	writeCommitLogData(t, setup, commitLogOpts, ns2SeriesMap, ns2, false)
    94  	log.Info("written data - ns2")
    95  
    96  	// Setup bootstrapper after writing data so filesystem inspection can find it
    97  	setupCommitLogBootstrapperWithFSInspection(t, setup, commitLogOpts)
    98  
    99  	later := now.Add(4 * ns1BlockSize)
   100  	setup.SetNowFn(later)
   101  	// Start the server with filesystem bootstrapper
   102  	require.NoError(t, setup.StartServer())
   103  	log.Debug("server is now up")
   104  
   105  	// Stop the server
   106  	defer func() {
   107  		require.NoError(t, setup.StopServer())
   108  		log.Debug("server is now down")
   109  	}()
   110  
   111  	log.Info("waiting until data is bootstrapped")
   112  	bootstrapped := waitUntil(func() bool { return setup.DB().IsBootstrapped() }, 20*time.Second)
   113  	require.True(t, bootstrapped)
   114  	log.Info("data bootstrapped")
   115  
   116  	// Verify in-memory data match what we expect
   117  	log.Info("verifying ns1 data")
   118  	verifySeriesMaps(t, setup, testNamespaces[0], ns1SeriesMap)
   119  	log.Info("verified ns1 data")
   120  
   121  	log.Info("verifying ns2 data")
   122  	verifySeriesMaps(t, setup, testNamespaces[1], ns2SeriesMap)
   123  	log.Info("verified ns2 data")
   124  }