github.com/m3db/m3@v1.5.0/src/dbnode/integration/commitlog_bootstrap_only_reads_required_files_test.go (about)

     1  // +build integration
     2  
     3  // Copyright (c) 2017 Uber Technologies, Inc.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  package integration
    24  
    25  import (
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/m3db/m3/src/dbnode/namespace"
    30  	"github.com/m3db/m3/src/dbnode/retention"
    31  
    32  	"github.com/stretchr/testify/require"
    33  )
    34  
    35  func TestCommitLogBootstrapOnlyReadsRequiredFiles(t *testing.T) {
    36  	// TODO(rartoul): Temporarily disabled until a subsequent P.R that will
    37  	// improve and simplify the commitlog bootstrapping logic. This is fine
    38  	// because this integration test protects against performance regressions
    39  	// not correctness.
    40  	// https://github.com/m3db/m3/issues/1383
    41  	t.SkipNow()
    42  
    43  	// Test setup
    44  	var (
    45  		ropts     = retention.NewOptions().SetRetentionPeriod(12 * time.Hour)
    46  		blockSize = ropts.BlockSize()
    47  	)
    48  	ns1, err := namespace.NewMetadata(testNamespaces[0], namespace.NewOptions().SetRetentionOptions(ropts))
    49  	require.NoError(t, err)
    50  	opts := NewTestOptions(t).
    51  		SetNamespaces([]namespace.Metadata{ns1})
    52  
    53  	setup, err := NewTestSetup(t, opts, nil)
    54  	require.NoError(t, err)
    55  	defer setup.Close()
    56  
    57  	commitLogOpts := setup.StorageOpts().CommitLogOptions().
    58  		SetFlushInterval(defaultIntegrationTestFlushInterval)
    59  	setup.SetStorageOpts(setup.StorageOpts().SetCommitLogOptions(commitLogOpts))
    60  
    61  	log := setup.StorageOpts().InstrumentOptions().Logger()
    62  	log.Info("commit log bootstrap test")
    63  
    64  	// Write test data
    65  	log.Info("generating data")
    66  	now := setup.NowFn()()
    67  	seriesMaps := generateSeriesMaps(30, nil, now.Add(-2*blockSize), now.Add(-blockSize))
    68  	log.Info("writing data")
    69  	writeCommitLogData(t, setup, commitLogOpts, seriesMaps, ns1, false)
    70  	log.Info("finished writing data")
    71  
    72  	// The datapoints in this generated data are within the retention period and
    73  	// would ordinarily be bootstrapped, however, we intentionally write them to a
    74  	// commitlog file that has a timestamp outside of the retention period. This
    75  	// allows us to verify the commitlog bootstrapping logic will not waste time
    76  	// reading commitlog files that are outside of the retention period.
    77  	log.Info("generating data")
    78  	seriesMapsExpiredCommitlog := generateSeriesMaps(30, nil, now.Add(-2*blockSize), now.Add(-blockSize))
    79  	log.Info("writing data to commitlog file with out of range timestamp")
    80  	writeCommitLogDataSpecifiedTS(
    81  		t,
    82  		setup,
    83  		commitLogOpts,
    84  		seriesMapsExpiredCommitlog,
    85  		ns1,
    86  		now.Add(-2*ropts.RetentionPeriod()),
    87  		false,
    88  	)
    89  	log.Info("finished writing data to commitlog file with out of range timestamp")
    90  
    91  	// Setup bootstrapper after writing data so filesystem inspection can find it.
    92  	setupCommitLogBootstrapperWithFSInspection(t, setup, commitLogOpts)
    93  
    94  	setup.SetNowFn(now)
    95  	// Start the server with filesystem bootstrapper
    96  	require.NoError(t, setup.StartServer())
    97  	log.Debug("server is now up")
    98  
    99  	// Stop the server
   100  	defer func() {
   101  		require.NoError(t, setup.StopServer())
   102  		log.Debug("server is now down")
   103  	}()
   104  
   105  	// Verify in-memory data match what we expect - all writes from seriesMaps
   106  	// should be present, and none of the writes from seriesMapsExpiredCommitlog
   107  	// should be present.
   108  	metadatasByShard := testSetupMetadatas(t, setup, testNamespaces[0], now.Add(-2*blockSize), now)
   109  	observedSeriesMaps := testSetupToSeriesMaps(t, setup, ns1, metadatasByShard)
   110  	verifySeriesMapsEqual(t, seriesMaps, observedSeriesMaps)
   111  }