github.com/m3db/m3@v1.5.0/src/dbnode/integration/disk_flush_multi_ns_test.go (about)

     1  // +build integration
     2  
     3  // Copyright (c) 2017 Uber Technologies, Inc.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  package integration
    24  
    25  import (
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/m3db/m3/src/dbnode/integration/generate"
    30  	"github.com/m3db/m3/src/dbnode/namespace"
    31  	"github.com/m3db/m3/src/dbnode/retention"
    32  	xtime "github.com/m3db/m3/src/x/time"
    33  
    34  	"github.com/stretchr/testify/require"
    35  	"go.uber.org/zap"
    36  )
    37  
    38  func TestDiskFlushMultipleNamespace(t *testing.T) {
    39  	if testing.Short() {
    40  		t.SkipNow() // Just skip if we're doing a short run
    41  	}
    42  
    43  	// Test setup
    44  	var (
    45  		rOpts        = retention.NewOptions().SetRetentionPeriod(18 * time.Hour)
    46  		ns1BlockSize = 2 * time.Hour
    47  		ns2BlockSize = 3 * time.Hour
    48  		ns1ROpts     = rOpts.SetBlockSize(ns1BlockSize)
    49  		ns2ROpts     = rOpts.SetBlockSize(ns2BlockSize)
    50  	)
    51  
    52  	ns1, err := namespace.NewMetadata(testNamespaces[0], namespace.NewOptions().SetRetentionOptions(ns1ROpts))
    53  	require.NoError(t, err)
    54  	ns2, err := namespace.NewMetadata(testNamespaces[1], namespace.NewOptions().SetRetentionOptions(ns2ROpts))
    55  	require.NoError(t, err)
    56  	opts := NewTestOptions(t).
    57  		SetNamespaces([]namespace.Metadata{ns1, ns2})
    58  
    59  	// Test setup
    60  	testSetup, err := NewTestSetup(t, opts, nil)
    61  	require.NoError(t, err)
    62  	defer testSetup.Close()
    63  
    64  	clOpts := testSetup.StorageOpts().CommitLogOptions()
    65  	filePathPrefix := clOpts.FilesystemOptions().FilePathPrefix()
    66  
    67  	// it's aligned to lcm of ns block sizes
    68  	now := testSetup.NowFn()()
    69  
    70  	// Start the server
    71  	log := testSetup.StorageOpts().InstrumentOptions().Logger()
    72  	log.Info("disk flush multiple namespaces test")
    73  	require.NoError(t, testSetup.StartServer())
    74  	log.Info("server is now up")
    75  
    76  	// Stop the server
    77  	defer func() {
    78  		require.NoError(t, testSetup.StopServer())
    79  		log.Info("server is now down")
    80  	}()
    81  
    82  	log.Info("generating test data")
    83  	// test data for ns1
    84  	ns1SeriesMaps := make(map[xtime.UnixNano]generate.SeriesBlock)
    85  	ns1InputData := []generate.BlockConfig{
    86  		{IDs: []string{"foo", "bar"}, NumPoints: 100, Start: now},
    87  		{IDs: []string{"foo", "baz"}, NumPoints: 50, Start: now.Add(ns1BlockSize)},
    88  	}
    89  
    90  	// test data for ns2
    91  	ns2SeriesMaps := make(map[xtime.UnixNano]generate.SeriesBlock)
    92  	ns2InputData := []generate.BlockConfig{
    93  		{IDs: []string{"foo", "bar"}, NumPoints: 20, Start: now},
    94  	}
    95  
    96  	for _, ns1Input := range ns1InputData {
    97  		// write the data for ns1, always
    98  		testSetup.SetNowFn(ns1Input.Start)
    99  		testData := generate.Block(ns1Input)
   100  		ns1SeriesMaps[ns1Input.Start] = testData
   101  		require.NoError(t, testSetup.WriteBatch(testNamespaces[0], testData))
   102  		log.Info("wrote ns1 for time", zap.Time("start", ns1Input.Start.ToTime()))
   103  
   104  		// when applicable, write the data for ns2, too
   105  		for _, ns2Input := range ns2InputData {
   106  			if ns1Input.Start != ns2Input.Start {
   107  				continue
   108  			}
   109  			testData = generate.Block(ns2Input)
   110  			ns2SeriesMaps[ns2Input.Start] = testData
   111  			log.Info("wrote ns2 for time", zap.Time("start", ns2Input.Start.ToTime()))
   112  			require.NoError(t, testSetup.WriteBatch(testNamespaces[1], testData))
   113  		}
   114  	}
   115  	log.Info("test data written successfully")
   116  
   117  	// Advance time to make sure all data are flushed. Because data
   118  	// are flushed to disk asynchronously, need to poll to check
   119  	// when data are written.
   120  	maxWaitTime := time.Minute
   121  	log.Info("waiting until data is flushed")
   122  	testSetup.SetNowFn(testSetup.NowFn()().Add(3 * ns1BlockSize))
   123  	require.NoError(t, waitUntilDataFilesFlushed(filePathPrefix, testSetup.ShardSet(), testNamespaces[0], ns1SeriesMaps, maxWaitTime))
   124  	require.NoError(t, waitUntilDataFilesFlushed(filePathPrefix, testSetup.ShardSet(), testNamespaces[1], ns2SeriesMaps, maxWaitTime))
   125  	log.Info("data has been flushed")
   126  
   127  	// Verify on-disk data match what we expect
   128  	log.Info("verifying flushed data")
   129  	verifyFlushedDataFiles(t, testSetup.ShardSet(), testSetup.StorageOpts(), testNamespaces[0], ns1SeriesMaps)
   130  	verifyFlushedDataFiles(t, testSetup.ShardSet(), testSetup.StorageOpts(), testNamespaces[1], ns2SeriesMaps)
   131  	log.Info("flushed data verified")
   132  }