github.com/m3db/m3@v1.5.0/src/dbnode/integration/disk_cleanup_index_test.go (about)

     1  // +build integration
     2  
     3  // Copyright (c) 2018 Uber Technologies, Inc.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  package integration
    24  
    25  import (
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/m3db/m3/src/dbnode/namespace"
    30  	"github.com/m3db/m3/src/dbnode/persist/fs"
    31  	"github.com/m3db/m3/src/dbnode/retention"
    32  	xclock "github.com/m3db/m3/src/x/clock"
    33  
    34  	"github.com/stretchr/testify/require"
    35  )
    36  
    37  func TestDiskCleanupIndex(t *testing.T) {
    38  	if testing.Short() {
    39  		t.SkipNow() // Just skip if we're doing a short run
    40  	}
    41  
    42  	// Test setup
    43  	var (
    44  		rOpts        = retention.NewOptions().SetRetentionPeriod(48 * time.Hour)
    45  		nsBlockSize  = time.Hour
    46  		idxBlockSize = 2 * time.Hour
    47  		nsROpts      = rOpts.SetBlockSize(nsBlockSize)
    48  	)
    49  
    50  	md, err := namespace.NewMetadata(testNamespaces[0], namespace.NewOptions().
    51  		SetCleanupEnabled(true).
    52  		SetRetentionOptions(nsROpts).SetIndexOptions(
    53  		namespace.NewIndexOptions().SetBlockSize(idxBlockSize).SetEnabled(true)))
    54  	require.NoError(t, err)
    55  
    56  	opts := NewTestOptions(t).
    57  		SetNamespaces([]namespace.Metadata{md})
    58  
    59  	// Test setup
    60  	setup, err := NewTestSetup(t, opts, nil)
    61  	require.NoError(t, err)
    62  	defer setup.Close()
    63  
    64  	retentionPeriod := md.Options().RetentionOptions().RetentionPeriod()
    65  	filePathPrefix := setup.StorageOpts().CommitLogOptions().FilesystemOptions().FilePathPrefix()
    66  
    67  	// Start the server
    68  	log := setup.StorageOpts().InstrumentOptions().Logger()
    69  	log.Debug("disk index cleanup test")
    70  	require.NoError(t, setup.StartServer())
    71  	log.Debug("server is now up")
    72  
    73  	// Stop the server
    74  	defer func() {
    75  		require.NoError(t, setup.StopServer())
    76  		log.Debug("server is now down")
    77  	}()
    78  
    79  	// Now create some fileset files
    80  	numTimes := 10
    81  	filesetsIdentifiers := make([]fs.FileSetFileIdentifier, numTimes)
    82  	now := setup.NowFn()().Truncate(idxBlockSize)
    83  	for i := 0; i < numTimes; i++ {
    84  		filesetsIdentifiers[i] = fs.FileSetFileIdentifier{
    85  			Namespace:  md.ID(),
    86  			BlockStart: now.Add(time.Duration(i) * idxBlockSize),
    87  		}
    88  	}
    89  	writeIndexFileSetFiles(t, setup.StorageOpts(), md, filesetsIdentifiers)
    90  
    91  	deltaNow := now.Add(time.Minute)
    92  	filesets, err := fs.IndexFileSetsBefore(filePathPrefix, md.ID(), deltaNow)
    93  	require.NoError(t, err)
    94  	require.NotEmpty(t, filesets)
    95  
    96  	// Move now forward by retentionPeriod + blockSize so fileset files at now will be deleted
    97  	newNow := now.Add(retentionPeriod).Add(idxBlockSize)
    98  	setup.SetNowFn(newNow)
    99  
   100  	// Check if files have been deleted
   101  	waitTimeout := 30 * time.Second
   102  	deleted := xclock.WaitUntil(func() bool {
   103  		filesets, err := fs.IndexFileSetsBefore(filePathPrefix, md.ID(), deltaNow)
   104  		require.NoError(t, err)
   105  		return len(filesets) == 0
   106  	}, waitTimeout)
   107  	require.True(t, deleted)
   108  }