github.com/m3db/m3@v1.5.0/src/dbnode/integration/index_warm_write_gap_test.go (about)

     1  // +build integration
     2  //
     3  // Copyright (c) 2020 Uber Technologies, Inc.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  package integration
    24  
    25  import (
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/m3db/m3/src/dbnode/namespace"
    30  	"github.com/m3db/m3/src/dbnode/retention"
    31  	"github.com/m3db/m3/src/dbnode/storage/index"
    32  	"github.com/m3db/m3/src/m3ninx/idx"
    33  	xclock "github.com/m3db/m3/src/x/clock"
    34  	xtime "github.com/m3db/m3/src/x/time"
    35  
    36  	"github.com/stretchr/testify/require"
    37  	"go.uber.org/zap"
    38  )
    39  
    40  /*
    41   * This test runs the following situation, Now is 1p, data blockSize is 1h, index blockSize is 2h,
    42   * retention period 2h, buffer past 10mins, and buffer future 20mins. We write & index 50 metrics
    43   * between (12p, 12.50p). We then ensure that index writes within this warm index start -> start of buffer
    44   * past gap are indexed.
    45   *
    46   */
    47  func TestWarmIndexWriteGap(t *testing.T) {
    48  	if testing.Short() {
    49  		t.SkipNow() // Just skip if we're doing a short run
    50  	}
    51  
    52  	var (
    53  		numWrites       = 50
    54  		numTags         = 10
    55  		retentionPeriod = 2 * time.Hour
    56  		dataBlockSize   = time.Hour
    57  		indexBlockSize  = 2 * time.Hour
    58  		bufferFuture    = 20 * time.Minute
    59  		bufferPast      = 10 * time.Minute
    60  	)
    61  
    62  	// Test setup
    63  	md, err := namespace.NewMetadata(testNamespaces[0],
    64  		namespace.NewOptions().
    65  			SetRetentionOptions(
    66  				retention.NewOptions().
    67  					SetRetentionPeriod(retentionPeriod).
    68  					SetBufferPast(bufferPast).
    69  					SetBufferFuture(bufferFuture).
    70  					SetBlockSize(dataBlockSize)).
    71  			SetIndexOptions(
    72  				namespace.NewIndexOptions().
    73  					SetBlockSize(indexBlockSize).SetEnabled(true)).
    74  			SetColdWritesEnabled(true))
    75  	require.NoError(t, err)
    76  
    77  	testOpts := NewTestOptions(t).
    78  		SetNamespaces([]namespace.Metadata{md}).
    79  		SetWriteNewSeriesAsync(true)
    80  	testSetup, err := NewTestSetup(t, testOpts, nil)
    81  	require.NoError(t, err)
    82  	defer testSetup.Close()
    83  
    84  	t0 := xtime.ToUnixNano(time.Date(2018, time.May, 6, 13, 0, 0, 0, time.UTC))
    85  	// Issue writes in the gap between warm index start and start of buffer past.
    86  	t1 := t0.Truncate(indexBlockSize)
    87  	t2 := t0.Truncate(dataBlockSize).Add(-bufferPast)
    88  	testSetup.SetNowFn(t0)
    89  
    90  	writesPeriod0 := GenerateTestIndexWrite(0, numWrites, numTags, t1, t2)
    91  
    92  	// Start the server
    93  	log := testSetup.StorageOpts().InstrumentOptions().Logger()
    94  	require.NoError(t, testSetup.StartServer())
    95  
    96  	// Stop the server
    97  	defer func() {
    98  		require.NoError(t, testSetup.StopServer())
    99  		log.Debug("server is now down")
   100  	}()
   101  
   102  	client := testSetup.M3DBClient()
   103  	session, err := client.DefaultSession()
   104  	require.NoError(t, err)
   105  
   106  	log.Info("starting data write")
   107  	start := time.Now()
   108  	writesPeriod0.Write(t, md.ID(), session)
   109  	log.Info("test data written", zap.Duration("took", time.Since(start)))
   110  
   111  	log.Info("waiting till data is indexed")
   112  	indexed := xclock.WaitUntil(func() bool {
   113  		indexPeriod0 := writesPeriod0.NumIndexed(t, md.ID(), session)
   114  		return indexPeriod0 == len(writesPeriod0)
   115  	}, 5*time.Second)
   116  	require.True(t, indexed)
   117  	log.Info("verifiied data is indexed", zap.Duration("took", time.Since(start)))
   118  
   119  	// "shared":"shared", is a common tag across all written metrics
   120  	query := index.Query{
   121  		Query: idx.NewTermQuery([]byte("shared"), []byte("shared"))}
   122  
   123  	// ensure all data is present
   124  	log.Info("querying period0 results")
   125  	period0Results, _, err := session.FetchTagged(ContextWithDefaultTimeout(),
   126  		md.ID(), query, index.QueryOptions{StartInclusive: t1, EndExclusive: t2})
   127  	require.NoError(t, err)
   128  	writesPeriod0.MatchesSeriesIters(t, period0Results)
   129  	log.Info("found period0 results")
   130  }