github.com/m3db/m3@v1.5.1-0.20231129193456-75a402aa583b/src/dbnode/storage/bootstrap/util.go (about)

     1  // Copyright (c) 2019 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package bootstrap
    22  
    23  import (
    24  	"bytes"
    25  	"fmt"
    26  	"io"
    27  	"math"
    28  	"sort"
    29  	"sync"
    30  
    31  	"github.com/m3db/m3/src/dbnode/encoding"
    32  	"github.com/m3db/m3/src/dbnode/encoding/m3tsz"
    33  	"github.com/m3db/m3/src/dbnode/namespace"
    34  	"github.com/m3db/m3/src/dbnode/persist/fs"
    35  	"github.com/m3db/m3/src/dbnode/storage/block"
    36  	"github.com/m3db/m3/src/dbnode/storage/bootstrap/result"
    37  	"github.com/m3db/m3/src/dbnode/storage/series"
    38  	"github.com/m3db/m3/src/dbnode/x/xio"
    39  	"github.com/m3db/m3/src/x/context"
    40  	"github.com/m3db/m3/src/x/ident"
    41  	"github.com/m3db/m3/src/x/pool"
    42  	xtest "github.com/m3db/m3/src/x/test"
    43  	xtime "github.com/m3db/m3/src/x/time"
    44  
    45  	"github.com/golang/mock/gomock"
    46  	"github.com/stretchr/testify/assert"
    47  	"github.com/stretchr/testify/require"
    48  )
    49  
    50  // ReadersForID is a slice of readers that share a series ID.
    51  type ReadersForID []ReaderAtTime
    52  
    53  // ReaderMap is a map containing all gathered block segment readers.
    54  type ReaderMap map[string]ReadersForID
    55  
    56  // Must implement NamespaceDataAccumulator.
    57  var _ NamespaceDataAccumulator = (*TestDataAccumulator)(nil)
    58  
    59  // TestDataAccumulator is a NamespaceDataAccumulator that captures any
    60  // series inserts for examination.
    61  type TestDataAccumulator struct {
    62  	sync.Mutex
    63  
    64  	t              require.TestingT
    65  	ctrl           *gomock.Controller
    66  	ns             string
    67  	pool           encoding.MultiReaderIteratorPool
    68  	loadedBlockMap ReaderMap
    69  	schema         namespace.SchemaDescr
    70  	// writeMap is a map to which values are written directly.
    71  	writeMap DecodedBlockMap
    72  	results  map[string]CheckoutSeriesResult
    73  }
    74  
    75  // DecodedValues is a slice of series datapoints.
    76  type DecodedValues []series.DecodedTestValue
    77  
    78  // DecodedBlockMap is a map of decoded datapoints per series ID.
    79  type DecodedBlockMap map[string]DecodedValues
    80  
    81  func testValuesEqual(
    82  	a series.DecodedTestValue,
    83  	b series.DecodedTestValue,
    84  ) bool {
    85  	return a.Timestamp.Equal(b.Timestamp) &&
    86  		math.Abs(a.Value-b.Value) < 0.000001 &&
    87  		a.Unit == b.Unit &&
    88  		bytes.Equal(a.Annotation, b.Annotation)
    89  }
    90  
    91  // VerifyEquals verifies that two DecodedBlockMap are equal; errors otherwise.
    92  func (m DecodedBlockMap) VerifyEquals(other DecodedBlockMap) error {
    93  	if len(m) != len(other) {
    94  		return fmt.Errorf("block maps of length %d and %d do not match",
    95  			len(m), len(other))
    96  	}
    97  
    98  	seen := make(map[string]struct{})
    99  	for k, v := range m {
   100  		otherSeries, found := other[k]
   101  		if !found {
   102  			return fmt.Errorf("series %s: values not found", k)
   103  		}
   104  
   105  		if len(otherSeries) != len(v) {
   106  			return fmt.Errorf("series %s: length of series %d does not match other %d",
   107  				k, len(v), len(otherSeries))
   108  		}
   109  
   110  		// NB: make a clone here to avoid mutating base data
   111  		// just in case any tests care about order.
   112  		thisVal := append([]series.DecodedTestValue(nil), v...)
   113  		otherVal := append([]series.DecodedTestValue(nil), otherSeries...)
   114  
   115  		sort.Sort(series.ValuesByTime(thisVal))
   116  		sort.Sort(series.ValuesByTime(otherVal))
   117  		for i, t := range thisVal {
   118  			o := otherVal[i]
   119  			if !testValuesEqual(t, o) {
   120  				return fmt.Errorf("series %s: value %+v does not match other value %+v",
   121  					k, t, o)
   122  			}
   123  		}
   124  
   125  		seen[k] = struct{}{}
   126  	}
   127  
   128  	for k := range other {
   129  		if _, beenFound := seen[k]; !beenFound {
   130  			return fmt.Errorf("series %s not found in this map", k)
   131  		}
   132  	}
   133  
   134  	return nil
   135  }
   136  
   137  // ReaderAtTime captures incoming block loads, including
   138  // their start times and tags.
   139  type ReaderAtTime struct {
   140  	// Start is the block start time.
   141  	Start xtime.UnixNano
   142  	// Reader is the block segment reader.
   143  	Reader xio.SegmentReader
   144  	// Tags is the list of tags in a basic string map format.
   145  	Tags map[string]string
   146  }
   147  
   148  // dumpLoadedBlocks decodes any accumulated values gathered from calls to
   149  // series.LoadBlock() and returns them as raw values.
   150  func (a *TestDataAccumulator) dumpLoadedBlocks() DecodedBlockMap {
   151  	if len(a.loadedBlockMap) == 0 {
   152  		return nil
   153  	}
   154  
   155  	decodedMap := make(DecodedBlockMap, len(a.loadedBlockMap))
   156  	iter := a.pool.Get()
   157  	defer iter.Close()
   158  	for k, v := range a.loadedBlockMap {
   159  		readers := make([]xio.SegmentReader, 0, len(v))
   160  		for _, r := range v {
   161  			readers = append(readers, r.Reader)
   162  		}
   163  
   164  		value, err := series.DecodeSegmentValues(readers, iter, a.schema)
   165  		if err != nil {
   166  			if err != io.EOF {
   167  				require.NoError(a.t, err)
   168  			}
   169  
   170  			// NB: print out that we encountered EOF here to assist debugging tests,
   171  			// but this is not necessarily a failure.
   172  			fmt.Println("EOF: segment had no values.")
   173  		}
   174  
   175  		sort.Sort(series.ValuesByTime(value))
   176  		decodedMap[k] = value
   177  	}
   178  
   179  	return decodedMap
   180  }
   181  
   182  // CheckoutSeriesWithLock will retrieve a series for writing to,
   183  // and when the accumulator is closed, it will ensure that the
   184  // series is released (with lock).
   185  func (a *TestDataAccumulator) CheckoutSeriesWithLock(
   186  	shardID uint32,
   187  	id ident.ID,
   188  	tags ident.TagIterator,
   189  ) (CheckoutSeriesResult, bool, error) {
   190  	a.Lock()
   191  	defer a.Unlock()
   192  	return a.checkoutSeriesWithLock(shardID, id, tags)
   193  }
   194  
   195  // CheckoutSeriesWithoutLock will retrieve a series for writing to,
   196  // and when the accumulator is closed, it will ensure that the
   197  // series is released (without lock).
   198  func (a *TestDataAccumulator) CheckoutSeriesWithoutLock(
   199  	shardID uint32,
   200  	id ident.ID,
   201  	tags ident.TagIterator,
   202  ) (CheckoutSeriesResult, bool, error) {
   203  	return a.checkoutSeriesWithLock(shardID, id, tags)
   204  }
   205  
   206  func (a *TestDataAccumulator) checkoutSeriesWithLock(
   207  	shardID uint32,
   208  	id ident.ID,
   209  	tags ident.TagIterator,
   210  ) (CheckoutSeriesResult, bool, error) {
   211  	var decodedTags map[string]string
   212  	if tags != nil {
   213  		decodedTags = make(map[string]string, tags.Len())
   214  		for tags.Next() {
   215  			tag := tags.Current()
   216  			name := tag.Name.String()
   217  			value := tag.Value.String()
   218  			if len(name) > 0 && len(value) > 0 {
   219  				decodedTags[name] = value
   220  			}
   221  		}
   222  
   223  		if err := tags.Err(); err != nil {
   224  			return CheckoutSeriesResult{}, false, err
   225  		}
   226  	} else {
   227  		// Ensure the decoded tags aren't nil.
   228  		decodedTags = make(map[string]string)
   229  	}
   230  
   231  	stringID := id.String()
   232  	if result, found := a.results[stringID]; found {
   233  		return result, true, nil
   234  	}
   235  
   236  	var streamErr error
   237  	mockSeries := series.NewMockDatabaseSeries(a.ctrl)
   238  
   239  	mockSeries.EXPECT().
   240  		LoadBlock(gomock.Any(), gomock.Any()).
   241  		DoAndReturn(func(bl block.DatabaseBlock, _ series.WriteType) error {
   242  			a.Lock()
   243  			defer a.Unlock()
   244  
   245  			reader, err := bl.Stream(context.NewBackground())
   246  			if err != nil {
   247  				streamErr = err
   248  				return err
   249  			}
   250  
   251  			a.loadedBlockMap[stringID] = append(a.loadedBlockMap[stringID],
   252  				ReaderAtTime{
   253  					Start:  bl.StartTime(),
   254  					Reader: reader,
   255  					Tags:   decodedTags,
   256  				})
   257  
   258  			return nil
   259  		}).AnyTimes()
   260  
   261  	mockSeries.EXPECT().Write(
   262  		gomock.Any(), gomock.Any(), gomock.Any(),
   263  		gomock.Any(), gomock.Any(), gomock.Any()).
   264  		DoAndReturn(
   265  			func(
   266  				_ context.Context,
   267  				ts xtime.UnixNano,
   268  				val float64,
   269  				unit xtime.Unit,
   270  				annotation []byte,
   271  				_ series.WriteOptions,
   272  			) (bool, series.WriteType, error) {
   273  				a.Lock()
   274  				a.writeMap[stringID] = append(
   275  					a.writeMap[stringID], series.DecodedTestValue{
   276  						Timestamp:  ts,
   277  						Value:      val,
   278  						Unit:       unit,
   279  						Annotation: annotation,
   280  					})
   281  				a.Unlock()
   282  				return true, series.WarmWrite, nil
   283  			}).AnyTimes()
   284  
   285  	result := CheckoutSeriesResult{
   286  		Shard:    shardID,
   287  		Resolver: &seriesStaticResolver{series: mockSeries},
   288  	}
   289  
   290  	a.results[stringID] = result
   291  	return result, true, streamErr
   292  }
   293  
   294  var _ SeriesRefResolver = (*seriesStaticResolver)(nil)
   295  
   296  type seriesStaticResolver struct {
   297  	series SeriesRef
   298  }
   299  
   300  func (r *seriesStaticResolver) SeriesRef() (SeriesRef, error) {
   301  	return r.series, nil
   302  }
   303  
   304  func (r *seriesStaticResolver) ReleaseRef() {}
   305  
   306  // Release is a no-op on the test accumulator.
   307  func (a *TestDataAccumulator) Release() {}
   308  
   309  // Close is a no-op on the test accumulator.
   310  func (a *TestDataAccumulator) Close() error { return nil }
   311  
   312  // NamespacesTester is a utility to assist testing bootstrapping.
   313  type NamespacesTester struct {
   314  	t    require.TestingT
   315  	ctrl *gomock.Controller
   316  	pool encoding.MultiReaderIteratorPool
   317  
   318  	// Accumulators are the accumulators which incoming blocks get loaded into.
   319  	// One per namespace.
   320  	Accumulators []*TestDataAccumulator
   321  
   322  	// Namespaces are the namespaces for this tester.
   323  	Namespaces Namespaces
   324  	// Cache is a snapshot of data useful during bootstrapping.
   325  	Cache Cache
   326  	// Results are the namespace results after bootstrapping.
   327  	Results NamespaceResults
   328  }
   329  
   330  func buildDefaultIterPool() encoding.MultiReaderIteratorPool {
   331  	iterPool := encoding.NewMultiReaderIteratorPool(pool.NewObjectPoolOptions())
   332  	iterPool.Init(m3tsz.DefaultReaderIteratorAllocFn(encoding.NewOptions()))
   333  	return iterPool
   334  }
   335  
   336  // BuildNamespacesTester builds a NamespacesTester.
   337  func BuildNamespacesTester(
   338  	t require.TestingT,
   339  	runOpts RunOptions,
   340  	ranges result.ShardTimeRanges,
   341  	mds ...namespace.Metadata,
   342  ) NamespacesTester {
   343  	return BuildNamespacesTesterWithReaderIteratorPool(
   344  		t,
   345  		runOpts,
   346  		ranges,
   347  		nil,
   348  		fs.NewOptions(),
   349  		mds...,
   350  	)
   351  }
   352  
   353  // BuildNamespacesTesterWithFilesystemOptions builds a NamespacesTester with fs.Options
   354  func BuildNamespacesTesterWithFilesystemOptions(
   355  	t require.TestingT,
   356  	runOpts RunOptions,
   357  	ranges result.ShardTimeRanges,
   358  	fsOpts fs.Options,
   359  	mds ...namespace.Metadata,
   360  ) NamespacesTester {
   361  	return BuildNamespacesTesterWithReaderIteratorPool(
   362  		t,
   363  		runOpts,
   364  		ranges,
   365  		nil,
   366  		fsOpts,
   367  		mds...,
   368  	)
   369  }
   370  
   371  // BuildNamespacesTesterWithReaderIteratorPool builds a NamespacesTester with a
   372  // given MultiReaderIteratorPool.
   373  func BuildNamespacesTesterWithReaderIteratorPool(
   374  	t require.TestingT,
   375  	runOpts RunOptions,
   376  	ranges result.ShardTimeRanges,
   377  	iterPool encoding.MultiReaderIteratorPool,
   378  	fsOpts fs.Options,
   379  	mds ...namespace.Metadata,
   380  ) NamespacesTester {
   381  	shards := make([]uint32, 0, ranges.Len())
   382  	for shard := range ranges.Iter() {
   383  		shards = append(shards, shard)
   384  	}
   385  
   386  	if iterPool == nil {
   387  		iterPool = buildDefaultIterPool()
   388  	}
   389  
   390  	ctrl := xtest.NewController(t)
   391  	namespacesMap := NewNamespacesMap(NamespacesMapOptions{})
   392  	accumulators := make([]*TestDataAccumulator, 0, len(mds))
   393  	finders := make([]NamespaceDetails, 0, len(mds))
   394  	for _, md := range mds {
   395  		nsCtx := namespace.NewContextFrom(md)
   396  		acc := &TestDataAccumulator{
   397  			t:              t,
   398  			ctrl:           ctrl,
   399  			pool:           iterPool,
   400  			ns:             md.ID().String(),
   401  			results:        make(map[string]CheckoutSeriesResult),
   402  			loadedBlockMap: make(ReaderMap),
   403  			writeMap:       make(DecodedBlockMap),
   404  			schema:         nsCtx.Schema,
   405  		}
   406  
   407  		accumulators = append(accumulators, acc)
   408  		namespacesMap.Set(md.ID(), Namespace{
   409  			Metadata:        md,
   410  			Shards:          shards,
   411  			DataAccumulator: acc,
   412  			DataRunOptions: NamespaceRunOptions{
   413  				ShardTimeRanges:       ranges.Copy(),
   414  				TargetShardTimeRanges: ranges.Copy(),
   415  				RunOptions:            runOpts,
   416  			},
   417  			IndexRunOptions: NamespaceRunOptions{
   418  				ShardTimeRanges:       ranges.Copy(),
   419  				TargetShardTimeRanges: ranges.Copy(),
   420  				RunOptions:            runOpts,
   421  			},
   422  		})
   423  		finders = append(finders, NamespaceDetails{
   424  			Namespace: md,
   425  			Shards:    shards,
   426  		})
   427  	}
   428  	cache, err := NewCache(NewCacheOptions().
   429  		SetFilesystemOptions(fsOpts).
   430  		SetInstrumentOptions(fsOpts.InstrumentOptions()).
   431  		SetNamespaceDetails(finders))
   432  	require.NoError(t, err)
   433  
   434  	return NamespacesTester{
   435  		t:            t,
   436  		ctrl:         ctrl,
   437  		pool:         iterPool,
   438  		Accumulators: accumulators,
   439  		Cache:        cache,
   440  		Namespaces: Namespaces{
   441  			Namespaces: namespacesMap,
   442  		},
   443  	}
   444  }
   445  
   446  // DecodedNamespaceMap is a map of decoded blocks per namespace ID.
   447  type DecodedNamespaceMap map[string]DecodedBlockMap
   448  
   449  // DumpLoadedBlocks dumps any loaded blocks as decoded series per namespace.
   450  func (nt *NamespacesTester) DumpLoadedBlocks() DecodedNamespaceMap {
   451  	nsMap := make(DecodedNamespaceMap, len(nt.Accumulators))
   452  	for _, acc := range nt.Accumulators {
   453  		block := acc.dumpLoadedBlocks()
   454  
   455  		if block != nil {
   456  			nsMap[acc.ns] = block
   457  		}
   458  	}
   459  
   460  	return nsMap
   461  }
   462  
   463  // EnsureDumpLoadedBlocksForNamespace dumps all loaded blocks as decoded series,
   464  // and fails if the namespace is not found.
   465  func (nt *NamespacesTester) EnsureDumpLoadedBlocksForNamespace(
   466  	md namespace.Metadata,
   467  ) DecodedBlockMap {
   468  	id := md.ID().String()
   469  	for _, acc := range nt.Accumulators {
   470  		if acc.ns == id {
   471  			return acc.dumpLoadedBlocks()
   472  		}
   473  	}
   474  
   475  	assert.FailNow(nt.t, fmt.Sprintf("namespace with id %s not found "+
   476  		"valid namespaces are %v", id, nt.Namespaces))
   477  	return nil
   478  }
   479  
   480  // EnsureNoLoadedBlocks ensures that no blocks have been loaded into any of this
   481  // testers accumulators.
   482  func (nt *NamespacesTester) EnsureNoLoadedBlocks() {
   483  	require.Equal(nt.t, 0, len(nt.DumpLoadedBlocks()))
   484  }
   485  
   486  // DumpWrites dumps the writes encountered for all namespaces.
   487  func (nt *NamespacesTester) DumpWrites() DecodedNamespaceMap {
   488  	nsMap := make(DecodedNamespaceMap, len(nt.Accumulators))
   489  	for _, acc := range nt.Accumulators {
   490  		if len(acc.writeMap) > 0 {
   491  			nsMap[acc.ns] = acc.writeMap
   492  		}
   493  	}
   494  
   495  	return nsMap
   496  }
   497  
   498  // EnsureDumpWritesForNamespace dumps the writes encountered for the
   499  // given namespace, and fails if the namespace is not found.
   500  func (nt *NamespacesTester) EnsureDumpWritesForNamespace(
   501  	md namespace.Metadata,
   502  ) DecodedBlockMap {
   503  	id := md.ID().String()
   504  	for _, acc := range nt.Accumulators {
   505  		if acc.ns == id {
   506  			return acc.writeMap
   507  		}
   508  	}
   509  
   510  	assert.FailNow(nt.t, fmt.Sprintf("namespace with id %s not found "+
   511  		"valid namespaces are %v", id, nt.Namespaces))
   512  	return nil
   513  }
   514  
   515  // EnsureNoWrites ensures that no writes have been written into any of this
   516  // testers accumulators.
   517  func (nt *NamespacesTester) EnsureNoWrites() {
   518  	require.Equal(nt.t, 0, len(nt.DumpWrites()))
   519  }
   520  
   521  // EnsureDumpAllForNamespace dumps all results for a single namespace, and
   522  // fails if the namespace is not found. The results are unsorted; if sorted
   523  // order is important for verification, they should be sorted afterwards.
   524  func (nt *NamespacesTester) EnsureDumpAllForNamespace(
   525  	md namespace.Metadata,
   526  ) (DecodedBlockMap, error) {
   527  	id := md.ID().String()
   528  	for _, acc := range nt.Accumulators {
   529  		if acc.ns != id {
   530  			continue
   531  		}
   532  
   533  		writeMap := acc.writeMap
   534  		loadedBlockMap := acc.dumpLoadedBlocks()
   535  		merged := make(DecodedBlockMap, len(writeMap)+len(loadedBlockMap))
   536  		for k, v := range writeMap {
   537  			merged[k] = v
   538  		}
   539  
   540  		for k, v := range loadedBlockMap {
   541  			if vals, found := merged[k]; found {
   542  				merged[k] = append(vals, v...)
   543  			} else {
   544  				merged[k] = v
   545  			}
   546  		}
   547  
   548  		return merged, nil
   549  	}
   550  
   551  	return nil, fmt.Errorf("namespace with id %s not found "+
   552  		"valid namespaces are %v", id, nt.Namespaces)
   553  }
   554  
   555  // EnsureDumpReadersForNamespace dumps the readers and their start times for a
   556  // given namespace, and fails if the namespace is not found.
   557  func (nt *NamespacesTester) EnsureDumpReadersForNamespace(
   558  	md namespace.Metadata,
   559  ) ReaderMap {
   560  	id := md.ID().String()
   561  	for _, acc := range nt.Accumulators {
   562  		if acc.ns == id {
   563  			return acc.loadedBlockMap
   564  		}
   565  	}
   566  
   567  	assert.FailNow(nt.t, fmt.Sprintf("namespace with id %s not found "+
   568  		"valid namespaces are %v", id, nt.Namespaces))
   569  	return nil
   570  }
   571  
   572  // ResultForNamespace gives the result for the given namespace, and fails if
   573  // the namespace is not found.
   574  func (nt *NamespacesTester) ResultForNamespace(id ident.ID) NamespaceResult {
   575  	result, found := nt.Results.Results.Get(id)
   576  	require.True(nt.t, found)
   577  	return result
   578  }
   579  
   580  // TestBootstrapWith bootstraps the current Namespaces with the
   581  // provided bootstrapper.
   582  func (nt *NamespacesTester) TestBootstrapWith(b Bootstrapper) {
   583  	ctx := context.NewBackground()
   584  	defer ctx.Close()
   585  	res, err := b.Bootstrap(ctx, nt.Namespaces, nt.Cache)
   586  	assert.NoError(nt.t, err)
   587  	nt.Results = res
   588  }
   589  
   590  // TestReadWith reads the current Namespaces with the
   591  // provided bootstrap source.
   592  func (nt *NamespacesTester) TestReadWith(s Source) {
   593  	ctx := context.NewBackground()
   594  	defer ctx.Close()
   595  	res, err := s.Read(ctx, nt.Namespaces, nt.Cache)
   596  	require.NoError(nt.t, err)
   597  	nt.Results = res
   598  }
   599  
   600  func validateRanges(ac xtime.Ranges, ex xtime.Ranges) error {
   601  	// Make range eclipses expected.
   602  	removedRange := ex.Clone()
   603  	removedRange.RemoveRanges(ac)
   604  	if !removedRange.IsEmpty() {
   605  		return fmt.Errorf("actual range %v does not match expected range %v "+
   606  			"diff: %v", ac, ex, removedRange)
   607  	}
   608  
   609  	// Now make sure no ranges outside of expected.
   610  	expectedWithAddedRanges := ex.Clone()
   611  	expectedWithAddedRanges.AddRanges(ac)
   612  	if ex.Len() != expectedWithAddedRanges.Len() {
   613  		return fmt.Errorf("expected with re-added ranges not equal")
   614  	}
   615  
   616  	iter := ex.Iter()
   617  	withAddedRangesIter := expectedWithAddedRanges.Iter()
   618  	for iter.Next() && withAddedRangesIter.Next() {
   619  		if !iter.Value().Equal(withAddedRangesIter.Value()) {
   620  			return fmt.Errorf("actual range %v does not match expected range %v",
   621  				ac, ex)
   622  		}
   623  	}
   624  
   625  	return nil
   626  }
   627  
   628  func validateShardTimeRanges(
   629  	r result.ShardTimeRanges,
   630  	ex result.ShardTimeRanges,
   631  ) error {
   632  	if ex.Len() != r.Len() {
   633  		return fmt.Errorf("expected %v and actual %v size mismatch", ex, r)
   634  	}
   635  
   636  	seen := make(map[uint32]struct{}, r.Len())
   637  	for k, val := range r.Iter() {
   638  		expectedVal, ok := ex.Get(k)
   639  		if !ok {
   640  			return fmt.Errorf("expected shard map %v does not have shard %d; "+
   641  				"actual: %v", ex, k, r)
   642  		}
   643  
   644  		if err := validateRanges(val, expectedVal); err != nil {
   645  			return err
   646  		}
   647  
   648  		seen[k] = struct{}{}
   649  	}
   650  
   651  	for k := range ex.Iter() {
   652  		if _, beenFound := seen[k]; !beenFound {
   653  			return fmt.Errorf("shard %d in actual not found in expected %v", k, ex)
   654  		}
   655  	}
   656  
   657  	return nil
   658  }
   659  
   660  // TestUnfulfilledForNamespace ensures the given namespace has the expected
   661  // range flagged as unfulfilled.
   662  func (nt *NamespacesTester) TestUnfulfilledForNamespace(
   663  	md namespace.Metadata,
   664  	ex result.ShardTimeRanges,
   665  	exIdx result.ShardTimeRanges,
   666  ) {
   667  	ns := nt.ResultForNamespace(md.ID())
   668  	actual := ns.DataResult.Unfulfilled()
   669  	require.NoError(nt.t, validateShardTimeRanges(actual, ex), "data")
   670  
   671  	if md.Options().IndexOptions().Enabled() {
   672  		actual := ns.IndexResult.Unfulfilled()
   673  		require.NoError(nt.t, validateShardTimeRanges(actual, exIdx), "index")
   674  	}
   675  }
   676  
   677  // TestIndexResultForNamespace verifies index result.
   678  func (nt *NamespacesTester) TestIndexResultForNamespace(
   679  	md namespace.Metadata,
   680  	expected result.IndexBootstrapResult,
   681  ) {
   682  	ns := nt.ResultForNamespace(md.ID())
   683  	require.Equal(nt.t, expected, ns.IndexResult)
   684  }
   685  
   686  // TestUnfulfilledForNamespaceIsEmpty ensures the given namespace has an empty
   687  // unfulfilled range.
   688  func (nt *NamespacesTester) TestUnfulfilledForNamespaceIsEmpty(
   689  	md namespace.Metadata,
   690  ) {
   691  	nt.TestUnfulfilledForIDIsEmpty(md.ID(), md.Options().IndexOptions().Enabled())
   692  }
   693  
   694  // TestUnfulfilledForIDIsEmpty ensures the given id has an empty
   695  // unfulfilled range.
   696  func (nt *NamespacesTester) TestUnfulfilledForIDIsEmpty(
   697  	id ident.ID,
   698  	useIndex bool,
   699  ) {
   700  	ns := nt.ResultForNamespace(id)
   701  	actual := ns.DataResult.Unfulfilled()
   702  	assert.True(nt.t, actual.IsEmpty(), fmt.Sprintf("data: not empty %v", actual))
   703  
   704  	if useIndex {
   705  		actual := ns.DataResult.Unfulfilled()
   706  		assert.True(nt.t, actual.IsEmpty(),
   707  			fmt.Sprintf("index: not empty %v", actual))
   708  	}
   709  }
   710  
   711  // Finish closes the namespaceTester and tests mocks for completion.
   712  func (nt *NamespacesTester) Finish() {
   713  	nt.ctrl.Finish()
   714  }
   715  
   716  // NamespaceMatcher is a matcher for namespaces.
   717  type NamespaceMatcher struct {
   718  	// Namespaces are the expected namespaces.
   719  	Namespaces Namespaces
   720  }
   721  
   722  // String describes what the matcher matches.
   723  func (m NamespaceMatcher) String() string { return "namespace query" }
   724  
   725  // Matches returns whether x is a match.
   726  func (m NamespaceMatcher) Matches(x interface{}) bool {
   727  	ns, ok := x.(Namespaces)
   728  	if !ok {
   729  		return false
   730  	}
   731  
   732  	equalRange := func(a, b TargetRange) bool {
   733  		return a.Range.Start.Equal(b.Range.Start) &&
   734  			a.Range.End.Equal(b.Range.End)
   735  	}
   736  
   737  	for _, v := range ns.Namespaces.Iter() {
   738  		other, found := m.Namespaces.Namespaces.Get(v.Key())
   739  		if !found {
   740  			return false
   741  		}
   742  
   743  		val := v.Value()
   744  		if !other.Metadata.Equal(val.Metadata) {
   745  			return false
   746  		}
   747  
   748  		if !equalRange(val.DataTargetRange, other.DataTargetRange) {
   749  			return false
   750  		}
   751  
   752  		if !equalRange(val.IndexTargetRange, other.IndexTargetRange) {
   753  			return false
   754  		}
   755  	}
   756  
   757  	return true
   758  }
   759  
   760  // NB: assert NamespaceMatcher is a gomock.Matcher
   761  var _ gomock.Matcher = (*NamespaceMatcher)(nil)
   762  
   763  // ShardTimeRangesMatcher is a matcher for ShardTimeRanges.
   764  type ShardTimeRangesMatcher struct {
   765  	// Ranges are the expected ranges.
   766  	Ranges result.ShardTimeRanges
   767  }
   768  
   769  // Matches returns whether x is a match.
   770  func (m ShardTimeRangesMatcher) Matches(x interface{}) bool {
   771  	actual, ok := x.(result.ShardTimeRanges)
   772  	if !ok {
   773  		return false
   774  	}
   775  
   776  	if err := validateShardTimeRanges(m.Ranges, actual); err != nil {
   777  		fmt.Println("shard time ranges do not match:", err.Error())
   778  		return false
   779  	}
   780  
   781  	return true
   782  }
   783  
   784  // String describes what the matcher matches.
   785  func (m ShardTimeRangesMatcher) String() string {
   786  	return "shardTimeRangesMatcher"
   787  }
   788  
   789  // NB: assert ShardTimeRangesMatcher is a gomock.Matcher
   790  var _ gomock.Matcher = (*ShardTimeRangesMatcher)(nil)