github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/store/nbs/journal_test.go (about)

     1  // Copyright 2022 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package nbs
    16  
    17  import (
    18  	"context"
    19  	"math/rand"
    20  	"os"
    21  	"testing"
    22  
    23  	"github.com/stretchr/testify/assert"
    24  	"github.com/stretchr/testify/require"
    25  	"github.com/stretchr/testify/suite"
    26  
    27  	"github.com/dolthub/dolt/go/libraries/utils/file"
    28  	"github.com/dolthub/dolt/go/store/chunks"
    29  	"github.com/dolthub/dolt/go/store/types"
    30  )
    31  
    32  func makeTestChunkJournal(t *testing.T) *ChunkJournal {
    33  	cacheOnce.Do(makeGlobalCaches)
    34  	ctx := context.Background()
    35  	dir, err := os.MkdirTemp("", "")
    36  	require.NoError(t, err)
    37  	t.Cleanup(func() { file.RemoveAll(dir) })
    38  	m, err := newJournalManifest(ctx, dir)
    39  	require.NoError(t, err)
    40  	q := NewUnlimitedMemQuotaProvider()
    41  	p := newFSTablePersister(dir, q)
    42  	nbf := types.Format_Default.VersionString()
    43  	j, err := newChunkJournal(ctx, nbf, dir, m, p.(*fsTablePersister))
    44  	require.NoError(t, err)
    45  	t.Cleanup(func() { j.Close() })
    46  	return j
    47  }
    48  
    49  func TestChunkJournalBlockStoreSuite(t *testing.T) {
    50  	cacheOnce.Do(makeGlobalCaches)
    51  	fn := func(ctx context.Context, dir string) (*NomsBlockStore, error) {
    52  		q := NewUnlimitedMemQuotaProvider()
    53  		nbf := types.Format_Default.VersionString()
    54  		return NewLocalJournalingStore(ctx, nbf, dir, q)
    55  	}
    56  	suite.Run(t, &BlockStoreSuite{
    57  		factory:        fn,
    58  		skipInterloper: true,
    59  	})
    60  }
    61  
    62  func TestChunkJournalPersist(t *testing.T) {
    63  	ctx := context.Background()
    64  	j := makeTestChunkJournal(t)
    65  	const iters = 64
    66  	stats := &Stats{}
    67  	haver := emptyChunkSource{}
    68  	for i := 0; i < iters; i++ {
    69  		memTbl, chunkMap := randomMemTable(16)
    70  		source, err := j.Persist(ctx, memTbl, haver, stats)
    71  		assert.NoError(t, err)
    72  
    73  		for h, ch := range chunkMap {
    74  			ok, err := source.has(h)
    75  			assert.NoError(t, err)
    76  			assert.True(t, ok)
    77  			data, err := source.get(ctx, h, stats)
    78  			assert.NoError(t, err)
    79  			assert.Equal(t, ch.Data(), data)
    80  		}
    81  
    82  		cs, err := j.Open(ctx, source.hash(), 16, stats)
    83  		assert.NotNil(t, cs)
    84  		assert.NoError(t, err)
    85  	}
    86  }
    87  
    88  func TestReadRecordRanges(t *testing.T) {
    89  	ctx := context.Background()
    90  	j := makeTestChunkJournal(t)
    91  
    92  	var buf []byte
    93  	mt, data := randomMemTable(256)
    94  	gets := make([]getRecord, 0, len(data))
    95  	for h := range data {
    96  		gets = append(gets, getRecord{a: &h, prefix: h.Prefix()})
    97  	}
    98  
    99  	jcs, err := j.Persist(ctx, mt, emptyChunkSource{}, &Stats{})
   100  	require.NoError(t, err)
   101  
   102  	rdr, sz, err := jcs.(journalChunkSource).journal.snapshot()
   103  	require.NoError(t, err)
   104  	defer rdr.Close()
   105  
   106  	buf = make([]byte, sz)
   107  	n, err := rdr.Read(buf)
   108  	require.NoError(t, err)
   109  	assert.Equal(t, int(sz), n)
   110  
   111  	ranges, err := jcs.getRecordRanges(gets)
   112  	require.NoError(t, err)
   113  
   114  	for h, rng := range ranges {
   115  		b, err := jcs.get(ctx, h, &Stats{})
   116  		assert.NoError(t, err)
   117  		ch1 := chunks.NewChunkWithHash(h, b)
   118  		assert.Equal(t, data[h], ch1)
   119  
   120  		start, stop := rng.Offset, uint32(rng.Offset)+rng.Length
   121  		cc2, err := NewCompressedChunk(h, buf[start:stop])
   122  		assert.NoError(t, err)
   123  		ch2, err := cc2.ToChunk()
   124  		assert.NoError(t, err)
   125  		assert.Equal(t, data[h], ch2)
   126  	}
   127  }
   128  
   129  func randBuf(n int) (b []byte) {
   130  	b = make([]byte, n)
   131  	rand.Read(b)
   132  	return
   133  }