github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/store/nbs/journal_index_record_test.go (about)

     1  // Copyright 2023 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package nbs
    16  
    17  import (
    18  	"bufio"
    19  	"bytes"
    20  	"hash/crc32"
    21  	"math"
    22  	"math/rand"
    23  	"testing"
    24  
    25  	"github.com/stretchr/testify/require"
    26  
    27  	"github.com/dolthub/dolt/go/store/hash"
    28  )
    29  
    30  func TestRoundTripIndexLookups(t *testing.T) {
    31  	// write lookups to a writer
    32  	buf := new(bytes.Buffer)
    33  	w := bufio.NewWriter(buf)
    34  	batches := 10
    35  	chunksPerBatch := 1000
    36  	start := uint64(math.MaxInt32)
    37  	var off int
    38  	for i := 0; i <= batches; i++ {
    39  		lookups, meta := newLookups(t, chunksPerBatch, start)
    40  		for _, l := range lookups {
    41  			err := writeIndexLookup(w, l)
    42  			require.NoError(t, err)
    43  		}
    44  		err := writeJournalIndexMeta(w, meta.latestHash, meta.batchStart, meta.batchEnd, meta.checkSum)
    45  		require.NoError(t, err)
    46  		start = uint64(meta.batchEnd)
    47  		off += (1+lookupSz)*chunksPerBatch + (1 + lookupMetaSz)
    48  	}
    49  
    50  	// read lookups from the buffer
    51  	lookupCnt := 0
    52  	metaCnt := 0
    53  
    54  	_, err := processIndexRecords(bufio.NewReader(buf), int64(off), func(meta lookupMeta, lookups []lookup, checksum uint32) error {
    55  		require.Equal(t, meta.checkSum, checksum)
    56  		lookupCnt += len(lookups)
    57  		metaCnt += 1
    58  		return nil
    59  	})
    60  	require.NoError(t, err)
    61  	require.Equal(t, batches*chunksPerBatch, lookupCnt)
    62  	require.Equal(t, batches, metaCnt)
    63  }
    64  
    65  func newLookups(t *testing.T, n int, start uint64) ([]lookup, lookupMeta) {
    66  	var lookups []lookup
    67  	var off uint64
    68  	var end uint64
    69  	var checksum uint32
    70  	hashes := genBytes(20, n)
    71  	for _, h := range hashes {
    72  		addr := toAddr16(hash.New(h))
    73  		length := (rand.Uint64() % 1024)
    74  		checksum = crc32.Update(checksum, crcTable, addr[:])
    75  		start = end
    76  		lookups = append(lookups, lookup{
    77  			a: toAddr16(hash.New(h)),
    78  			r: Range{Offset: off, Length: uint32(length)},
    79  		})
    80  		off += length
    81  		end = start + (rand.Uint64() % 1024)
    82  	}
    83  	return lookups, lookupMeta{
    84  		batchStart: int64(start),
    85  		batchEnd:   int64(end),
    86  		checkSum:   checksum,
    87  		latestHash: hash.New(hashes[len(hashes)-1]),
    88  	}
    89  }
    90  
    91  func genBytes(size, count int) (keys [][]byte) {
    92  	src := rand.New(rand.NewSource(int64(size * count)))
    93  	letters := []byte("123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
    94  	r := make([]byte, size*count)
    95  	for i := range r {
    96  		r[i] = letters[src.Intn(len(letters))]
    97  	}
    98  	keys = make([][]byte, count)
    99  	for i := range keys {
   100  		keys[i] = r[:size]
   101  		r = r[size:]
   102  	}
   103  	return
   104  }
   105  
   106  func TestRoundTripIndexLookupMeta(t *testing.T) {
   107  	// write metadata to buffer
   108  	// read from buffer
   109  }
   110  
   111  func TestRoundTripIndexLookupsMeta(t *testing.T) {
   112  	// create writer
   113  	// add lookups through ranges.put
   114  	// flush with flushIndexRecord
   115  	// do a bunch of iters
   116  	// use processIndexRecords2 to read back, make sure roots/checksums are consistent, counts, etc
   117  }
   118  
   119  func makeLookups(cnt int) (lookups []lookup) {
   120  	lookups = make([]lookup, cnt)
   121  	buf := make([]byte, cnt*hash.ByteLen)
   122  	rand.Read(buf)
   123  	var off uint64
   124  	for i := range lookups {
   125  		copy(lookups[i].a[:], buf)
   126  		buf = buf[hash.ByteLen:]
   127  		lookups[i].r.Offset = off
   128  		l := rand.Uint32() % 1024
   129  		lookups[i].r.Length = l
   130  		off += uint64(l)
   131  	}
   132  	return
   133  }