github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/store/nbs/util.go (about)

     1  // Copyright 2021 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package nbs
    16  
    17  import (
    18  	"context"
    19  	"io"
    20  	"math"
    21  
    22  	"github.com/dolthub/dolt/go/libraries/utils/iohelp"
    23  
    24  	"github.com/dolthub/dolt/go/store/chunks"
    25  	"github.com/dolthub/dolt/go/store/hash"
    26  )
    27  
    28  func IterChunks(ctx context.Context, rd io.ReadSeeker, cb func(chunk chunks.Chunk) (stop bool, err error)) error {
    29  	idx, err := readTableIndexByCopy(ctx, rd, &UnlimitedQuotaProvider{})
    30  	if err != nil {
    31  		return err
    32  	}
    33  
    34  	defer idx.Close()
    35  
    36  	seen := make(map[hash.Hash]struct{})
    37  	for i := uint32(0); i < idx.chunkCount(); i++ {
    38  		var h hash.Hash
    39  		ie, err := idx.indexEntry(i, &h)
    40  		if err != nil {
    41  			return err
    42  		}
    43  		if _, ok := seen[h]; !ok {
    44  			seen[h] = struct{}{}
    45  			chunkBytes, err := readNFrom(rd, ie.Offset(), ie.Length())
    46  			if err != nil {
    47  				return err
    48  			}
    49  
    50  			cmpChnk, err := NewCompressedChunk(h, chunkBytes)
    51  			if err != nil {
    52  				return err
    53  			}
    54  
    55  			chunk, err := cmpChnk.ToChunk()
    56  			if err != nil {
    57  				return err
    58  			}
    59  
    60  			stop, err := cb(chunk)
    61  			if err != nil {
    62  				return err
    63  			} else if stop {
    64  				break
    65  			}
    66  		}
    67  	}
    68  
    69  	return nil
    70  }
    71  
    72  func GetTableIndexPrefixes(ctx context.Context, rd io.ReadSeeker) (prefixes []uint64, err error) {
    73  	idx, err := readTableIndexByCopy(ctx, rd, &UnlimitedQuotaProvider{})
    74  	if err != nil {
    75  		return nil, err
    76  	}
    77  	defer func() {
    78  		cerr := idx.Close()
    79  		if err == nil {
    80  			err = cerr
    81  		}
    82  	}()
    83  
    84  	prefixes, err = idx.prefixes()
    85  	if err != nil {
    86  		return nil, err
    87  	}
    88  	return
    89  }
    90  
    91  func GuessPrefixOrdinal(prefix uint64, n uint32) int {
    92  	hi := prefix >> 32
    93  	return int((hi * uint64(n)) / uint64(math.MaxUint32))
    94  }
    95  
    96  func readNFrom(rd io.ReadSeeker, offset uint64, length uint32) ([]byte, error) {
    97  	_, err := rd.Seek(int64(offset), io.SeekStart)
    98  
    99  	if err != nil {
   100  		return nil, err
   101  	}
   102  
   103  	return iohelp.ReadNBytes(rd, int(length))
   104  }