github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/ledger/complete/wal/checkpoint_v6_leaf_reader.go (about)

     1  package wal
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  
     7  	"github.com/rs/zerolog"
     8  
     9  	"github.com/onflow/flow-go/ledger"
    10  	"github.com/onflow/flow-go/ledger/common/hash"
    11  	"github.com/onflow/flow-go/ledger/complete/mtrie/flattener"
    12  	"github.com/onflow/flow-go/ledger/complete/mtrie/node"
    13  )
    14  
    15  type LeafNode struct {
    16  	Hash    hash.Hash
    17  	Path    ledger.Path
    18  	Payload *ledger.Payload
    19  }
    20  
    21  func nodeToLeaf(leaf *node.Node) *LeafNode {
    22  	return &LeafNode{
    23  		Hash:    leaf.Hash(),
    24  		Path:    *leaf.Path(),
    25  		Payload: leaf.Payload(),
    26  	}
    27  }
    28  
    29  // OpenAndReadLeafNodesFromCheckpointV6 takes a channel for pushing the leaf nodes that are read from
    30  // the given checkpoint file specified by dir and fileName.
    31  // It returns when finish reading the checkpoint file and the input channel can be closed.
    32  func OpenAndReadLeafNodesFromCheckpointV6(allLeafNodesCh chan<- *LeafNode, dir string, fileName string, logger zerolog.Logger) (errToReturn error) {
    33  	// we are the only sender of the channel, closing it after done
    34  	defer func() {
    35  		close(allLeafNodesCh)
    36  	}()
    37  
    38  	filepath := filePathCheckpointHeader(dir, fileName)
    39  
    40  	f, err := os.Open(filepath)
    41  	if err != nil {
    42  		return fmt.Errorf("could not open file %v: %w", filepath, err)
    43  	}
    44  	defer func(file *os.File) {
    45  		errToReturn = closeAndMergeError(file, errToReturn)
    46  	}(f)
    47  
    48  	subtrieChecksums, _, err := readCheckpointHeader(filepath, logger)
    49  	if err != nil {
    50  		return fmt.Errorf("could not read header: %w", err)
    51  	}
    52  
    53  	// ensure all checkpoint part file exists, might return os.ErrNotExist error
    54  	// if a file is missing
    55  	err = allPartFileExist(dir, fileName, len(subtrieChecksums))
    56  	if err != nil {
    57  		return fmt.Errorf("fail to check all checkpoint part file exist: %w", err)
    58  	}
    59  
    60  	// push leaf nodes to allLeafNodesCh
    61  	for i, checksum := range subtrieChecksums {
    62  		err := readCheckpointSubTrieLeafNodes(allLeafNodesCh, dir, fileName, i, checksum, logger)
    63  		if err != nil {
    64  			return fmt.Errorf("fail to read checkpoint leaf nodes from %v-th subtrie file: %w", i, err)
    65  		}
    66  	}
    67  
    68  	return nil
    69  }
    70  
    71  func readCheckpointSubTrieLeafNodes(leafNodesCh chan<- *LeafNode, dir string, fileName string, index int, checksum uint32, logger zerolog.Logger) error {
    72  	return processCheckpointSubTrie(dir, fileName, index, checksum, logger,
    73  		func(reader *Crc32Reader, nodesCount uint64) error {
    74  			scratch := make([]byte, 1024*4) // must not be less than 1024
    75  
    76  			logging := logProgress(fmt.Sprintf("reading %v-th sub trie roots", index), int(nodesCount), logger)
    77  			dummyChild := &node.Node{}
    78  			for i := uint64(1); i <= nodesCount; i++ {
    79  				node, err := flattener.ReadNode(reader, scratch, func(nodeIndex uint64) (*node.Node, error) {
    80  					if nodeIndex >= i {
    81  						return nil, fmt.Errorf("sequence of serialized nodes does not satisfy Descendents-First-Relationship")
    82  					}
    83  					return dummyChild, nil
    84  				})
    85  				if err != nil {
    86  					return fmt.Errorf("cannot read node %d: %w", i, err)
    87  				}
    88  				if node.IsLeaf() {
    89  					leafNodesCh <- nodeToLeaf(node)
    90  				}
    91  
    92  				logging(i)
    93  			}
    94  			return nil
    95  		})
    96  }