github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/core/ledger/kvledger/txmgmt/rwsetutil/query_results_helper.go (about) 1 /* 2 Copyright hechain. All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package rwsetutil 8 9 import ( 10 "fmt" 11 12 "github.com/golang/protobuf/proto" 13 "github.com/hyperledger/fabric-protos-go/ledger/rwset/kvrwset" 14 "github.com/pkg/errors" 15 ) 16 17 // MerkleTreeLevel used for representing a level of the merkle tree 18 type MerkleTreeLevel uint32 19 20 // Hash represents bytes of a hash 21 type Hash []byte 22 23 const ( 24 leafLevel = MerkleTreeLevel(1) 25 ) 26 27 // HashFunc - the function signature for the hash function that is used in building and validating 28 // the merkle tree in the rwset, for the range query results 29 type HashFunc func(data []byte) (hashsum []byte, err error) 30 31 // RangeQueryResultsHelper helps preparing range query results for phantom items detection during validation. 32 // The results are expected to be fed as they are being iterated over. 33 // If the `hashingEnabled` is set to true, a merkle tree is built of the hashes over the results. 34 // The merkle tree helps reducing the size of the RWSet which otherwise would need to store all the raw KVReads 35 // 36 // The mental model of the tree can be described as below: 37 // All the results are treated as leaf nodes (level 0) of the tree. Next up level of the tree is built by collecting 'maxDegree + 1' 38 // items from the previous level and hashing the entire collection. 39 // Further upper levels of the tree are built in similar manner however the only difference is that unlike level-0 40 // (where collection consists of raw KVReads), collection at level 1 and above, consists of the hashes 41 // (of the collection of previous level). 42 // This is repeated until we reach at a level where we are left with the number of items less than or equals to `maxDegree`. 43 // In the last collection, the number of items can be less than 'maxDegree' (except if this is the only collection at the given level). 44 // 45 // As a result, if the number of total input results are less than or equals to 'maxDegree', no hashing is performed at all. 46 // And the final output of the computation is either the collection of raw results (if less that or equals to 'maxDegree') or 47 // a collection of hashes (that or equals to 'maxDegree') at some level in the tree. 48 // 49 // `AddResult` function should be invoke to supply the next result and at the end `Done` function should be invoked. 50 // The `Done` function does the final processing and returns the final output 51 type RangeQueryResultsHelper struct { 52 pendingResults []*kvrwset.KVRead 53 mt *merkleTree 54 maxDegree uint32 55 hashingEnabled bool 56 hashFunc HashFunc 57 } 58 59 // NewRangeQueryResultsHelper constructs a RangeQueryResultsHelper 60 func NewRangeQueryResultsHelper(enableHashing bool, maxDegree uint32, hashFunc HashFunc) (*RangeQueryResultsHelper, error) { 61 helper := &RangeQueryResultsHelper{ 62 pendingResults: nil, 63 hashingEnabled: enableHashing, 64 maxDegree: maxDegree, 65 mt: nil, 66 hashFunc: hashFunc, 67 } 68 if enableHashing { 69 var err error 70 if helper.mt, err = newMerkleTree(maxDegree, hashFunc); err != nil { 71 return nil, err 72 } 73 } 74 return helper, nil 75 } 76 77 // AddResult adds a new query result for processing. 78 // Put the result into the list of pending results. If the number of pending results exceeds `maxDegree`, 79 // consume the results for incrementally update the merkle tree 80 func (helper *RangeQueryResultsHelper) AddResult(kvRead *kvrwset.KVRead) error { 81 logger.Debug("Adding a result") 82 helper.pendingResults = append(helper.pendingResults, kvRead) 83 if helper.hashingEnabled && uint32(len(helper.pendingResults)) > helper.maxDegree { 84 logger.Debug("Processing the accumulated results") 85 if err := helper.processPendingResults(); err != nil { 86 return err 87 } 88 } 89 return nil 90 } 91 92 // Done processes any pending results if needed 93 // This returns the final pending results (i.e., []*KVRead) and hashes of the results (i.e., *MerkleSummary) 94 // Only one of these two will be non-nil (except when no results are ever added). 95 // `MerkleSummary` will be nil if and only if either `enableHashing` is set to false 96 // or the number of total results are less than `maxDegree` 97 func (helper *RangeQueryResultsHelper) Done() ([]*kvrwset.KVRead, *kvrwset.QueryReadsMerkleSummary, error) { 98 // The merkle tree will be empty if total results are less than or equals to 'maxDegree' 99 // i.e., not even once the results were processed for hashing 100 if !helper.hashingEnabled || helper.mt.isEmpty() { 101 return helper.pendingResults, nil, nil 102 } 103 if len(helper.pendingResults) != 0 { 104 logger.Debug("Processing the pending results") 105 if err := helper.processPendingResults(); err != nil { 106 return helper.pendingResults, nil, err 107 } 108 } 109 if err := helper.mt.done(); err != nil { 110 return nil, nil, err 111 } 112 return helper.pendingResults, helper.mt.getSummery(), nil 113 } 114 115 // GetMerkleSummary return the current state of the MerkleSummary 116 // This intermediate state of the merkle tree helps during validation to detect a mismatch early on. 117 // That helps by not requiring to build the complete merkle tree during validation 118 // if there is a mismatch in early portion of the result-set. 119 func (helper *RangeQueryResultsHelper) GetMerkleSummary() *kvrwset.QueryReadsMerkleSummary { 120 if !helper.hashingEnabled { 121 return nil 122 } 123 return helper.mt.getSummery() 124 } 125 126 func (helper *RangeQueryResultsHelper) processPendingResults() error { 127 var b []byte 128 var err error 129 if b, err = serializeKVReads(helper.pendingResults); err != nil { 130 return err 131 } 132 helper.pendingResults = nil 133 hash, err := helper.hashFunc(b) 134 if err != nil { 135 return err 136 } 137 return helper.mt.update(hash) 138 } 139 140 func serializeKVReads(kvReads []*kvrwset.KVRead) ([]byte, error) { 141 return proto.Marshal(&kvrwset.QueryReads{KvReads: kvReads}) 142 } 143 144 //////////// Merkle tree building code /////// 145 146 type merkleTree struct { 147 tree map[MerkleTreeLevel][]Hash 148 maxLevel MerkleTreeLevel 149 maxDegree uint32 150 hashFunc HashFunc 151 } 152 153 func newMerkleTree(maxDegree uint32, hashFunc HashFunc) (*merkleTree, error) { 154 if maxDegree < 2 { 155 return nil, errors.Errorf("maxDegree [%d] should not be less than 2 in the merkle tree", maxDegree) 156 } 157 return &merkleTree{ 158 make(map[MerkleTreeLevel][]Hash), 159 1, 160 maxDegree, 161 hashFunc, 162 }, nil 163 } 164 165 // update takes a hash that forms the next leaf level (level-1) node in the merkle tree. 166 // Also, complete the merkle tree as much as possible with the addition of this new leaf node - 167 // i.e. recursively build the higher level nodes and delete the underlying sub-tree. 168 func (m *merkleTree) update(nextLeafLevelHash Hash) error { 169 logger.Debugf("Before update() = %s", m) 170 defer logger.Debugf("After update() = %s", m) 171 m.tree[leafLevel] = append(m.tree[leafLevel], nextLeafLevelHash) 172 currentLevel := leafLevel 173 for { 174 currentLevelHashes := m.tree[currentLevel] 175 if uint32(len(currentLevelHashes)) <= m.maxDegree { 176 return nil 177 } 178 nextLevelHash, err := computeCombinedHash(currentLevelHashes, m.hashFunc) 179 if err != nil { 180 return err 181 } 182 delete(m.tree, currentLevel) 183 nextLevel := currentLevel + 1 184 m.tree[nextLevel] = append(m.tree[nextLevel], nextLevelHash) 185 if nextLevel > m.maxLevel { 186 m.maxLevel = nextLevel 187 } 188 currentLevel = nextLevel 189 } 190 } 191 192 // done completes the merkle tree. 193 // There may have been some nodes that are at the levels lower than the maxLevel (maximum level seen by the tree so far). 194 // Make the parent nodes out of such nodes till we complete the tree at the level of maxLevel (or maxLevel+1). 195 func (m *merkleTree) done() error { 196 logger.Debugf("Before done() = %s", m) 197 defer logger.Debugf("After done() = %s", m) 198 currentLevel := leafLevel 199 var h Hash 200 var err error 201 for currentLevel < m.maxLevel { 202 currentLevelHashes := m.tree[currentLevel] 203 switch len(currentLevelHashes) { 204 case 0: 205 currentLevel++ 206 continue 207 case 1: 208 h = currentLevelHashes[0] 209 default: 210 if h, err = computeCombinedHash(currentLevelHashes, m.hashFunc); err != nil { 211 return err 212 } 213 } 214 delete(m.tree, currentLevel) 215 currentLevel++ 216 m.tree[currentLevel] = append(m.tree[currentLevel], h) 217 } 218 219 finalHashes := m.tree[m.maxLevel] 220 if uint32(len(finalHashes)) > m.maxDegree { 221 delete(m.tree, m.maxLevel) 222 m.maxLevel++ 223 combinedHash, err := computeCombinedHash(finalHashes, m.hashFunc) 224 if err != nil { 225 return err 226 } 227 m.tree[m.maxLevel] = []Hash{combinedHash} 228 } 229 return nil 230 } 231 232 func (m *merkleTree) getSummery() *kvrwset.QueryReadsMerkleSummary { 233 return &kvrwset.QueryReadsMerkleSummary{ 234 MaxDegree: m.maxDegree, 235 MaxLevel: uint32(m.getMaxLevel()), 236 MaxLevelHashes: hashesToBytes(m.getMaxLevelHashes()), 237 } 238 } 239 240 func (m *merkleTree) getMaxLevel() MerkleTreeLevel { 241 return m.maxLevel 242 } 243 244 func (m *merkleTree) getMaxLevelHashes() []Hash { 245 return m.tree[m.maxLevel] 246 } 247 248 func (m *merkleTree) isEmpty() bool { 249 return m.maxLevel == 1 && len(m.tree[m.maxLevel]) == 0 250 } 251 252 func (m *merkleTree) String() string { 253 return fmt.Sprintf("tree := %#v", m.tree) 254 } 255 256 func computeCombinedHash(hashes []Hash, hashFunc HashFunc) (Hash, error) { 257 combinedHash := []byte{} 258 for _, h := range hashes { 259 combinedHash = append(combinedHash, h...) 260 } 261 return hashFunc(combinedHash) 262 } 263 264 func hashesToBytes(hashes []Hash) [][]byte { 265 b := [][]byte{} 266 for _, hash := range hashes { 267 b = append(b, hash) 268 } 269 return b 270 }