github.com/tenywen/fabric@v1.0.0-beta.0.20170620030522-a5b1ed380643/core/ledger/kvledger/txmgmt/rwsetutil/query_results_helper.go (about) 1 /* 2 Copyright IBM Corp. 2016 All Rights Reserved. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package rwsetutil 18 19 import ( 20 "fmt" 21 22 "github.com/golang/protobuf/proto" 23 "github.com/hyperledger/fabric/bccsp" 24 bccspfactory "github.com/hyperledger/fabric/bccsp/factory" 25 "github.com/hyperledger/fabric/protos/ledger/rwset/kvrwset" 26 ) 27 28 // MerkleTreeLevel used for representing a level of the merkle tree 29 type MerkleTreeLevel uint32 30 31 // Hash represents bytes of a hash 32 type Hash []byte 33 34 const ( 35 leafLevel = MerkleTreeLevel(1) 36 ) 37 38 var ( 39 hashOpts = &bccsp.SHA256Opts{} 40 ) 41 42 // RangeQueryResultsHelper helps preparing range query results for phantom items detection during validation. 43 // The results are expected to be fed as they are being iterated over. 44 // If the `hashingEnabled` is set to true, a merkle tree is built of the hashes over the results. 45 // The merkle tree helps reducing the size of the RWSet which otherwise would need to store all the raw KVReads 46 // 47 // The mental model of the tree can be described as below: 48 // All the results are treated as leaf nodes (level 0) of the tree. Next up level of the tree is built by collecting 'maxDegree + 1' 49 // items from the previous level and hashing the entire collection. 50 // Further upper levels of the tree are built in similar manner however the only difference is that unlike level-0 51 // (where collection consists of raw KVReads), collection at level 1 and above, consists of the hashes 52 // (of the collection of previous level). 53 // This is repeated until we reach at a level where we are left with the number of items less than or equals to `maxDegree`. 54 // In the last collection, the number of items can be less than 'maxDegree' (except if this is the only collection at the given level). 55 // 56 // As a result, if the number of total input results are less than or equals to 'maxDegree', no hashing is performed at all. 57 // And the final output of the computation is either the collection of raw results (if less that or equals to 'maxDegree') or 58 // a collection of hashes (that or equals to 'maxDegree') at some level in the tree. 59 // 60 // `AddResult` function should be invoke to supply the next result and at the end `Done` function should be invoked. 61 // The `Done` function does the final processing and returns the final output 62 type RangeQueryResultsHelper struct { 63 pendingResults []*kvrwset.KVRead 64 mt *merkleTree 65 maxDegree uint32 66 hashingEnabled bool 67 } 68 69 // NewRangeQueryResultsHelper constructs a RangeQueryResultsHelper 70 func NewRangeQueryResultsHelper(enableHashing bool, maxDegree uint32) (*RangeQueryResultsHelper, error) { 71 helper := &RangeQueryResultsHelper{pendingResults: nil, 72 hashingEnabled: enableHashing, 73 maxDegree: maxDegree, 74 mt: nil} 75 if enableHashing { 76 var err error 77 if helper.mt, err = newMerkleTree(maxDegree); err != nil { 78 return nil, err 79 } 80 } 81 return helper, nil 82 } 83 84 // AddResult adds a new query result for processing. 85 // Put the result into the list of pending results. If the number of pending results exceeds `maxDegree`, 86 // consume the results for incrementally update the merkle tree 87 func (helper *RangeQueryResultsHelper) AddResult(kvRead *kvrwset.KVRead) error { 88 logger.Debug("Adding a result") 89 helper.pendingResults = append(helper.pendingResults, kvRead) 90 if helper.hashingEnabled && uint32(len(helper.pendingResults)) > helper.maxDegree { 91 logger.Debug("Processing the accumulated results") 92 if err := helper.processPendingResults(); err != nil { 93 return err 94 } 95 } 96 return nil 97 } 98 99 // Done processes any pending results if needed 100 // This returns the final pending results (i.e., []*KVRead) and hashes of the results (i.e., *MerkleSummary) 101 // Only one of these two will be non-nil (except when no results are ever added). 102 // `MerkleSummary` will be nil if and only if either `enableHashing` is set to false 103 // or the number of total results are less than `maxDegree` 104 func (helper *RangeQueryResultsHelper) Done() ([]*kvrwset.KVRead, *kvrwset.QueryReadsMerkleSummary, error) { 105 // The merkle tree will be empty if total results are less than or equals to 'maxDegree' 106 // i.e., not even once the results were processed for hashing 107 if !helper.hashingEnabled || helper.mt.isEmpty() { 108 return helper.pendingResults, nil, nil 109 } 110 if len(helper.pendingResults) != 0 { 111 logger.Debug("Processing the pending results") 112 if err := helper.processPendingResults(); err != nil { 113 return helper.pendingResults, nil, err 114 } 115 } 116 helper.mt.done() 117 return helper.pendingResults, helper.mt.getSummery(), nil 118 } 119 120 // GetMerkleSummary return the current state of the MerkleSummary 121 // This intermediate state of the merkle tree helps during validation to detect a mismatch early on. 122 // That helps by not requiring to build the complete merkle tree during validation 123 // if there is a mismatch in early portion of the result-set. 124 func (helper *RangeQueryResultsHelper) GetMerkleSummary() *kvrwset.QueryReadsMerkleSummary { 125 if !helper.hashingEnabled { 126 return nil 127 } 128 return helper.mt.getSummery() 129 } 130 131 func (helper *RangeQueryResultsHelper) processPendingResults() error { 132 var b []byte 133 var err error 134 if b, err = serializeKVReads(helper.pendingResults); err != nil { 135 return err 136 } 137 helper.pendingResults = nil 138 hash, err := bccspfactory.GetDefault().Hash(b, hashOpts) 139 if err != nil { 140 return err 141 } 142 helper.mt.update(hash) 143 return nil 144 } 145 146 func serializeKVReads(kvReads []*kvrwset.KVRead) ([]byte, error) { 147 return proto.Marshal(&kvrwset.QueryReads{KvReads: kvReads}) 148 } 149 150 //////////// Merkle tree building code /////// 151 152 type merkleTree struct { 153 tree map[MerkleTreeLevel][]Hash 154 maxLevel MerkleTreeLevel 155 maxDegree uint32 156 } 157 158 func newMerkleTree(maxDegree uint32) (*merkleTree, error) { 159 if maxDegree < 2 { 160 return nil, fmt.Errorf("maxDegree [is %d] should not be less than 2 in the merkle tree", maxDegree) 161 } 162 return &merkleTree{make(map[MerkleTreeLevel][]Hash), 1, maxDegree}, nil 163 } 164 165 // update takes a hash that forms the next leaf level (level-1) node in the merkle tree. 166 // Also, complete the merkle tree as much as possible with the addition of this new leaf node - 167 // i.e. recursively build the higher level nodes and delete the underlying sub-tree. 168 func (m *merkleTree) update(nextLeafLevelHash Hash) error { 169 logger.Debugf("Before update() = %s", m) 170 defer logger.Debugf("After update() = %s", m) 171 m.tree[leafLevel] = append(m.tree[leafLevel], nextLeafLevelHash) 172 currentLevel := leafLevel 173 for { 174 currentLevelHashes := m.tree[currentLevel] 175 if uint32(len(currentLevelHashes)) <= m.maxDegree { 176 return nil 177 } 178 nextLevelHash, err := computeCombinedHash(currentLevelHashes) 179 if err != nil { 180 return err 181 } 182 delete(m.tree, currentLevel) 183 nextLevel := currentLevel + 1 184 m.tree[nextLevel] = append(m.tree[nextLevel], nextLevelHash) 185 if nextLevel > m.maxLevel { 186 m.maxLevel = nextLevel 187 } 188 currentLevel = nextLevel 189 } 190 } 191 192 // done completes the merkle tree. 193 // There may have been some nodes that are at the levels lower than the maxLevel (maximum level seen by the tree so far). 194 // Make the parent nodes out of such nodes till we complete the tree at the level of maxLevel (or maxLevel+1). 195 func (m *merkleTree) done() error { 196 logger.Debugf("Before done() = %s", m) 197 defer logger.Debugf("After done() = %s", m) 198 currentLevel := leafLevel 199 var h Hash 200 var err error 201 for currentLevel < m.maxLevel { 202 currentLevelHashes := m.tree[currentLevel] 203 switch len(currentLevelHashes) { 204 case 0: 205 currentLevel++ 206 continue 207 case 1: 208 h = currentLevelHashes[0] 209 default: 210 if h, err = computeCombinedHash(currentLevelHashes); err != nil { 211 return err 212 } 213 } 214 delete(m.tree, currentLevel) 215 currentLevel++ 216 m.tree[currentLevel] = append(m.tree[currentLevel], h) 217 } 218 219 finalHashes := m.tree[m.maxLevel] 220 if uint32(len(finalHashes)) > m.maxDegree { 221 delete(m.tree, m.maxLevel) 222 m.maxLevel++ 223 combinedHash, err := computeCombinedHash(finalHashes) 224 if err != nil { 225 return err 226 } 227 m.tree[m.maxLevel] = []Hash{combinedHash} 228 } 229 return nil 230 } 231 232 func (m *merkleTree) getSummery() *kvrwset.QueryReadsMerkleSummary { 233 return &kvrwset.QueryReadsMerkleSummary{MaxDegree: m.maxDegree, 234 MaxLevel: uint32(m.getMaxLevel()), 235 MaxLevelHashes: hashesToBytes(m.getMaxLevelHashes())} 236 } 237 238 func (m *merkleTree) getMaxLevel() MerkleTreeLevel { 239 return m.maxLevel 240 } 241 242 func (m *merkleTree) getMaxLevelHashes() []Hash { 243 return m.tree[m.maxLevel] 244 } 245 246 func (m *merkleTree) isEmpty() bool { 247 return m.maxLevel == 1 && len(m.tree[m.maxLevel]) == 0 248 } 249 250 func (m *merkleTree) String() string { 251 return fmt.Sprintf("tree := %#v", m.tree) 252 } 253 254 func computeCombinedHash(hashes []Hash) (Hash, error) { 255 combinedHash := []byte{} 256 for _, h := range hashes { 257 combinedHash = append(combinedHash, h...) 258 } 259 return bccspfactory.GetDefault().Hash(combinedHash, hashOpts) 260 } 261 262 func hashesToBytes(hashes []Hash) [][]byte { 263 b := [][]byte{} 264 for _, hash := range hashes { 265 b = append(b, hash) 266 } 267 return b 268 }