github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/core/ledger/pvtdatastorage/kv_encoding.go (about) 1 /* 2 Copyright hechain. All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package pvtdatastorage 8 9 import ( 10 "bytes" 11 "encoding/binary" 12 "math" 13 14 "github.com/golang/protobuf/proto" 15 "github.com/hechain20/hechain/core/ledger/internal/version" 16 "github.com/hyperledger/fabric-protos-go/ledger/rwset" 17 "github.com/pkg/errors" 18 "github.com/willf/bitset" 19 ) 20 21 var ( 22 pendingCommitKey = []byte{0} 23 lastCommittedBlkkey = []byte{1} 24 pvtDataKeyPrefix = []byte{2} 25 expiryKeyPrefix = []byte{3} 26 elgPrioritizedMissingDataGroup = []byte{4} 27 inelgMissingDataGroup = []byte{5} 28 collElgKeyPrefix = []byte{6} 29 lastUpdatedOldBlocksKey = []byte{7} 30 elgDeprioritizedMissingDataGroup = []byte{8} 31 bootKVHashesKeyPrefix = []byte{9} 32 lastBlockInBootSnapshotKey = []byte{'a'} 33 34 nilByte = byte(0) 35 emptyValue = []byte{} 36 ) 37 38 func getDataKeysForRangeScanByBlockNum(blockNum uint64) ([]byte, []byte) { 39 startKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, 0).ToBytes()...) 40 endKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum+1, 0).ToBytes()...) 41 return startKey, endKey 42 } 43 44 func getExpiryKeysForRangeScan(minBlkNum, maxBlkNum uint64) ([]byte, []byte) { 45 startKey := append(expiryKeyPrefix, version.NewHeight(minBlkNum, 0).ToBytes()...) 46 endKey := append(expiryKeyPrefix, version.NewHeight(maxBlkNum+1, 0).ToBytes()...) 47 return startKey, endKey 48 } 49 50 func encodeLastCommittedBlockVal(blockNum uint64) []byte { 51 return proto.EncodeVarint(blockNum) 52 } 53 54 func decodeLastCommittedBlockVal(blockNumBytes []byte) uint64 { 55 s, _ := proto.DecodeVarint(blockNumBytes) 56 return s 57 } 58 59 func encodeDataKey(key *dataKey) []byte { 60 dataKeyBytes := append(pvtDataKeyPrefix, version.NewHeight(key.blkNum, key.txNum).ToBytes()...) 61 dataKeyBytes = append(dataKeyBytes, []byte(key.ns)...) 62 dataKeyBytes = append(dataKeyBytes, nilByte) 63 return append(dataKeyBytes, []byte(key.coll)...) 64 } 65 66 func encodeDataValue(collData *rwset.CollectionPvtReadWriteSet) ([]byte, error) { 67 return proto.Marshal(collData) 68 } 69 70 func encodeExpiryKey(expiryKey *expiryKey) []byte { 71 // reusing version encoding scheme here 72 return append(expiryKeyPrefix, version.NewHeight(expiryKey.expiringBlk, expiryKey.committingBlk).ToBytes()...) 73 } 74 75 func encodeExpiryValue(expiryData *ExpiryData) ([]byte, error) { 76 return proto.Marshal(expiryData) 77 } 78 79 func decodeExpiryKey(expiryKeyBytes []byte) (*expiryKey, error) { 80 height, _, err := version.NewHeightFromBytes(expiryKeyBytes[1:]) 81 if err != nil { 82 return nil, err 83 } 84 return &expiryKey{expiringBlk: height.BlockNum, committingBlk: height.TxNum}, nil 85 } 86 87 func decodeExpiryValue(expiryValueBytes []byte) (*ExpiryData, error) { 88 expiryData := &ExpiryData{} 89 err := proto.Unmarshal(expiryValueBytes, expiryData) 90 return expiryData, errors.Wrap(err, "error while decoding expiry value") 91 } 92 93 func decodeDatakey(datakeyBytes []byte) (*dataKey, error) { 94 v, n, err := version.NewHeightFromBytes(datakeyBytes[1:]) 95 if err != nil { 96 return nil, err 97 } 98 blkNum := v.BlockNum 99 tranNum := v.TxNum 100 remainingBytes := datakeyBytes[n+1:] 101 nilByteIndex := bytes.IndexByte(remainingBytes, nilByte) 102 ns := string(remainingBytes[:nilByteIndex]) 103 coll := string(remainingBytes[nilByteIndex+1:]) 104 return &dataKey{nsCollBlk{ns, coll, blkNum}, tranNum}, nil 105 } 106 107 func decodeDataValue(datavalueBytes []byte) (*rwset.CollectionPvtReadWriteSet, error) { 108 collPvtdata := &rwset.CollectionPvtReadWriteSet{} 109 err := proto.Unmarshal(datavalueBytes, collPvtdata) 110 return collPvtdata, err 111 } 112 113 func encodeElgPrioMissingDataKey(key *missingDataKey) []byte { 114 // When missing pvtData reconciler asks for missing data info, 115 // it is necessary to pass the missing pvtdata info associated with 116 // the most recent block so that missing pvtdata in the state db can 117 // be fixed sooner to reduce the "private data matching public hash version 118 // is not available" error during endorserments. In order to give priority 119 // to missing pvtData in the most recent block, we use reverse order 120 // preserving encoding for the missing data key. This simplifies the 121 // implementation of GetMissingPvtDataInfoForMostRecentBlocks(). 122 encKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(key.blkNum)...) 123 encKey = append(encKey, []byte(key.ns)...) 124 encKey = append(encKey, nilByte) 125 return append(encKey, []byte(key.coll)...) 126 } 127 128 func encodeElgDeprioMissingDataKey(key *missingDataKey) []byte { 129 encKey := append(elgDeprioritizedMissingDataGroup, encodeReverseOrderVarUint64(key.blkNum)...) 130 encKey = append(encKey, []byte(key.ns)...) 131 encKey = append(encKey, nilByte) 132 return append(encKey, []byte(key.coll)...) 133 } 134 135 func decodeElgMissingDataKey(keyBytes []byte) *missingDataKey { 136 key := &missingDataKey{nsCollBlk: nsCollBlk{}} 137 blkNum, numBytesConsumed := decodeReverseOrderVarUint64(keyBytes[1:]) 138 splittedKey := bytes.Split(keyBytes[numBytesConsumed+1:], []byte{nilByte}) 139 key.ns = string(splittedKey[0]) 140 key.coll = string(splittedKey[1]) 141 key.blkNum = blkNum 142 return key 143 } 144 145 func encodeInelgMissingDataKey(key *missingDataKey) []byte { 146 encKey := append(inelgMissingDataGroup, []byte(key.ns)...) 147 encKey = append(encKey, nilByte) 148 encKey = append(encKey, []byte(key.coll)...) 149 encKey = append(encKey, nilByte) 150 return append(encKey, []byte(encodeReverseOrderVarUint64(key.blkNum))...) 151 } 152 153 func decodeInelgMissingDataKey(keyBytes []byte) *missingDataKey { 154 key := &missingDataKey{nsCollBlk: nsCollBlk{}} 155 splittedKey := bytes.SplitN(keyBytes[1:], []byte{nilByte}, 3) // encoded bytes for blknum may contain empty bytes 156 key.ns = string(splittedKey[0]) 157 key.coll = string(splittedKey[1]) 158 key.blkNum, _ = decodeReverseOrderVarUint64(splittedKey[2]) 159 return key 160 } 161 162 func encodeMissingDataValue(bitmap *bitset.BitSet) ([]byte, error) { 163 return bitmap.MarshalBinary() 164 } 165 166 func decodeMissingDataValue(bitmapBytes []byte) (*bitset.BitSet, error) { 167 bitmap := &bitset.BitSet{} 168 if err := bitmap.UnmarshalBinary(bitmapBytes); err != nil { 169 return nil, errors.Wrap(err, "error while decoding missing data value") 170 } 171 return bitmap, nil 172 } 173 174 func encodeCollElgKey(blkNum uint64) []byte { 175 return append(collElgKeyPrefix, encodeReverseOrderVarUint64(blkNum)...) 176 } 177 178 func decodeCollElgKey(b []byte) uint64 { 179 blkNum, _ := decodeReverseOrderVarUint64(b[1:]) 180 return blkNum 181 } 182 183 func encodeCollElgVal(m *CollElgInfo) ([]byte, error) { 184 return proto.Marshal(m) 185 } 186 187 func decodeCollElgVal(b []byte) (*CollElgInfo, error) { 188 m := &CollElgInfo{} 189 if err := proto.Unmarshal(b, m); err != nil { 190 return nil, errors.WithStack(err) 191 } 192 return m, nil 193 } 194 195 func encodeBootKVHashesKey(key *bootKVHashesKey) []byte { 196 k := append(bootKVHashesKeyPrefix, version.NewHeight(key.blkNum, key.txNum).ToBytes()...) 197 k = append(k, []byte(key.ns)...) 198 k = append(k, nilByte) 199 return append(k, []byte(key.coll)...) 200 } 201 202 func encodeBootKVHashesVal(val *BootKVHashes) ([]byte, error) { 203 b, err := proto.Marshal(val) 204 if err != nil { 205 return nil, errors.Wrap(err, "error while marshalling BootKVHashes") 206 } 207 return b, nil 208 } 209 210 func decodeBootKVHashesVal(b []byte) (*BootKVHashes, error) { 211 val := &BootKVHashes{} 212 if err := proto.Unmarshal(b, val); err != nil { 213 return nil, errors.Wrap(err, "error while unmarshalling bytes for BootKVHashes") 214 } 215 return val, nil 216 } 217 218 func encodeLastBlockInBootSnapshotVal(blockNum uint64) []byte { 219 return proto.EncodeVarint(blockNum) 220 } 221 222 func decodeLastBlockInBootSnapshotVal(blockNumBytes []byte) (uint64, error) { 223 s, n := proto.DecodeVarint(blockNumBytes) 224 if n == 0 { 225 return 0, errors.New("unexpected bytes for interpreting as varint") 226 } 227 return s, nil 228 } 229 230 func createRangeScanKeysForElgMissingData(blkNum uint64, group []byte) ([]byte, []byte) { 231 startKey := append(group, encodeReverseOrderVarUint64(blkNum)...) 232 endKey := append(group, encodeReverseOrderVarUint64(0)...) 233 234 return startKey, endKey 235 } 236 237 func createRangeScanKeysForInelgMissingData(maxBlkNum uint64, ns, coll string) ([]byte, []byte) { 238 startKey := encodeInelgMissingDataKey( 239 &missingDataKey{ 240 nsCollBlk: nsCollBlk{ 241 ns: ns, 242 coll: coll, 243 blkNum: maxBlkNum, 244 }, 245 }, 246 ) 247 endKey := encodeInelgMissingDataKey( 248 &missingDataKey{ 249 nsCollBlk: nsCollBlk{ 250 ns: ns, 251 coll: coll, 252 blkNum: 0, 253 }, 254 }, 255 ) 256 257 return startKey, endKey 258 } 259 260 func createRangeScanKeysForCollElg() (startKey, endKey []byte) { 261 return encodeCollElgKey(math.MaxUint64), 262 encodeCollElgKey(0) 263 } 264 265 func datakeyRange(blockNum uint64) ([]byte, []byte) { 266 startKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, 0).ToBytes()...) 267 endKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, math.MaxUint64).ToBytes()...) 268 return startKey, endKey 269 } 270 271 func eligibleMissingdatakeyRange(blkNum uint64) ([]byte, []byte) { 272 startKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(blkNum)...) 273 endKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(blkNum-1)...) 274 return startKey, endKey 275 } 276 277 // encodeReverseOrderVarUint64 returns a byte-representation for a uint64 number such that 278 // the number is first subtracted from MaxUint64 and then all the leading 0xff bytes 279 // are trimmed and replaced by the number of such trimmed bytes. This helps in reducing the size. 280 // In the byte order comparison this encoding ensures that EncodeReverseOrderVarUint64(A) > EncodeReverseOrderVarUint64(B), 281 // If B > A 282 func encodeReverseOrderVarUint64(number uint64) []byte { 283 bytes := make([]byte, 8) 284 binary.BigEndian.PutUint64(bytes, math.MaxUint64-number) 285 numFFBytes := 0 286 for _, b := range bytes { 287 if b != 0xff { 288 break 289 } 290 numFFBytes++ 291 } 292 size := 8 - numFFBytes 293 encodedBytes := make([]byte, size+1) 294 encodedBytes[0] = proto.EncodeVarint(uint64(numFFBytes))[0] 295 copy(encodedBytes[1:], bytes[numFFBytes:]) 296 return encodedBytes 297 } 298 299 // decodeReverseOrderVarUint64 decodes the number from the bytes obtained from function 'EncodeReverseOrderVarUint64'. 300 // Also, returns the number of bytes that are consumed in the process 301 func decodeReverseOrderVarUint64(bytes []byte) (uint64, int) { 302 s, _ := proto.DecodeVarint(bytes) 303 numFFBytes := int(s) 304 decodedBytes := make([]byte, 8) 305 realBytesNum := 8 - numFFBytes 306 copy(decodedBytes[numFFBytes:], bytes[1:realBytesNum+1]) 307 numBytesConsumed := realBytesNum + 1 308 for i := 0; i < numFFBytes; i++ { 309 decodedBytes[i] = 0xff 310 } 311 return (math.MaxUint64 - binary.BigEndian.Uint64(decodedBytes)), numBytesConsumed 312 }