github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/core/ledger/pvtdatastorage/kv_encoding.go (about)

     1  /*
     2  Copyright hechain. All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package pvtdatastorage
     8  
     9  import (
    10  	"bytes"
    11  	"encoding/binary"
    12  	"math"
    13  
    14  	"github.com/golang/protobuf/proto"
    15  	"github.com/hechain20/hechain/core/ledger/internal/version"
    16  	"github.com/hyperledger/fabric-protos-go/ledger/rwset"
    17  	"github.com/pkg/errors"
    18  	"github.com/willf/bitset"
    19  )
    20  
    21  var (
    22  	pendingCommitKey                 = []byte{0}
    23  	lastCommittedBlkkey              = []byte{1}
    24  	pvtDataKeyPrefix                 = []byte{2}
    25  	expiryKeyPrefix                  = []byte{3}
    26  	elgPrioritizedMissingDataGroup   = []byte{4}
    27  	inelgMissingDataGroup            = []byte{5}
    28  	collElgKeyPrefix                 = []byte{6}
    29  	lastUpdatedOldBlocksKey          = []byte{7}
    30  	elgDeprioritizedMissingDataGroup = []byte{8}
    31  	bootKVHashesKeyPrefix            = []byte{9}
    32  	lastBlockInBootSnapshotKey       = []byte{'a'}
    33  
    34  	nilByte    = byte(0)
    35  	emptyValue = []byte{}
    36  )
    37  
    38  func getDataKeysForRangeScanByBlockNum(blockNum uint64) ([]byte, []byte) {
    39  	startKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, 0).ToBytes()...)
    40  	endKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum+1, 0).ToBytes()...)
    41  	return startKey, endKey
    42  }
    43  
    44  func getExpiryKeysForRangeScan(minBlkNum, maxBlkNum uint64) ([]byte, []byte) {
    45  	startKey := append(expiryKeyPrefix, version.NewHeight(minBlkNum, 0).ToBytes()...)
    46  	endKey := append(expiryKeyPrefix, version.NewHeight(maxBlkNum+1, 0).ToBytes()...)
    47  	return startKey, endKey
    48  }
    49  
    50  func encodeLastCommittedBlockVal(blockNum uint64) []byte {
    51  	return proto.EncodeVarint(blockNum)
    52  }
    53  
    54  func decodeLastCommittedBlockVal(blockNumBytes []byte) uint64 {
    55  	s, _ := proto.DecodeVarint(blockNumBytes)
    56  	return s
    57  }
    58  
    59  func encodeDataKey(key *dataKey) []byte {
    60  	dataKeyBytes := append(pvtDataKeyPrefix, version.NewHeight(key.blkNum, key.txNum).ToBytes()...)
    61  	dataKeyBytes = append(dataKeyBytes, []byte(key.ns)...)
    62  	dataKeyBytes = append(dataKeyBytes, nilByte)
    63  	return append(dataKeyBytes, []byte(key.coll)...)
    64  }
    65  
    66  func encodeDataValue(collData *rwset.CollectionPvtReadWriteSet) ([]byte, error) {
    67  	return proto.Marshal(collData)
    68  }
    69  
    70  func encodeExpiryKey(expiryKey *expiryKey) []byte {
    71  	// reusing version encoding scheme here
    72  	return append(expiryKeyPrefix, version.NewHeight(expiryKey.expiringBlk, expiryKey.committingBlk).ToBytes()...)
    73  }
    74  
    75  func encodeExpiryValue(expiryData *ExpiryData) ([]byte, error) {
    76  	return proto.Marshal(expiryData)
    77  }
    78  
    79  func decodeExpiryKey(expiryKeyBytes []byte) (*expiryKey, error) {
    80  	height, _, err := version.NewHeightFromBytes(expiryKeyBytes[1:])
    81  	if err != nil {
    82  		return nil, err
    83  	}
    84  	return &expiryKey{expiringBlk: height.BlockNum, committingBlk: height.TxNum}, nil
    85  }
    86  
    87  func decodeExpiryValue(expiryValueBytes []byte) (*ExpiryData, error) {
    88  	expiryData := &ExpiryData{}
    89  	err := proto.Unmarshal(expiryValueBytes, expiryData)
    90  	return expiryData, errors.Wrap(err, "error while decoding expiry value")
    91  }
    92  
    93  func decodeDatakey(datakeyBytes []byte) (*dataKey, error) {
    94  	v, n, err := version.NewHeightFromBytes(datakeyBytes[1:])
    95  	if err != nil {
    96  		return nil, err
    97  	}
    98  	blkNum := v.BlockNum
    99  	tranNum := v.TxNum
   100  	remainingBytes := datakeyBytes[n+1:]
   101  	nilByteIndex := bytes.IndexByte(remainingBytes, nilByte)
   102  	ns := string(remainingBytes[:nilByteIndex])
   103  	coll := string(remainingBytes[nilByteIndex+1:])
   104  	return &dataKey{nsCollBlk{ns, coll, blkNum}, tranNum}, nil
   105  }
   106  
   107  func decodeDataValue(datavalueBytes []byte) (*rwset.CollectionPvtReadWriteSet, error) {
   108  	collPvtdata := &rwset.CollectionPvtReadWriteSet{}
   109  	err := proto.Unmarshal(datavalueBytes, collPvtdata)
   110  	return collPvtdata, err
   111  }
   112  
   113  func encodeElgPrioMissingDataKey(key *missingDataKey) []byte {
   114  	// When missing pvtData reconciler asks for missing data info,
   115  	// it is necessary to pass the missing pvtdata info associated with
   116  	// the most recent block so that missing pvtdata in the state db can
   117  	// be fixed sooner to reduce the "private data matching public hash version
   118  	// is not available" error during endorserments. In order to give priority
   119  	// to missing pvtData in the most recent block, we use reverse order
   120  	// preserving encoding for the missing data key. This simplifies the
   121  	// implementation of GetMissingPvtDataInfoForMostRecentBlocks().
   122  	encKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(key.blkNum)...)
   123  	encKey = append(encKey, []byte(key.ns)...)
   124  	encKey = append(encKey, nilByte)
   125  	return append(encKey, []byte(key.coll)...)
   126  }
   127  
   128  func encodeElgDeprioMissingDataKey(key *missingDataKey) []byte {
   129  	encKey := append(elgDeprioritizedMissingDataGroup, encodeReverseOrderVarUint64(key.blkNum)...)
   130  	encKey = append(encKey, []byte(key.ns)...)
   131  	encKey = append(encKey, nilByte)
   132  	return append(encKey, []byte(key.coll)...)
   133  }
   134  
   135  func decodeElgMissingDataKey(keyBytes []byte) *missingDataKey {
   136  	key := &missingDataKey{nsCollBlk: nsCollBlk{}}
   137  	blkNum, numBytesConsumed := decodeReverseOrderVarUint64(keyBytes[1:])
   138  	splittedKey := bytes.Split(keyBytes[numBytesConsumed+1:], []byte{nilByte})
   139  	key.ns = string(splittedKey[0])
   140  	key.coll = string(splittedKey[1])
   141  	key.blkNum = blkNum
   142  	return key
   143  }
   144  
   145  func encodeInelgMissingDataKey(key *missingDataKey) []byte {
   146  	encKey := append(inelgMissingDataGroup, []byte(key.ns)...)
   147  	encKey = append(encKey, nilByte)
   148  	encKey = append(encKey, []byte(key.coll)...)
   149  	encKey = append(encKey, nilByte)
   150  	return append(encKey, []byte(encodeReverseOrderVarUint64(key.blkNum))...)
   151  }
   152  
   153  func decodeInelgMissingDataKey(keyBytes []byte) *missingDataKey {
   154  	key := &missingDataKey{nsCollBlk: nsCollBlk{}}
   155  	splittedKey := bytes.SplitN(keyBytes[1:], []byte{nilByte}, 3) // encoded bytes for blknum may contain empty bytes
   156  	key.ns = string(splittedKey[0])
   157  	key.coll = string(splittedKey[1])
   158  	key.blkNum, _ = decodeReverseOrderVarUint64(splittedKey[2])
   159  	return key
   160  }
   161  
   162  func encodeMissingDataValue(bitmap *bitset.BitSet) ([]byte, error) {
   163  	return bitmap.MarshalBinary()
   164  }
   165  
   166  func decodeMissingDataValue(bitmapBytes []byte) (*bitset.BitSet, error) {
   167  	bitmap := &bitset.BitSet{}
   168  	if err := bitmap.UnmarshalBinary(bitmapBytes); err != nil {
   169  		return nil, errors.Wrap(err, "error while decoding missing data value")
   170  	}
   171  	return bitmap, nil
   172  }
   173  
   174  func encodeCollElgKey(blkNum uint64) []byte {
   175  	return append(collElgKeyPrefix, encodeReverseOrderVarUint64(blkNum)...)
   176  }
   177  
   178  func decodeCollElgKey(b []byte) uint64 {
   179  	blkNum, _ := decodeReverseOrderVarUint64(b[1:])
   180  	return blkNum
   181  }
   182  
   183  func encodeCollElgVal(m *CollElgInfo) ([]byte, error) {
   184  	return proto.Marshal(m)
   185  }
   186  
   187  func decodeCollElgVal(b []byte) (*CollElgInfo, error) {
   188  	m := &CollElgInfo{}
   189  	if err := proto.Unmarshal(b, m); err != nil {
   190  		return nil, errors.WithStack(err)
   191  	}
   192  	return m, nil
   193  }
   194  
   195  func encodeBootKVHashesKey(key *bootKVHashesKey) []byte {
   196  	k := append(bootKVHashesKeyPrefix, version.NewHeight(key.blkNum, key.txNum).ToBytes()...)
   197  	k = append(k, []byte(key.ns)...)
   198  	k = append(k, nilByte)
   199  	return append(k, []byte(key.coll)...)
   200  }
   201  
   202  func encodeBootKVHashesVal(val *BootKVHashes) ([]byte, error) {
   203  	b, err := proto.Marshal(val)
   204  	if err != nil {
   205  		return nil, errors.Wrap(err, "error while marshalling BootKVHashes")
   206  	}
   207  	return b, nil
   208  }
   209  
   210  func decodeBootKVHashesVal(b []byte) (*BootKVHashes, error) {
   211  	val := &BootKVHashes{}
   212  	if err := proto.Unmarshal(b, val); err != nil {
   213  		return nil, errors.Wrap(err, "error while unmarshalling bytes for BootKVHashes")
   214  	}
   215  	return val, nil
   216  }
   217  
   218  func encodeLastBlockInBootSnapshotVal(blockNum uint64) []byte {
   219  	return proto.EncodeVarint(blockNum)
   220  }
   221  
   222  func decodeLastBlockInBootSnapshotVal(blockNumBytes []byte) (uint64, error) {
   223  	s, n := proto.DecodeVarint(blockNumBytes)
   224  	if n == 0 {
   225  		return 0, errors.New("unexpected bytes for interpreting as varint")
   226  	}
   227  	return s, nil
   228  }
   229  
   230  func createRangeScanKeysForElgMissingData(blkNum uint64, group []byte) ([]byte, []byte) {
   231  	startKey := append(group, encodeReverseOrderVarUint64(blkNum)...)
   232  	endKey := append(group, encodeReverseOrderVarUint64(0)...)
   233  
   234  	return startKey, endKey
   235  }
   236  
   237  func createRangeScanKeysForInelgMissingData(maxBlkNum uint64, ns, coll string) ([]byte, []byte) {
   238  	startKey := encodeInelgMissingDataKey(
   239  		&missingDataKey{
   240  			nsCollBlk: nsCollBlk{
   241  				ns:     ns,
   242  				coll:   coll,
   243  				blkNum: maxBlkNum,
   244  			},
   245  		},
   246  	)
   247  	endKey := encodeInelgMissingDataKey(
   248  		&missingDataKey{
   249  			nsCollBlk: nsCollBlk{
   250  				ns:     ns,
   251  				coll:   coll,
   252  				blkNum: 0,
   253  			},
   254  		},
   255  	)
   256  
   257  	return startKey, endKey
   258  }
   259  
   260  func createRangeScanKeysForCollElg() (startKey, endKey []byte) {
   261  	return encodeCollElgKey(math.MaxUint64),
   262  		encodeCollElgKey(0)
   263  }
   264  
   265  func datakeyRange(blockNum uint64) ([]byte, []byte) {
   266  	startKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, 0).ToBytes()...)
   267  	endKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, math.MaxUint64).ToBytes()...)
   268  	return startKey, endKey
   269  }
   270  
   271  func eligibleMissingdatakeyRange(blkNum uint64) ([]byte, []byte) {
   272  	startKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(blkNum)...)
   273  	endKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(blkNum-1)...)
   274  	return startKey, endKey
   275  }
   276  
   277  // encodeReverseOrderVarUint64 returns a byte-representation for a uint64 number such that
   278  // the number is first subtracted from MaxUint64 and then all the leading 0xff bytes
   279  // are trimmed and replaced by the number of such trimmed bytes. This helps in reducing the size.
   280  // In the byte order comparison this encoding ensures that EncodeReverseOrderVarUint64(A) > EncodeReverseOrderVarUint64(B),
   281  // If B > A
   282  func encodeReverseOrderVarUint64(number uint64) []byte {
   283  	bytes := make([]byte, 8)
   284  	binary.BigEndian.PutUint64(bytes, math.MaxUint64-number)
   285  	numFFBytes := 0
   286  	for _, b := range bytes {
   287  		if b != 0xff {
   288  			break
   289  		}
   290  		numFFBytes++
   291  	}
   292  	size := 8 - numFFBytes
   293  	encodedBytes := make([]byte, size+1)
   294  	encodedBytes[0] = proto.EncodeVarint(uint64(numFFBytes))[0]
   295  	copy(encodedBytes[1:], bytes[numFFBytes:])
   296  	return encodedBytes
   297  }
   298  
   299  // decodeReverseOrderVarUint64 decodes the number from the bytes obtained from function 'EncodeReverseOrderVarUint64'.
   300  // Also, returns the number of bytes that are consumed in the process
   301  func decodeReverseOrderVarUint64(bytes []byte) (uint64, int) {
   302  	s, _ := proto.DecodeVarint(bytes)
   303  	numFFBytes := int(s)
   304  	decodedBytes := make([]byte, 8)
   305  	realBytesNum := 8 - numFFBytes
   306  	copy(decodedBytes[numFFBytes:], bytes[1:realBytesNum+1])
   307  	numBytesConsumed := realBytesNum + 1
   308  	for i := 0; i < numFFBytes; i++ {
   309  		decodedBytes[i] = 0xff
   310  	}
   311  	return (math.MaxUint64 - binary.BigEndian.Uint64(decodedBytes)), numBytesConsumed
   312  }