github.com/osdi23p228/fabric@v0.0.0-20221218062954-77808885f5db/core/ledger/pvtdatastorage/kv_encoding.go (about)

     1  /*
     2  Copyright IBM Corp. All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package pvtdatastorage
     8  
     9  import (
    10  	"bytes"
    11  	"encoding/binary"
    12  	"math"
    13  
    14  	"github.com/golang/protobuf/proto"
    15  	"github.com/hyperledger/fabric-protos-go/ledger/rwset"
    16  	"github.com/osdi23p228/fabric/core/ledger/internal/version"
    17  	"github.com/pkg/errors"
    18  	"github.com/willf/bitset"
    19  )
    20  
    21  var (
    22  	pendingCommitKey                 = []byte{0}
    23  	lastCommittedBlkkey              = []byte{1}
    24  	pvtDataKeyPrefix                 = []byte{2}
    25  	expiryKeyPrefix                  = []byte{3}
    26  	elgPrioritizedMissingDataGroup   = []byte{4}
    27  	inelgMissingDataGroup            = []byte{5}
    28  	collElgKeyPrefix                 = []byte{6}
    29  	lastUpdatedOldBlocksKey          = []byte{7}
    30  	elgDeprioritizedMissingDataGroup = []byte{8}
    31  
    32  	nilByte    = byte(0)
    33  	emptyValue = []byte{}
    34  )
    35  
    36  func getDataKeysForRangeScanByBlockNum(blockNum uint64) ([]byte, []byte) {
    37  	startKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, 0).ToBytes()...)
    38  	endKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum+1, 0).ToBytes()...)
    39  	return startKey, endKey
    40  }
    41  
    42  func getExpiryKeysForRangeScan(minBlkNum, maxBlkNum uint64) ([]byte, []byte) {
    43  	startKey := append(expiryKeyPrefix, version.NewHeight(minBlkNum, 0).ToBytes()...)
    44  	endKey := append(expiryKeyPrefix, version.NewHeight(maxBlkNum+1, 0).ToBytes()...)
    45  	return startKey, endKey
    46  }
    47  
    48  func encodeLastCommittedBlockVal(blockNum uint64) []byte {
    49  	return proto.EncodeVarint(blockNum)
    50  }
    51  
    52  func decodeLastCommittedBlockVal(blockNumBytes []byte) uint64 {
    53  	s, _ := proto.DecodeVarint(blockNumBytes)
    54  	return s
    55  }
    56  
    57  func encodeDataKey(key *dataKey) []byte {
    58  	dataKeyBytes := append(pvtDataKeyPrefix, version.NewHeight(key.blkNum, key.txNum).ToBytes()...)
    59  	dataKeyBytes = append(dataKeyBytes, []byte(key.ns)...)
    60  	dataKeyBytes = append(dataKeyBytes, nilByte)
    61  	return append(dataKeyBytes, []byte(key.coll)...)
    62  }
    63  
    64  func encodeDataValue(collData *rwset.CollectionPvtReadWriteSet) ([]byte, error) {
    65  	return proto.Marshal(collData)
    66  }
    67  
    68  func encodeExpiryKey(expiryKey *expiryKey) []byte {
    69  	// reusing version encoding scheme here
    70  	return append(expiryKeyPrefix, version.NewHeight(expiryKey.expiringBlk, expiryKey.committingBlk).ToBytes()...)
    71  }
    72  
    73  func encodeExpiryValue(expiryData *ExpiryData) ([]byte, error) {
    74  	return proto.Marshal(expiryData)
    75  }
    76  
    77  func decodeExpiryKey(expiryKeyBytes []byte) (*expiryKey, error) {
    78  	height, _, err := version.NewHeightFromBytes(expiryKeyBytes[1:])
    79  	if err != nil {
    80  		return nil, err
    81  	}
    82  	return &expiryKey{expiringBlk: height.BlockNum, committingBlk: height.TxNum}, nil
    83  }
    84  
    85  func decodeExpiryValue(expiryValueBytes []byte) (*ExpiryData, error) {
    86  	expiryData := &ExpiryData{}
    87  	err := proto.Unmarshal(expiryValueBytes, expiryData)
    88  	return expiryData, err
    89  }
    90  
    91  func decodeDatakey(datakeyBytes []byte) (*dataKey, error) {
    92  	v, n, err := version.NewHeightFromBytes(datakeyBytes[1:])
    93  	if err != nil {
    94  		return nil, err
    95  	}
    96  	blkNum := v.BlockNum
    97  	tranNum := v.TxNum
    98  	remainingBytes := datakeyBytes[n+1:]
    99  	nilByteIndex := bytes.IndexByte(remainingBytes, nilByte)
   100  	ns := string(remainingBytes[:nilByteIndex])
   101  	coll := string(remainingBytes[nilByteIndex+1:])
   102  	return &dataKey{nsCollBlk{ns, coll, blkNum}, tranNum}, nil
   103  }
   104  
   105  func decodeDataValue(datavalueBytes []byte) (*rwset.CollectionPvtReadWriteSet, error) {
   106  	collPvtdata := &rwset.CollectionPvtReadWriteSet{}
   107  	err := proto.Unmarshal(datavalueBytes, collPvtdata)
   108  	return collPvtdata, err
   109  }
   110  
   111  func encodeElgPrioMissingDataKey(key *missingDataKey) []byte {
   112  	// When missing pvtData reconciler asks for missing data info,
   113  	// it is necessary to pass the missing pvtdata info associated with
   114  	// the most recent block so that missing pvtdata in the state db can
   115  	// be fixed sooner to reduce the "private data matching public hash version
   116  	// is not available" error during endorserments. In order to give priority
   117  	// to missing pvtData in the most recent block, we use reverse order
   118  	// preserving encoding for the missing data key. This simplifies the
   119  	// implementation of GetMissingPvtDataInfoForMostRecentBlocks().
   120  	encKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(key.blkNum)...)
   121  	encKey = append(encKey, []byte(key.ns)...)
   122  	encKey = append(encKey, nilByte)
   123  	return append(encKey, []byte(key.coll)...)
   124  }
   125  
   126  func encodeElgDeprioMissingDataKey(key *missingDataKey) []byte {
   127  	encKey := append(elgDeprioritizedMissingDataGroup, encodeReverseOrderVarUint64(key.blkNum)...)
   128  	encKey = append(encKey, []byte(key.ns)...)
   129  	encKey = append(encKey, nilByte)
   130  	return append(encKey, []byte(key.coll)...)
   131  }
   132  
   133  func decodeElgMissingDataKey(keyBytes []byte) *missingDataKey {
   134  	key := &missingDataKey{nsCollBlk: nsCollBlk{}}
   135  	blkNum, numBytesConsumed := decodeReverseOrderVarUint64(keyBytes[1:])
   136  	splittedKey := bytes.Split(keyBytes[numBytesConsumed+1:], []byte{nilByte})
   137  	key.ns = string(splittedKey[0])
   138  	key.coll = string(splittedKey[1])
   139  	key.blkNum = blkNum
   140  	return key
   141  }
   142  
   143  func encodeInelgMissingDataKey(key *missingDataKey) []byte {
   144  	encKey := append(inelgMissingDataGroup, []byte(key.ns)...)
   145  	encKey = append(encKey, nilByte)
   146  	encKey = append(encKey, []byte(key.coll)...)
   147  	encKey = append(encKey, nilByte)
   148  	return append(encKey, []byte(encodeReverseOrderVarUint64(key.blkNum))...)
   149  }
   150  
   151  func decodeInelgMissingDataKey(keyBytes []byte) *missingDataKey {
   152  	key := &missingDataKey{nsCollBlk: nsCollBlk{}}
   153  	splittedKey := bytes.SplitN(keyBytes[1:], []byte{nilByte}, 3) //encoded bytes for blknum may contain empty bytes
   154  	key.ns = string(splittedKey[0])
   155  	key.coll = string(splittedKey[1])
   156  	key.blkNum, _ = decodeReverseOrderVarUint64(splittedKey[2])
   157  	return key
   158  }
   159  
   160  func encodeMissingDataValue(bitmap *bitset.BitSet) ([]byte, error) {
   161  	return bitmap.MarshalBinary()
   162  }
   163  
   164  func decodeMissingDataValue(bitmapBytes []byte) (*bitset.BitSet, error) {
   165  	bitmap := &bitset.BitSet{}
   166  	if err := bitmap.UnmarshalBinary(bitmapBytes); err != nil {
   167  		return nil, err
   168  	}
   169  	return bitmap, nil
   170  }
   171  
   172  func encodeCollElgKey(blkNum uint64) []byte {
   173  	return append(collElgKeyPrefix, encodeReverseOrderVarUint64(blkNum)...)
   174  }
   175  
   176  func decodeCollElgKey(b []byte) uint64 {
   177  	blkNum, _ := decodeReverseOrderVarUint64(b[1:])
   178  	return blkNum
   179  }
   180  
   181  func encodeCollElgVal(m *CollElgInfo) ([]byte, error) {
   182  	return proto.Marshal(m)
   183  }
   184  
   185  func decodeCollElgVal(b []byte) (*CollElgInfo, error) {
   186  	m := &CollElgInfo{}
   187  	if err := proto.Unmarshal(b, m); err != nil {
   188  		return nil, errors.WithStack(err)
   189  	}
   190  	return m, nil
   191  }
   192  
   193  func createRangeScanKeysForElgMissingData(blkNum uint64, group []byte) ([]byte, []byte) {
   194  	startKey := append(group, encodeReverseOrderVarUint64(blkNum)...)
   195  	endKey := append(group, encodeReverseOrderVarUint64(0)...)
   196  
   197  	return startKey, endKey
   198  }
   199  
   200  func createRangeScanKeysForInelgMissingData(maxBlkNum uint64, ns, coll string) ([]byte, []byte) {
   201  	startKey := encodeInelgMissingDataKey(
   202  		&missingDataKey{
   203  			nsCollBlk: nsCollBlk{
   204  				ns:     ns,
   205  				coll:   coll,
   206  				blkNum: maxBlkNum,
   207  			},
   208  		},
   209  	)
   210  	endKey := encodeInelgMissingDataKey(
   211  		&missingDataKey{
   212  			nsCollBlk: nsCollBlk{
   213  				ns:     ns,
   214  				coll:   coll,
   215  				blkNum: 0,
   216  			},
   217  		},
   218  	)
   219  
   220  	return startKey, endKey
   221  }
   222  
   223  func createRangeScanKeysForCollElg() (startKey, endKey []byte) {
   224  	return encodeCollElgKey(math.MaxUint64),
   225  		encodeCollElgKey(0)
   226  }
   227  
   228  func datakeyRange(blockNum uint64) ([]byte, []byte) {
   229  	startKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, 0).ToBytes()...)
   230  	endKey := append(pvtDataKeyPrefix, version.NewHeight(blockNum, math.MaxUint64).ToBytes()...)
   231  	return startKey, endKey
   232  }
   233  
   234  func eligibleMissingdatakeyRange(blkNum uint64) ([]byte, []byte) {
   235  	startKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(blkNum)...)
   236  	endKey := append(elgPrioritizedMissingDataGroup, encodeReverseOrderVarUint64(blkNum-1)...)
   237  	return startKey, endKey
   238  }
   239  
   240  // encodeReverseOrderVarUint64 returns a byte-representation for a uint64 number such that
   241  // the number is first subtracted from MaxUint64 and then all the leading 0xff bytes
   242  // are trimmed and replaced by the number of such trimmed bytes. This helps in reducing the size.
   243  // In the byte order comparison this encoding ensures that EncodeReverseOrderVarUint64(A) > EncodeReverseOrderVarUint64(B),
   244  // If B > A
   245  func encodeReverseOrderVarUint64(number uint64) []byte {
   246  	bytes := make([]byte, 8)
   247  	binary.BigEndian.PutUint64(bytes, math.MaxUint64-number)
   248  	numFFBytes := 0
   249  	for _, b := range bytes {
   250  		if b != 0xff {
   251  			break
   252  		}
   253  		numFFBytes++
   254  	}
   255  	size := 8 - numFFBytes
   256  	encodedBytes := make([]byte, size+1)
   257  	encodedBytes[0] = proto.EncodeVarint(uint64(numFFBytes))[0]
   258  	copy(encodedBytes[1:], bytes[numFFBytes:])
   259  	return encodedBytes
   260  }
   261  
   262  // decodeReverseOrderVarUint64 decodes the number from the bytes obtained from function 'EncodeReverseOrderVarUint64'.
   263  // Also, returns the number of bytes that are consumed in the process
   264  func decodeReverseOrderVarUint64(bytes []byte) (uint64, int) {
   265  	s, _ := proto.DecodeVarint(bytes)
   266  	numFFBytes := int(s)
   267  	decodedBytes := make([]byte, 8)
   268  	realBytesNum := 8 - numFFBytes
   269  	copy(decodedBytes[numFFBytes:], bytes[1:realBytesNum+1])
   270  	numBytesConsumed := realBytesNum + 1
   271  	for i := 0; i < numFFBytes; i++ {
   272  		decodedBytes[i] = 0xff
   273  	}
   274  	return (math.MaxUint64 - binary.BigEndian.Uint64(decodedBytes)), numBytesConsumed
   275  }