github.com/matrixorigin/matrixone@v1.2.0/pkg/objectio/meta.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package objectio
    16  
    17  import (
    18  	"bytes"
    19  	"unsafe"
    20  
    21  	"github.com/matrixorigin/matrixone/pkg/container/types"
    22  )
    23  
    24  var EmptyZm = [64]byte{}
    25  
    26  const FooterSize = 64
    27  const HeaderSize = 64
    28  
    29  type objectDataMetaV1 []byte
    30  
    31  func buildObjectDataMetaV1(count uint16) objectDataMetaV1 {
    32  	length := headerLen + uint32(count)*colMetaLen
    33  	buf := make([]byte, length)
    34  	return buf[:]
    35  }
    36  
    37  func (o objectDataMetaV1) BlockHeader() BlockHeader {
    38  	return BlockHeader(o[:headerLen])
    39  }
    40  
    41  func (o objectDataMetaV1) MustGetColumn(seqnum uint16) ColumnMeta {
    42  	if seqnum > o.BlockHeader().MaxSeqnum() {
    43  		return BuildObjectColumnMeta()
    44  	}
    45  	return GetObjectColumnMeta(seqnum, o[headerLen:])
    46  }
    47  
    48  func (o objectDataMetaV1) AddColumnMeta(idx uint16, col ColumnMeta) {
    49  	offset := headerLen + uint32(idx)*colMetaLen
    50  	copy(o[offset:offset+colMetaLen], col)
    51  }
    52  
    53  func (o objectDataMetaV1) Length() uint32 {
    54  	return headerLen + uint32(o.BlockHeader().MetaColumnCount())*colMetaLen
    55  }
    56  
    57  func (o objectDataMetaV1) BlockCount() uint32 {
    58  	return uint32(o.BlockHeader().Sequence())
    59  }
    60  
    61  func (o objectDataMetaV1) BlockIndex() BlockIndex {
    62  	offset := o.Length()
    63  	return BlockIndex(o[offset:])
    64  }
    65  
    66  func (o objectDataMetaV1) GetBlockMeta(id uint32) BlockObject {
    67  	BlockID := id - uint32(o.BlockHeader().StartID())
    68  	offset, length := o.BlockIndex().BlockMetaPos(BlockID)
    69  	return BlockObject(o[offset : offset+length])
    70  }
    71  
    72  func (o objectDataMetaV1) GetColumnMeta(blk uint32, seqnum uint16) ColumnMeta {
    73  	return o.GetBlockMeta(blk).MustGetColumn(seqnum)
    74  }
    75  
    76  func (o objectDataMetaV1) IsEmpty() bool {
    77  	return len(o) == 0
    78  }
    79  
    80  const (
    81  	blockCountLen = 4
    82  	blockOffset   = 4
    83  	blockLen      = 4
    84  	posLen        = blockOffset + blockLen
    85  )
    86  
    87  type BlockIndex []byte
    88  
    89  func BuildBlockIndex(count uint32) BlockIndex {
    90  	length := blockCountLen + uint32(count)*posLen
    91  	buf := make([]byte, length)
    92  	return buf[:]
    93  }
    94  
    95  func (oh BlockIndex) BlockCount() uint32 {
    96  	return types.DecodeUint32(oh[:blockCountLen])
    97  }
    98  
    99  func (oh BlockIndex) SetBlockCount(cnt uint32) {
   100  	copy(oh[:blockCountLen], types.EncodeUint32(&cnt))
   101  }
   102  
   103  func (oh BlockIndex) BlockMetaPos(BlockID uint32) (uint32, uint32) {
   104  	offStart := blockCountLen + BlockID*posLen
   105  	offEnd := blockCountLen + BlockID*posLen + blockOffset
   106  	return types.DecodeUint32(oh[offStart:offEnd]), types.DecodeUint32(oh[offStart+blockLen : offEnd+blockLen])
   107  }
   108  
   109  func (oh BlockIndex) SetBlockMetaPos(BlockID uint32, offset, length uint32) {
   110  	offStart := blockCountLen + BlockID*posLen
   111  	offEnd := blockCountLen + BlockID*posLen + blockOffset
   112  	copy(oh[offStart:offEnd], types.EncodeUint32(&offset))
   113  	copy(oh[offStart+blockLen:offEnd+blockLen], types.EncodeUint32(&length))
   114  }
   115  
   116  func (oh BlockIndex) Length() uint32 {
   117  	return oh.BlockCount()*posLen + blockCountLen
   118  }
   119  
   120  // caller makes sure the data has column meta fot the given seqnum
   121  func GetObjectColumnMeta(seqnum uint16, data []byte) ColumnMeta {
   122  	offset := uint32(seqnum) * colMetaLen
   123  	return data[offset : offset+colMetaLen]
   124  }
   125  
   126  func BuildObjectColumnMeta() ColumnMeta {
   127  	var buf [colMetaLen]byte
   128  	return buf[:]
   129  }
   130  
   131  const (
   132  	sequenceLen        = 2
   133  	dbIDLen            = 8
   134  	tableIDOff         = dbIDLen
   135  	tableIDLen         = 8
   136  	blockIDOff         = tableIDOff + tableIDLen
   137  	blockIDLen         = types.BlockidSize
   138  	rowsOff            = blockIDOff + blockIDLen
   139  	rowsLen            = 4
   140  	columnCountOff     = rowsOff + rowsLen
   141  	columnCountLen     = 2
   142  	metaLocationOff    = columnCountOff + columnCountLen
   143  	metaLocationLen    = ExtentSize
   144  	bloomFilterOff     = metaLocationOff + metaLocationLen
   145  	bloomFilterLen     = ExtentSize
   146  	bloomCheckSumOff   = bloomFilterOff + bloomFilterLen
   147  	bloomCheckSumLen   = 4
   148  	zoneMapAreaOff     = bloomCheckSumOff + bloomCheckSumLen
   149  	zoneMapAreaLen     = ZoneMapSize
   150  	zoneMapCheckSumOff = zoneMapAreaOff + zoneMapAreaLen
   151  	zoneMapCheckSumLen = 4
   152  	metaColCntOff      = zoneMapCheckSumOff + zoneMapCheckSumLen
   153  	metaColCntLen      = 2
   154  	maxSeqOff          = metaColCntOff + metaColCntLen
   155  	maxSeqLen          = 2
   156  	startIDOff         = maxSeqOff + maxSeqLen
   157  	startIDLen         = 2
   158  	appendableOff      = startIDOff + startIDLen
   159  	appendableLen      = 1
   160  	sortKeyOff         = appendableOff + appendableLen
   161  	sortKeyLen         = 2
   162  	headerDummyOff     = sortKeyOff + sortKeyLen
   163  	headerDummyLen     = 30
   164  	headerLen          = headerDummyOff + headerDummyLen
   165  )
   166  
   167  type BlockHeader []byte
   168  
   169  func BuildBlockHeader() BlockHeader {
   170  	var buf [headerLen]byte
   171  	return buf[:]
   172  }
   173  
   174  func (bh BlockHeader) TableID() uint64 {
   175  	return types.DecodeUint64(bh[tableIDOff:])
   176  }
   177  
   178  func (bh BlockHeader) SetTableID(id uint64) {
   179  	copy(bh[tableIDOff:], types.EncodeUint64(&id))
   180  }
   181  
   182  func (bh BlockHeader) BlockID() *Blockid {
   183  	return (*Blockid)(unsafe.Pointer(&bh[blockIDOff]))
   184  }
   185  
   186  func (bh BlockHeader) SetBlockID(id *Blockid) {
   187  	copy(bh[blockIDOff:blockIDOff+blockIDLen], id[:])
   188  }
   189  
   190  func (bh BlockHeader) ShortName() *ObjectNameShort {
   191  	return (*ObjectNameShort)(unsafe.Pointer(&bh[blockIDOff]))
   192  }
   193  
   194  func (bh BlockHeader) Sequence() uint16 {
   195  	return types.DecodeUint16(bh[rowsOff-sequenceLen : rowsOff])
   196  }
   197  
   198  func (bh BlockHeader) SetSequence(seq uint16) {
   199  	copy(bh[rowsOff-sequenceLen:rowsOff], types.EncodeUint16(&seq))
   200  }
   201  
   202  func (bh BlockHeader) Rows() uint32 {
   203  	return types.DecodeUint32(bh[rowsOff : rowsOff+rowsLen])
   204  }
   205  
   206  func (bh BlockHeader) SetRows(rows uint32) {
   207  	copy(bh[rowsOff:rowsOff+rowsLen], types.EncodeUint32(&rows))
   208  }
   209  
   210  func (bh BlockHeader) ColumnCount() uint16 {
   211  	return types.DecodeUint16(bh[columnCountOff : columnCountOff+columnCountLen])
   212  }
   213  
   214  func (bh BlockHeader) SetColumnCount(count uint16) {
   215  	copy(bh[columnCountOff:columnCountOff+columnCountLen], types.EncodeUint16(&count))
   216  }
   217  
   218  func (bh BlockHeader) MetaColumnCount() uint16 {
   219  	return types.DecodeUint16(bh[metaColCntOff : metaColCntOff+metaColCntLen])
   220  }
   221  
   222  func (bh BlockHeader) SetMetaColumnCount(count uint16) {
   223  	copy(bh[metaColCntOff:metaColCntOff+metaColCntLen], types.EncodeUint16(&count))
   224  }
   225  
   226  func (bh BlockHeader) MaxSeqnum() uint16 {
   227  	return types.DecodeUint16(bh[maxSeqOff : maxSeqOff+maxSeqLen])
   228  }
   229  
   230  func (bh BlockHeader) SetMaxSeqnum(seqnum uint16) {
   231  	copy(bh[maxSeqOff:maxSeqOff+maxSeqLen], types.EncodeUint16(&seqnum))
   232  }
   233  
   234  func (bh BlockHeader) StartID() uint16 {
   235  	return types.DecodeUint16(bh[startIDOff : startIDOff+startIDLen])
   236  }
   237  
   238  func (bh BlockHeader) SetStartID(id uint16) {
   239  	copy(bh[startIDOff:startIDOff+startIDLen], types.EncodeUint16(&id))
   240  }
   241  
   242  func (bh BlockHeader) MetaLocation() Extent {
   243  	return Extent(bh[metaLocationOff : metaLocationOff+metaLocationLen])
   244  }
   245  
   246  func (bh BlockHeader) SetMetaLocation(location Extent) {
   247  	copy(bh[metaLocationOff:metaLocationOff+metaLocationLen], location)
   248  }
   249  
   250  func (bh BlockHeader) ZoneMapArea() Extent {
   251  	return Extent(bh[zoneMapAreaOff : zoneMapAreaOff+zoneMapAreaLen])
   252  }
   253  
   254  func (bh BlockHeader) SetZoneMapArea(location Extent) {
   255  	copy(bh[zoneMapAreaOff:zoneMapAreaOff+zoneMapAreaLen], location)
   256  }
   257  
   258  func (bh BlockHeader) BFExtent() Extent {
   259  	return Extent(bh[bloomFilterOff : bloomFilterOff+bloomFilterLen])
   260  }
   261  
   262  func (bh BlockHeader) SetBFExtent(location Extent) {
   263  	copy(bh[bloomFilterOff:bloomFilterOff+bloomFilterLen], location)
   264  }
   265  
   266  func (bh BlockHeader) SetAppendable(appendable bool) {
   267  	copy(bh[appendableOff:appendableOff+appendableLen], types.EncodeBool(&appendable))
   268  }
   269  
   270  func (bh BlockHeader) Appendable() bool {
   271  	return types.DecodeBool(bh[appendableOff : appendableOff+appendableLen])
   272  }
   273  
   274  func (bh BlockHeader) SetSortKey(idx uint16) {
   275  	copy(bh[sortKeyOff:sortKeyOff+sortKeyLen], types.EncodeUint16(&idx))
   276  }
   277  
   278  func (bh BlockHeader) SortKey() uint16 {
   279  	return types.DecodeUint16(bh[sortKeyOff : sortKeyOff+sortKeyLen])
   280  }
   281  
   282  func (bh BlockHeader) IsEmpty() bool {
   283  	return len(bh) == 0
   284  }
   285  
   286  type BloomFilter []byte
   287  
   288  func (bf BloomFilter) Size() int {
   289  	return len(bf)
   290  }
   291  
   292  func (bf BloomFilter) BlockCount() uint32 {
   293  	return types.DecodeUint32(bf[:blockCountLen])
   294  }
   295  
   296  func (bf BloomFilter) GetBloomFilter(BlockID uint32) []byte {
   297  	offStart := blockCountLen + BlockID*posLen
   298  	offEnd := blockCountLen + BlockID*posLen + blockOffset
   299  	offset := types.DecodeUint32(bf[offStart:offEnd])
   300  	length := types.DecodeUint32(bf[offStart+blockLen : offEnd+blockLen])
   301  	return bf[offset : offset+length]
   302  }
   303  
   304  func (bf BloomFilter) GetObjectBloomFilter() []byte {
   305  	return bf.GetBloomFilter(bf.BlockCount())
   306  }
   307  
   308  type ZoneMapArea []byte
   309  
   310  func (zma ZoneMapArea) BlockCount() uint32 {
   311  	return types.DecodeUint32(zma[:blockCountLen])
   312  }
   313  
   314  func (zma ZoneMapArea) GetZoneMap(idx uint16, BlockID uint32) ZoneMap {
   315  	offStart := blockCountLen + BlockID*posLen
   316  	offEnd := blockCountLen + BlockID*posLen + blockOffset
   317  	blockOff := types.DecodeUint32(zma[offStart:offEnd])
   318  	blockLength := types.DecodeUint32(zma[offStart+blockLen : offEnd+blockLen])
   319  	offset := blockOff + uint32(idx)*ZoneMapSize
   320  	return ZoneMap(zma[offset : offset+blockLength])
   321  
   322  }
   323  
   324  type Header []byte
   325  
   326  func BuildHeader() Header {
   327  	var buf [HeaderSize]byte
   328  	magic := uint64(Magic)
   329  	version := uint16(Version)
   330  	copy(buf[:8], types.EncodeUint64(&magic))
   331  	copy(buf[8:8+2], types.EncodeUint16(&version))
   332  	return buf[:]
   333  }
   334  
   335  func (h Header) SetExtent(location Extent) {
   336  	copy(h[8+2:8+2+ExtentSize], location)
   337  }
   338  
   339  func (h Header) Extent() Extent {
   340  	return Extent(h[8+2 : 8+2+ExtentSize])
   341  }
   342  
   343  func (h Header) SetSchemaVersion(ver uint32) {
   344  	copy(h[8+2+ExtentSize:8+2+ExtentSize+4], types.EncodeUint32(&ver))
   345  }
   346  
   347  func (h Header) SchemaVersion(ver uint32) {
   348  	types.DecodeUint32(h[8+2+ExtentSize : 8+2+ExtentSize+4])
   349  }
   350  
   351  type Footer struct {
   352  	dummy      [37]byte
   353  	checksum   uint32
   354  	metaExtent Extent
   355  	version    uint16
   356  	magic      uint64
   357  }
   358  
   359  func (f Footer) Marshal() []byte {
   360  	return unsafe.Slice((*byte)(unsafe.Pointer(&f)), FooterSize)
   361  }
   362  
   363  func IsSameObjectLocVsMeta(location Location, meta ObjectDataMeta) bool {
   364  	if len(location) == 0 || len(meta) == 0 {
   365  		return false
   366  	}
   367  	return location.ShortName().Equal(meta.BlockHeader().ShortName()[:])
   368  }
   369  
   370  func IsSameObjectLocVsShort(location Location, short *ObjectNameShort) bool {
   371  	if len(location) == 0 || short == nil {
   372  		return false
   373  	}
   374  	return location.ShortName().Equal(short[:])
   375  }
   376  
   377  // test used
   378  func BuildMetaData(blkCount, colCount uint16) objectDataMetaV1 {
   379  	var meta bytes.Buffer
   380  	length := uint32(0)
   381  	seqnum := NewSeqnums(nil)
   382  	seqnum.InitWithColCnt(int(colCount))
   383  	objectMeta := BuildObjectMeta(colCount)
   384  	objectMeta.BlockHeader().SetColumnCount(colCount)
   385  	objectMeta.BlockHeader().SetMetaColumnCount(colCount)
   386  	objectMeta.BlockHeader().SetSequence(blkCount)
   387  	length += objectMeta.Length()
   388  	blockIndex := BuildBlockIndex(uint32(blkCount))
   389  	blockIndex.SetBlockCount(uint32(blkCount))
   390  	length += blockIndex.Length()
   391  	var blkMetaBuf bytes.Buffer
   392  	for i := uint16(0); i < blkCount; i++ {
   393  		blkMeta := NewBlock(seqnum)
   394  		blkMeta.BlockHeader().SetSequence(i)
   395  		n := uint32(len(blkMeta))
   396  		blockIndex.SetBlockMetaPos(uint32(i), length, n)
   397  		length += n
   398  		blkMetaBuf.Write(blkMeta)
   399  	}
   400  	meta.Write(objectMeta)
   401  	meta.Write(blockIndex)
   402  	meta.Write(blkMetaBuf.Bytes())
   403  	return meta.Bytes()
   404  }