github.com/matrixorigin/matrixone@v0.7.0/pkg/objectio/meta.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package objectio
    16  
    17  import (
    18  	"bytes"
    19  	"encoding/binary"
    20  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    21  )
    22  
    23  const HeaderSize = 64
    24  const ColumnMetaSize = 128
    25  
    26  // +---------------------------------------------------------------------------------------------+
    27  // |                                           Header                                            |
    28  // +-------------+---------------+--------------+---------------+---------------+----------------+
    29  // | TableID(8B) | SegmentID(8B) | BlockID(8B)  | ColumnCnt(2B) | Reserved(34B) |  Chksum(4B)    |
    30  // +-------------+---------------+--------------+---------------+---------------+----------------+
    31  // |                                         ColumnMeta                                          |
    32  // +---------------------------------------------------------------------------------------------+
    33  // |                                         ColumnMeta                                          |
    34  // +---------------------------------------------------------------------------------------------+
    35  // |                                         ColumnMeta                                          |
    36  // +---------------------------------------------------------------------------------------------+
    37  // |                                         ..........                                          |
    38  // +---------------------------------------------------------------------------------------------+
    39  // Header Size = 64B
    40  // TableID = Table ID for Block
    41  // SegmentID = Segment ID
    42  // BlockID = Block ID
    43  // ColumnCnt = The number of column in the block
    44  // Chksum = Block metadata checksum
    45  // Reserved = 34 bytes reserved space
    46  type BlockMeta struct {
    47  	header BlockHeader
    48  	name   string
    49  }
    50  
    51  func (bm *BlockMeta) GetName() string {
    52  	return bm.name
    53  }
    54  
    55  func (bm *BlockMeta) GetHeader() BlockHeader {
    56  	return bm.header
    57  }
    58  
    59  type BlockHeader struct {
    60  	tableId     uint64
    61  	segmentId   uint64
    62  	blockId     uint64
    63  	columnCount uint16
    64  	dummy       [34]byte
    65  	checksum    uint32
    66  }
    67  
    68  func (bh *BlockHeader) GetTableId() uint64 {
    69  	return bh.tableId
    70  }
    71  func (bh *BlockHeader) GetSegmentId() uint64 {
    72  	return bh.segmentId
    73  }
    74  func (bh *BlockHeader) GetBlockId() uint64 {
    75  	return bh.blockId
    76  }
    77  func (bh *BlockHeader) GetColumnCount() uint16 {
    78  	return bh.columnCount
    79  }
    80  
    81  // +---------------------------------------------------------------------------------------------------------------+
    82  // |                                                    ColumnMeta                                                 |
    83  // +--------+-------+----------+--------+---------+--------+--------+------------+---------+----------+------------+
    84  // |Type(1B)|Idx(2B)| Algo(1B) |Offset(4B)|Size(4B)|oSize(4B)|Min(32B)|Max(32B)|BFoffset(4b)|BFlen(4b)|BFoSize(4B) |
    85  // +--------+-------+----------+--------+---------+--------+--------+------------+---------+----------+------------+
    86  // |                                        Reserved(32B)                                             | Chksum(4B) |
    87  // +---------------------------------------------------------------------------------------------------------------+
    88  // ColumnMeta Size = 128B
    89  // Type = Metadata type, always 0, representing column meta, used for extension.
    90  // Idx = Column index
    91  // Algo = Type of compression algorithm for column data
    92  // Offset = Offset of column data
    93  // Size = Size of column data
    94  // oSize = Original data size
    95  // Min = Column min value
    96  // Max = Column Max value
    97  // BFoffset = Bloomfilter data offset
    98  // Bflen = Bloomfilter data size
    99  // BFoSize = Bloomfilter original data size
   100  // Chksum = Data checksum
   101  // Reserved = 32 bytes reserved space
   102  type ColumnMeta struct {
   103  	typ         uint8
   104  	idx         uint16
   105  	alg         uint8
   106  	location    Extent
   107  	zoneMap     ZoneMap
   108  	bloomFilter Extent
   109  	dummy       [32]byte
   110  	checksum    uint32
   111  }
   112  
   113  func (cm *ColumnMeta) GetType() uint8 {
   114  	return cm.typ
   115  }
   116  
   117  func (cm *ColumnMeta) GetIdx() uint16 {
   118  	return cm.idx
   119  }
   120  
   121  func (cm *ColumnMeta) GetAlg() uint8 {
   122  	return cm.alg
   123  }
   124  
   125  func (cm *ColumnMeta) GetLocation() Extent {
   126  	return cm.location
   127  }
   128  
   129  func (cm *ColumnMeta) GetZoneMap() ZoneMap {
   130  	return cm.zoneMap
   131  }
   132  
   133  func (cm *ColumnMeta) GetBloomFilter() Extent {
   134  	return cm.bloomFilter
   135  }
   136  
   137  type Header struct {
   138  	magic   uint64
   139  	version uint16
   140  	dummy   [22]byte
   141  }
   142  
   143  type Footer struct {
   144  	magic      uint64
   145  	blockCount uint32
   146  	extents    []Extent
   147  }
   148  
   149  func (f *Footer) UnMarshalFooter(data []byte) error {
   150  	var err error
   151  	footer := data[len(data)-FooterSize:]
   152  	FooterCache := bytes.NewBuffer(footer)
   153  	if err = binary.Read(FooterCache, endian, &f.blockCount); err != nil {
   154  		return err
   155  	}
   156  	if err = binary.Read(FooterCache, endian, &f.magic); err != nil {
   157  		return err
   158  	}
   159  	if f.magic != uint64(Magic) {
   160  		return moerr.NewInternalErrorNoCtx("object io: invalid footer")
   161  	}
   162  	if f.blockCount*ExtentTypeSize+FooterSize > uint32(len(data)) {
   163  		return nil
   164  	} else {
   165  		f.extents = make([]Extent, f.blockCount)
   166  	}
   167  	extents := data[len(data)-int(FooterSize+f.blockCount*ExtentTypeSize):]
   168  	ExtentsCache := bytes.NewBuffer(extents)
   169  	size := uint32(0)
   170  	for i := 0; i < int(f.blockCount); i++ {
   171  		f.extents[i].id = uint32(i)
   172  		if err = binary.Read(ExtentsCache, endian, &f.extents[i].offset); err != nil {
   173  			return err
   174  		}
   175  		if err = binary.Read(ExtentsCache, endian, &f.extents[i].length); err != nil {
   176  			return err
   177  		}
   178  		if err = binary.Read(ExtentsCache, endian, &f.extents[i].originSize); err != nil {
   179  			return err
   180  		}
   181  		size += f.extents[i].originSize
   182  	}
   183  
   184  	for i := 0; i < int(f.blockCount); i++ {
   185  		f.extents[i].offset = f.extents[0].offset
   186  		f.extents[i].length = size
   187  		f.extents[i].originSize = size
   188  	}
   189  
   190  	return err
   191  }