github.com/matrixorigin/matrixone@v1.2.0/pkg/objectio/types.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package objectio
    16  
    17  import (
    18  	"context"
    19  	"math"
    20  
    21  	"github.com/matrixorigin/matrixone/pkg/common/mpool"
    22  	"github.com/matrixorigin/matrixone/pkg/container/batch"
    23  	"github.com/matrixorigin/matrixone/pkg/fileservice"
    24  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index"
    25  )
    26  
    27  const (
    28  	SEQNUM_UPPER    = math.MaxUint16 - 5 // reserved 5 column for special committs、committs etc.
    29  	SEQNUM_ROWID    = math.MaxUint16
    30  	SEQNUM_ABORT    = math.MaxUint16 - 1
    31  	SEQNUM_COMMITTS = math.MaxUint16 - 2
    32  )
    33  
    34  type WriteType int8
    35  
    36  const (
    37  	WriteTS WriteType = iota
    38  )
    39  
    40  const ZoneMapSize = index.ZMSize
    41  
    42  type ZoneMap = index.ZM
    43  type StaticFilter = index.StaticFilter
    44  
    45  var NewZM = index.NewZM
    46  var BuildZM = index.BuildZM
    47  
    48  type ColumnMetaFetcher interface {
    49  	MustGetColumn(seqnum uint16) ColumnMeta
    50  }
    51  
    52  type WriteOptions struct {
    53  	Type WriteType
    54  	Val  any
    55  }
    56  
    57  type ReadBlockOptions struct {
    58  	Id       uint16
    59  	DataType uint16
    60  	Idxes    map[uint16]bool
    61  }
    62  
    63  // Writer is to virtualize batches into multiple blocks
    64  // and write them into filefservice at one time
    65  type Writer interface {
    66  	// Write writes one batch to the Buffer at a time,
    67  	// one batch corresponds to a virtual block,
    68  	// and returns the handle of the block.
    69  	Write(batch *batch.Batch) (BlockObject, error)
    70  
    71  	// Write metadata for every column of all blocks
    72  	WriteObjectMeta(ctx context.Context, totalRow uint32, metas []ColumnMeta)
    73  
    74  	// WriteEnd is to write multiple batches written to
    75  	// the buffer to the fileservice at one time
    76  	WriteEnd(ctx context.Context, items ...WriteOptions) ([]BlockObject, error)
    77  }
    78  
    79  // Reader is to read data from fileservice
    80  type Reader interface {
    81  	// Read is to read columns data of a block from fileservice at one time
    82  	// extent is location of the block meta
    83  	// idxs is the column serial number of the data to be read
    84  	Read(ctx context.Context,
    85  		extent *Extent, idxs []uint16,
    86  		id uint32,
    87  		m *mpool.MPool,
    88  		readFunc CacheConstructorFactory) (*fileservice.IOVector, error)
    89  
    90  	ReadAll(
    91  		ctx context.Context,
    92  		extent *Extent,
    93  		idxs []uint16,
    94  		m *mpool.MPool,
    95  		readFunc CacheConstructorFactory,
    96  	) (*fileservice.IOVector, error)
    97  
    98  	ReadBlocks(ctx context.Context,
    99  		extent *Extent,
   100  		ids map[uint32]*ReadBlockOptions,
   101  		m *mpool.MPool,
   102  		readFunc CacheConstructorFactory) (*fileservice.IOVector, error)
   103  
   104  	// ReadMeta is the meta that reads a block
   105  	// extent is location of the block meta
   106  	ReadMeta(ctx context.Context, extent *Extent, m *mpool.MPool) (ObjectDataMeta, error)
   107  
   108  	// ReadAllMeta is read the meta of all blocks in an object
   109  	ReadAllMeta(ctx context.Context, m *mpool.MPool) (ObjectDataMeta, error)
   110  
   111  	GetObject() *Object
   112  }
   113  
   114  type ObjectMeta interface {
   115  	MustGetMeta(metaType DataMetaType) ObjectDataMeta
   116  
   117  	HeaderLength() uint32
   118  
   119  	DataMetaCount() uint16
   120  	TombstoneMetaCount() uint16
   121  
   122  	DataMeta() (ObjectDataMeta, bool)
   123  
   124  	MustDataMeta() ObjectDataMeta
   125  
   126  	TombstoneMeta() (ObjectDataMeta, bool)
   127  
   128  	MustTombstoneMeta() ObjectDataMeta
   129  
   130  	SetDataMetaCount(count uint16)
   131  
   132  	SetDataMetaOffset(offset uint32)
   133  
   134  	SetTombstoneMetaCount(count uint16)
   135  
   136  	SetTombstoneMetaOffset(offset uint32)
   137  
   138  	SubMeta(pos uint16) (ObjectDataMeta, bool)
   139  
   140  	SubMetaCount() uint16
   141  
   142  	SubMetaIndex() SubMetaIndex
   143  
   144  	SubMetaTypes() []uint16
   145  }