github.com/matrixorigin/matrixone@v0.7.0/pkg/objectio/type.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package objectio
    16  
    17  import (
    18  	"context"
    19  	"encoding/binary"
    20  	"github.com/matrixorigin/matrixone/pkg/common/mpool"
    21  	"github.com/matrixorigin/matrixone/pkg/container/batch"
    22  	"github.com/matrixorigin/matrixone/pkg/fileservice"
    23  )
    24  
    25  type WriteType int8
    26  
    27  const (
    28  	WriteTS WriteType = iota
    29  )
    30  
    31  type WriteOptions struct {
    32  	Type WriteType
    33  	Val  any
    34  }
    35  
    36  // Writer is to virtualize batches into multiple blocks
    37  // and write them into filefservice at one time
    38  type Writer interface {
    39  	// Write writes one batch to the Buffer at a time,
    40  	// one batch corresponds to a virtual block,
    41  	// and returns the handle of the block.
    42  	Write(batch *batch.Batch) (BlockObject, error)
    43  
    44  	// WriteIndex is the index of the column in the block written to the block's handle.
    45  	// block is the handle of the block
    46  	// idx is the column to which the index is written
    47  	// buf is the data to write to the index
    48  	WriteIndex(block BlockObject, index IndexData) error
    49  
    50  	// WriteEnd is to write multiple batches written to
    51  	// the buffer to the fileservice at one time
    52  	WriteEnd(ctx context.Context, items ...WriteOptions) ([]BlockObject, error)
    53  }
    54  
    55  // Reader is to read data from fileservice
    56  type Reader interface {
    57  	// Read is to read columns data of a block from fileservice at one time
    58  	// extent is location of the block meta
    59  	// idxs is the column serial number of the data to be read
    60  	Read(ctx context.Context, extents Extent, idxs []uint16, m *mpool.MPool) (*fileservice.IOVector, error)
    61  
    62  	// ReadMeta is the meta that reads a block
    63  	// extent is location of the block meta
    64  	ReadMeta(ctx context.Context, extent []Extent, m *mpool.MPool) ([]BlockObject, error)
    65  
    66  	// ReadIndex is the index data of the read columns
    67  	ReadIndex(ctx context.Context, extent Extent, idxs []uint16, dataType IndexDataType, m *mpool.MPool) ([]IndexData, error)
    68  
    69  	// ReadAllMeta is read the meta of all blocks in an object
    70  	ReadAllMeta(ctx context.Context, fileSize int64, m *mpool.MPool) ([]BlockObject, error)
    71  }
    72  
    73  // BlockObject is a batch written to fileservice
    74  type BlockObject interface {
    75  	// GetColumn gets a ColumnObject with idx
    76  	GetColumn(idx uint16) (ColumnObject, error)
    77  
    78  	// GetRows gets the rows of the BlockObject
    79  	GetRows() (uint32, error)
    80  
    81  	// GetMeta gets the meta of the BlockObject
    82  	GetMeta() BlockMeta
    83  
    84  	// GetExtent gets the metadata location of BlockObject in fileservice
    85  	GetExtent() Extent
    86  
    87  	// GetID is to get the serial number of the block in the object
    88  	GetID() uint32
    89  }
    90  
    91  // ColumnObject is a vector in a batch written to fileservice
    92  type ColumnObject interface {
    93  	// GetData gets the data of ColumnObject
    94  	// Returns an IOVector, the caller needs to traverse the IOVector
    95  	// to get all the structures required for data generation
    96  	GetData(ctx context.Context, m *mpool.MPool) (*fileservice.IOVector, error)
    97  
    98  	// GetIndex gets the index of ColumnObject
    99  	GetIndex(ctx context.Context, dataType IndexDataType, m *mpool.MPool) (IndexData, error)
   100  
   101  	// GetMeta gets the metadata of ColumnObject
   102  	GetMeta() *ColumnMeta
   103  }
   104  
   105  var (
   106  	endian = binary.LittleEndian
   107  )