github.com/matrixorigin/matrixone@v0.7.0/pkg/vm/engine/tae/tables/evictable/deltadata.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package evictable
    16  
    17  import (
    18  	"bytes"
    19  	"context"
    20  	"github.com/matrixorigin/matrixone/pkg/objectio"
    21  
    22  	"github.com/matrixorigin/matrixone/pkg/container/types"
    23  	"github.com/matrixorigin/matrixone/pkg/container/vector"
    24  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/buffer"
    25  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/buffer/base"
    26  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
    27  )
    28  
    29  type DeltaDataNode struct {
    30  	*buffer.Node
    31  
    32  	Data containers.Vector
    33  
    34  	deltaDataKey string
    35  
    36  	colidx uint16
    37  	typ    types.Type
    38  
    39  	metaKey          string
    40  	mgr              base.INodeManager
    41  	deltaMetaFactory EvictableNodeFactory
    42  }
    43  
    44  func NewDeltaDataNode(mgr base.INodeManager, deltaDataKey, metaKey string, fs *objectio.ObjectFS, deltaloc string, colidx uint16, typ types.Type) (node *DeltaDataNode, err error) {
    45  	node = &DeltaDataNode{
    46  		deltaDataKey:     deltaDataKey,
    47  		metaKey:          metaKey,
    48  		mgr:              mgr,
    49  		colidx:           colidx,
    50  		typ:              typ,
    51  		deltaMetaFactory: func() (base.INode, error) { return NewDeltaMetaNode(mgr, metaKey, fs, deltaloc), nil },
    52  	}
    53  	// For disk, size is zero, do not cache, read directly when GetData
    54  	var size uint32 = 0
    55  	if StorageBackend == S3 {
    56  		// on s3, fetch coldata to get data size
    57  		h, pinerr := PinEvictableNode(mgr, metaKey, node.deltaMetaFactory)
    58  		if pinerr != nil {
    59  			return nil, pinerr
    60  		}
    61  		defer h.Close()
    62  		meta := h.GetNode().(*DeltaMetaNode)
    63  		col, err := meta.GetColumn(colidx)
    64  		if err != nil {
    65  			return nil, err
    66  		}
    67  		size = col.GetMeta().GetLocation().OriginSize()
    68  	}
    69  
    70  	node.Node = buffer.NewNode(node, mgr, deltaDataKey, uint64(size))
    71  	node.LoadFunc = node.onLoad
    72  	node.UnloadFunc = node.onUnload
    73  	node.HardEvictableFunc = func() bool { return true }
    74  	return
    75  }
    76  
    77  func (n *DeltaDataNode) onLoad() {
    78  	switch StorageBackend {
    79  	case S3:
    80  		// fetch data via s3 and cache it
    81  		// TODOa: error handling
    82  		data, _ := n.fetchData()
    83  		n.Data = data
    84  	case Disk:
    85  		// for disk, do nothing when load
    86  	}
    87  }
    88  
    89  func (n *DeltaDataNode) fetchData() (containers.Vector, error) {
    90  	var h base.INodeHandle
    91  	var err error
    92  	h, err = PinEvictableNode(n.mgr, n.metaKey, n.deltaMetaFactory)
    93  	if err != nil {
    94  		return nil, err
    95  	}
    96  	defer h.Close()
    97  
    98  	meta := h.GetNode().(*DeltaMetaNode)
    99  	col, err := meta.GetColumn(n.colidx)
   100  	if err != nil {
   101  		return nil, err
   102  	}
   103  
   104  	// Do IO, fetch data buf
   105  	fsVector, err := col.GetData(context.Background(), nil)
   106  	if err != nil {
   107  		return nil, err
   108  	}
   109  
   110  	//srcBuf := fsVector.Entries[0].Object.([]byte)
   111  	srcBuf := make([]byte, len(fsVector.Entries[0].Object.([]byte)))
   112  	copy(srcBuf, fsVector.Entries[0].Object.([]byte))
   113  	v := vector.New(n.typ)
   114  	v.Read(srcBuf)
   115  	return containers.NewVectorWithSharedMemory(v, false /* rowid committs abort are all non-nullable */), nil
   116  }
   117  
   118  func (n *DeltaDataNode) GetData(buf *bytes.Buffer) (containers.Vector, error) {
   119  	// after loading, for s3, its data is n.Data
   120  	switch StorageBackend {
   121  	case Disk:
   122  		// for disk, read directly
   123  		return n.fetchData()
   124  	case S3:
   125  		return copyVector(n.Data, buf), nil
   126  	}
   127  	return nil, nil
   128  }
   129  
   130  func (n *DeltaDataNode) onUnload() {
   131  	n.Data = nil
   132  }
   133  
   134  func FetchDeltaData(buf *bytes.Buffer, mgr base.INodeManager, fs *objectio.ObjectFS, deltaloc string, colidx uint16, typ types.Type) (res containers.Vector, err error) {
   135  	deltaDataKey := EncodeDeltaDataKey(colidx, deltaloc)
   136  	factory := func() (base.INode, error) {
   137  		return NewDeltaDataNode(mgr, deltaDataKey, EncodeDeltaMetaKey(deltaloc), fs, deltaloc, colidx, typ)
   138  	}
   139  	h, err := PinEvictableNode(mgr, deltaDataKey, factory)
   140  	if err != nil {
   141  		return nil, err
   142  	}
   143  	defer h.Close()
   144  	node := h.GetNode().(*DeltaDataNode)
   145  	return node.GetData(buf)
   146  }