github.com/matrixorigin/matrixone@v0.7.0/pkg/vm/engine/tae/tables/indexwrapper/bfnode.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  // http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package indexwrapper
    16  
    17  import (
    18  	"github.com/RoaringBitmap/roaring"
    19  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    20  	"github.com/matrixorigin/matrixone/pkg/container/types"
    21  	"github.com/matrixorigin/matrixone/pkg/objectio"
    22  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/buffer/base"
    23  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
    24  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers"
    25  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index"
    26  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/tables/evictable"
    27  )
    28  
    29  type BfReader struct {
    30  	bfKey     string
    31  	mgr       base.INodeManager
    32  	bfFacotry evictable.EvictableNodeFactory
    33  }
    34  
    35  func newBfReader(
    36  	id *common.ID,
    37  	typ types.Type,
    38  	metaloc string,
    39  	mgr base.INodeManager,
    40  	fs *objectio.ObjectFS,
    41  ) *BfReader {
    42  	metaKey := evictable.EncodeColMetaKey(id.Idx, metaloc)
    43  	bfKey := evictable.EncodeColBfKey(id.Idx, metaloc)
    44  
    45  	return &BfReader{
    46  		mgr:   mgr,
    47  		bfKey: bfKey,
    48  		bfFacotry: func() (base.INode, error) {
    49  			return evictable.NewBfNode(
    50  				id.Idx,
    51  				typ,
    52  				metaloc,
    53  				bfKey,
    54  				metaKey,
    55  				mgr,
    56  				fs,
    57  			)
    58  		},
    59  	}
    60  }
    61  
    62  func (r *BfReader) MayContainsKey(key any) (b bool, err error) {
    63  	h, err := evictable.PinEvictableNode(r.mgr, r.bfKey, r.bfFacotry)
    64  	if err != nil {
    65  		// TODOa: Error Handling?
    66  		return
    67  	}
    68  	defer h.Close()
    69  	bfNode := h.GetNode().(*evictable.BfNode)
    70  	return bfNode.Bf.MayContainsKey(key)
    71  }
    72  
    73  func (r *BfReader) MayContainsAnyKeys(keys containers.Vector, visibility *roaring.Bitmap) (b bool, m *roaring.Bitmap, err error) {
    74  	h, err := evictable.PinEvictableNode(r.mgr, r.bfKey, r.bfFacotry)
    75  	if err != nil {
    76  		// TODOa: Error Handling?
    77  		return
    78  	}
    79  	defer h.Close()
    80  	bfNode := h.GetNode().(*evictable.BfNode)
    81  	return bfNode.Bf.MayContainsAnyKeys(keys, visibility)
    82  }
    83  
    84  func (r *BfReader) Destroy() error { return nil }
    85  
    86  type BFWriter struct {
    87  	cType       common.CompressType
    88  	writer      objectio.Writer
    89  	block       objectio.BlockObject
    90  	impl        index.StaticFilter
    91  	data        containers.Vector
    92  	colIdx      uint16
    93  	internalIdx uint16
    94  }
    95  
    96  func NewBFWriter() *BFWriter {
    97  	return &BFWriter{}
    98  }
    99  
   100  func (writer *BFWriter) Init(wr objectio.Writer, block objectio.BlockObject, cType common.CompressType, colIdx uint16, internalIdx uint16) error {
   101  	writer.writer = wr
   102  	writer.block = block
   103  	writer.cType = cType
   104  	writer.colIdx = colIdx
   105  	writer.internalIdx = internalIdx
   106  	return nil
   107  }
   108  
   109  func (writer *BFWriter) Finalize() (*IndexMeta, error) {
   110  	if writer.impl != nil {
   111  		panic("formerly finalized filter not cleared yet")
   112  	}
   113  	sf, err := index.NewBinaryFuseFilter(writer.data)
   114  	if err != nil {
   115  		return nil, err
   116  	}
   117  	writer.impl = sf
   118  	writer.data = nil
   119  
   120  	appender := writer.writer
   121  	meta := NewEmptyIndexMeta()
   122  	meta.SetIndexType(StaticFilterIndex)
   123  	meta.SetCompressType(writer.cType)
   124  	meta.SetIndexedColumn(writer.colIdx)
   125  	meta.SetInternalIndex(writer.internalIdx)
   126  
   127  	//var startOffset uint32
   128  	iBuf, err := writer.impl.Marshal()
   129  	if err != nil {
   130  		return nil, err
   131  	}
   132  	bf := objectio.NewBloomFilter(writer.colIdx, uint8(writer.cType), iBuf)
   133  	rawSize := uint32(len(iBuf))
   134  	compressed := common.Compress(iBuf, writer.cType)
   135  	exactSize := uint32(len(compressed))
   136  	meta.SetSize(rawSize, exactSize)
   137  
   138  	err = appender.WriteIndex(writer.block, bf)
   139  	if err != nil {
   140  		return nil, err
   141  	}
   142  	//meta.SetStartOffset(startOffset)
   143  	writer.impl = nil
   144  	return meta, nil
   145  }
   146  
   147  func (writer *BFWriter) AddValues(values containers.Vector) error {
   148  	if writer.data == nil {
   149  		writer.data = values
   150  		return nil
   151  	}
   152  	if writer.data.GetType() != values.GetType() {
   153  		return moerr.NewInternalErrorNoCtx("wrong type")
   154  	}
   155  	writer.data.Extend(values)
   156  	return nil
   157  }
   158  
   159  // Query is only used for testing or debugging
   160  func (writer *BFWriter) Query(key any) (bool, error) {
   161  	return writer.impl.MayContainsKey(key)
   162  }