github.com/matrixorigin/matrixone@v0.7.0/pkg/container/index/dict/reverse_index.go (about)

     1  // Copyright 2021 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package dict
    16  
    17  import (
    18  	"unsafe"
    19  
    20  	"github.com/matrixorigin/matrixone/pkg/common/mpool"
    21  	"github.com/matrixorigin/matrixone/pkg/container/hashtable"
    22  )
    23  
    24  const (
    25  	unitLimit = 256
    26  )
    27  
    28  var (
    29  	zeroUint64 []uint64
    30  )
    31  
    32  func init() {
    33  	zeroUint64 = make([]uint64, unitLimit)
    34  }
    35  
    36  type reverseIndex interface {
    37  	insert(keys any) ([]uint64, error)
    38  	find(keys any) []uint64
    39  	free()
    40  }
    41  
    42  type fixedReverseIndex struct {
    43  	m  *mpool.MPool
    44  	ht *hashtable.Int64HashMap
    45  }
    46  
    47  func newFixedReverseIndex(m *mpool.MPool) (*fixedReverseIndex, error) {
    48  	ht := &hashtable.Int64HashMap{}
    49  	if err := ht.Init(m); err != nil {
    50  		ht.Free(m)
    51  		return nil, err
    52  	}
    53  	return &fixedReverseIndex{
    54  		m:  m,
    55  		ht: ht,
    56  	}, nil
    57  }
    58  
    59  func (idx *fixedReverseIndex) insert(keys any) ([]uint64, error) {
    60  	ks := keys.([]uint64)
    61  	count := len(ks)
    62  	hashes := make([]uint64, unitLimit)
    63  	values := make([]uint64, count)
    64  	for i := 0; i < count; i += unitLimit {
    65  		n := count - i
    66  		if n > unitLimit {
    67  			n = unitLimit
    68  		}
    69  		copy(hashes[:n], zeroUint64[:n])
    70  		if err := idx.ht.InsertBatch(n, hashes[:n], unsafe.Pointer(&ks[i]), values[i:i+n], idx.m); err != nil {
    71  			return nil, err
    72  		}
    73  	}
    74  	return values, nil
    75  }
    76  
    77  func (idx *fixedReverseIndex) find(keys any) []uint64 {
    78  	ks := keys.([]uint64)
    79  	count := len(ks)
    80  	hashes := make([]uint64, unitLimit)
    81  	values := make([]uint64, count)
    82  	for i := 0; i < count; i += unitLimit {
    83  		n := count - i
    84  		if n > unitLimit {
    85  			n = unitLimit
    86  		}
    87  		copy(hashes[:n], zeroUint64[:n])
    88  		idx.ht.FindBatch(n, hashes[:n], unsafe.Pointer(&ks[i]), values[i:i+n])
    89  	}
    90  	return values
    91  }
    92  
    93  func (idx *fixedReverseIndex) free() {
    94  	idx.ht.Free(idx.m)
    95  }
    96  
    97  type varReverseIndex struct {
    98  	m          *mpool.MPool
    99  	ht         *hashtable.StringHashMap
   100  	hashStates [][3]uint64
   101  }
   102  
   103  func newVarReverseIndex(m *mpool.MPool) (*varReverseIndex, error) {
   104  	ht := &hashtable.StringHashMap{}
   105  	if err := ht.Init(m); err != nil {
   106  		ht.Free(m)
   107  		return nil, err
   108  	}
   109  	return &varReverseIndex{
   110  		m:          m,
   111  		ht:         ht,
   112  		hashStates: make([][3]uint64, unitLimit),
   113  	}, nil
   114  }
   115  
   116  func (idx *varReverseIndex) insert(keys any) ([]uint64, error) {
   117  	ks := checkPadding(keys.([][]byte))
   118  	count := len(ks)
   119  	values := make([]uint64, count)
   120  	for i := 0; i < count; i += unitLimit {
   121  		n := count - i
   122  		if n > unitLimit {
   123  			n = unitLimit
   124  		}
   125  		if err := idx.ht.InsertStringBatch(idx.hashStates, ks[i:i+n], values[i:i+n], idx.m); err != nil {
   126  			return nil, err
   127  		}
   128  	}
   129  	return values, nil
   130  }
   131  
   132  func (idx *varReverseIndex) find(keys any) []uint64 {
   133  	ks := checkPadding(keys.([][]byte))
   134  	count := len(ks)
   135  	values := make([]uint64, count)
   136  	for i := 0; i < count; i += unitLimit {
   137  		n := count - i
   138  		if n > unitLimit {
   139  			n = unitLimit
   140  		}
   141  		idx.ht.FindStringBatch(idx.hashStates, ks[i:i+n], values[i:i+n])
   142  	}
   143  	return values
   144  }
   145  
   146  func (idx *varReverseIndex) free() {
   147  	idx.ht.Free(idx.m)
   148  }
   149  
   150  // checkPadding checks if the length of each key is less than 16.
   151  func checkPadding(keys [][]byte) [][]byte {
   152  	ks := make([][]byte, len(keys))
   153  	for i := range ks {
   154  		if len(keys[i]) < 16 {
   155  			dst := make([]byte, 16)
   156  			copy(dst, keys[i])
   157  			ks[i] = dst
   158  		} else {
   159  			ks[i] = keys[i]
   160  		}
   161  	}
   162  	return ks
   163  }