github.com/zuoyebang/bitalosdb@v1.1.1-0.20240516111551-79a8c4d8ce20/bitable/db.go (about)

     1  // Copyright 2021 The Bitalosdb author(hustxrb@163.com) and other contributors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bitable
    16  
    17  import (
    18  	"bytes"
    19  	"fmt"
    20  	"io"
    21  
    22  	"github.com/zuoyebang/bitalosdb/bithash"
    23  	"github.com/zuoyebang/bitalosdb/internal/base"
    24  	"github.com/zuoyebang/bitalosdb/internal/hash"
    25  	"github.com/zuoyebang/bitalosdb/internal/humanize"
    26  	"github.com/zuoyebang/bitalosdb/internal/options"
    27  	bt "github.com/zuoyebang/bitalostable"
    28  	"github.com/zuoyebang/bitalostable/bloom"
    29  )
    30  
    31  type Bitable struct {
    32  	db    *bt.DB
    33  	index int
    34  	wo    *bt.WriteOptions
    35  	bhash *bithash.Bithash
    36  	opts  *options.BitableOptions
    37  }
    38  
    39  func Open(bhash *bithash.Bithash, dirname string, opts *options.BitableOptions) (b *Bitable, err error) {
    40  	b = &Bitable{
    41  		index: opts.Index,
    42  		wo:    bt.NoSync,
    43  		bhash: bhash,
    44  		opts:  opts,
    45  	}
    46  
    47  	l0Size := opts.L0FileSize
    48  	lopts := make([]bt.LevelOptions, 7)
    49  	for l := 0; l < 7; l++ {
    50  		lopts[l] = bt.LevelOptions{
    51  			Compression:    bt.SnappyCompression,
    52  			BlockSize:      32 * 1024,
    53  			TargetFileSize: l0Size,
    54  			FilterPolicy:   bloom.FilterPolicy(10),
    55  		}
    56  		l0Size = l0Size * 2
    57  	}
    58  
    59  	btOpts := &bt.Options{
    60  		MemTableSize:                opts.MemTableSize,
    61  		MemTableStopWritesThreshold: opts.MemTableStopWritesThreshold,
    62  		L0CompactionFileThreshold:   opts.L0CompactionFileThreshold,
    63  		L0CompactionThreshold:       opts.L0CompactionThreshold,
    64  		L0StopWritesThreshold:       opts.L0StopWritesThreshold,
    65  		LBaseMaxBytes:               opts.LBaseMaxBytes,
    66  		MaxOpenFiles:                opts.MaxOpenFiles,
    67  		Levels:                      lopts,
    68  		Logger:                      opts.Logger,
    69  		Verbose:                     true,
    70  		LogTag:                      fmt.Sprintf("[bitable/%d]", opts.Index),
    71  		KvCheckExpireFunc:           opts.CheckExpireCB,
    72  	}
    73  
    74  	btCache := bt.NewCache(opts.CacheSize)
    75  	defer btCache.Unref()
    76  	btOpts.Cache = btCache
    77  
    78  	b.db, err = bt.Open(dirname, btOpts)
    79  	if err != nil {
    80  		return nil, err
    81  	}
    82  
    83  	return b, nil
    84  }
    85  
    86  func (b *Bitable) Close() error {
    87  	return b.db.Close()
    88  }
    89  
    90  func (b *Bitable) bithashGet(key []byte, fn uint32) ([]byte, func(), error) {
    91  	khash := hash.Crc32(key)
    92  	return b.bhash.Get(key, khash, bithash.FileNum(fn))
    93  }
    94  
    95  func (b *Bitable) Get(key []byte) ([]byte, io.Closer, error) {
    96  	v, closer, err := b.db.Get(key)
    97  	if err == bt.ErrNotFound {
    98  		return nil, nil, base.ErrNotFound
    99  	}
   100  	return v, closer, err
   101  }
   102  
   103  func (b *Bitable) Delete(key []byte) error {
   104  	return b.db.Delete(key, b.wo)
   105  }
   106  
   107  func (b *Bitable) Checkpoint(destDir string) error {
   108  	return b.db.Checkpoint(destDir)
   109  }
   110  
   111  func (b *Bitable) Compact(start, end []byte, parallelize bool) error {
   112  	return b.db.Compact(start, end, parallelize)
   113  }
   114  
   115  func (b *Bitable) Metrics() string {
   116  	return b.db.Metrics().String()
   117  }
   118  
   119  func (b *Bitable) NewIter(o *options.IterOptions) *BitableIterator {
   120  	btIterOpts := &bt.IterOptions{
   121  		LowerBound: o.GetLowerBound(),
   122  		UpperBound: o.GetUpperBound(),
   123  	}
   124  
   125  	iter := &BitableIterator{
   126  		btable: b,
   127  		iter:   b.db.NewIter(btIterOpts),
   128  	}
   129  	return iter
   130  }
   131  
   132  func (b *Bitable) OpenAutomaticCompactions() {
   133  	b.db.SetOptsDisableAutomaticCompactions(false)
   134  }
   135  
   136  func (b *Bitable) CloseAutomaticCompactions() {
   137  	b.db.SetOptsDisableAutomaticCompactions(true)
   138  }
   139  
   140  func hitRate(hits, misses int64) float64 {
   141  	sum := hits + misses
   142  	if sum == 0 {
   143  		return 0
   144  	}
   145  	return 100 * float64(hits) / float64(sum)
   146  }
   147  
   148  func (b *Bitable) DebugInfo(dataType string) string {
   149  	metrics := b.db.Metrics()
   150  	buf := new(bytes.Buffer)
   151  
   152  	fmt.Fprintf(buf, "%s-bitable%d: bcache(count=%s size=%s hitRate=%.1f%%) levels(files=%s %s %s %s %s %s %s)\n",
   153  		dataType, b.index,
   154  		humanize.SI.Int64(metrics.BlockCache.Count),
   155  		humanize.IEC.Int64(metrics.BlockCache.Size),
   156  		hitRate(metrics.BlockCache.Hits, metrics.BlockCache.Misses),
   157  		humanize.SI.Int64(metrics.Levels[0].NumFiles),
   158  		humanize.SI.Int64(metrics.Levels[1].NumFiles),
   159  		humanize.SI.Int64(metrics.Levels[2].NumFiles),
   160  		humanize.SI.Int64(metrics.Levels[3].NumFiles),
   161  		humanize.SI.Int64(metrics.Levels[4].NumFiles),
   162  		humanize.SI.Int64(metrics.Levels[5].NumFiles),
   163  		humanize.SI.Int64(metrics.Levels[6].NumFiles),
   164  	)
   165  
   166  	return buf.String()
   167  }