github.com/scottcagno/storage@v1.8.0/pkg/lsmt/lsmtree_test.go (about)

     1  package lsmt
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/binary"
     6  	"errors"
     7  	"fmt"
     8  	binary2 "github.com/scottcagno/storage/pkg/lsmt/binary"
     9  	"github.com/scottcagno/storage/pkg/util"
    10  	"log"
    11  	"math"
    12  	"os"
    13  	"path/filepath"
    14  	"runtime"
    15  	"strconv"
    16  	"testing"
    17  	"time"
    18  )
    19  
    20  func makeKey(i int) string {
    21  	//return fmt.Sprintf("key-%06d", i)
    22  	hexa := strconv.FormatInt(int64(i), 16)
    23  	return fmt.Sprintf("%s%06s", "key-", hexa)
    24  }
    25  
    26  func makeVal(i int) []byte {
    27  	return []byte(fmt.Sprintf("value-%08d", i))
    28  }
    29  func makeCustomKey(format string, i int) string {
    30  	return fmt.Sprintf(format, i)
    31  }
    32  
    33  func makeCustomVal(i int, suffix string) []byte {
    34  	return []byte(fmt.Sprintf("value-%08d-%s", i, suffix))
    35  }
    36  
    37  func logit(s string) {
    38  	log.SetPrefix("[INFO] ")
    39  	log.Printf("%s\n", s)
    40  }
    41  
    42  var conf = &LSMConfig{
    43  	BaseDir:         "lsm-testing",
    44  	FlushThreshold:  -1,
    45  	SyncOnWrite:     false,
    46  	BloomFilterSize: 1 << 16,
    47  }
    48  
    49  func TestLSMChecksum(t *testing.T) {
    50  
    51  	db, err := OpenLSMTree(conf)
    52  	if err != nil {
    53  		t.Fatalf("open: %v\n", err)
    54  	}
    55  
    56  	err = db.Close()
    57  	if err != nil {
    58  		t.Fatalf("open: %v\n", err)
    59  	}
    60  }
    61  
    62  func TestLSMTLogging(t *testing.T) {
    63  
    64  	level := LevelInfo
    65  	l := NewLogger(level)
    66  	fmt.Println(LevelText(level))
    67  	l.Debug("foo")
    68  	l.Debug("foo with args: %d\n", 4)
    69  	l.Info("foo")
    70  	l.Info("foo with args: %d\n", 4)
    71  	l.Warn("foo")
    72  	l.Warn("foo with args: %d\n", 4)
    73  	l.Error("foo")
    74  	l.Error("foo with args: %d\n", 4)
    75  	for i := LevelOff; i < LevelFatal; i++ {
    76  
    77  	}
    78  
    79  }
    80  
    81  func TestLSMTreeKeyOverride(t *testing.T) {
    82  
    83  	db, err := OpenLSMTree(conf)
    84  	if err != nil {
    85  		t.Errorf("open: %v\n", err)
    86  	}
    87  
    88  	err = db.Put("Hi!", []byte("Hello world, LSMTree!"))
    89  	if err != nil {
    90  		panic(fmt.Errorf("failed to put: %w", err))
    91  	}
    92  
    93  	err = db.Put("Does it override key?", []byte("No!"))
    94  	if err != nil {
    95  		panic(fmt.Errorf("failed to put: %w", err))
    96  	}
    97  
    98  	err = db.Put("Does it override key?", []byte("Yes, absolutely! The key has been overridden."))
    99  	if err != nil {
   100  		panic(fmt.Errorf("failed to put: %w", err))
   101  	}
   102  
   103  	err = db.Close()
   104  	if err != nil {
   105  		t.Errorf("close: %v\n", err)
   106  	}
   107  
   108  	db, err = OpenLSMTree(conf)
   109  	if err != nil {
   110  		t.Errorf("open: %v\n", err)
   111  	}
   112  
   113  	key := "Hi!"
   114  	val, err := db.Get(key)
   115  	if err != nil {
   116  		panic(fmt.Errorf("failed to get value: %w", err))
   117  	}
   118  	fmt.Printf("get(%q)=%q\n", key, val)
   119  
   120  	key = "Does it override key?"
   121  	val, err = db.Get(key)
   122  	if err != nil {
   123  		panic(fmt.Errorf("failed to get value: %w", err))
   124  	}
   125  	fmt.Printf("get(%q)=%q\n", key, val)
   126  
   127  	err = db.Close()
   128  	if err != nil {
   129  		t.Errorf("close: %v\n", err)
   130  	}
   131  
   132  	// Expected output:
   133  	// Hello world, LSMTree!
   134  	// Yes, absolutely! The key has been overridden.
   135  }
   136  
   137  var (
   138  	ErrKeyRequired   = errors.New("")
   139  	ErrValueRequired = errors.New("")
   140  )
   141  
   142  func TestPrintMaxSizes(t *testing.T) {
   143  
   144  	getSize := func(s string, size uint) string {
   145  		out := fmt.Sprintf("%s: %d B", s, size)
   146  		if size >= 1<<10 {
   147  			out += fmt.Sprintf(", %d KB", size/1000)
   148  		}
   149  		if size >= 1<<20 {
   150  			out += fmt.Sprintf(", %d MB", size/1000/1000)
   151  		}
   152  		if size >= 1<<30 {
   153  			out += fmt.Sprintf(", %d GB", size/1000/1000/1000)
   154  		}
   155  		if size >= 1<<40 {
   156  			out += fmt.Sprintf(", %d TB", size/1000/1000/1000/1000)
   157  		}
   158  		if size >= 1<<50 {
   159  			out += fmt.Sprintf(", %d PB", size/1000/1000/1000/1000/1000)
   160  		}
   161  		return out
   162  	}
   163  
   164  	fmt.Println(util.Sizeof(len([]byte{})))
   165  	var s0 string
   166  	fmt.Println(util.Sizeof(s0))
   167  
   168  	var s1 string = "xxxxxxxx"
   169  	fmt.Println(util.Sizeof(s1))
   170  
   171  	fmt.Printf("%s\n", getSize("MaxUint8", math.MaxUint8))
   172  	fmt.Printf("%s\n", getSize("MaxUint16", math.MaxUint16))
   173  	fmt.Printf("%s\n", getSize("MaxUint32", math.MaxUint32))
   174  	fmt.Printf("%s\n", getSize("MaxUint64", math.MaxUint64))
   175  
   176  }
   177  
   178  func TestLSMTreeLogger(t *testing.T) {
   179  
   180  	db, err := OpenLSMTree(conf)
   181  	if err != nil {
   182  		t.Fatalf("open: %v", err)
   183  	}
   184  
   185  	err = db.Put("jkldafdsa", nil)
   186  	if err != nil {
   187  		t.Fatalf("put: %v", err)
   188  	}
   189  
   190  	err = db.Close()
   191  	if err != nil {
   192  		t.Fatalf("open: %v", err)
   193  	}
   194  }
   195  
   196  func TestPutForErrors(t *testing.T) {
   197  
   198  	origPath := conf.BaseDir
   199  	tempPath := filepath.Join(conf.BaseDir, "put-for-errors")
   200  	conf.BaseDir = tempPath
   201  
   202  	defer func() {
   203  		if err := os.RemoveAll(conf.BaseDir); err != nil {
   204  			panic(fmt.Errorf("failed to remove %s: %w", conf.BaseDir, err))
   205  		}
   206  		conf.BaseDir = origPath
   207  	}()
   208  
   209  	logit("opening")
   210  	db, err := OpenLSMTree(conf)
   211  	if err != nil {
   212  		panic(fmt.Errorf("failed to open LSM tree %s: %w", conf.BaseDir, err))
   213  	}
   214  
   215  	logit("checking put empty key")
   216  	err = db.Put("", []byte("some value"))
   217  	if err != nil {
   218  		t.Errorf("empty key: %v\n", err)
   219  	}
   220  
   221  	logit("checking put nil value")
   222  	err = db.Put("some key", nil)
   223  	if err != nil {
   224  		t.Errorf("nil val: %v\n", err)
   225  	}
   226  
   227  	logit("checking put empty value")
   228  	err = db.Put("some key", []byte{})
   229  	if err != nil {
   230  		t.Errorf("empty val slice: %v\n", err)
   231  	}
   232  
   233  	logit("checking put large key (65,536 bytes)")
   234  	var largeKey [65536]byte
   235  	err = db.Put(string(largeKey[:]), []byte("some value"))
   236  	if err != nil {
   237  		t.Errorf("large key: %v\n", err)
   238  	}
   239  
   240  	logit("checking put large value (4,294,967,296 bytes)")
   241  	var largeValue [4294967296]byte
   242  	err = db.Put("some key", largeValue[:])
   243  	if err != nil {
   244  		t.Errorf("large val: %v\n", err)
   245  	}
   246  
   247  	logit("close")
   248  	err = db.Close()
   249  	if err != nil {
   250  		t.Errorf("close: %v\n", err)
   251  	}
   252  }
   253  
   254  func TestOpenAndCloseNoWrite(t *testing.T) {
   255  
   256  	db, err := OpenLSMTree(conf)
   257  	if err != nil {
   258  		t.Errorf("open: %v\n", err)
   259  	}
   260  
   261  	err = db.Close()
   262  	if err != nil {
   263  		t.Errorf("close: %v\n", err)
   264  	}
   265  
   266  	db, err = OpenLSMTree(conf)
   267  	if err != nil {
   268  		t.Errorf("open: %v\n", err)
   269  	}
   270  
   271  	err = db.Close()
   272  	if err != nil {
   273  		t.Errorf("open: %v\n", err)
   274  	}
   275  }
   276  
   277  func TestLSMTreeReadEmptyDir(t *testing.T) {
   278  	sanitize := func(base string) (string, error) {
   279  		base, err := filepath.Abs(base)
   280  		if err != nil {
   281  			return "", err
   282  		}
   283  		base = filepath.ToSlash(base)
   284  		return base, nil
   285  	}
   286  	base, err := sanitize("testing-empty-dir")
   287  	if err != nil {
   288  		t.Errorf("sanitize: %v\n", err)
   289  	}
   290  	_, err = os.ReadDir(base)
   291  	if err != nil {
   292  		t.Errorf("read dir: %T %v\n", err, err)
   293  	}
   294  }
   295  
   296  func testSSTableBehavior(t *testing.T) {
   297  
   298  	origPath := conf.BaseDir
   299  	tempPath := filepath.Join(conf.BaseDir, "sstables")
   300  	conf.BaseDir = tempPath
   301  
   302  	// open lsm tree
   303  	logit("opening lsm tree")
   304  	lsm, err := OpenLSMTree(conf)
   305  	if err != nil {
   306  		t.Errorf("open: %v\n", err)
   307  	}
   308  
   309  	// note: write data until ss-table flush is triggered
   310  	// note: close tree, and re-open and "delete" one of
   311  	// note: the records that is now on disk. check and
   312  	// note: make sure that it takes the "deleted" record
   313  	// note: over the one on disk.
   314  
   315  	// write data
   316  	logit("writing data")
   317  	ts1 := time.Now()
   318  	for i := 0; i < 1024; i++ {
   319  		err = lsm.Put(makeKey(i), makeCustomVal(i, lgVal))
   320  		if err != nil {
   321  			t.Errorf("error type: %T, put: %v\n", err, err)
   322  		}
   323  	}
   324  	ts2 := time.Now()
   325  	fmt.Println(util.FormatTime("writing entries", ts1, ts2))
   326  
   327  	// get lsm-tree status
   328  	logit("getting lsm-tree stats")
   329  	st, err := lsm.Stats()
   330  	if err != nil {
   331  		t.Errorf("stats [text]: %v\n", err)
   332  	}
   333  	dat, err := st.JSON()
   334  	if err != nil {
   335  		t.Errorf("stats [json]: %v\n", err)
   336  	}
   337  	fmt.Printf("stats:\n%s\n\njson:\n%s\n", st, dat)
   338  
   339  	// close
   340  	logit("closing lsm tree")
   341  	err = lsm.Close()
   342  	if err != nil {
   343  		t.Errorf("close: %v\n", err)
   344  	}
   345  
   346  	// open lsm tree
   347  	logit("opening lsm tree")
   348  	lsm, err = OpenLSMTree(conf)
   349  	if err != nil {
   350  		t.Errorf("open: %v\n", err)
   351  	}
   352  	util.DEBUG(">>>>>>>>> has 500: %v\n", lsm.Has(makeKey(500)))
   353  
   354  	// delete record(s)
   355  	logit("deleting record(s) [500,501,502,503 and 505]")
   356  	err = lsm.Del(makeKey(500))
   357  	if err != nil {
   358  		t.Errorf("delete: %v\n", err)
   359  	}
   360  	err = lsm.Del(makeKey(501))
   361  	if err != nil {
   362  		t.Errorf("delete: %v\n", err)
   363  	}
   364  	err = lsm.Del(makeKey(502))
   365  	if err != nil {
   366  		t.Errorf("delete: %v\n", err)
   367  	}
   368  	err = lsm.Del(makeKey(503))
   369  	if err != nil {
   370  		t.Errorf("delete: %v\n", err)
   371  	}
   372  	err = lsm.Del(makeKey(505))
   373  	if err != nil {
   374  		t.Errorf("delete: %v\n", err)
   375  	}
   376  
   377  	// close
   378  	logit("closing lsm tree")
   379  	err = lsm.Close()
   380  	if err != nil {
   381  		t.Errorf("close: %v\n", err)
   382  	}
   383  
   384  	// open lsm tree
   385  	logit("opening lsm tree")
   386  	lsm, err = OpenLSMTree(conf)
   387  	if err != nil {
   388  		t.Errorf("open: %v\n", err)
   389  	}
   390  
   391  	// checking for records
   392  	logit("checking for records [475-512]")
   393  	for i := 475; i < 512; i++ {
   394  		key := makeKey(i)
   395  		ok := lsm.Has(key)
   396  		log.Printf("[record: %d] has(%s): %v\n", i, key, ok)
   397  	}
   398  
   399  	// close
   400  	logit("closing lsm tree")
   401  	err = lsm.Close()
   402  	if err != nil {
   403  		t.Errorf("close: %v\n", err)
   404  	}
   405  
   406  	conf.BaseDir = origPath
   407  }
   408  
   409  func testLSMTreeHasAndBatches(t *testing.T) {
   410  
   411  	origPath := conf.BaseDir
   412  	tempPath := filepath.Join(conf.BaseDir, "batches")
   413  	conf.BaseDir = tempPath
   414  
   415  	logit(fmt.Sprintf("bloom fileter size: %d\n", conf.BloomFilterSize))
   416  
   417  	// open lsm tree
   418  	logit("opening lsm tree")
   419  	lsm, err := OpenLSMTree(conf)
   420  	if err != nil {
   421  		t.Errorf("open: %v\n", err)
   422  	}
   423  
   424  	count := 32500
   425  
   426  	// batch some entries
   427  	logit("batching some entries")
   428  	ts1 := time.Now()
   429  	batch := binary2.NewBatch()
   430  	for i := 0; i < count; i++ {
   431  		batch.Write(makeKey(i), makeVal(i))
   432  	}
   433  	ts2 := time.Now()
   434  	fmt.Println(util.FormatTime("batching entries", ts1, ts2))
   435  
   436  	// write batch
   437  	logit("write batch")
   438  	ts1 = time.Now()
   439  	err = lsm.PutBatch(batch)
   440  	if err != nil {
   441  		t.Errorf("put batch: %v\n", err)
   442  	}
   443  	ts2 = time.Now()
   444  	fmt.Println(util.FormatTime("writing batch", ts1, ts2))
   445  
   446  	// manual sync
   447  	logit("manual sync")
   448  	err = lsm.Sync()
   449  	if err != nil {
   450  		t.Errorf("manual sync: %v\n", err)
   451  	}
   452  
   453  	// close
   454  	logit("closing lsm tree")
   455  	err = lsm.Close()
   456  	if err != nil {
   457  		t.Errorf("close: %v\n", err)
   458  	}
   459  
   460  	// open lsm tree
   461  	logit("opening lsm tree")
   462  	lsm, err = OpenLSMTree(conf)
   463  	if err != nil {
   464  		t.Errorf("open: %v\n", err)
   465  	}
   466  
   467  	// check has
   468  	logit("checking has")
   469  	for i := range batch.Entries {
   470  		if i%500 == 0 {
   471  			logit(fmt.Sprintf("checking entry: %d", i))
   472  			// get entry
   473  			entry := batch.Entries[i]
   474  			// check for valid key
   475  			if ok := lsm.Has(string(entry.Key)); !ok {
   476  				t.Errorf("has(%q) should be true, got: %v\n", entry.Key, ok)
   477  			}
   478  			// check invalid key also
   479  			invalid := makeCustomKey("%d-poopoo", i)
   480  			if ok := lsm.Has(invalid); ok {
   481  				t.Errorf("has(%q) should be false, got: %v\n", invalid, ok)
   482  			}
   483  		} else {
   484  			// skip some entries
   485  			continue
   486  		}
   487  	}
   488  
   489  	// check get batch
   490  	logit("checkin get batch")
   491  	var keys []string
   492  	for i := range batch.Entries {
   493  		if i%500 == 0 {
   494  			keys = append(keys, string(batch.Entries[i].Key))
   495  		} else {
   496  			continue
   497  		}
   498  	}
   499  	_, err = lsm.GetBatch(keys...)
   500  	if err != nil && err != ErrNotFound {
   501  		t.Errorf("getbatch: %v\n", err)
   502  	}
   503  
   504  	// add a few more records just to ensure the segment file is working properly
   505  	err = lsm.Put("foo-1", []byte("bar-1"))
   506  	if err != nil {
   507  		t.Errorf("putting: %v\n", err)
   508  	}
   509  
   510  	err = lsm.Put("key-big-1", []byte(lgVal))
   511  	if err != nil {
   512  		t.Errorf("putting: %v\n", err)
   513  	}
   514  
   515  	err = lsm.Put("foo-2", []byte("bar-2"))
   516  	if err != nil {
   517  		t.Errorf("putting: %v\n", err)
   518  	}
   519  
   520  	err = lsm.Put("key-big-2", []byte(lgVal))
   521  	if err != nil {
   522  		t.Errorf("putting: %v\n", err)
   523  	}
   524  
   525  	err = lsm.Put("foo-3", []byte("bar-3"))
   526  	if err != nil {
   527  		t.Errorf("putting: %v\n", err)
   528  	}
   529  
   530  	v, err := lsm.Get("key-big-2")
   531  	if err != nil {
   532  		t.Errorf("getting: %v\n", err)
   533  	}
   534  	if v == nil || !bytes.Equal(v, []byte(lgVal)) {
   535  		t.Errorf("getting val, expected lgVal but got: %v\n", v)
   536  	}
   537  
   538  	// close
   539  	logit("closing lsm tree")
   540  	err = lsm.Close()
   541  	if err != nil {
   542  		t.Errorf("close: %v\n", err)
   543  	}
   544  
   545  	conf.BaseDir = origPath
   546  }
   547  
   548  func TestLSMTree_Search_vs_LinearSearch(t *testing.T) {
   549  
   550  	count := 50000
   551  
   552  	// open lsm tree
   553  	logit("opening lsm tree")
   554  	lsm, err := OpenLSMTree(conf)
   555  	if err != nil {
   556  		t.Errorf("open: %v\n", err)
   557  	}
   558  
   559  	// write Entries
   560  	logit("writing data")
   561  	for i := 0; i < count; i++ {
   562  		err := lsm.Put(makeKey(i), makeVal(i))
   563  		if err != nil {
   564  			t.Errorf("put: %v\n", err)
   565  		}
   566  	}
   567  
   568  	// close
   569  	logit("closing lsm tree")
   570  	err = lsm.Close()
   571  	if err != nil {
   572  		t.Errorf("close: %v\n", err)
   573  	}
   574  
   575  	// open lsm tree
   576  	logit("opening lsm tree")
   577  	lsm, err = OpenLSMTree(conf)
   578  	if err != nil {
   579  		t.Errorf("open: %v\n", err)
   580  	}
   581  
   582  	// reading entries linear scanner
   583  	logit("reading entries [scanner]")
   584  	err = lsm.Scan(0, func(e *binary2.Entry) bool {
   585  		if e.Key != nil && e.Value != nil {
   586  			fmt.Printf("--> %s\n", e)
   587  			return true
   588  		}
   589  		return false
   590  	})
   591  	if err != nil {
   592  		t.Errorf("scanning: %v\n", err)
   593  	}
   594  
   595  	// close
   596  	logit("closing lsm tree")
   597  	err = lsm.Close()
   598  	if err != nil {
   599  		t.Errorf("close: %v\n", err)
   600  	}
   601  
   602  	// open lsm tree
   603  	logit("opening lsm tree")
   604  	lsm, err = OpenLSMTree(conf)
   605  	if err != nil {
   606  		t.Errorf("open: %v\n", err)
   607  	}
   608  
   609  	// reading entries search
   610  	logit("reading entries [search]")
   611  	for i := 0; i < count; i += 1000 {
   612  		ts1 := time.Now()
   613  		k := makeKey(i)
   614  		v, err := lsm.Get(k)
   615  		if err != nil || v == nil {
   616  			t.Errorf("reading: %v\n", err)
   617  		}
   618  		ts2 := time.Now()
   619  		fmt.Println(util.FormatTime("reading entries [search]", ts1, ts2))
   620  		if i%1000 == 0 {
   621  			fmt.Printf("get(%q) -> %q\n", k, v)
   622  		}
   623  	}
   624  
   625  	// close
   626  	logit("closing lsm tree")
   627  	err = lsm.Close()
   628  	if err != nil {
   629  		t.Errorf("close: %v\n", err)
   630  	}
   631  
   632  	// open lsm tree
   633  	logit("opening lsm tree")
   634  	lsm, err = OpenLSMTree(conf)
   635  	if err != nil {
   636  		t.Errorf("open: %v\n", err)
   637  	}
   638  
   639  	// reading entries linear search
   640  	logit("reading entries [linear search]")
   641  	for i := 0; i < count; i += 1000 {
   642  		ts1 := time.Now()
   643  		k := makeKey(i)
   644  		v, err := lsm.GetLinear(k)
   645  		if err != nil || v == nil {
   646  			t.Errorf("reading: %v\n", err)
   647  		}
   648  		ts2 := time.Now()
   649  		fmt.Println(util.FormatTime("reading entries [linear search]", ts1, ts2))
   650  		if i%1000 == 0 {
   651  			fmt.Printf("get(%q) -> %q\n", k, v)
   652  		}
   653  	}
   654  
   655  	// close
   656  	logit("closing lsm tree")
   657  	err = lsm.Close()
   658  	if err != nil {
   659  		t.Errorf("close: %v\n", err)
   660  	}
   661  }
   662  
   663  func TestLSMTree(t *testing.T) {
   664  
   665  	// sstable tests
   666  	testSSTableBehavior(t)
   667  
   668  	// test has and batching
   669  	testLSMTreeHasAndBatches(t)
   670  
   671  	max := 100000
   672  	for i := 10; i <= max; i *= 10 {
   673  		log.Printf("running tests with count: %d\n", i)
   674  		testingLSMTreeN(i, t)
   675  		runtime.GC()
   676  		time.Sleep(3)
   677  	}
   678  	doClean := false
   679  	if doClean {
   680  		err := os.RemoveAll(conf.BaseDir)
   681  		if err != nil {
   682  			t.Errorf("remove: %s, err: %v\n", conf.BaseDir, err)
   683  		}
   684  	}
   685  }
   686  
   687  func testingLSMTreeN(count int, t *testing.T) {
   688  
   689  	strt := 0
   690  	stop := strt + count
   691  
   692  	origPath := conf.BaseDir
   693  	tempPath := filepath.Join(conf.BaseDir, strconv.Itoa(count))
   694  	conf.BaseDir = tempPath
   695  
   696  	n, err := ReadLastSequenceNumber(conf.BaseDir)
   697  	if n > 0 && err == nil {
   698  		strt = int(n)
   699  		stop = strt + count
   700  	}
   701  	util.DEBUG("start: %d, stop: %d, count: %d\n", strt, stop, count)
   702  
   703  	// open lsm tree
   704  	logit("opening lsm tree")
   705  	lsm, err := OpenLSMTree(conf)
   706  	if err != nil {
   707  		t.Errorf("open: %v\n", err)
   708  	}
   709  
   710  	// write Entries
   711  	logit("writing data")
   712  	ts1 := time.Now()
   713  	for i := strt; i < stop; i++ {
   714  		err := lsm.Put(makeKey(i), makeVal(i))
   715  		if err != nil {
   716  			t.Errorf("put: %v\n", err)
   717  		}
   718  	}
   719  	ts2 := time.Now()
   720  	fmt.Println(util.FormatTime("writing entries", ts1, ts2))
   721  
   722  	// close
   723  	logit("closing lsm tree")
   724  	err = lsm.Close()
   725  	if err != nil {
   726  		t.Errorf("close: %v\n", err)
   727  	}
   728  
   729  	// open
   730  	logit("opening lsm tree")
   731  	lsm, err = OpenLSMTree(conf)
   732  	if err != nil {
   733  		t.Errorf("open: %v\n", err)
   734  	}
   735  
   736  	doPrintAllReads := false
   737  	doPrintSomeReads := true
   738  
   739  	// read Entries
   740  	logit("reading data")
   741  	ts1 = time.Now()
   742  	var step int
   743  	if count >= 100 {
   744  		step = count / 100
   745  	} else {
   746  		step = 1
   747  	}
   748  	for i := strt; i < stop; i += step {
   749  		v, err := lsm.Get(makeKey(i))
   750  		if err != nil && err == ErrNotFound {
   751  			// skip, we don't care if it's not found
   752  			continue
   753  		}
   754  		if err != nil {
   755  			t.Errorf("get: %v\n", err)
   756  		}
   757  		if doPrintAllReads {
   758  			fmt.Printf("get(%q) -> %q\n", makeKey(i), v)
   759  		} else if doPrintSomeReads {
   760  			if i%step == 0 {
   761  				fmt.Printf("get(%q) -> %q\n", makeKey(i), v)
   762  			}
   763  		}
   764  	}
   765  	ts2 = time.Now()
   766  	fmt.Println(util.FormatTime("reading Entries", ts1, ts2))
   767  
   768  	doDelete := true
   769  	if doDelete {
   770  		// remove Entries
   771  		logit("removing data (only odds)")
   772  		ts1 = time.Now()
   773  		for i := strt; i < stop; i++ {
   774  			if i%2 != 0 {
   775  				key := makeKey(i)
   776  				err = lsm.Del(key)
   777  				if err != nil {
   778  					t.Errorf("del: %v\n", err)
   779  				}
   780  			}
   781  		}
   782  		ts2 = time.Now()
   783  		fmt.Println(util.FormatTime("removing Entries", ts1, ts2))
   784  	}
   785  
   786  	// close
   787  	logit("closing lsm tree")
   788  	err = lsm.Close()
   789  	if err != nil {
   790  		t.Errorf("close: %v\n", err)
   791  	}
   792  
   793  	// open
   794  	logit("opening lsm tree")
   795  	lsm, err = OpenLSMTree(conf)
   796  	if err != nil {
   797  		t.Errorf("open: %v\n", err)
   798  	}
   799  
   800  	// read Entries
   801  	logit("reading data")
   802  	ts1 = time.Now()
   803  	for i := strt; i < stop; i += step {
   804  		v, err := lsm.Get(makeKey(i))
   805  		if err != nil && err == ErrNotFound {
   806  			// skip, we don't care if it's not found
   807  			continue
   808  		}
   809  		if err != nil {
   810  			t.Errorf("get: %v\n", err)
   811  		}
   812  		if doPrintAllReads {
   813  			fmt.Printf("get(%q) -> %q\n", makeKey(i), v)
   814  		} else if doPrintSomeReads {
   815  			if i%step == 0 {
   816  				fmt.Printf("get(%q) -> %q\n", makeKey(i), v)
   817  			}
   818  		}
   819  	}
   820  	ts2 = time.Now()
   821  	fmt.Println(util.FormatTime("reading data", ts1, ts2))
   822  
   823  	_ = WriteLastSequenceNumber(int64(stop-1), conf.BaseDir)
   824  
   825  	//
   826  	err = lsm.sstm.CompactAllSSTables()
   827  	if err != nil {
   828  		t.Errorf("lsm.compact error: %s\n", err)
   829  	}
   830  
   831  	// close
   832  	logit("closing lsm tree")
   833  	err = lsm.Close()
   834  	if err != nil {
   835  		t.Errorf("close: %v\n", err)
   836  	}
   837  
   838  	conf.BaseDir = origPath
   839  
   840  }
   841  
   842  func TestLSMTree_Put(t *testing.T) {
   843  }
   844  
   845  func TestLSMTree_Get(t *testing.T) {
   846  
   847  }
   848  
   849  func TestLSMTree_Del(t *testing.T) {
   850  
   851  }
   852  
   853  func TestLSMTree_Close(t *testing.T) {
   854  
   855  }
   856  
   857  func WriteLastSequenceNumber(n int64, base string) error {
   858  	dat := make([]byte, binary.MaxVarintLen64)
   859  	binary.PutVarint(dat, n)
   860  	file := filepath.Join(base, "last-seq.dat")
   861  	err := os.WriteFile(file, dat, 0666)
   862  	if err != nil {
   863  		return err
   864  	}
   865  	return nil
   866  }
   867  
   868  func ReadLastSequenceNumber(base string) (int64, error) {
   869  	file := filepath.Join(base, "last-seq.dat")
   870  	dat, err := os.ReadFile(file)
   871  	if err != nil {
   872  		return -1, err
   873  	}
   874  	err = os.RemoveAll(file)
   875  	n, _ := binary.Varint(dat)
   876  	return n, nil
   877  }
   878  
   879  var smVal = `Praesent efficitur, ante eget eleifend scelerisque, neque erat malesuada neque, vel euismod 
   880  dui leo a nisl. Donec a eleifend dui. Maecenas necleo odio. In maximus convallis ligula eget sodales.`
   881  
   882  var mdVal = `Quisque bibendum tellus ac odio dictum vulputate. Sed imperdiet enim eget tortor vehicula, 
   883  nec vehicula erat lacinia. Praesent et bibendum turpis. Mauris ac blandit nulla, ac dignissim 
   884  quam. Ut ut est placerat quam suscipit sodales a quis lacus. Praesent hendrerit mattis diam et 
   885  sodales. In a augue sit amet odio iaculis tempus sed a erat. Donec quis nisi tellus. Nam hendrerit 
   886  purus ligula, id bibendum metus pulvinar sed. Nulla eu neque lobortis, porta elit quis, luctus 
   887  purus. Vestibulum et ultrices nulla. Curabitur sagittis, sem sed elementum aliquam, dui mauris 
   888  interdum libero, ullamcorper convallis urna tortor ornare metus. Integer non nibh id diam accumsan 
   889  tincidunt. Quisque sed felis aliquet, luctus dolor vitae, porta nibh. Vestibulum ac est mollis, 
   890  sodales erat et, pharetra nibh. Maecenas porta diam in elit venenatis, sed bibendum orci 
   891  feugiat. Suspendisse diam enim, dictum quis magna sed, aliquet porta turpis. Etiam scelerisque 
   892  aliquam neque, vel iaculis nibh laoreet ac. Sed placerat, arcu eu feugiat ullamcorper, massa 
   893  justo aliquet lorem, id imperdiet neque ipsum id diam. Vestibulum semper felis urna, sit amet 
   894  volutpat est porttitor nec. Phasellus lacinia volutpat orci, id eleifend ipsum semper non. 
   895  `
   896  
   897  var lgVal = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent libero turpis, aliquam quis 
   898  consequat ac, volutpat et arcu. Nullam varius, ligula eu venenatis dignissim, lectus ligula 
   899  ullamcorper odio, in rhoncus nisi nisl congue sem. In hac habitasse platea dictumst. Donec 
   900  sem est, rutrum ut libero nec, placerat vehicula neque. Nulla mollis dictum nunc, ut viverra 
   901  ex. Nam ac lacus at quam rhoncus finibus. Praesent efficitur, ante eget eleifend scelerisque, 
   902  neque erat malesuada neque, vel euismod dui leo a nisl. Donec a eleifend dui. Maecenas nec 
   903  leo odio. In maximus convallis ligula eget sodales. Nullam a mi hendrerit, finibus dolor eu, 
   904  pellentesque ligula. Proin ultricies vitae neque sit amet tempus. Sed a purus enim. Maecenas 
   905  maximus placerat risus, at commodo libero consectetur sed. Nullam pulvinar lobortis augue in 
   906  pulvinar. Aliquam erat volutpat. Vestibulum eget felis egestas, sollicitudin sem eu, venenatis 
   907  metus. Nam ac eros vel sem suscipit facilisis in ut ligula. Nulla porta eros eu arcu efficitur 
   908  molestie. Proin tristique eget quam quis ullamcorper. Integer pretium tellus non sapien euismod, 
   909  et ultrices leo placerat. Suspendisse potenti. Aenean pulvinar pretium diam, lobortis pretium 
   910  sapien congue quis. Fusce tempor, diam id commodo maximus, mi turpis rhoncus orci, ut blandit 
   911  ipsum turpis congue dolor. Aenean lobortis, turpis nec dignissim pulvinar, sem massa bibendum 
   912  lorem, ut scelerisque nibh odio sed odio. Sed sed nulla lectus. Donec vitae ipsum dolor. Donec 
   913  eu gravida lectus. In tempor ultrices malesuada. Cras sodales in lacus et volutpat. Vivamus 
   914  nibh ante, egestas vitae faucibus id, consectetur at augue. Pellentesque habitant morbi tristique 
   915  senectus et netus et malesuada fames ac turpis egestas. Pellentesque quis velit non quam convallis 
   916  molestie sit amet sit amet metus. Aenean eget sapien nisl. Lorem ipsum dolor sit amet, consectetur 
   917  adipiscing elit. Donec maximus nisi in nunc pellentesque imperdiet. Aliquam erat volutpat. 
   918  Quisque bibendum tellus ac odio dictum vulputate. Sed imperdiet enim eget tortor vehicula, nec 
   919  vehicula erat lacinia. Praesent et bibendum turpis. Mauris ac blandit nulla, ac dignissim quam. 
   920  Ut ut est placerat quam suscipit sodales a quis lacus. Praesent hendrerit mattis diam et sodales. 
   921  In a augue sit amet odio iaculis tempus sed a erat. Donec quis nisi tellus. Nam hendrerit purus 
   922  ligula, id bibendum metus pulvinar sed. Nulla eu neque lobortis, porta elit quis, luctus purus. 
   923  Vestibulum et ultrices nulla. Curabitur sagittis, sem sed elementum aliquam, dui mauris interdum 
   924  libero, ullamcorper convallis urna tortor ornare metus. Integer non nibh id diam accumsan 
   925  tincidunt. Quisque sed felis aliquet, luctus dolor vitae, porta nibh. Vestibulum ac est mollis, 
   926  sodales erat et, pharetra nibh. Maecenas porta diam in elit venenatis, sed bibendum orci 
   927  feugiat. Suspendisse diam enim, dictum quis magna sed, aliquet porta turpis. Etiam scelerisque 
   928  aliquam neque, vel iaculis nibh laoreet ac. Sed placerat, arcu eu feugiat ullamcorper, massa 
   929  justo aliquet lorem, id imperdiet neque ipsum id diam. Vestibulum semper felis urna, sit amet 
   930  volutpat est porttitor nec. Phasellus lacinia volutpat orci, id eleifend ipsum semper non. 
   931  Pellentesque quis velit non quam convallis molestie sit amet sit amet metus. Aenean eget sapien 
   932  nisl. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec maximus nisi in nunc 
   933  pellentesque imperdiet. Aliquam erat volutpat. Quisque bibendum tellus ac odio dictum vulputate. 
   934  Sed imperdiet enim eget tortor vehicula, nec vehicula erat lacinia. Praesent et bibendum turpis. 
   935  Mauris ac blandit nulla, ac dignissim quam. Ut ut est placerat quam suscipit sodales a quis 
   936  lacus. Praesent hendrerit mattis diam et sodales. In a augue sit amet odio iaculis tempus sed 
   937  a erat. Donec quis nisi tellus. Nam hendrerit purus ligula, id bibendum metus pulvinar sed. 
   938  Nulla eu neque lobortis, porta elit quis, luctus purus. Vestibulum et ultrices nulla.`