github.com/rosedblabs/rosedb/v2@v2.3.7-0.20240423093736-a89ea823e5b9/merge_test.go (about)

     1  package rosedb
     2  
     3  import (
     4  	"math/rand"
     5  	"os"
     6  	"sync"
     7  	"testing"
     8  
     9  	"github.com/rosedblabs/rosedb/v2/utils"
    10  	"github.com/stretchr/testify/assert"
    11  )
    12  
    13  func TestDB_Merge_1_Empty(t *testing.T) {
    14  	options := DefaultOptions
    15  	db, err := Open(options)
    16  	assert.Nil(t, err)
    17  	defer destroyDB(db)
    18  
    19  	err = db.Merge(false)
    20  	assert.Nil(t, err)
    21  }
    22  
    23  func TestDB_Merge_2_All_Invalid(t *testing.T) {
    24  	options := DefaultOptions
    25  	db, err := Open(options)
    26  	assert.Nil(t, err)
    27  	defer destroyDB(db)
    28  
    29  	for i := 0; i < 100000; i++ {
    30  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
    31  		assert.Nil(t, err)
    32  	}
    33  	for i := 0; i < 100000; i++ {
    34  		err := db.Delete(utils.GetTestKey(i))
    35  		assert.Nil(t, err)
    36  	}
    37  
    38  	err = db.Merge(false)
    39  	assert.Nil(t, err)
    40  
    41  	_ = db.Close()
    42  	db2, err := Open(options)
    43  	assert.Nil(t, err)
    44  	defer func() {
    45  		_ = db2.Close()
    46  	}()
    47  
    48  	stat := db2.Stat()
    49  	assert.Equal(t, 0, stat.KeysNum)
    50  }
    51  
    52  func TestDB_Merge_3_All_Valid(t *testing.T) {
    53  	options := DefaultOptions
    54  	db, err := Open(options)
    55  	assert.Nil(t, err)
    56  	defer destroyDB(db)
    57  
    58  	for i := 0; i < 100000; i++ {
    59  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
    60  		assert.Nil(t, err)
    61  	}
    62  
    63  	err = db.Merge(false)
    64  	assert.Nil(t, err)
    65  
    66  	_ = db.Close()
    67  	db2, err := Open(options)
    68  	assert.Nil(t, err)
    69  	defer func() {
    70  		_ = db2.Close()
    71  	}()
    72  
    73  	for i := 0; i < 100000; i++ {
    74  		val, err := db2.Get(utils.GetTestKey(i))
    75  		assert.Nil(t, err)
    76  		assert.NotNil(t, val)
    77  	}
    78  }
    79  
    80  func TestDB_Merge_4_Twice(t *testing.T) {
    81  	options := DefaultOptions
    82  	db, err := Open(options)
    83  	assert.Nil(t, err)
    84  	defer destroyDB(db)
    85  
    86  	for i := 0; i < 100000; i++ {
    87  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
    88  		assert.Nil(t, err)
    89  	}
    90  
    91  	err = db.Merge(false)
    92  	assert.Nil(t, err)
    93  	err = db.Merge(false)
    94  	assert.Nil(t, err)
    95  
    96  	_ = db.Close()
    97  	db2, err := Open(options)
    98  	assert.Nil(t, err)
    99  	defer func() {
   100  		_ = db2.Close()
   101  	}()
   102  
   103  	for i := 0; i < 100000; i++ {
   104  		val, err := db2.Get(utils.GetTestKey(i))
   105  		assert.Nil(t, err)
   106  		assert.NotNil(t, val)
   107  	}
   108  }
   109  
   110  func TestDB_Merge_5_Mixed(t *testing.T) {
   111  	options := DefaultOptions
   112  	db, err := Open(options)
   113  	assert.Nil(t, err)
   114  	defer destroyDB(db)
   115  
   116  	for i := 0; i < 100000; i++ {
   117  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
   118  		assert.Nil(t, err)
   119  	}
   120  	for i := 0; i < 100000; i++ {
   121  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
   122  		assert.Nil(t, err)
   123  	}
   124  	for i := 100000; i < 300000; i++ {
   125  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
   126  		assert.Nil(t, err)
   127  	}
   128  	for i := 100000; i < 200000; i++ {
   129  		err := db.Delete(utils.GetTestKey(i))
   130  		assert.Nil(t, err)
   131  	}
   132  
   133  	err = db.Merge(false)
   134  	assert.Nil(t, err)
   135  
   136  	_ = db.Close()
   137  	db2, err := Open(options)
   138  	assert.Nil(t, err)
   139  	defer func() {
   140  		_ = db2.Close()
   141  	}()
   142  	stat := db2.Stat()
   143  	assert.Equal(t, 200000, stat.KeysNum)
   144  }
   145  
   146  func TestDB_Merge_6_Appending(t *testing.T) {
   147  	options := DefaultOptions
   148  	db, err := Open(options)
   149  	assert.Nil(t, err)
   150  	defer destroyDB(db)
   151  
   152  	for i := 0; i < 100000; i++ {
   153  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
   154  		assert.Nil(t, err)
   155  	}
   156  	for i := 0; i < 100000; i++ {
   157  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
   158  		assert.Nil(t, err)
   159  	}
   160  	for i := 100000; i < 300000; i++ {
   161  		err := db.Put(utils.GetTestKey(i), utils.RandomValue(128))
   162  		assert.Nil(t, err)
   163  	}
   164  	for i := 100000; i < 200000; i++ {
   165  		err := db.Delete(utils.GetTestKey(i))
   166  		assert.Nil(t, err)
   167  	}
   168  
   169  	wg := sync.WaitGroup{}
   170  	m := sync.Map{}
   171  	wg.Add(10)
   172  	for i := 0; i < 10; i++ {
   173  		go func() {
   174  			defer wg.Done()
   175  			for i := 0; i < 10000; i++ {
   176  				key := utils.GetTestKey(rand.Int())
   177  				m.Store(string(key), struct{}{})
   178  				e := db.Put(key, utils.RandomValue(128))
   179  				assert.Nil(t, e)
   180  			}
   181  		}()
   182  	}
   183  
   184  	err = db.Merge(false)
   185  	assert.Nil(t, err)
   186  
   187  	wg.Wait()
   188  
   189  	_ = db.Close()
   190  	db2, err := Open(options)
   191  	assert.Nil(t, err)
   192  	defer func() {
   193  		_ = db2.Close()
   194  	}()
   195  	stat := db2.Stat()
   196  	var count int
   197  	m.Range(func(key, value any) bool {
   198  		count++
   199  		return true
   200  	})
   201  	assert.Equal(t, 200000+count, stat.KeysNum)
   202  }
   203  
   204  func TestDB_Multi_Open_Merge(t *testing.T) {
   205  	options := DefaultOptions
   206  	kvs := make(map[string][]byte)
   207  	for i := 0; i < 5; i++ {
   208  		db, err := Open(options)
   209  		assert.Nil(t, err)
   210  
   211  		for i := 0; i < 10000; i++ {
   212  			key := utils.GetTestKey(rand.Int())
   213  			value := utils.RandomValue(128)
   214  			kvs[string(key)] = value
   215  			err = db.Put(key, value)
   216  			assert.Nil(t, err)
   217  		}
   218  
   219  		err = db.Merge(false)
   220  		assert.Nil(t, err)
   221  		err = db.Close()
   222  		assert.Nil(t, err)
   223  	}
   224  	db, err := Open(options)
   225  	assert.Nil(t, err)
   226  	defer destroyDB(db)
   227  
   228  	for key, value := range kvs {
   229  		v, err := db.Get([]byte(key))
   230  		assert.Nil(t, err)
   231  		assert.Equal(t, value, v)
   232  	}
   233  	assert.Equal(t, len(kvs), db.index.Size())
   234  }
   235  
   236  func TestDB_Merge_ReopenAfterDone(t *testing.T) {
   237  	options := DefaultOptions
   238  	db, err := Open(options)
   239  	assert.Nil(t, err)
   240  	defer destroyDB(db)
   241  
   242  	kvs := make(map[string][]byte)
   243  	for i := 0; i < 200000; i++ {
   244  		key := utils.GetTestKey(i)
   245  		value := utils.RandomValue(128)
   246  		kvs[string(key)] = value
   247  		err := db.Put(key, value)
   248  		assert.Nil(t, err)
   249  	}
   250  
   251  	err = db.Merge(true)
   252  	assert.Nil(t, err)
   253  	_, err = os.Stat(mergeDirPath(options.DirPath))
   254  	assert.Equal(t, true, os.IsNotExist(err))
   255  
   256  	for key, value := range kvs {
   257  		v, err := db.Get([]byte(key))
   258  		assert.Nil(t, err)
   259  		assert.Equal(t, value, v)
   260  	}
   261  	assert.Equal(t, len(kvs), db.index.Size())
   262  }
   263  
   264  func TestDB_Merge_Concurrent_Put(t *testing.T) {
   265  	options := DefaultOptions
   266  	db, err := Open(options)
   267  	assert.Nil(t, err)
   268  	defer destroyDB(db)
   269  
   270  	wg := sync.WaitGroup{}
   271  	m := sync.Map{}
   272  	wg.Add(11)
   273  	for i := 0; i < 10; i++ {
   274  		go func() {
   275  			defer wg.Done()
   276  			for i := 0; i < 10000; i++ {
   277  				key := utils.GetTestKey(rand.Int())
   278  				value := utils.RandomValue(128)
   279  				m.Store(string(key), value)
   280  				e := db.Put(key, value)
   281  				assert.Nil(t, e)
   282  			}
   283  		}()
   284  	}
   285  	go func() {
   286  		defer wg.Done()
   287  		err = db.Merge(true)
   288  		assert.Nil(t, err)
   289  	}()
   290  	wg.Wait()
   291  
   292  	_, err = os.Stat(mergeDirPath(options.DirPath))
   293  	assert.Equal(t, true, os.IsNotExist(err))
   294  
   295  	var count int
   296  	m.Range(func(key, value any) bool {
   297  		v, err := db.Get([]byte(key.(string)))
   298  		assert.Nil(t, err)
   299  		assert.Equal(t, value, v)
   300  		count++
   301  		return true
   302  	})
   303  	assert.Equal(t, count, db.index.Size())
   304  
   305  }