github.com/zuoyebang/bitalostable@v1.0.1-0.20240229032404-e3b99a834294/bitable_test.go (about) 1 // Copyright 2019 The Bitalostored Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package bitalostable 6 7 import ( 8 "bytes" 9 "encoding/binary" 10 "fmt" 11 "os" 12 "testing" 13 "time" 14 15 "github.com/stretchr/testify/require" 16 "golang.org/x/exp/rand" 17 ) 18 19 const ( 20 testDir = "./data" 21 22 maxLogFileSize = 128 << 20 23 memTableSize = 1 << 20 24 maxWriteBufferNumber = 3 25 26 valrandStr = "1qaz2wsx3edc4rfv5tgb6yhn7ujm8ik9ol0p" 27 ) 28 29 type BitableDB struct { 30 db *DB 31 ro *IterOptions 32 wo *WriteOptions 33 opts *Options 34 } 35 36 func testRandBytes(n int, randstr string) []byte { 37 b := make([]byte, n) 38 for i := range b { 39 b[i] = randstr[rand.Int63()%int64(len(randstr))] 40 } 41 return b 42 } 43 44 func openBitable(dir string, walDir string) (*BitableDB, error) { 45 opts := &Options{ 46 MaxManifestFileSize: maxLogFileSize, 47 MemTableSize: memTableSize, 48 MemTableStopWritesThreshold: maxWriteBufferNumber, 49 Verbose: true, 50 } 51 return openBitableByOpts(dir, walDir, opts) 52 } 53 54 func openBitableByOpts(dir string, walDir string, opts *Options) (*BitableDB, error) { 55 cache := NewCache(0) 56 opts.Cache = cache 57 if len(walDir) > 0 { 58 _, err := os.Stat(walDir) 59 if nil != err && !os.IsExist(err) { 60 err = os.MkdirAll(walDir, 0775) 61 if nil != err { 62 return nil, err 63 } 64 opts.WALDir = walDir 65 } 66 } 67 _, err := os.Stat(dir) 68 if nil != err && !os.IsExist(err) { 69 err = os.MkdirAll(dir, 0775) 70 if nil != err { 71 return nil, err 72 } 73 opts.WALDir = walDir 74 } 75 pdb, err := Open(dir, opts) 76 if err != nil { 77 return nil, err 78 } 79 cache.Unref() 80 return &BitableDB{ 81 db: pdb, 82 ro: &IterOptions{}, 83 wo: NoSync, 84 opts: opts, 85 }, nil 86 } 87 88 func TestBitable_MemIterator(t *testing.T) { 89 defer os.RemoveAll(testDir) 90 bitalostableDB, err := openBitable(testDir, "") 91 if err != nil { 92 panic(err) 93 } 94 defer func() { 95 require.NoError(t, bitalostableDB.db.Close()) 96 }() 97 98 for i := 0; i < 100; i++ { 99 newKey := []byte(fmt.Sprintf("quota:host:province:succ:total_%d", i)) 100 for j := 0; j < 100; j++ { 101 err = bitalostableDB.db.Set(newKey, []byte(fmt.Sprintf("%d", j)), bitalostableDB.wo) 102 if err != nil { 103 t.Fatal(err) 104 } 105 } 106 } 107 108 it := bitalostableDB.db.NewIter(nil) 109 defer it.Close() 110 for it.First(); it.Valid(); it.Next() { 111 //fmt.Println("iter", string(it.Key()), string(it.Value())) 112 } 113 stats := it.Stats() 114 fmt.Printf("stats: %s\n", stats.String()) 115 } 116 117 func TestBitable_MemGet(t *testing.T) { 118 defer os.RemoveAll(testDir) 119 bitalostableDB, err := openBitable(testDir, "") 120 if err != nil { 121 panic(err) 122 } 123 defer func() { 124 require.NoError(t, bitalostableDB.db.Close()) 125 }() 126 127 newKey := []byte(fmt.Sprintf("quota:host:province:succ:total_1")) 128 for j := 0; j < 100; j++ { 129 err = bitalostableDB.db.Set(newKey, []byte(fmt.Sprintf("%d", j)), bitalostableDB.wo) 130 if err != nil { 131 t.Fatal(err) 132 } 133 } 134 135 require.NoError(t, bitalostableDB.db.Flush()) 136 137 val, i, err := bitalostableDB.db.Get(newKey) 138 require.Equal(t, []byte(fmt.Sprintf("%d", 99)), val) 139 defer func() { 140 if i != nil { 141 require.NoError(t, i.Close()) 142 } 143 }() 144 145 it := i.(*Iterator) 146 stats := it.Stats() 147 fmt.Printf("stats: %s\n", stats.String()) 148 } 149 150 func TestBitable_NewFlushBatch_DisableCompact(t *testing.T) { 151 for _, isCompact := range []bool{false} { 152 fmt.Println("start-------------", isCompact) 153 func() { 154 defer os.RemoveAll(testDir) 155 os.RemoveAll(testDir) 156 157 opts := &Options{ 158 MaxManifestFileSize: maxLogFileSize, 159 MemTableSize: memTableSize, 160 MemTableStopWritesThreshold: 8, 161 L0CompactionFileThreshold: 2, 162 L0CompactionThreshold: 2, 163 L0StopWritesThreshold: 128, 164 Verbose: true, 165 } 166 bitalostableDB, err := openBitableByOpts(testDir, "", opts) 167 require.NoError(t, err) 168 defer func() { 169 require.NoError(t, bitalostableDB.db.Close()) 170 }() 171 172 inum := 1 173 jnum := 10000 174 value := testRandBytes(2048, valrandStr) 175 176 writeData := func() { 177 batchSize := 5 << 20 178 batch := bitalostableDB.db.NewFlushBatch(batchSize) 179 for i := 0; i < inum; i++ { 180 for j := 0; j < jnum; j++ { 181 newKey := []byte(fmt.Sprintf("key_%d_%d", i, j)) 182 err = batch.Set(newKey, value, bitalostableDB.wo) 183 if err != nil { 184 t.Fatal(err) 185 } 186 } 187 fmt.Println("batch.Commit") 188 err = batch.Commit(bitalostableDB.wo) 189 if err != nil { 190 t.Fatal(err) 191 } 192 require.NoError(t, batch.Close()) 193 batch = bitalostableDB.db.NewFlushBatch(batchSize) 194 } 195 require.NoError(t, batch.Close()) 196 } 197 198 readData := func() { 199 for i := 0; i < inum; i++ { 200 for j := 0; j < jnum; j++ { 201 newKey := []byte(fmt.Sprintf("key_%d_%d", i, j)) 202 val, closer, err := bitalostableDB.db.Get(newKey) 203 if err != nil { 204 t.Error("get err", err, string(newKey)) 205 } else if !bytes.Equal(value, val) { 206 t.Error("get val err", string(newKey)) 207 } 208 if closer != nil { 209 require.NoError(t, closer.Close()) 210 } 211 } 212 } 213 } 214 215 bitalostableDB.db.SetOptsDisableAutomaticCompactions(true) 216 writeData() 217 readData() 218 time.Sleep(time.Second) 219 fmt.Println(bitalostableDB.db.Metrics().String()) 220 221 bitalostableDB.db.SetOptsDisableAutomaticCompactions(false) 222 223 if isCompact { 224 require.NoError(t, bitalostableDB.db.Compact(nil, []byte("\xff"), false)) 225 } else { 226 writeData() 227 time.Sleep(time.Second) 228 fmt.Println(bitalostableDB.db.Metrics().String()) 229 } 230 231 readData() 232 }() 233 } 234 } 235 236 func TestBitable_NewBatch(t *testing.T) { 237 defer os.RemoveAll(testDir) 238 os.RemoveAll(testDir) 239 bitalostableDB, err := openBitable(testDir, "") 240 require.NoError(t, err) 241 defer func() { 242 require.NoError(t, bitalostableDB.db.Close()) 243 }() 244 245 inum := 1 246 jnum := 1000 247 248 batch := bitalostableDB.db.NewBatch() 249 value := testRandBytes(2048, valrandStr) 250 for i := 0; i < inum; i++ { 251 for j := 0; j < jnum; j++ { 252 newKey := []byte(fmt.Sprintf("key_%d_%d", i, j)) 253 err = batch.Set(newKey, value, bitalostableDB.wo) 254 if err != nil { 255 t.Fatal(err) 256 } 257 } 258 err = batch.Commit(bitalostableDB.wo) 259 if err != nil { 260 t.Fatal(err) 261 } 262 batch.Close() 263 batch = bitalostableDB.db.NewBatch() 264 } 265 batch.Close() 266 267 fmt.Println(bitalostableDB.db.Metrics().String()) 268 269 for i := 0; i < inum; i++ { 270 for j := 0; j < jnum; j++ { 271 newKey := []byte(fmt.Sprintf("key_%d_%d", i, j)) 272 val, closer, err := bitalostableDB.db.Get(newKey) 273 if err != nil { 274 t.Error("get err", err, string(newKey)) 275 } else if len(val) <= 0 { 276 t.Error("get val len err") 277 } else if !bytes.Equal(value, val) { 278 t.Error("get val err", string(newKey)) 279 } 280 if closer != nil { 281 require.NoError(t, closer.Close()) 282 } 283 } 284 } 285 } 286 287 func TestBitable_BatchSetMulti(t *testing.T) { 288 defer os.RemoveAll(testDir) 289 os.RemoveAll(testDir) 290 bitalostableDB, err := openBitable(testDir, "") 291 require.NoError(t, err) 292 293 kvList := make(map[string][]byte, 0) 294 295 for i := 0; i < 100; i++ { 296 b := bitalostableDB.db.NewBatch() 297 key := []byte(fmt.Sprintf("key_%d", i)) 298 if i%2 == 0 { 299 val := testRandBytes(100, valrandStr) 300 _ = b.Set(key, val, bitalostableDB.wo) 301 kvList[string(key)] = val 302 } else { 303 val1 := testRandBytes(100, valrandStr) 304 val2 := testRandBytes(100, valrandStr) 305 _ = b.SetMultiValue(key, val1, val2) 306 var val []byte 307 val = append(val, val1...) 308 val = append(val, val2...) 309 kvList[string(key)] = val 310 } 311 require.NoError(t, b.Commit(bitalostableDB.wo)) 312 require.NoError(t, b.Close()) 313 } 314 315 for i := 0; i < 100; i++ { 316 key := []byte(fmt.Sprintf("key_%d", i)) 317 v, vcloser, err := bitalostableDB.db.Get(key) 318 require.NoError(t, err) 319 require.Equal(t, kvList[string(key)], v) 320 require.NoError(t, vcloser.Close()) 321 } 322 323 require.NoError(t, bitalostableDB.db.Close()) 324 } 325 326 func TestBitable_Compact_CheckExpire(t *testing.T) { 327 defer os.RemoveAll(testDir) 328 os.RemoveAll(testDir) 329 330 opts := &Options{ 331 MaxManifestFileSize: maxLogFileSize, 332 MemTableSize: memTableSize, 333 MemTableStopWritesThreshold: maxWriteBufferNumber, 334 L0CompactionFileThreshold: 8, 335 L0CompactionThreshold: 8, 336 L0StopWritesThreshold: 16, 337 Verbose: true, 338 KvCheckExpireFunc: func(k, v []byte) bool { 339 if v != nil && uint8(v[0]) == 1 { 340 timestamp := binary.BigEndian.Uint64(v[1:9]) 341 if timestamp == 0 { 342 return false 343 } 344 now := uint64(time.Now().UnixMilli()) 345 return timestamp <= now 346 } 347 return false 348 }, 349 } 350 bitalostableDB, err := openBitableByOpts(testDir, "", opts) 351 require.NoError(t, err) 352 defer func() { 353 require.NoError(t, bitalostableDB.db.Close()) 354 }() 355 356 now := uint64(time.Now().UnixMilli()) 357 fmt.Println("now time", now) 358 makeValue := func(i int, valBytes []byte) []byte { 359 var val []byte 360 var ttl uint64 361 if i%5 == 0 { 362 ttl = now + 2000 363 } else { 364 ttl = now + 100000 365 } 366 val = make([]byte, len(valBytes)+9) 367 val[0] = 1 368 binary.BigEndian.PutUint64(val[1:9], ttl) 369 copy(val[9:], valBytes) 370 return val 371 } 372 373 num := 10000 374 value := testRandBytes(1024, valrandStr) 375 376 writeData := func() { 377 for j := 0; j < num; j++ { 378 newKey := []byte(fmt.Sprintf("key_%d", j)) 379 err = bitalostableDB.db.Set(newKey, makeValue(j, value), bitalostableDB.wo) 380 if err != nil { 381 t.Fatal(err) 382 } 383 } 384 } 385 386 readData := func(isDel bool) { 387 for i := 0; i < num; i++ { 388 newKey := []byte(fmt.Sprintf("key_%d", i)) 389 val, closer, err := bitalostableDB.db.Get(newKey) 390 if isDel && i%5 == 0 { 391 if err != ErrNotFound { 392 t.Fatal("find expire key", string(newKey)) 393 } 394 } else { 395 if err != nil { 396 t.Fatal("find not expire key err", string(newKey), err) 397 } else if !bytes.Equal(makeValue(i, value), val) { 398 t.Fatal("find not expire key val err", string(newKey)) 399 } 400 } 401 if closer != nil { 402 require.NoError(t, closer.Close()) 403 } 404 } 405 } 406 407 writeData() 408 readData(false) 409 fmt.Println("---------wr 1") 410 time.Sleep(2 * time.Second) 411 require.NoError(t, bitalostableDB.db.Compact(nil, []byte("\xff"), false)) 412 readData(true) 413 }