github.com/zuoyebang/bitalosdb@v1.1.1-0.20240516111551-79a8c4d8ce20/batch_test.go (about) 1 // Copyright 2021 The Bitalosdb author(hustxrb@163.com) and other contributors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package bitalosdb 16 17 import ( 18 "encoding/binary" 19 "errors" 20 "fmt" 21 "math" 22 "os" 23 "strings" 24 "testing" 25 26 "github.com/stretchr/testify/require" 27 ) 28 29 func TestBatchSetDelete(t *testing.T) { 30 defer os.RemoveAll(testDirname) 31 os.RemoveAll(testDirname) 32 33 db := openTestDB(testDirname, nil) 34 defer func() { 35 require.NoError(t, db.Close()) 36 }() 37 38 keyIndex := 0 39 val := testRandBytes(100) 40 41 for i := 0; i < 10; i++ { 42 b := db.NewBatch() 43 for j := 0; j < 100; j++ { 44 key := makeTestIntKey(keyIndex) 45 require.NoError(t, b.Set(key, val, nil)) 46 if keyIndex%10 == 0 { 47 require.NoError(t, b.Delete(key, nil)) 48 } 49 keyIndex++ 50 } 51 require.NoError(t, b.Commit(NoSync)) 52 require.NoError(t, b.Close()) 53 } 54 55 for i := 0; i < 1000; i++ { 56 key := makeTestIntKey(i) 57 if i%10 == 0 { 58 require.NoError(t, verifyGetNotFound(db, key)) 59 } else { 60 require.NoError(t, verifyGet(db, key, val)) 61 } 62 } 63 } 64 65 func TestBatchDelete(t *testing.T) { 66 defer os.RemoveAll(testDirname) 67 os.RemoveAll(testDirname) 68 69 db := openTestDB(testDirname, nil) 70 defer func() { 71 require.NoError(t, db.Close()) 72 }() 73 74 keyIndex := 0 75 val := testRandBytes(100) 76 77 for i := 0; i < 10; i++ { 78 b := db.NewBatch() 79 for j := 0; j < 100; j++ { 80 key := makeTestIntKey(keyIndex) 81 require.NoError(t, b.Set(key, val, nil)) 82 keyIndex++ 83 } 84 require.NoError(t, b.Commit(NoSync)) 85 require.NoError(t, b.Close()) 86 } 87 88 for i := 0; i < 1000; i++ { 89 key := makeTestIntKey(i) 90 require.NoError(t, verifyGet(db, key, val)) 91 } 92 93 keyIndex = 0 94 for i := 0; i < 10; i++ { 95 b := db.NewBatch() 96 for j := 0; j < 10; j++ { 97 key := makeTestIntKey(keyIndex) 98 require.NoError(t, b.Delete(key, nil)) 99 keyIndex++ 100 } 101 require.NoError(t, b.Commit(NoSync)) 102 require.NoError(t, b.Close()) 103 } 104 105 for i := 0; i < 1000; i++ { 106 key := makeTestIntKey(i) 107 if i < 100 { 108 require.NoError(t, verifyGetNotFound(db, key)) 109 } else { 110 require.NoError(t, verifyGet(db, key, val)) 111 } 112 } 113 } 114 115 func TestBatchSetMultiValue(t *testing.T) { 116 defer os.RemoveAll(testDirname) 117 os.RemoveAll(testDirname) 118 119 db := openTestDB(testDirname, nil) 120 defer func() { 121 require.NoError(t, db.Close()) 122 }() 123 124 var val []byte 125 keyIndex := 0 126 val1 := testRandBytes(100) 127 val2 := testRandBytes(10) 128 val3 := testRandBytes(10) 129 val = append(val, val1...) 130 val = append(val, val2...) 131 val = append(val, val3...) 132 133 for i := 0; i < 10; i++ { 134 b := db.NewBatch() 135 for j := 0; j < 100; j++ { 136 key := makeTestIntKey(keyIndex) 137 require.NoError(t, b.SetMultiValue(key, val1, val2, val3)) 138 keyIndex++ 139 } 140 require.NoError(t, b.Commit(NoSync)) 141 require.NoError(t, b.Close()) 142 } 143 144 for i := 0; i < 1000; i++ { 145 key := makeTestIntKey(i) 146 require.NoError(t, verifyGet(db, key, val)) 147 } 148 } 149 150 func TestBatchSetPrefixDeleteKey(t *testing.T) { 151 defer os.RemoveAll(testDirname) 152 os.RemoveAll(testDirname) 153 154 db := openTestDB(testDirname, nil) 155 defer func() { 156 require.NoError(t, db.Close()) 157 }() 158 159 b := db.NewBatch() 160 for i := 0; i < 10; i++ { 161 key := makeTestIntKey(i) 162 require.NoError(t, b.PrefixDeleteKeySet(key, nil)) 163 } 164 require.NoError(t, b.Commit(NoSync)) 165 require.NoError(t, b.Close()) 166 167 for i := 0; i < 10; i++ { 168 key := makeTestIntKey(i) 169 require.NoError(t, verifyGet(db, key, []byte(""))) 170 } 171 } 172 173 func TestBatchCommitEmpty(t *testing.T) { 174 dir := testDirname 175 defer os.RemoveAll(dir) 176 os.RemoveAll(dir) 177 db := openTestDB(testDirname, nil) 178 defer db.Close() 179 180 b := db.NewBatch() 181 require.NoError(t, b.Commit(NoSync)) 182 require.NoError(t, b.Close()) 183 } 184 185 func TestBatchBitower(t *testing.T) { 186 type testCase struct { 187 kind InternalKeyKind 188 key, value string 189 } 190 191 verifyTestCases := func(b *BatchBitower, testCases []testCase) { 192 r := b.Reader() 193 194 for _, tc := range testCases { 195 kind, k, v, ok := r.Next() 196 if !ok { 197 t.Fatalf("next returned !ok: test case = %v", tc) 198 } 199 key, value := string(k), string(v) 200 if kind != tc.kind || key != tc.key || value != tc.value { 201 t.Errorf("got (%d, %q, %q), want (%d, %q, %q)", 202 kind, key, value, tc.kind, tc.key, tc.value) 203 } 204 } 205 if len(r) != 0 { 206 t.Errorf("reader was not exhausted: remaining bytes = %q", r) 207 } 208 } 209 210 testCases := []testCase{ 211 {InternalKeyKindSet, "roses", "red"}, 212 {InternalKeyKindSet, "violets", "blue"}, 213 {InternalKeyKindDelete, "roses", ""}, 214 {InternalKeyKindSet, "", ""}, 215 {InternalKeyKindSet, "", "non-empty"}, 216 {InternalKeyKindDelete, "", ""}, 217 {InternalKeyKindSet, "grass", "green"}, 218 {InternalKeyKindSet, "grass", "greener"}, 219 {InternalKeyKindSet, "eleventy", strings.Repeat("!!11!", 100)}, 220 {InternalKeyKindDelete, "nosuchkey", ""}, 221 {InternalKeyKindSet, "binarydata", "\x00"}, 222 {InternalKeyKindSet, "binarydata", "\xff"}, 223 {InternalKeyKindLogData, "logdata", ""}, 224 {InternalKeyKindLogData, "", ""}, 225 } 226 var b BatchBitower 227 for _, tc := range testCases { 228 switch tc.kind { 229 case InternalKeyKindSet: 230 _ = b.Set([]byte(tc.key), []byte(tc.value), nil) 231 case InternalKeyKindDelete: 232 _ = b.Delete([]byte(tc.key), nil) 233 case InternalKeyKindLogData: 234 _ = b.LogData([]byte(tc.key), nil) 235 } 236 } 237 verifyTestCases(&b, testCases) 238 239 b.Reset() 240 for _, tc := range testCases { 241 key := []byte(tc.key) 242 value := []byte(tc.value) 243 switch tc.kind { 244 case InternalKeyKindSet: 245 d := b.SetDeferred(len(key), len(value)) 246 copy(d.Key, key) 247 copy(d.Value, value) 248 d.Finish() 249 case InternalKeyKindDelete: 250 d := b.DeleteDeferred(len(key)) 251 copy(d.Key, key) 252 copy(d.Value, value) 253 d.Finish() 254 case InternalKeyKindLogData: 255 _ = b.LogData([]byte(tc.key), nil) 256 } 257 } 258 verifyTestCases(&b, testCases) 259 } 260 261 func TestBatchBitowerEmpty(t *testing.T) { 262 var b BatchBitower 263 require.True(t, b.Empty()) 264 265 b.Set(nil, nil, nil) 266 require.False(t, b.Empty()) 267 b.Reset() 268 require.True(t, b.Empty()) 269 b = BatchBitower{} 270 271 b.Delete(nil, nil) 272 require.False(t, b.Empty()) 273 b.Reset() 274 require.True(t, b.Empty()) 275 b = BatchBitower{} 276 277 b.LogData(nil, nil) 278 require.False(t, b.Empty()) 279 b.Reset() 280 require.True(t, b.Empty()) 281 b = BatchBitower{} 282 283 _ = b.Reader() 284 require.True(t, b.Empty()) 285 b.Reset() 286 require.True(t, b.Empty()) 287 b = BatchBitower{} 288 289 require.Equal(t, uint64(0), b.SeqNum()) 290 require.True(t, b.Empty()) 291 b.Reset() 292 require.True(t, b.Empty()) 293 } 294 295 func TestBatchBitowerCommitEmpty(t *testing.T) { 296 dir := testDirname 297 defer os.RemoveAll(dir) 298 os.RemoveAll(dir) 299 db := openTestDB(testDirname, nil) 300 defer db.Close() 301 302 b := db.NewBatchBitower() 303 require.NoError(t, b.Commit(nil)) 304 require.NoError(t, b.Close()) 305 } 306 307 func TestBatchBitowerReset(t *testing.T) { 308 dir := testDirname 309 defer os.RemoveAll(dir) 310 os.RemoveAll(dir) 311 db := openTestDB(testDirname, nil) 312 defer db.Close() 313 key := makeTestSlotKey([]byte("test-key")) 314 value := []byte("test-value") 315 b := db.NewBatchBitower() 316 b.Set(key, value, nil) 317 b.Delete(key, nil) 318 b.setSeqNum(100) 319 b.applied = 1 320 b.commitErr = errors.New("test-error") 321 b.commit.Add(1) 322 require.Equal(t, uint32(2), b.Count()) 323 require.True(t, len(b.data) > 0) 324 require.True(t, b.SeqNum() > 0) 325 require.True(t, b.memTableSize > 0) 326 require.NotEqual(t, b.deferredOp, DeferredBatchOp{}) 327 b.Reset() 328 require.Equal(t, db, b.db) 329 require.Equal(t, uint32(0), b.applied) 330 require.Nil(t, b.commitErr) 331 require.Equal(t, uint32(0), b.Count()) 332 require.Equal(t, batchHeaderLen, len(b.data)) 333 require.Equal(t, uint64(0), b.SeqNum()) 334 require.Equal(t, uint64(0), b.memTableSize) 335 require.Equal(t, b.deferredOp, DeferredBatchOp{}) 336 var expected BatchBitower 337 expected.SetRepr(b.data) 338 expected.db = db 339 b.Set(key, value, nil) 340 require.NoError(t, db.ApplyBitower(b, nil)) 341 require.NoError(t, verifyGet(db, key, value)) 342 } 343 344 func TestBatchBitowerIncrement(t *testing.T) { 345 testCases := []uint32{ 346 0x00000000, 347 0x00000001, 348 0x00000002, 349 0x0000007f, 350 0x00000080, 351 0x000000fe, 352 0x000000ff, 353 0x00000100, 354 0x00000101, 355 0x000001ff, 356 0x00000200, 357 0x00000fff, 358 0x00001234, 359 0x0000fffe, 360 0x0000ffff, 361 0x00010000, 362 0x00010001, 363 0x000100fe, 364 0x000100ff, 365 0x00020100, 366 0x03fffffe, 367 0x03ffffff, 368 0x04000000, 369 0x04000001, 370 0x7fffffff, 371 0xfffffffe, 372 } 373 for _, tc := range testCases { 374 var buf [batchHeaderLen]byte 375 binary.LittleEndian.PutUint32(buf[8:12], tc) 376 var b BatchBitower 377 b.SetRepr(buf[:]) 378 b.count++ 379 got := binary.LittleEndian.Uint32(b.Repr()[8:12]) 380 want := tc + 1 381 if got != want { 382 t.Errorf("input=%d: got %d, want %d", tc, got, want) 383 } 384 _, count := ReadBatchBitower(b.Repr()) 385 if got != want { 386 t.Errorf("input=%d: got %d, want %d", tc, count, want) 387 } 388 } 389 } 390 391 func TestBatchBitowerMemTableSizeOverflow(t *testing.T) { 392 dir := testDirname 393 defer os.RemoveAll(dir) 394 os.RemoveAll(dir) 395 396 db := openTestDB(testDirname, nil) 397 bigValue := make([]byte, 1000) 398 b := db.NewBatchBitower() 399 400 b.memTableSize = math.MaxUint32 - 50 401 for i := 0; i < 10; i++ { 402 k := fmt.Sprintf("key-%05d", i) 403 require.NoError(t, b.Set([]byte(k), bigValue, nil)) 404 } 405 require.Greater(t, b.memTableSize, uint64(math.MaxUint32)) 406 require.NoError(t, b.Close()) 407 require.NoError(t, db.Close()) 408 } 409 410 func BenchmarkBatchBitowerSet(b *testing.B) { 411 value := make([]byte, 10) 412 for i := range value { 413 value[i] = byte(i) 414 } 415 key := make([]byte, 8) 416 batch := newBatchBitower(nil) 417 418 b.ResetTimer() 419 420 const batchSize = 1000 421 for i := 0; i < b.N; i += batchSize { 422 end := i + batchSize 423 if end > b.N { 424 end = b.N 425 } 426 427 for j := i; j < end; j++ { 428 binary.BigEndian.PutUint64(key, uint64(j)) 429 batch.Set(key, value, nil) 430 } 431 batch.Reset() 432 } 433 434 b.StopTimer() 435 } 436 437 func BenchmarkBatchBitowerSetDeferred(b *testing.B) { 438 value := make([]byte, 10) 439 for i := range value { 440 value[i] = byte(i) 441 } 442 key := make([]byte, 8) 443 batch := newBatchBitower(nil) 444 445 b.ResetTimer() 446 447 const batchSize = 1000 448 for i := 0; i < b.N; i += batchSize { 449 end := i + batchSize 450 if end > b.N { 451 end = b.N 452 } 453 454 for j := i; j < end; j++ { 455 binary.BigEndian.PutUint64(key, uint64(j)) 456 deferredOp := batch.SetDeferred(len(key), len(value)) 457 458 copy(deferredOp.Key, key) 459 copy(deferredOp.Value, value) 460 461 deferredOp.Finish() 462 } 463 batch.Reset() 464 } 465 466 b.StopTimer() 467 }