github.com/newbtp/btp@v0.0.0-20190709081714-e4aafa07224e/core/rawdb/freezer_table_test.go (about) 1 // Copyright 2018 The go-btpereum Authors 2 // This file is part of the go-btpereum library. 3 // 4 // The go-btpereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-btpereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-btpereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "bytes" 21 "fmt" 22 "math/rand" 23 "os" 24 "path/filepath" 25 "testing" 26 "time" 27 28 "github.com/btpereum/go-btpereum/metrics" 29 ) 30 31 func init() { 32 rand.Seed(time.Now().Unix()) 33 } 34 35 // Gets a chunk of data, filled with 'b' 36 func getChunk(size int, b int) []byte { 37 data := make([]byte, size) 38 for i := range data { 39 data[i] = byte(b) 40 } 41 return data 42 } 43 44 func print(t *testing.T, f *freezerTable, item uint64) { 45 a, err := f.Retrieve(item) 46 if err != nil { 47 t.Fatal(err) 48 } 49 fmt.Printf("db[%d] = %x\n", item, a) 50 } 51 52 // TestFreezerBasics test initializing a freezertable from scratch, writing to the table, 53 // and reading it back. 54 func TestFreezerBasics(t *testing.T) { 55 t.Parallel() 56 // set cutoff at 50 bytes 57 f, err := newCustomTable(os.TempDir(), 58 fmt.Sprintf("unittest-%d", rand.Uint64()), 59 metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter(), 50, true) 60 if err != nil { 61 t.Fatal(err) 62 } 63 defer f.Close() 64 // Write 15 bytes 255 times, results in 85 files 65 for x := 0; x < 255; x++ { 66 data := getChunk(15, x) 67 f.Append(uint64(x), data) 68 } 69 70 //print(t, f, 0) 71 //print(t, f, 1) 72 //print(t, f, 2) 73 // 74 //db[0] = 000000000000000000000000000000 75 //db[1] = 010101010101010101010101010101 76 //db[2] = 020202020202020202020202020202 77 78 for y := 0; y < 255; y++ { 79 exp := getChunk(15, y) 80 got, err := f.Retrieve(uint64(y)) 81 if err != nil { 82 t.Fatal(err) 83 } 84 if !bytes.Equal(got, exp) { 85 t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) 86 } 87 } 88 // Check that we cannot read too far 89 _, err = f.Retrieve(uint64(255)) 90 if err != errOutOfBounds { 91 t.Fatal(err) 92 } 93 } 94 95 // TestFreezerBasicsClosing tests same as TestFreezerBasics, but also closes and reopens the freezer between 96 // every operation 97 func TestFreezerBasicsClosing(t *testing.T) { 98 t.Parallel() 99 // set cutoff at 50 bytes 100 var ( 101 fname = fmt.Sprintf("basics-close-%d", rand.Uint64()) 102 rm, wm, sc = metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 103 f *freezerTable 104 err error 105 ) 106 f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 107 if err != nil { 108 t.Fatal(err) 109 } 110 // Write 15 bytes 255 times, results in 85 files 111 for x := 0; x < 255; x++ { 112 data := getChunk(15, x) 113 f.Append(uint64(x), data) 114 f.Close() 115 f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 116 } 117 defer f.Close() 118 119 for y := 0; y < 255; y++ { 120 exp := getChunk(15, y) 121 got, err := f.Retrieve(uint64(y)) 122 if err != nil { 123 t.Fatal(err) 124 } 125 if !bytes.Equal(got, exp) { 126 t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) 127 } 128 f.Close() 129 f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 130 if err != nil { 131 t.Fatal(err) 132 } 133 } 134 } 135 136 // TestFreezerRepairDanglingHead tests that we can recover if index entries are removed 137 func TestFreezerRepairDanglingHead(t *testing.T) { 138 t.Parallel() 139 rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 140 fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64()) 141 142 { // Fill table 143 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 144 if err != nil { 145 t.Fatal(err) 146 } 147 // Write 15 bytes 255 times 148 for x := 0; x < 255; x++ { 149 data := getChunk(15, x) 150 f.Append(uint64(x), data) 151 } 152 // The last item should be there 153 if _, err = f.Retrieve(0xfe); err != nil { 154 t.Fatal(err) 155 } 156 f.Close() 157 } 158 // open the index 159 idxFile, err := os.OpenFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s.ridx", fname)), os.O_RDWR, 0644) 160 if err != nil { 161 t.Fatalf("Failed to open index file: %v", err) 162 } 163 // Remove 4 bytes 164 stat, err := idxFile.Stat() 165 if err != nil { 166 t.Fatalf("Failed to stat index file: %v", err) 167 } 168 idxFile.Truncate(stat.Size() - 4) 169 idxFile.Close() 170 // Now open it again 171 { 172 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 173 // The last item should be missing 174 if _, err = f.Retrieve(0xff); err == nil { 175 t.Errorf("Expected error for missing index entry") 176 } 177 // The one before should still be there 178 if _, err = f.Retrieve(0xfd); err != nil { 179 t.Fatalf("Expected no error, got %v", err) 180 } 181 } 182 } 183 184 // TestFreezerRepairDanglingHeadLarge tests that we can recover if very many index entries are removed 185 func TestFreezerRepairDanglingHeadLarge(t *testing.T) { 186 t.Parallel() 187 rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 188 fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64()) 189 190 { // Fill a table and close it 191 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 192 if err != nil { 193 t.Fatal(err) 194 } 195 // Write 15 bytes 255 times 196 for x := 0; x < 0xff; x++ { 197 data := getChunk(15, x) 198 f.Append(uint64(x), data) 199 } 200 // The last item should be there 201 if _, err = f.Retrieve(f.items - 1); err == nil { 202 if err != nil { 203 t.Fatal(err) 204 } 205 } 206 f.Close() 207 } 208 // open the index 209 idxFile, err := os.OpenFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s.ridx", fname)), os.O_RDWR, 0644) 210 if err != nil { 211 t.Fatalf("Failed to open index file: %v", err) 212 } 213 // Remove everything but the first item, and leave data unaligned 214 // 0-indexEntry, 1-indexEntry, corrupt-indexEntry 215 idxFile.Truncate(indexEntrySize + indexEntrySize + indexEntrySize/2) 216 idxFile.Close() 217 // Now open it again 218 { 219 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 220 // The first item should be there 221 if _, err = f.Retrieve(0); err != nil { 222 t.Fatal(err) 223 } 224 // The second item should be missing 225 if _, err = f.Retrieve(1); err == nil { 226 t.Errorf("Expected error for missing index entry") 227 } 228 // We should now be able to store items again, from item = 1 229 for x := 1; x < 0xff; x++ { 230 data := getChunk(15, ^x) 231 f.Append(uint64(x), data) 232 } 233 f.Close() 234 } 235 // And if we open it, we should now be able to read all of them (new values) 236 { 237 f, _ := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 238 for y := 1; y < 255; y++ { 239 exp := getChunk(15, ^y) 240 got, err := f.Retrieve(uint64(y)) 241 if err != nil { 242 t.Fatal(err) 243 } 244 if !bytes.Equal(got, exp) { 245 t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) 246 } 247 } 248 } 249 } 250 251 // TestSnappyDetection tests that we fail to open a snappy database and vice versa 252 func TestSnappyDetection(t *testing.T) { 253 t.Parallel() 254 rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 255 fname := fmt.Sprintf("snappytest-%d", rand.Uint64()) 256 // Open with snappy 257 { 258 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 259 if err != nil { 260 t.Fatal(err) 261 } 262 // Write 15 bytes 255 times 263 for x := 0; x < 0xff; x++ { 264 data := getChunk(15, x) 265 f.Append(uint64(x), data) 266 } 267 f.Close() 268 } 269 // Open without snappy 270 { 271 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, false) 272 if _, err = f.Retrieve(0); err == nil { 273 f.Close() 274 t.Fatalf("expected empty table") 275 } 276 } 277 278 // Open with snappy 279 { 280 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 281 // There should be 255 items 282 if _, err = f.Retrieve(0xfe); err != nil { 283 f.Close() 284 t.Fatalf("expected no error, got %v", err) 285 } 286 } 287 288 } 289 func assertFileSize(f string, size int64) error { 290 stat, err := os.Stat(f) 291 if err != nil { 292 return err 293 } 294 if stat.Size() != size { 295 return fmt.Errorf("error, expected size %d, got %d", size, stat.Size()) 296 } 297 return nil 298 299 } 300 301 // TestFreezerRepairDanglingIndex checks that if the index has more entries than there are data, 302 // the index is repaired 303 func TestFreezerRepairDanglingIndex(t *testing.T) { 304 t.Parallel() 305 rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 306 fname := fmt.Sprintf("dangling_indextest-%d", rand.Uint64()) 307 308 { // Fill a table and close it 309 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 310 if err != nil { 311 t.Fatal(err) 312 } 313 // Write 15 bytes 9 times : 150 bytes 314 for x := 0; x < 9; x++ { 315 data := getChunk(15, x) 316 f.Append(uint64(x), data) 317 } 318 // The last item should be there 319 if _, err = f.Retrieve(f.items - 1); err != nil { 320 f.Close() 321 t.Fatal(err) 322 } 323 f.Close() 324 // File sizes should be 45, 45, 45 : items[3, 3, 3) 325 } 326 // Crop third file 327 fileToCrop := filepath.Join(os.TempDir(), fmt.Sprintf("%s.0002.rdat", fname)) 328 // Truncate third file: 45 ,45, 20 329 { 330 if err := assertFileSize(fileToCrop, 45); err != nil { 331 t.Fatal(err) 332 } 333 file, err := os.OpenFile(fileToCrop, os.O_RDWR, 0644) 334 if err != nil { 335 t.Fatal(err) 336 } 337 file.Truncate(20) 338 file.Close() 339 } 340 // Open db it again 341 // It should restore the file(s) to 342 // 45, 45, 15 343 // with 3+3+1 items 344 { 345 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 346 if err != nil { 347 t.Fatal(err) 348 } 349 if f.items != 7 { 350 f.Close() 351 t.Fatalf("expected %d items, got %d", 7, f.items) 352 } 353 if err := assertFileSize(fileToCrop, 15); err != nil { 354 t.Fatal(err) 355 } 356 } 357 } 358 359 func TestFreezerTruncate(t *testing.T) { 360 361 t.Parallel() 362 rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 363 fname := fmt.Sprintf("truncation-%d", rand.Uint64()) 364 365 { // Fill table 366 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 367 if err != nil { 368 t.Fatal(err) 369 } 370 // Write 15 bytes 30 times 371 for x := 0; x < 30; x++ { 372 data := getChunk(15, x) 373 f.Append(uint64(x), data) 374 } 375 // The last item should be there 376 if _, err = f.Retrieve(f.items - 1); err != nil { 377 t.Fatal(err) 378 } 379 f.Close() 380 } 381 // Reopen, truncate 382 { 383 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 384 if err != nil { 385 t.Fatal(err) 386 } 387 defer f.Close() 388 f.truncate(10) // 150 bytes 389 if f.items != 10 { 390 t.Fatalf("expected %d items, got %d", 10, f.items) 391 } 392 // 45, 45, 45, 15 -- bytes should be 15 393 if f.headBytes != 15 { 394 t.Fatalf("expected %d bytes, got %d", 15, f.headBytes) 395 } 396 397 } 398 399 } 400 401 // TestFreezerRepairFirstFile tests a head file with the very first item only half-written. 402 // That will rewind the index, and _should_ truncate the head file 403 func TestFreezerRepairFirstFile(t *testing.T) { 404 t.Parallel() 405 rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 406 fname := fmt.Sprintf("truncationfirst-%d", rand.Uint64()) 407 { // Fill table 408 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 409 if err != nil { 410 t.Fatal(err) 411 } 412 // Write 80 bytes, splitting out into two files 413 f.Append(0, getChunk(40, 0xFF)) 414 f.Append(1, getChunk(40, 0xEE)) 415 // The last item should be there 416 if _, err = f.Retrieve(f.items - 1); err != nil { 417 t.Fatal(err) 418 } 419 f.Close() 420 } 421 // Truncate the file in half 422 fileToCrop := filepath.Join(os.TempDir(), fmt.Sprintf("%s.0001.rdat", fname)) 423 { 424 if err := assertFileSize(fileToCrop, 40); err != nil { 425 t.Fatal(err) 426 } 427 file, err := os.OpenFile(fileToCrop, os.O_RDWR, 0644) 428 if err != nil { 429 t.Fatal(err) 430 } 431 file.Truncate(20) 432 file.Close() 433 } 434 // Reopen 435 { 436 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 437 if err != nil { 438 t.Fatal(err) 439 } 440 if f.items != 1 { 441 f.Close() 442 t.Fatalf("expected %d items, got %d", 0, f.items) 443 } 444 // Write 40 bytes 445 f.Append(1, getChunk(40, 0xDD)) 446 f.Close() 447 // Should have been truncated down to zero and then 40 written 448 if err := assertFileSize(fileToCrop, 40); err != nil { 449 t.Fatal(err) 450 } 451 } 452 } 453 454 // TestFreezerReadAndTruncate tests: 455 // - we have a table open 456 // - do some reads, so files are open in readonly 457 // - truncate so those files are 'removed' 458 // - check that we did not keep the rdonly file descriptors 459 func TestFreezerReadAndTruncate(t *testing.T) { 460 t.Parallel() 461 rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 462 fname := fmt.Sprintf("read_truncate-%d", rand.Uint64()) 463 { // Fill table 464 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 465 if err != nil { 466 t.Fatal(err) 467 } 468 // Write 15 bytes 30 times 469 for x := 0; x < 30; x++ { 470 data := getChunk(15, x) 471 f.Append(uint64(x), data) 472 } 473 // The last item should be there 474 if _, err = f.Retrieve(f.items - 1); err != nil { 475 t.Fatal(err) 476 } 477 f.Close() 478 } 479 // Reopen and read all files 480 { 481 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true) 482 if err != nil { 483 t.Fatal(err) 484 } 485 if f.items != 30 { 486 f.Close() 487 t.Fatalf("expected %d items, got %d", 0, f.items) 488 } 489 for y := byte(0); y < 30; y++ { 490 f.Retrieve(uint64(y)) 491 } 492 // Now, truncate back to zero 493 f.truncate(0) 494 // Write the data again 495 for x := 0; x < 30; x++ { 496 data := getChunk(15, ^x) 497 if err := f.Append(uint64(x), data); err != nil { 498 t.Fatalf("error %v", err) 499 } 500 } 501 f.Close() 502 } 503 } 504 505 func TestOffset(t *testing.T) { 506 t.Parallel() 507 rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter() 508 fname := fmt.Sprintf("offset-%d", rand.Uint64()) 509 { // Fill table 510 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 40, true) 511 if err != nil { 512 t.Fatal(err) 513 } 514 // Write 6 x 20 bytes, splitting out into three files 515 f.Append(0, getChunk(20, 0xFF)) 516 f.Append(1, getChunk(20, 0xEE)) 517 518 f.Append(2, getChunk(20, 0xdd)) 519 f.Append(3, getChunk(20, 0xcc)) 520 521 f.Append(4, getChunk(20, 0xbb)) 522 f.Append(5, getChunk(20, 0xaa)) 523 f.printIndex() 524 f.Close() 525 } 526 // Now crop it. 527 { 528 // delete files 0 and 1 529 for i := 0; i < 2; i++ { 530 p := filepath.Join(os.TempDir(), fmt.Sprintf("%v.%04d.rdat", fname, i)) 531 if err := os.Remove(p); err != nil { 532 t.Fatal(err) 533 } 534 } 535 // Read the index file 536 p := filepath.Join(os.TempDir(), fmt.Sprintf("%v.ridx", fname)) 537 indexFile, err := os.OpenFile(p, os.O_RDWR, 0644) 538 if err != nil { 539 t.Fatal(err) 540 } 541 indexBuf := make([]byte, 7*indexEntrySize) 542 indexFile.Read(indexBuf) 543 544 // Update the index file, so that we store 545 // [ file = 2, offset = 4 ] at index zero 546 547 tailId := uint32(2) // First file is 2 548 itemOffset := uint32(4) // We have removed four items 549 zeroIndex := indexEntry{ 550 offset: tailId, 551 filenum: itemOffset, 552 } 553 buf := zeroIndex.marshallBinary() 554 // Overwrite index zero 555 copy(indexBuf, buf) 556 // Remove the four next indices by overwriting 557 copy(indexBuf[indexEntrySize:], indexBuf[indexEntrySize*5:]) 558 indexFile.WriteAt(indexBuf, 0) 559 // Need to truncate the moved index items 560 indexFile.Truncate(indexEntrySize * (1 + 2)) 561 indexFile.Close() 562 563 } 564 // Now open again 565 { 566 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 40, true) 567 if err != nil { 568 t.Fatal(err) 569 } 570 f.printIndex() 571 // It should allow writing item 6 572 f.Append(6, getChunk(20, 0x99)) 573 574 // It should be fine to fetch 4,5,6 575 if got, err := f.Retrieve(4); err != nil { 576 t.Fatal(err) 577 } else if exp := getChunk(20, 0xbb); !bytes.Equal(got, exp) { 578 t.Fatalf("expected %x got %x", exp, got) 579 } 580 if got, err := f.Retrieve(5); err != nil { 581 t.Fatal(err) 582 } else if exp := getChunk(20, 0xaa); !bytes.Equal(got, exp) { 583 t.Fatalf("expected %x got %x", exp, got) 584 } 585 if got, err := f.Retrieve(6); err != nil { 586 t.Fatal(err) 587 } else if exp := getChunk(20, 0x99); !bytes.Equal(got, exp) { 588 t.Fatalf("expected %x got %x", exp, got) 589 } 590 591 // It should error at 0, 1,2,3 592 for i := 0; i < 4; i++ { 593 if _, err := f.Retrieve(uint64(i)); err == nil { 594 t.Fatal("expected err") 595 } 596 } 597 } 598 } 599 600 // TODO (?) 601 // - test that if we remove several head-files, aswell as data last data-file, 602 // the index is truncated accordingly 603 // Right now, the freezer would fail on these conditions: 604 // 1. have data files d0, d1, d2, d3 605 // 2. remove d2,d3 606 // 607 // However, all 'normal' failure modes arising due to failing to sync() or save a file should be 608 // handled already, and the case described above can only (?) happen if an external process/user 609 // deletes files from the filesystem.