github.com/gnc-project/gnc@v1.0.0/core/rawdb/freezer_table_test.go (about) 1 // Copyright 2019 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "bytes" 21 "fmt" 22 "math/rand" 23 "os" 24 "path/filepath" 25 "testing" 26 "time" 27 28 "github.com/gnc-project/gnc/metrics" 29 ) 30 31 func init() { 32 rand.Seed(time.Now().Unix()) 33 } 34 35 // Gets a chunk of data, filled with 'b' 36 func getChunk(size int, b int) []byte { 37 data := make([]byte, size) 38 for i := range data { 39 data[i] = byte(b) 40 } 41 return data 42 } 43 44 // TestFreezerBasics test initializing a freezertable from scratch, writing to the table, 45 // and reading it back. 46 func TestFreezerBasics(t *testing.T) { 47 t.Parallel() 48 // set cutoff at 50 bytes 49 f, err := newCustomTable(os.TempDir(), 50 fmt.Sprintf("unittest-%d", rand.Uint64()), 51 metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true) 52 if err != nil { 53 t.Fatal(err) 54 } 55 defer f.Close() 56 // Write 15 bytes 255 times, results in 85 files 57 for x := 0; x < 255; x++ { 58 data := getChunk(15, x) 59 f.Append(uint64(x), data) 60 } 61 62 //print(t, f, 0) 63 //print(t, f, 1) 64 //print(t, f, 2) 65 // 66 //db[0] = 000000000000000000000000000000 67 //db[1] = 010101010101010101010101010101 68 //db[2] = 020202020202020202020202020202 69 70 for y := 0; y < 255; y++ { 71 exp := getChunk(15, y) 72 got, err := f.Retrieve(uint64(y)) 73 if err != nil { 74 t.Fatal(err) 75 } 76 if !bytes.Equal(got, exp) { 77 t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) 78 } 79 } 80 // Check that we cannot read too far 81 _, err = f.Retrieve(uint64(255)) 82 if err != errOutOfBounds { 83 t.Fatal(err) 84 } 85 } 86 87 // TestFreezerBasicsClosing tests same as TestFreezerBasics, but also closes and reopens the freezer between 88 // every operation 89 func TestFreezerBasicsClosing(t *testing.T) { 90 t.Parallel() 91 // set cutoff at 50 bytes 92 var ( 93 fname = fmt.Sprintf("basics-close-%d", rand.Uint64()) 94 rm, wm, sg = metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 95 f *freezerTable 96 err error 97 ) 98 f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 99 if err != nil { 100 t.Fatal(err) 101 } 102 // Write 15 bytes 255 times, results in 85 files 103 for x := 0; x < 255; x++ { 104 data := getChunk(15, x) 105 f.Append(uint64(x), data) 106 f.Close() 107 f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 108 if err != nil { 109 t.Fatal(err) 110 } 111 } 112 defer f.Close() 113 114 for y := 0; y < 255; y++ { 115 exp := getChunk(15, y) 116 got, err := f.Retrieve(uint64(y)) 117 if err != nil { 118 t.Fatal(err) 119 } 120 if !bytes.Equal(got, exp) { 121 t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) 122 } 123 f.Close() 124 f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 125 if err != nil { 126 t.Fatal(err) 127 } 128 } 129 } 130 131 // TestFreezerRepairDanglingHead tests that we can recover if index entries are removed 132 func TestFreezerRepairDanglingHead(t *testing.T) { 133 t.Parallel() 134 rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 135 fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64()) 136 137 { // Fill table 138 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 139 if err != nil { 140 t.Fatal(err) 141 } 142 // Write 15 bytes 255 times 143 for x := 0; x < 255; x++ { 144 data := getChunk(15, x) 145 f.Append(uint64(x), data) 146 } 147 // The last item should be there 148 if _, err = f.Retrieve(0xfe); err != nil { 149 t.Fatal(err) 150 } 151 f.Close() 152 } 153 // open the index 154 idxFile, err := os.OpenFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s.ridx", fname)), os.O_RDWR, 0644) 155 if err != nil { 156 t.Fatalf("Failed to open index file: %v", err) 157 } 158 // Remove 4 bytes 159 stat, err := idxFile.Stat() 160 if err != nil { 161 t.Fatalf("Failed to stat index file: %v", err) 162 } 163 idxFile.Truncate(stat.Size() - 4) 164 idxFile.Close() 165 // Now open it again 166 { 167 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 168 if err != nil { 169 t.Fatal(err) 170 } 171 // The last item should be missing 172 if _, err = f.Retrieve(0xff); err == nil { 173 t.Errorf("Expected error for missing index entry") 174 } 175 // The one before should still be there 176 if _, err = f.Retrieve(0xfd); err != nil { 177 t.Fatalf("Expected no error, got %v", err) 178 } 179 } 180 } 181 182 // TestFreezerRepairDanglingHeadLarge tests that we can recover if very many index entries are removed 183 func TestFreezerRepairDanglingHeadLarge(t *testing.T) { 184 t.Parallel() 185 rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 186 fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64()) 187 188 { // Fill a table and close it 189 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 190 if err != nil { 191 t.Fatal(err) 192 } 193 // Write 15 bytes 255 times 194 for x := 0; x < 0xff; x++ { 195 data := getChunk(15, x) 196 f.Append(uint64(x), data) 197 } 198 // The last item should be there 199 if _, err = f.Retrieve(f.items - 1); err == nil { 200 if err != nil { 201 t.Fatal(err) 202 } 203 } 204 f.Close() 205 } 206 // open the index 207 idxFile, err := os.OpenFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s.ridx", fname)), os.O_RDWR, 0644) 208 if err != nil { 209 t.Fatalf("Failed to open index file: %v", err) 210 } 211 // Remove everything but the first item, and leave data unaligned 212 // 0-indexEntry, 1-indexEntry, corrupt-indexEntry 213 idxFile.Truncate(indexEntrySize + indexEntrySize + indexEntrySize/2) 214 idxFile.Close() 215 // Now open it again 216 { 217 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 218 if err != nil { 219 t.Fatal(err) 220 } 221 // The first item should be there 222 if _, err = f.Retrieve(0); err != nil { 223 t.Fatal(err) 224 } 225 // The second item should be missing 226 if _, err = f.Retrieve(1); err == nil { 227 t.Errorf("Expected error for missing index entry") 228 } 229 // We should now be able to store items again, from item = 1 230 for x := 1; x < 0xff; x++ { 231 data := getChunk(15, ^x) 232 f.Append(uint64(x), data) 233 } 234 f.Close() 235 } 236 // And if we open it, we should now be able to read all of them (new values) 237 { 238 f, _ := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 239 for y := 1; y < 255; y++ { 240 exp := getChunk(15, ^y) 241 got, err := f.Retrieve(uint64(y)) 242 if err != nil { 243 t.Fatal(err) 244 } 245 if !bytes.Equal(got, exp) { 246 t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) 247 } 248 } 249 } 250 } 251 252 // TestSnappyDetection tests that we fail to open a snappy database and vice versa 253 func TestSnappyDetection(t *testing.T) { 254 t.Parallel() 255 rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 256 fname := fmt.Sprintf("snappytest-%d", rand.Uint64()) 257 // Open with snappy 258 { 259 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 260 if err != nil { 261 t.Fatal(err) 262 } 263 // Write 15 bytes 255 times 264 for x := 0; x < 0xff; x++ { 265 data := getChunk(15, x) 266 f.Append(uint64(x), data) 267 } 268 f.Close() 269 } 270 // Open without snappy 271 { 272 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, false) 273 if err != nil { 274 t.Fatal(err) 275 } 276 if _, err = f.Retrieve(0); err == nil { 277 f.Close() 278 t.Fatalf("expected empty table") 279 } 280 } 281 282 // Open with snappy 283 { 284 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 285 if err != nil { 286 t.Fatal(err) 287 } 288 // There should be 255 items 289 if _, err = f.Retrieve(0xfe); err != nil { 290 f.Close() 291 t.Fatalf("expected no error, got %v", err) 292 } 293 } 294 295 } 296 func assertFileSize(f string, size int64) error { 297 stat, err := os.Stat(f) 298 if err != nil { 299 return err 300 } 301 if stat.Size() != size { 302 return fmt.Errorf("error, expected size %d, got %d", size, stat.Size()) 303 } 304 return nil 305 306 } 307 308 // TestFreezerRepairDanglingIndex checks that if the index has more entries than there are data, 309 // the index is repaired 310 func TestFreezerRepairDanglingIndex(t *testing.T) { 311 t.Parallel() 312 rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 313 fname := fmt.Sprintf("dangling_indextest-%d", rand.Uint64()) 314 315 { // Fill a table and close it 316 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 317 if err != nil { 318 t.Fatal(err) 319 } 320 // Write 15 bytes 9 times : 150 bytes 321 for x := 0; x < 9; x++ { 322 data := getChunk(15, x) 323 f.Append(uint64(x), data) 324 } 325 // The last item should be there 326 if _, err = f.Retrieve(f.items - 1); err != nil { 327 f.Close() 328 t.Fatal(err) 329 } 330 f.Close() 331 // File sizes should be 45, 45, 45 : items[3, 3, 3) 332 } 333 // Crop third file 334 fileToCrop := filepath.Join(os.TempDir(), fmt.Sprintf("%s.0002.rdat", fname)) 335 // Truncate third file: 45 ,45, 20 336 { 337 if err := assertFileSize(fileToCrop, 45); err != nil { 338 t.Fatal(err) 339 } 340 file, err := os.OpenFile(fileToCrop, os.O_RDWR, 0644) 341 if err != nil { 342 t.Fatal(err) 343 } 344 file.Truncate(20) 345 file.Close() 346 } 347 // Open db it again 348 // It should restore the file(s) to 349 // 45, 45, 15 350 // with 3+3+1 items 351 { 352 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 353 if err != nil { 354 t.Fatal(err) 355 } 356 if f.items != 7 { 357 f.Close() 358 t.Fatalf("expected %d items, got %d", 7, f.items) 359 } 360 if err := assertFileSize(fileToCrop, 15); err != nil { 361 t.Fatal(err) 362 } 363 } 364 } 365 366 func TestFreezerTruncate(t *testing.T) { 367 368 t.Parallel() 369 rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 370 fname := fmt.Sprintf("truncation-%d", rand.Uint64()) 371 372 { // Fill table 373 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 374 if err != nil { 375 t.Fatal(err) 376 } 377 // Write 15 bytes 30 times 378 for x := 0; x < 30; x++ { 379 data := getChunk(15, x) 380 f.Append(uint64(x), data) 381 } 382 // The last item should be there 383 if _, err = f.Retrieve(f.items - 1); err != nil { 384 t.Fatal(err) 385 } 386 f.Close() 387 } 388 // Reopen, truncate 389 { 390 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 391 if err != nil { 392 t.Fatal(err) 393 } 394 defer f.Close() 395 f.truncate(10) // 150 bytes 396 if f.items != 10 { 397 t.Fatalf("expected %d items, got %d", 10, f.items) 398 } 399 // 45, 45, 45, 15 -- bytes should be 15 400 if f.headBytes != 15 { 401 t.Fatalf("expected %d bytes, got %d", 15, f.headBytes) 402 } 403 404 } 405 406 } 407 408 // TestFreezerRepairFirstFile tests a head file with the very first item only half-written. 409 // That will rewind the index, and _should_ truncate the head file 410 func TestFreezerRepairFirstFile(t *testing.T) { 411 t.Parallel() 412 rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 413 fname := fmt.Sprintf("truncationfirst-%d", rand.Uint64()) 414 { // Fill table 415 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 416 if err != nil { 417 t.Fatal(err) 418 } 419 // Write 80 bytes, splitting out into two files 420 f.Append(0, getChunk(40, 0xFF)) 421 f.Append(1, getChunk(40, 0xEE)) 422 // The last item should be there 423 if _, err = f.Retrieve(f.items - 1); err != nil { 424 t.Fatal(err) 425 } 426 f.Close() 427 } 428 // Truncate the file in half 429 fileToCrop := filepath.Join(os.TempDir(), fmt.Sprintf("%s.0001.rdat", fname)) 430 { 431 if err := assertFileSize(fileToCrop, 40); err != nil { 432 t.Fatal(err) 433 } 434 file, err := os.OpenFile(fileToCrop, os.O_RDWR, 0644) 435 if err != nil { 436 t.Fatal(err) 437 } 438 file.Truncate(20) 439 file.Close() 440 } 441 // Reopen 442 { 443 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 444 if err != nil { 445 t.Fatal(err) 446 } 447 if f.items != 1 { 448 f.Close() 449 t.Fatalf("expected %d items, got %d", 0, f.items) 450 } 451 // Write 40 bytes 452 f.Append(1, getChunk(40, 0xDD)) 453 f.Close() 454 // Should have been truncated down to zero and then 40 written 455 if err := assertFileSize(fileToCrop, 40); err != nil { 456 t.Fatal(err) 457 } 458 } 459 } 460 461 // TestFreezerReadAndTruncate tests: 462 // - we have a table open 463 // - do some reads, so files are open in readonly 464 // - truncate so those files are 'removed' 465 // - check that we did not keep the rdonly file descriptors 466 func TestFreezerReadAndTruncate(t *testing.T) { 467 t.Parallel() 468 rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 469 fname := fmt.Sprintf("read_truncate-%d", rand.Uint64()) 470 { // Fill table 471 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 472 if err != nil { 473 t.Fatal(err) 474 } 475 // Write 15 bytes 30 times 476 for x := 0; x < 30; x++ { 477 data := getChunk(15, x) 478 f.Append(uint64(x), data) 479 } 480 // The last item should be there 481 if _, err = f.Retrieve(f.items - 1); err != nil { 482 t.Fatal(err) 483 } 484 f.Close() 485 } 486 // Reopen and read all files 487 { 488 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) 489 if err != nil { 490 t.Fatal(err) 491 } 492 if f.items != 30 { 493 f.Close() 494 t.Fatalf("expected %d items, got %d", 0, f.items) 495 } 496 for y := byte(0); y < 30; y++ { 497 f.Retrieve(uint64(y)) 498 } 499 // Now, truncate back to zero 500 f.truncate(0) 501 // Write the data again 502 for x := 0; x < 30; x++ { 503 data := getChunk(15, ^x) 504 if err := f.Append(uint64(x), data); err != nil { 505 t.Fatalf("error %v", err) 506 } 507 } 508 f.Close() 509 } 510 } 511 512 func TestOffset(t *testing.T) { 513 t.Parallel() 514 rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() 515 fname := fmt.Sprintf("offset-%d", rand.Uint64()) 516 { // Fill table 517 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true) 518 if err != nil { 519 t.Fatal(err) 520 } 521 // Write 6 x 20 bytes, splitting out into three files 522 f.Append(0, getChunk(20, 0xFF)) 523 f.Append(1, getChunk(20, 0xEE)) 524 525 f.Append(2, getChunk(20, 0xdd)) 526 f.Append(3, getChunk(20, 0xcc)) 527 528 f.Append(4, getChunk(20, 0xbb)) 529 f.Append(5, getChunk(20, 0xaa)) 530 f.printIndex() 531 f.Close() 532 } 533 // Now crop it. 534 { 535 // delete files 0 and 1 536 for i := 0; i < 2; i++ { 537 p := filepath.Join(os.TempDir(), fmt.Sprintf("%v.%04d.rdat", fname, i)) 538 if err := os.Remove(p); err != nil { 539 t.Fatal(err) 540 } 541 } 542 // Read the index file 543 p := filepath.Join(os.TempDir(), fmt.Sprintf("%v.ridx", fname)) 544 indexFile, err := os.OpenFile(p, os.O_RDWR, 0644) 545 if err != nil { 546 t.Fatal(err) 547 } 548 indexBuf := make([]byte, 7*indexEntrySize) 549 indexFile.Read(indexBuf) 550 551 // Update the index file, so that we store 552 // [ file = 2, offset = 4 ] at index zero 553 554 tailId := uint32(2) // First file is 2 555 itemOffset := uint32(4) // We have removed four items 556 zeroIndex := indexEntry{ 557 offset: tailId, 558 filenum: itemOffset, 559 } 560 buf := zeroIndex.marshallBinary() 561 // Overwrite index zero 562 copy(indexBuf, buf) 563 // Remove the four next indices by overwriting 564 copy(indexBuf[indexEntrySize:], indexBuf[indexEntrySize*5:]) 565 indexFile.WriteAt(indexBuf, 0) 566 // Need to truncate the moved index items 567 indexFile.Truncate(indexEntrySize * (1 + 2)) 568 indexFile.Close() 569 570 } 571 // Now open again 572 { 573 f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true) 574 if err != nil { 575 t.Fatal(err) 576 } 577 f.printIndex() 578 // It should allow writing item 6 579 f.Append(6, getChunk(20, 0x99)) 580 581 // It should be fine to fetch 4,5,6 582 if got, err := f.Retrieve(4); err != nil { 583 t.Fatal(err) 584 } else if exp := getChunk(20, 0xbb); !bytes.Equal(got, exp) { 585 t.Fatalf("expected %x got %x", exp, got) 586 } 587 if got, err := f.Retrieve(5); err != nil { 588 t.Fatal(err) 589 } else if exp := getChunk(20, 0xaa); !bytes.Equal(got, exp) { 590 t.Fatalf("expected %x got %x", exp, got) 591 } 592 if got, err := f.Retrieve(6); err != nil { 593 t.Fatal(err) 594 } else if exp := getChunk(20, 0x99); !bytes.Equal(got, exp) { 595 t.Fatalf("expected %x got %x", exp, got) 596 } 597 598 // It should error at 0, 1,2,3 599 for i := 0; i < 4; i++ { 600 if _, err := f.Retrieve(uint64(i)); err == nil { 601 t.Fatal("expected err") 602 } 603 } 604 } 605 } 606 607 // TODO (?) 608 // - test that if we remove several head-files, aswell as data last data-file, 609 // the index is truncated accordingly 610 // Right now, the freezer would fail on these conditions: 611 // 1. have data files d0, d1, d2, d3 612 // 2. remove d2,d3 613 // 614 // However, all 'normal' failure modes arising due to failing to sync() or save a file should be 615 // handled already, and the case described above can only (?) happen if an external process/user 616 // deletes files from the filesystem.