github.com/EgonCoin/EgonChain@v1.10.16/core/rawdb/freezer_table.go (about) 1 // Copyright 2019 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "bytes" 21 "encoding/binary" 22 "errors" 23 "fmt" 24 "io" 25 "os" 26 "path/filepath" 27 "sync" 28 "sync/atomic" 29 30 "github.com/EgonCoin/EgonChain/common" 31 "github.com/EgonCoin/EgonChain/log" 32 "github.com/EgonCoin/EgonChain/metrics" 33 "github.com/golang/snappy" 34 ) 35 36 var ( 37 // errClosed is returned if an operation attempts to read from or write to the 38 // freezer table after it has already been closed. 39 errClosed = errors.New("closed") 40 41 // errOutOfBounds is returned if the item requested is not contained within the 42 // freezer table. 43 errOutOfBounds = errors.New("out of bounds") 44 45 // errNotSupported is returned if the database doesn't support the required operation. 46 errNotSupported = errors.New("this operation is not supported") 47 ) 48 49 // indexEntry contains the number/id of the file that the data resides in, aswell as the 50 // offset within the file to the end of the data 51 // In serialized form, the filenum is stored as uint16. 52 type indexEntry struct { 53 filenum uint32 // stored as uint16 ( 2 bytes) 54 offset uint32 // stored as uint32 ( 4 bytes) 55 } 56 57 const indexEntrySize = 6 58 59 // unmarshalBinary deserializes binary b into the rawIndex entry. 60 func (i *indexEntry) unmarshalBinary(b []byte) error { 61 i.filenum = uint32(binary.BigEndian.Uint16(b[:2])) 62 i.offset = binary.BigEndian.Uint32(b[2:6]) 63 return nil 64 } 65 66 // append adds the encoded entry to the end of b. 67 func (i *indexEntry) append(b []byte) []byte { 68 offset := len(b) 69 out := append(b, make([]byte, indexEntrySize)...) 70 binary.BigEndian.PutUint16(out[offset:], uint16(i.filenum)) 71 binary.BigEndian.PutUint32(out[offset+2:], i.offset) 72 return out 73 } 74 75 // bounds returns the start- and end- offsets, and the file number of where to 76 // read there data item marked by the two index entries. The two entries are 77 // assumed to be sequential. 78 func (start *indexEntry) bounds(end *indexEntry) (startOffset, endOffset, fileId uint32) { 79 if start.filenum != end.filenum { 80 // If a piece of data 'crosses' a data-file, 81 // it's actually in one piece on the second data-file. 82 // We return a zero-indexEntry for the second file as start 83 return 0, end.offset, end.filenum 84 } 85 return start.offset, end.offset, end.filenum 86 } 87 88 // freezerTable represents a single chained data table within the freezer (e.g. blocks). 89 // It consists of a data file (snappy encoded arbitrary data blobs) and an indexEntry 90 // file (uncompressed 64 bit indices into the data file). 91 type freezerTable struct { 92 // WARNING: The `items` field is accessed atomically. On 32 bit platforms, only 93 // 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned, 94 // so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG). 95 items uint64 // Number of items stored in the table (including items removed from tail) 96 97 noCompression bool // if true, disables snappy compression. Note: does not work retroactively 98 readonly bool 99 maxFileSize uint32 // Max file size for data-files 100 name string 101 path string 102 103 head *os.File // File descriptor for the data head of the table 104 files map[uint32]*os.File // open files 105 headId uint32 // number of the currently active head file 106 tailId uint32 // number of the earliest file 107 index *os.File // File descriptor for the indexEntry file of the table 108 109 // In the case that old items are deleted (from the tail), we use itemOffset 110 // to count how many historic items have gone missing. 111 itemOffset uint32 // Offset (number of discarded items) 112 113 headBytes int64 // Number of bytes written to the head file 114 readMeter metrics.Meter // Meter for measuring the effective amount of data read 115 writeMeter metrics.Meter // Meter for measuring the effective amount of data written 116 sizeGauge metrics.Gauge // Gauge for tracking the combined size of all freezer tables 117 118 logger log.Logger // Logger with database path and table name ambedded 119 lock sync.RWMutex // Mutex protecting the data file descriptors 120 } 121 122 // NewFreezerTable opens the given path as a freezer table. 123 func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { 124 return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly) 125 } 126 127 // openFreezerFileForAppend opens a freezer table file and seeks to the end 128 func openFreezerFileForAppend(filename string) (*os.File, error) { 129 // Open the file without the O_APPEND flag 130 // because it has differing behaviour during Truncate operations 131 // on different OS's 132 file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) 133 if err != nil { 134 return nil, err 135 } 136 // Seek to end for append 137 if _, err = file.Seek(0, io.SeekEnd); err != nil { 138 return nil, err 139 } 140 return file, nil 141 } 142 143 // openFreezerFileForReadOnly opens a freezer table file for read only access 144 func openFreezerFileForReadOnly(filename string) (*os.File, error) { 145 return os.OpenFile(filename, os.O_RDONLY, 0644) 146 } 147 148 // openFreezerFileTruncated opens a freezer table making sure it is truncated 149 func openFreezerFileTruncated(filename string) (*os.File, error) { 150 return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) 151 } 152 153 // truncateFreezerFile resizes a freezer table file and seeks to the end 154 func truncateFreezerFile(file *os.File, size int64) error { 155 if err := file.Truncate(size); err != nil { 156 return err 157 } 158 // Seek to end for append 159 if _, err := file.Seek(0, io.SeekEnd); err != nil { 160 return err 161 } 162 return nil 163 } 164 165 // newTable opens a freezer table, creating the data and index files if they are 166 // non existent. Both files are truncated to the shortest common length to ensure 167 // they don't go out of sync. 168 func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression, readonly bool) (*freezerTable, error) { 169 // Ensure the containing directory exists and open the indexEntry file 170 if err := os.MkdirAll(path, 0755); err != nil { 171 return nil, err 172 } 173 var idxName string 174 if noCompression { 175 // Raw idx 176 idxName = fmt.Sprintf("%s.ridx", name) 177 } else { 178 // Compressed idx 179 idxName = fmt.Sprintf("%s.cidx", name) 180 } 181 var ( 182 err error 183 offsets *os.File 184 ) 185 if readonly { 186 // Will fail if table doesn't exist 187 offsets, err = openFreezerFileForReadOnly(filepath.Join(path, idxName)) 188 } else { 189 offsets, err = openFreezerFileForAppend(filepath.Join(path, idxName)) 190 } 191 if err != nil { 192 return nil, err 193 } 194 // Create the table and repair any past inconsistency 195 tab := &freezerTable{ 196 index: offsets, 197 files: make(map[uint32]*os.File), 198 readMeter: readMeter, 199 writeMeter: writeMeter, 200 sizeGauge: sizeGauge, 201 name: name, 202 path: path, 203 logger: log.New("database", path, "table", name), 204 noCompression: noCompression, 205 readonly: readonly, 206 maxFileSize: maxFilesize, 207 } 208 if err := tab.repair(); err != nil { 209 tab.Close() 210 return nil, err 211 } 212 // Initialize the starting size counter 213 size, err := tab.sizeNolock() 214 if err != nil { 215 tab.Close() 216 return nil, err 217 } 218 tab.sizeGauge.Inc(int64(size)) 219 220 return tab, nil 221 } 222 223 // repair cross checks the head and the index file and truncates them to 224 // be in sync with each other after a potential crash / data loss. 225 func (t *freezerTable) repair() error { 226 // Create a temporary offset buffer to init files with and read indexEntry into 227 buffer := make([]byte, indexEntrySize) 228 229 // If we've just created the files, initialize the index with the 0 indexEntry 230 stat, err := t.index.Stat() 231 if err != nil { 232 return err 233 } 234 if stat.Size() == 0 { 235 if _, err := t.index.Write(buffer); err != nil { 236 return err 237 } 238 } 239 // Ensure the index is a multiple of indexEntrySize bytes 240 if overflow := stat.Size() % indexEntrySize; overflow != 0 { 241 truncateFreezerFile(t.index, stat.Size()-overflow) // New file can't trigger this path 242 } 243 // Retrieve the file sizes and prepare for truncation 244 if stat, err = t.index.Stat(); err != nil { 245 return err 246 } 247 offsetsSize := stat.Size() 248 249 // Open the head file 250 var ( 251 firstIndex indexEntry 252 lastIndex indexEntry 253 contentSize int64 254 contentExp int64 255 ) 256 // Read index zero, determine what file is the earliest 257 // and what item offset to use 258 t.index.ReadAt(buffer, 0) 259 firstIndex.unmarshalBinary(buffer) 260 261 t.tailId = firstIndex.filenum 262 t.itemOffset = firstIndex.offset 263 264 t.index.ReadAt(buffer, offsetsSize-indexEntrySize) 265 lastIndex.unmarshalBinary(buffer) 266 if t.readonly { 267 t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForReadOnly) 268 } else { 269 t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForAppend) 270 } 271 if err != nil { 272 return err 273 } 274 if stat, err = t.head.Stat(); err != nil { 275 return err 276 } 277 contentSize = stat.Size() 278 279 // Keep truncating both files until they come in sync 280 contentExp = int64(lastIndex.offset) 281 282 for contentExp != contentSize { 283 // Truncate the head file to the last offset pointer 284 if contentExp < contentSize { 285 t.logger.Warn("Truncating dangling head", "indexed", common.StorageSize(contentExp), "stored", common.StorageSize(contentSize)) 286 if err := truncateFreezerFile(t.head, contentExp); err != nil { 287 return err 288 } 289 contentSize = contentExp 290 } 291 // Truncate the index to point within the head file 292 if contentExp > contentSize { 293 t.logger.Warn("Truncating dangling indexes", "indexed", common.StorageSize(contentExp), "stored", common.StorageSize(contentSize)) 294 if err := truncateFreezerFile(t.index, offsetsSize-indexEntrySize); err != nil { 295 return err 296 } 297 offsetsSize -= indexEntrySize 298 t.index.ReadAt(buffer, offsetsSize-indexEntrySize) 299 var newLastIndex indexEntry 300 newLastIndex.unmarshalBinary(buffer) 301 // We might have slipped back into an earlier head-file here 302 if newLastIndex.filenum != lastIndex.filenum { 303 // Release earlier opened file 304 t.releaseFile(lastIndex.filenum) 305 if t.head, err = t.openFile(newLastIndex.filenum, openFreezerFileForAppend); err != nil { 306 return err 307 } 308 if stat, err = t.head.Stat(); err != nil { 309 // TODO, anything more we can do here? 310 // A data file has gone missing... 311 return err 312 } 313 contentSize = stat.Size() 314 } 315 lastIndex = newLastIndex 316 contentExp = int64(lastIndex.offset) 317 } 318 } 319 // Sync() fails for read-only files on windows. 320 if !t.readonly { 321 // Ensure all reparation changes have been written to disk 322 if err := t.index.Sync(); err != nil { 323 return err 324 } 325 if err := t.head.Sync(); err != nil { 326 return err 327 } 328 } 329 // Update the item and byte counters and return 330 t.items = uint64(t.itemOffset) + uint64(offsetsSize/indexEntrySize-1) // last indexEntry points to the end of the data file 331 t.headBytes = contentSize 332 t.headId = lastIndex.filenum 333 334 // Close opened files and preopen all files 335 if err := t.preopen(); err != nil { 336 return err 337 } 338 t.logger.Debug("Chain freezer table opened", "items", t.items, "size", common.StorageSize(t.headBytes)) 339 return nil 340 } 341 342 // preopen opens all files that the freezer will need. This method should be called from an init-context, 343 // since it assumes that it doesn't have to bother with locking 344 // The rationale for doing preopen is to not have to do it from within Retrieve, thus not needing to ever 345 // obtain a write-lock within Retrieve. 346 func (t *freezerTable) preopen() (err error) { 347 // The repair might have already opened (some) files 348 t.releaseFilesAfter(0, false) 349 // Open all except head in RDONLY 350 for i := t.tailId; i < t.headId; i++ { 351 if _, err = t.openFile(i, openFreezerFileForReadOnly); err != nil { 352 return err 353 } 354 } 355 if t.readonly { 356 t.head, err = t.openFile(t.headId, openFreezerFileForReadOnly) 357 } else { 358 // Open head in read/write 359 t.head, err = t.openFile(t.headId, openFreezerFileForAppend) 360 } 361 return err 362 } 363 364 // truncate discards any recent data above the provided threshold number. 365 func (t *freezerTable) truncate(items uint64) error { 366 t.lock.Lock() 367 defer t.lock.Unlock() 368 369 // If our item count is correct, don't do anything 370 existing := atomic.LoadUint64(&t.items) 371 if existing <= items { 372 return nil 373 } 374 // We need to truncate, save the old size for metrics tracking 375 oldSize, err := t.sizeNolock() 376 if err != nil { 377 return err 378 } 379 // Something's out of sync, truncate the table's offset index 380 log := t.logger.Debug 381 if existing > items+1 { 382 log = t.logger.Warn // Only loud warn if we delete multiple items 383 } 384 log("Truncating freezer table", "items", existing, "limit", items) 385 if err := truncateFreezerFile(t.index, int64(items+1)*indexEntrySize); err != nil { 386 return err 387 } 388 // Calculate the new expected size of the data file and truncate it 389 buffer := make([]byte, indexEntrySize) 390 if _, err := t.index.ReadAt(buffer, int64(items*indexEntrySize)); err != nil { 391 return err 392 } 393 var expected indexEntry 394 expected.unmarshalBinary(buffer) 395 396 // We might need to truncate back to older files 397 if expected.filenum != t.headId { 398 // If already open for reading, force-reopen for writing 399 t.releaseFile(expected.filenum) 400 newHead, err := t.openFile(expected.filenum, openFreezerFileForAppend) 401 if err != nil { 402 return err 403 } 404 // Release any files _after the current head -- both the previous head 405 // and any files which may have been opened for reading 406 t.releaseFilesAfter(expected.filenum, true) 407 // Set back the historic head 408 t.head = newHead 409 t.headId = expected.filenum 410 } 411 if err := truncateFreezerFile(t.head, int64(expected.offset)); err != nil { 412 return err 413 } 414 // All data files truncated, set internal counters and return 415 t.headBytes = int64(expected.offset) 416 atomic.StoreUint64(&t.items, items) 417 418 // Retrieve the new size and update the total size counter 419 newSize, err := t.sizeNolock() 420 if err != nil { 421 return err 422 } 423 t.sizeGauge.Dec(int64(oldSize - newSize)) 424 425 return nil 426 } 427 428 // Close closes all opened files. 429 func (t *freezerTable) Close() error { 430 t.lock.Lock() 431 defer t.lock.Unlock() 432 433 var errs []error 434 if err := t.index.Close(); err != nil { 435 errs = append(errs, err) 436 } 437 t.index = nil 438 439 for _, f := range t.files { 440 if err := f.Close(); err != nil { 441 errs = append(errs, err) 442 } 443 } 444 t.head = nil 445 446 if errs != nil { 447 return fmt.Errorf("%v", errs) 448 } 449 return nil 450 } 451 452 // openFile assumes that the write-lock is held by the caller 453 func (t *freezerTable) openFile(num uint32, opener func(string) (*os.File, error)) (f *os.File, err error) { 454 var exist bool 455 if f, exist = t.files[num]; !exist { 456 var name string 457 if t.noCompression { 458 name = fmt.Sprintf("%s.%04d.rdat", t.name, num) 459 } else { 460 name = fmt.Sprintf("%s.%04d.cdat", t.name, num) 461 } 462 f, err = opener(filepath.Join(t.path, name)) 463 if err != nil { 464 return nil, err 465 } 466 t.files[num] = f 467 } 468 return f, err 469 } 470 471 // releaseFile closes a file, and removes it from the open file cache. 472 // Assumes that the caller holds the write lock 473 func (t *freezerTable) releaseFile(num uint32) { 474 if f, exist := t.files[num]; exist { 475 delete(t.files, num) 476 f.Close() 477 } 478 } 479 480 // releaseFilesAfter closes all open files with a higher number, and optionally also deletes the files 481 func (t *freezerTable) releaseFilesAfter(num uint32, remove bool) { 482 for fnum, f := range t.files { 483 if fnum > num { 484 delete(t.files, fnum) 485 f.Close() 486 if remove { 487 os.Remove(f.Name()) 488 } 489 } 490 } 491 } 492 493 // getIndices returns the index entries for the given from-item, covering 'count' items. 494 // N.B: The actual number of returned indices for N items will always be N+1 (unless an 495 // error is returned). 496 // OBS: This method assumes that the caller has already verified (and/or trimmed) the range 497 // so that the items are within bounds. If this method is used to read out of bounds, 498 // it will return error. 499 func (t *freezerTable) getIndices(from, count uint64) ([]*indexEntry, error) { 500 // Apply the table-offset 501 from = from - uint64(t.itemOffset) 502 // For reading N items, we need N+1 indices. 503 buffer := make([]byte, (count+1)*indexEntrySize) 504 if _, err := t.index.ReadAt(buffer, int64(from*indexEntrySize)); err != nil { 505 return nil, err 506 } 507 var ( 508 indices []*indexEntry 509 offset int 510 ) 511 for i := from; i <= from+count; i++ { 512 index := new(indexEntry) 513 index.unmarshalBinary(buffer[offset:]) 514 offset += indexEntrySize 515 indices = append(indices, index) 516 } 517 if from == 0 { 518 // Special case if we're reading the first item in the freezer. We assume that 519 // the first item always start from zero(regarding the deletion, we 520 // only support deletion by files, so that the assumption is held). 521 // This means we can use the first item metadata to carry information about 522 // the 'global' offset, for the deletion-case 523 indices[0].offset = 0 524 indices[0].filenum = indices[1].filenum 525 } 526 return indices, nil 527 } 528 529 // Retrieve looks up the data offset of an item with the given number and retrieves 530 // the raw binary blob from the data file. 531 func (t *freezerTable) Retrieve(item uint64) ([]byte, error) { 532 items, err := t.RetrieveItems(item, 1, 0) 533 if err != nil { 534 return nil, err 535 } 536 return items[0], nil 537 } 538 539 // RetrieveItems returns multiple items in sequence, starting from the index 'start'. 540 // It will return at most 'max' items, but will abort earlier to respect the 541 // 'maxBytes' argument. However, if the 'maxBytes' is smaller than the size of one 542 // item, it _will_ return one element and possibly overflow the maxBytes. 543 func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, error) { 544 // First we read the 'raw' data, which might be compressed. 545 diskData, sizes, err := t.retrieveItems(start, count, maxBytes) 546 if err != nil { 547 return nil, err 548 } 549 var ( 550 output = make([][]byte, 0, count) 551 offset int // offset for reading 552 outputSize int // size of uncompressed data 553 ) 554 // Now slice up the data and decompress. 555 for i, diskSize := range sizes { 556 item := diskData[offset : offset+diskSize] 557 offset += diskSize 558 decompressedSize := diskSize 559 if !t.noCompression { 560 decompressedSize, _ = snappy.DecodedLen(item) 561 } 562 if i > 0 && uint64(outputSize+decompressedSize) > maxBytes { 563 break 564 } 565 if !t.noCompression { 566 data, err := snappy.Decode(nil, item) 567 if err != nil { 568 return nil, err 569 } 570 output = append(output, data) 571 } else { 572 output = append(output, item) 573 } 574 outputSize += decompressedSize 575 } 576 return output, nil 577 } 578 579 // retrieveItems reads up to 'count' items from the table. It reads at least 580 // one item, but otherwise avoids reading more than maxBytes bytes. 581 // It returns the (potentially compressed) data, and the sizes. 582 func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []int, error) { 583 t.lock.RLock() 584 defer t.lock.RUnlock() 585 586 // Ensure the table and the item is accessible 587 if t.index == nil || t.head == nil { 588 return nil, nil, errClosed 589 } 590 itemCount := atomic.LoadUint64(&t.items) // max number 591 // Ensure the start is written, not deleted from the tail, and that the 592 // caller actually wants something 593 if itemCount <= start || uint64(t.itemOffset) > start || count == 0 { 594 return nil, nil, errOutOfBounds 595 } 596 if start+count > itemCount { 597 count = itemCount - start 598 } 599 var ( 600 output = make([]byte, maxBytes) // Buffer to read data into 601 outputSize int // Used size of that buffer 602 ) 603 // readData is a helper method to read a single data item from disk. 604 readData := func(fileId, start uint32, length int) error { 605 // In case a small limit is used, and the elements are large, may need to 606 // realloc the read-buffer when reading the first (and only) item. 607 if len(output) < length { 608 output = make([]byte, length) 609 } 610 dataFile, exist := t.files[fileId] 611 if !exist { 612 return fmt.Errorf("missing data file %d", fileId) 613 } 614 if _, err := dataFile.ReadAt(output[outputSize:outputSize+length], int64(start)); err != nil { 615 return err 616 } 617 outputSize += length 618 return nil 619 } 620 // Read all the indexes in one go 621 indices, err := t.getIndices(start, count) 622 if err != nil { 623 return nil, nil, err 624 } 625 var ( 626 sizes []int // The sizes for each element 627 totalSize = 0 // The total size of all data read so far 628 readStart = indices[0].offset // Where, in the file, to start reading 629 unreadSize = 0 // The size of the as-yet-unread data 630 ) 631 632 for i, firstIndex := range indices[:len(indices)-1] { 633 secondIndex := indices[i+1] 634 // Determine the size of the item. 635 offset1, offset2, _ := firstIndex.bounds(secondIndex) 636 size := int(offset2 - offset1) 637 // Crossing a file boundary? 638 if secondIndex.filenum != firstIndex.filenum { 639 // If we have unread data in the first file, we need to do that read now. 640 if unreadSize > 0 { 641 if err := readData(firstIndex.filenum, readStart, unreadSize); err != nil { 642 return nil, nil, err 643 } 644 unreadSize = 0 645 } 646 readStart = 0 647 } 648 if i > 0 && uint64(totalSize+size) > maxBytes { 649 // About to break out due to byte limit being exceeded. We don't 650 // read this last item, but we need to do the deferred reads now. 651 if unreadSize > 0 { 652 if err := readData(secondIndex.filenum, readStart, unreadSize); err != nil { 653 return nil, nil, err 654 } 655 } 656 break 657 } 658 // Defer the read for later 659 unreadSize += size 660 totalSize += size 661 sizes = append(sizes, size) 662 if i == len(indices)-2 || uint64(totalSize) > maxBytes { 663 // Last item, need to do the read now 664 if err := readData(secondIndex.filenum, readStart, unreadSize); err != nil { 665 return nil, nil, err 666 } 667 break 668 } 669 } 670 return output[:outputSize], sizes, nil 671 } 672 673 // has returns an indicator whether the specified number data 674 // exists in the freezer table. 675 func (t *freezerTable) has(number uint64) bool { 676 return atomic.LoadUint64(&t.items) > number 677 } 678 679 // size returns the total data size in the freezer table. 680 func (t *freezerTable) size() (uint64, error) { 681 t.lock.RLock() 682 defer t.lock.RUnlock() 683 684 return t.sizeNolock() 685 } 686 687 // sizeNolock returns the total data size in the freezer table without obtaining 688 // the mutex first. 689 func (t *freezerTable) sizeNolock() (uint64, error) { 690 stat, err := t.index.Stat() 691 if err != nil { 692 return 0, err 693 } 694 total := uint64(t.maxFileSize)*uint64(t.headId-t.tailId) + uint64(t.headBytes) + uint64(stat.Size()) 695 return total, nil 696 } 697 698 // advanceHead should be called when the current head file would outgrow the file limits, 699 // and a new file must be opened. The caller of this method must hold the write-lock 700 // before calling this method. 701 func (t *freezerTable) advanceHead() error { 702 t.lock.Lock() 703 defer t.lock.Unlock() 704 705 // We open the next file in truncated mode -- if this file already 706 // exists, we need to start over from scratch on it. 707 nextID := t.headId + 1 708 newHead, err := t.openFile(nextID, openFreezerFileTruncated) 709 if err != nil { 710 return err 711 } 712 713 // Close old file, and reopen in RDONLY mode. 714 t.releaseFile(t.headId) 715 t.openFile(t.headId, openFreezerFileForReadOnly) 716 717 // Swap out the current head. 718 t.head = newHead 719 t.headBytes = 0 720 t.headId = nextID 721 return nil 722 } 723 724 // Sync pushes any pending data from memory out to disk. This is an expensive 725 // operation, so use it with care. 726 func (t *freezerTable) Sync() error { 727 if err := t.index.Sync(); err != nil { 728 return err 729 } 730 return t.head.Sync() 731 } 732 733 // DumpIndex is a debug print utility function, mainly for testing. It can also 734 // be used to analyse a live freezer table index. 735 func (t *freezerTable) DumpIndex(start, stop int64) { 736 t.dumpIndex(os.Stdout, start, stop) 737 } 738 739 func (t *freezerTable) dumpIndexString(start, stop int64) string { 740 var out bytes.Buffer 741 out.WriteString("\n") 742 t.dumpIndex(&out, start, stop) 743 return out.String() 744 } 745 746 func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { 747 buf := make([]byte, indexEntrySize) 748 749 fmt.Fprintf(w, "| number | fileno | offset |\n") 750 fmt.Fprintf(w, "|--------|--------|--------|\n") 751 752 for i := uint64(start); ; i++ { 753 if _, err := t.index.ReadAt(buf, int64(i*indexEntrySize)); err != nil { 754 break 755 } 756 var entry indexEntry 757 entry.unmarshalBinary(buf) 758 fmt.Fprintf(w, "| %03d | %03d | %03d | \n", i, entry.filenum, entry.offset) 759 if stop > 0 && i >= uint64(stop) { 760 break 761 } 762 } 763 fmt.Fprintf(w, "|--------------------------|\n") 764 }