github.com/palisadeinc/bor@v0.0.0-20230615125219-ab7196213d15/core/rawdb/freezer.go (about)

     1  // Copyright 2019 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package rawdb
    18  
    19  import (
    20  	"errors"
    21  	"fmt"
    22  	"io/ioutil"
    23  	"math"
    24  	"os"
    25  	"path/filepath"
    26  	"sync"
    27  	"sync/atomic"
    28  	"time"
    29  
    30  	"github.com/ethereum/go-ethereum/common"
    31  	"github.com/ethereum/go-ethereum/ethdb"
    32  	"github.com/ethereum/go-ethereum/log"
    33  	"github.com/ethereum/go-ethereum/metrics"
    34  	"github.com/ethereum/go-ethereum/params"
    35  	"github.com/prometheus/tsdb/fileutil"
    36  )
    37  
    38  var (
    39  	// errReadOnly is returned if the freezer is opened in read only mode. All the
    40  	// mutations are disallowed.
    41  	errReadOnly = errors.New("read only")
    42  
    43  	// errUnknownTable is returned if the user attempts to read from a table that is
    44  	// not tracked by the freezer.
    45  	errUnknownTable = errors.New("unknown table")
    46  
    47  	// errOutOrderInsertion is returned if the user attempts to inject out-of-order
    48  	// binary blobs into the freezer.
    49  	errOutOrderInsertion = errors.New("the append operation is out-order")
    50  
    51  	// errSymlinkDatadir is returned if the ancient directory specified by user
    52  	// is a symbolic link.
    53  	errSymlinkDatadir = errors.New("symbolic link datadir is not supported")
    54  )
    55  
    56  const (
    57  	// freezerRecheckInterval is the frequency to check the key-value database for
    58  	// chain progression that might permit new blocks to be frozen into immutable
    59  	// storage.
    60  	freezerRecheckInterval = time.Minute
    61  
    62  	// freezerBatchLimit is the maximum number of blocks to freeze in one batch
    63  	// before doing an fsync and deleting it from the key-value store.
    64  	freezerBatchLimit = 30000
    65  
    66  	// freezerTableSize defines the maximum size of freezer data files.
    67  	freezerTableSize = 2 * 1000 * 1000 * 1000
    68  )
    69  
    70  // freezer is a memory mapped append-only database to store immutable chain data
    71  // into flat files:
    72  //
    73  // - The append only nature ensures that disk writes are minimized.
    74  // - The memory mapping ensures we can max out system memory for caching without
    75  //   reserving it for go-ethereum. This would also reduce the memory requirements
    76  //   of Geth, and thus also GC overhead.
    77  type freezer struct {
    78  	// WARNING: The `frozen` field is accessed atomically. On 32 bit platforms, only
    79  	// 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned,
    80  	// so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG).
    81  	frozen    uint64 // Number of blocks already frozen
    82  	tail      uint64 // Number of the first stored item in the freezer
    83  	threshold uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests)
    84  
    85  	// This lock synchronizes writers and the truncate operation, as well as
    86  	// the "atomic" (batched) read operations.
    87  	writeLock  sync.RWMutex
    88  	writeBatch *freezerBatch
    89  
    90  	readonly     bool
    91  	tables       map[string]*freezerTable // Data tables for storing everything
    92  	instanceLock fileutil.Releaser        // File-system lock to prevent double opens
    93  
    94  	trigger chan chan struct{} // Manual blocking freeze trigger, test determinism
    95  
    96  	quit      chan struct{}
    97  	wg        sync.WaitGroup
    98  	closeOnce sync.Once
    99  }
   100  
   101  // newFreezer creates a chain freezer that moves ancient chain data into
   102  // append-only flat file containers.
   103  //
   104  // The 'tables' argument defines the data tables. If the value of a map
   105  // entry is true, snappy compression is disabled for the table.
   106  func newFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*freezer, error) {
   107  	// Create the initial freezer object
   108  	var (
   109  		readMeter  = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
   110  		writeMeter = metrics.NewRegisteredMeter(namespace+"ancient/write", nil)
   111  		sizeGauge  = metrics.NewRegisteredGauge(namespace+"ancient/size", nil)
   112  	)
   113  	// Ensure the datadir is not a symbolic link if it exists.
   114  	if info, err := os.Lstat(datadir); !os.IsNotExist(err) {
   115  		if info.Mode()&os.ModeSymlink != 0 {
   116  			log.Warn("Symbolic link ancient database is not supported", "path", datadir)
   117  			return nil, errSymlinkDatadir
   118  		}
   119  	}
   120  	// Leveldb uses LOCK as the filelock filename. To prevent the
   121  	// name collision, we use FLOCK as the lock name.
   122  	lock, _, err := fileutil.Flock(filepath.Join(datadir, "FLOCK"))
   123  	if err != nil {
   124  		return nil, err
   125  	}
   126  	// Open all the supported data tables
   127  	freezer := &freezer{
   128  		readonly:     readonly,
   129  		threshold:    params.FullImmutabilityThreshold,
   130  		tables:       make(map[string]*freezerTable),
   131  		instanceLock: lock,
   132  		trigger:      make(chan chan struct{}),
   133  		quit:         make(chan struct{}),
   134  	}
   135  
   136  	// Create the tables.
   137  	for name, disableSnappy := range tables {
   138  		table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy, readonly)
   139  		if err != nil {
   140  			for _, table := range freezer.tables {
   141  				table.Close()
   142  			}
   143  			lock.Release()
   144  			return nil, err
   145  		}
   146  		freezer.tables[name] = table
   147  	}
   148  
   149  	// Adjust table length for bor-receipt freezer for already synced nodes.
   150  	//
   151  	// Since, table only supports sequential data, this will fill empty-data upto current
   152  	// synced block (till current total header number).
   153  	//
   154  	// This way they don't have to sync again from block 0 and still be compatible
   155  	// for block logs for future blocks. Note that already synced nodes
   156  	// won't have past block logs. Newly synced node will have all the data.
   157  	if _, ok := freezer.tables[freezerBorReceiptTable]; ok {
   158  		if err := freezer.tables[freezerBorReceiptTable].Fill(freezer.tables[freezerHeaderTable].items); err != nil {
   159  			return nil, err
   160  		}
   161  	}
   162  	if freezer.readonly {
   163  		// In readonly mode only validate, don't truncate.
   164  		// validate also sets `freezer.frozen`.
   165  		err = freezer.validate()
   166  	} else {
   167  		// Truncate all tables to common length.
   168  		err = freezer.repair()
   169  	}
   170  	if err != nil {
   171  		for _, table := range freezer.tables {
   172  			table.Close()
   173  		}
   174  		lock.Release()
   175  		return nil, err
   176  	}
   177  
   178  	// Create the write batch.
   179  	freezer.writeBatch = newFreezerBatch(freezer)
   180  
   181  	log.Info("Opened ancient database", "database", datadir, "readonly", readonly)
   182  	return freezer, nil
   183  }
   184  
   185  // Close terminates the chain freezer, unmapping all the data files.
   186  func (f *freezer) Close() error {
   187  	f.writeLock.Lock()
   188  	defer f.writeLock.Unlock()
   189  
   190  	var errs []error
   191  	f.closeOnce.Do(func() {
   192  		close(f.quit)
   193  		// Wait for any background freezing to stop
   194  		f.wg.Wait()
   195  		for _, table := range f.tables {
   196  			if err := table.Close(); err != nil {
   197  				errs = append(errs, err)
   198  			}
   199  		}
   200  		if err := f.instanceLock.Release(); err != nil {
   201  			errs = append(errs, err)
   202  		}
   203  	})
   204  	if errs != nil {
   205  		return fmt.Errorf("%v", errs)
   206  	}
   207  	return nil
   208  }
   209  
   210  // HasAncient returns an indicator whether the specified ancient data exists
   211  // in the freezer.
   212  func (f *freezer) HasAncient(kind string, number uint64) (bool, error) {
   213  	if table := f.tables[kind]; table != nil {
   214  		return table.has(number), nil
   215  	}
   216  	return false, nil
   217  }
   218  
   219  // Ancient retrieves an ancient binary blob from the append-only immutable files.
   220  func (f *freezer) Ancient(kind string, number uint64) ([]byte, error) {
   221  	if table := f.tables[kind]; table != nil {
   222  		return table.Retrieve(number)
   223  	}
   224  	return nil, errUnknownTable
   225  }
   226  
   227  // AncientRange retrieves multiple items in sequence, starting from the index 'start'.
   228  // It will return
   229  //  - at most 'max' items,
   230  //  - at least 1 item (even if exceeding the maxByteSize), but will otherwise
   231  //   return as many items as fit into maxByteSize.
   232  func (f *freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
   233  	if table := f.tables[kind]; table != nil {
   234  		return table.RetrieveItems(start, count, maxBytes)
   235  	}
   236  	return nil, errUnknownTable
   237  }
   238  
   239  // Ancients returns the length of the frozen items.
   240  func (f *freezer) Ancients() (uint64, error) {
   241  	return atomic.LoadUint64(&f.frozen), nil
   242  }
   243  
   244  // Tail returns the number of first stored item in the freezer.
   245  func (f *freezer) Tail() (uint64, error) {
   246  	return atomic.LoadUint64(&f.tail), nil
   247  }
   248  
   249  // AncientSize returns the ancient size of the specified category.
   250  func (f *freezer) AncientSize(kind string) (uint64, error) {
   251  	// This needs the write lock to avoid data races on table fields.
   252  	// Speed doesn't matter here, AncientSize is for debugging.
   253  	f.writeLock.RLock()
   254  	defer f.writeLock.RUnlock()
   255  
   256  	if table := f.tables[kind]; table != nil {
   257  		return table.size()
   258  	}
   259  	return 0, errUnknownTable
   260  }
   261  
   262  // ReadAncients runs the given read operation while ensuring that no writes take place
   263  // on the underlying freezer.
   264  func (f *freezer) ReadAncients(fn func(ethdb.AncientReader) error) (err error) {
   265  	f.writeLock.RLock()
   266  	defer f.writeLock.RUnlock()
   267  	return fn(f)
   268  }
   269  
   270  // ModifyAncients runs the given write operation.
   271  func (f *freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize int64, err error) {
   272  	if f.readonly {
   273  		return 0, errReadOnly
   274  	}
   275  	f.writeLock.Lock()
   276  	defer f.writeLock.Unlock()
   277  
   278  	// Roll back all tables to the starting position in case of error.
   279  	prevItem := f.frozen
   280  	defer func() {
   281  		if err != nil {
   282  			// The write operation has failed. Go back to the previous item position.
   283  			for name, table := range f.tables {
   284  				err := table.truncateHead(prevItem)
   285  				if err != nil {
   286  					log.Error("Freezer table roll-back failed", "table", name, "index", prevItem, "err", err)
   287  				}
   288  			}
   289  		}
   290  	}()
   291  
   292  	f.writeBatch.reset()
   293  	if err := fn(f.writeBatch); err != nil {
   294  		return 0, err
   295  	}
   296  	item, writeSize, err := f.writeBatch.commit()
   297  	if err != nil {
   298  		return 0, err
   299  	}
   300  	atomic.StoreUint64(&f.frozen, item)
   301  	return writeSize, nil
   302  }
   303  
   304  // TruncateHead discards any recent data above the provided threshold number.
   305  func (f *freezer) TruncateHead(items uint64) error {
   306  	if f.readonly {
   307  		return errReadOnly
   308  	}
   309  	f.writeLock.Lock()
   310  	defer f.writeLock.Unlock()
   311  
   312  	if atomic.LoadUint64(&f.frozen) <= items {
   313  		return nil
   314  	}
   315  	for _, table := range f.tables {
   316  		if err := table.truncateHead(items); err != nil {
   317  			return err
   318  		}
   319  	}
   320  	atomic.StoreUint64(&f.frozen, items)
   321  	return nil
   322  }
   323  
   324  // TruncateTail discards any recent data below the provided threshold number.
   325  func (f *freezer) TruncateTail(tail uint64) error {
   326  	if f.readonly {
   327  		return errReadOnly
   328  	}
   329  	f.writeLock.Lock()
   330  	defer f.writeLock.Unlock()
   331  
   332  	if atomic.LoadUint64(&f.tail) >= tail {
   333  		return nil
   334  	}
   335  	for _, table := range f.tables {
   336  		if err := table.truncateTail(tail); err != nil {
   337  			return err
   338  		}
   339  	}
   340  	atomic.StoreUint64(&f.tail, tail)
   341  	return nil
   342  }
   343  
   344  // Sync flushes all data tables to disk.
   345  func (f *freezer) Sync() error {
   346  	var errs []error
   347  	for _, table := range f.tables {
   348  		if err := table.Sync(); err != nil {
   349  			errs = append(errs, err)
   350  		}
   351  	}
   352  	if errs != nil {
   353  		return fmt.Errorf("%v", errs)
   354  	}
   355  	return nil
   356  }
   357  
   358  // validate checks that every table has the same length.
   359  // Used instead of `repair` in readonly mode.
   360  func (f *freezer) validate() error {
   361  	if len(f.tables) == 0 {
   362  		return nil
   363  	}
   364  	var (
   365  		length uint64
   366  		name   string
   367  	)
   368  	// Hack to get length of any table
   369  	for kind, table := range f.tables {
   370  		length = atomic.LoadUint64(&table.items)
   371  		name = kind
   372  		break
   373  	}
   374  	// Now check every table against that length
   375  	for kind, table := range f.tables {
   376  		items := atomic.LoadUint64(&table.items)
   377  		if length != items {
   378  			return fmt.Errorf("freezer tables %s and %s have differing lengths: %d != %d", kind, name, items, length)
   379  		}
   380  	}
   381  	atomic.StoreUint64(&f.frozen, length)
   382  	return nil
   383  }
   384  
   385  // repair truncates all data tables to the same length.
   386  func (f *freezer) repair() error {
   387  	var (
   388  		head = uint64(math.MaxUint64)
   389  		tail = uint64(0)
   390  	)
   391  	for _, table := range f.tables {
   392  		items := atomic.LoadUint64(&table.items)
   393  		if head > items {
   394  			head = items
   395  		}
   396  		hidden := atomic.LoadUint64(&table.itemHidden)
   397  		if hidden > tail {
   398  			tail = hidden
   399  		}
   400  	}
   401  	for _, table := range f.tables {
   402  		if err := table.truncateHead(head); err != nil {
   403  			return err
   404  		}
   405  		if err := table.truncateTail(tail); err != nil {
   406  			return err
   407  		}
   408  	}
   409  	atomic.StoreUint64(&f.frozen, head)
   410  	atomic.StoreUint64(&f.tail, tail)
   411  	return nil
   412  }
   413  
   414  // freeze is a background thread that periodically checks the blockchain for any
   415  // import progress and moves ancient data from the fast database into the freezer.
   416  //
   417  // This functionality is deliberately broken off from block importing to avoid
   418  // incurring additional data shuffling delays on block propagation.
   419  func (f *freezer) freeze(db ethdb.KeyValueStore) {
   420  	nfdb := &nofreezedb{KeyValueStore: db}
   421  
   422  	var (
   423  		backoff   bool
   424  		triggered chan struct{} // Used in tests
   425  	)
   426  	for {
   427  		select {
   428  		case <-f.quit:
   429  			log.Info("Freezer shutting down")
   430  			return
   431  		default:
   432  		}
   433  		if backoff {
   434  			// If we were doing a manual trigger, notify it
   435  			if triggered != nil {
   436  				triggered <- struct{}{}
   437  				triggered = nil
   438  			}
   439  			select {
   440  			case <-time.NewTimer(freezerRecheckInterval).C:
   441  				backoff = false
   442  			case triggered = <-f.trigger:
   443  				backoff = false
   444  			case <-f.quit:
   445  				return
   446  			}
   447  		}
   448  		// Retrieve the freezing threshold.
   449  		hash := ReadHeadBlockHash(nfdb)
   450  		if hash == (common.Hash{}) {
   451  			log.Debug("Current full block hash unavailable") // new chain, empty database
   452  			backoff = true
   453  			continue
   454  		}
   455  		number := ReadHeaderNumber(nfdb, hash)
   456  		threshold := atomic.LoadUint64(&f.threshold)
   457  
   458  		switch {
   459  		case number == nil:
   460  			log.Error("Current full block number unavailable", "hash", hash)
   461  			backoff = true
   462  			continue
   463  
   464  		case *number < threshold:
   465  			log.Debug("Current full block not old enough", "number", *number, "hash", hash, "delay", threshold)
   466  			backoff = true
   467  			continue
   468  
   469  		case *number-threshold <= f.frozen:
   470  			log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", f.frozen)
   471  			backoff = true
   472  			continue
   473  		}
   474  		head := ReadHeader(nfdb, hash, *number)
   475  		if head == nil {
   476  			log.Error("Current full block unavailable", "number", *number, "hash", hash)
   477  			backoff = true
   478  			continue
   479  		}
   480  
   481  		// Seems we have data ready to be frozen, process in usable batches
   482  		var (
   483  			start    = time.Now()
   484  			first, _ = f.Ancients()
   485  			limit    = *number - threshold
   486  		)
   487  		if limit-first > freezerBatchLimit {
   488  			limit = first + freezerBatchLimit
   489  		}
   490  		ancients, err := f.freezeRange(nfdb, first, limit)
   491  		if err != nil {
   492  			log.Error("Error in block freeze operation", "err", err)
   493  			backoff = true
   494  			continue
   495  		}
   496  
   497  		// Batch of blocks have been frozen, flush them before wiping from leveldb
   498  		if err := f.Sync(); err != nil {
   499  			log.Crit("Failed to flush frozen tables", "err", err)
   500  		}
   501  
   502  		// Wipe out all data from the active database
   503  		batch := db.NewBatch()
   504  		for i := 0; i < len(ancients); i++ {
   505  			// Always keep the genesis block in active database
   506  			if first+uint64(i) != 0 {
   507  				DeleteBlockWithoutNumber(batch, ancients[i], first+uint64(i))
   508  				DeleteCanonicalHash(batch, first+uint64(i))
   509  			}
   510  		}
   511  		if err := batch.Write(); err != nil {
   512  			log.Crit("Failed to delete frozen canonical blocks", "err", err)
   513  		}
   514  		batch.Reset()
   515  
   516  		// Wipe out side chains also and track dangling side chains
   517  		var dangling []common.Hash
   518  		for number := first; number < f.frozen; number++ {
   519  			// Always keep the genesis block in active database
   520  			if number != 0 {
   521  				dangling = ReadAllHashes(db, number)
   522  				for _, hash := range dangling {
   523  					log.Trace("Deleting side chain", "number", number, "hash", hash)
   524  					DeleteBlock(batch, hash, number)
   525  				}
   526  			}
   527  		}
   528  		if err := batch.Write(); err != nil {
   529  			log.Crit("Failed to delete frozen side blocks", "err", err)
   530  		}
   531  		batch.Reset()
   532  
   533  		// Step into the future and delete and dangling side chains
   534  		if f.frozen > 0 {
   535  			tip := f.frozen
   536  			for len(dangling) > 0 {
   537  				drop := make(map[common.Hash]struct{})
   538  				for _, hash := range dangling {
   539  					log.Debug("Dangling parent from freezer", "number", tip-1, "hash", hash)
   540  					drop[hash] = struct{}{}
   541  				}
   542  				children := ReadAllHashes(db, tip)
   543  				for i := 0; i < len(children); i++ {
   544  					// Dig up the child and ensure it's dangling
   545  					child := ReadHeader(nfdb, children[i], tip)
   546  					if child == nil {
   547  						log.Error("Missing dangling header", "number", tip, "hash", children[i])
   548  						continue
   549  					}
   550  					if _, ok := drop[child.ParentHash]; !ok {
   551  						children = append(children[:i], children[i+1:]...)
   552  						i--
   553  						continue
   554  					}
   555  					// Delete all block data associated with the child
   556  					log.Debug("Deleting dangling block", "number", tip, "hash", children[i], "parent", child.ParentHash)
   557  					DeleteBlock(batch, children[i], tip)
   558  				}
   559  				dangling = children
   560  				tip++
   561  			}
   562  			if err := batch.Write(); err != nil {
   563  				log.Crit("Failed to delete dangling side blocks", "err", err)
   564  			}
   565  		}
   566  
   567  		// Log something friendly for the user
   568  		context := []interface{}{
   569  			"blocks", f.frozen - first, "elapsed", common.PrettyDuration(time.Since(start)), "number", f.frozen - 1,
   570  		}
   571  		if n := len(ancients); n > 0 {
   572  			context = append(context, []interface{}{"hash", ancients[n-1]}...)
   573  		}
   574  		log.Info("Deep froze chain segment", context...)
   575  
   576  		// Avoid database thrashing with tiny writes
   577  		if f.frozen-first < freezerBatchLimit {
   578  			backoff = true
   579  		}
   580  	}
   581  }
   582  
   583  func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) {
   584  	hashes = make([]common.Hash, 0, limit-number)
   585  
   586  	_, err = f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
   587  		for ; number <= limit; number++ {
   588  			// Retrieve all the components of the canonical block.
   589  			hash := ReadCanonicalHash(nfdb, number)
   590  			if hash == (common.Hash{}) {
   591  				return fmt.Errorf("canonical hash missing, can't freeze block %d", number)
   592  			}
   593  			header := ReadHeaderRLP(nfdb, hash, number)
   594  			if len(header) == 0 {
   595  				return fmt.Errorf("block header missing, can't freeze block %d", number)
   596  			}
   597  			body := ReadBodyRLP(nfdb, hash, number)
   598  			if len(body) == 0 {
   599  				return fmt.Errorf("block body missing, can't freeze block %d", number)
   600  			}
   601  			receipts := ReadReceiptsRLP(nfdb, hash, number)
   602  			if len(receipts) == 0 {
   603  				return fmt.Errorf("block receipts missing, can't freeze block %d", number)
   604  			}
   605  			td := ReadTdRLP(nfdb, hash, number)
   606  			if len(td) == 0 {
   607  				return fmt.Errorf("total difficulty missing, can't freeze block %d", number)
   608  			}
   609  
   610  			// Write to the batch.
   611  			if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil {
   612  				return fmt.Errorf("can't write hash to freezer: %v", err)
   613  			}
   614  			if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil {
   615  				return fmt.Errorf("can't write header to freezer: %v", err)
   616  			}
   617  			if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil {
   618  				return fmt.Errorf("can't write body to freezer: %v", err)
   619  			}
   620  			if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil {
   621  				return fmt.Errorf("can't write receipts to freezer: %v", err)
   622  			}
   623  			if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil {
   624  				return fmt.Errorf("can't write td to freezer: %v", err)
   625  			}
   626  
   627  			// bor block receipt
   628  			borBlockReceipt := ReadBorReceiptRLP(nfdb, hash, number)
   629  			if err := op.AppendRaw(freezerBorReceiptTable, number, borBlockReceipt); err != nil {
   630  				return fmt.Errorf("can't write bor-receipt to freezer: %v", err)
   631  			}
   632  
   633  			hashes = append(hashes, hash)
   634  		}
   635  		return nil
   636  	})
   637  
   638  	return hashes, err
   639  }
   640  
   641  // convertLegacyFn takes a raw freezer entry in an older format and
   642  // returns it in the new format.
   643  type convertLegacyFn = func([]byte) ([]byte, error)
   644  
   645  // MigrateTable processes the entries in a given table in sequence
   646  // converting them to a new format if they're of an old format.
   647  func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error {
   648  	if f.readonly {
   649  		return errReadOnly
   650  	}
   651  	f.writeLock.Lock()
   652  	defer f.writeLock.Unlock()
   653  
   654  	table, ok := f.tables[kind]
   655  	if !ok {
   656  		return errUnknownTable
   657  	}
   658  	// forEach iterates every entry in the table serially and in order, calling `fn`
   659  	// with the item as argument. If `fn` returns an error the iteration stops
   660  	// and that error will be returned.
   661  	forEach := func(t *freezerTable, offset uint64, fn func(uint64, []byte) error) error {
   662  		var (
   663  			items     = atomic.LoadUint64(&t.items)
   664  			batchSize = uint64(1024)
   665  			maxBytes  = uint64(1024 * 1024)
   666  		)
   667  		for i := offset; i < items; {
   668  			if i+batchSize > items {
   669  				batchSize = items - i
   670  			}
   671  			data, err := t.RetrieveItems(i, batchSize, maxBytes)
   672  			if err != nil {
   673  				return err
   674  			}
   675  			for j, item := range data {
   676  				if err := fn(i+uint64(j), item); err != nil {
   677  					return err
   678  				}
   679  			}
   680  			i += uint64(len(data))
   681  		}
   682  		return nil
   683  	}
   684  	// TODO(s1na): This is a sanity-check since as of now no process does tail-deletion. But the migration
   685  	// process assumes no deletion at tail and needs to be modified to account for that.
   686  	if table.itemOffset > 0 || table.itemHidden > 0 {
   687  		return fmt.Errorf("migration not supported for tail-deleted freezers")
   688  	}
   689  	ancientsPath := filepath.Dir(table.index.Name())
   690  	// Set up new dir for the migrated table, the content of which
   691  	// we'll at the end move over to the ancients dir.
   692  	migrationPath := filepath.Join(ancientsPath, "migration")
   693  	newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind], false)
   694  	if err != nil {
   695  		return err
   696  	}
   697  	var (
   698  		batch  = newTable.newBatch()
   699  		out    []byte
   700  		start  = time.Now()
   701  		logged = time.Now()
   702  		offset = newTable.items
   703  	)
   704  	if offset > 0 {
   705  		log.Info("found previous migration attempt", "migrated", offset)
   706  	}
   707  	// Iterate through entries and transform them
   708  	if err := forEach(table, offset, func(i uint64, blob []byte) error {
   709  		if i%10000 == 0 && time.Since(logged) > 16*time.Second {
   710  			log.Info("Processing legacy elements", "count", i, "elapsed", common.PrettyDuration(time.Since(start)))
   711  			logged = time.Now()
   712  		}
   713  		out, err = convert(blob)
   714  		if err != nil {
   715  			return err
   716  		}
   717  		if err := batch.AppendRaw(i, out); err != nil {
   718  			return err
   719  		}
   720  		return nil
   721  	}); err != nil {
   722  		return err
   723  	}
   724  	if err := batch.commit(); err != nil {
   725  		return err
   726  	}
   727  	log.Info("Replacing old table files with migrated ones", "elapsed", common.PrettyDuration(time.Since(start)))
   728  	// Release and delete old table files. Note this won't
   729  	// delete the index file.
   730  	table.releaseFilesAfter(0, true)
   731  
   732  	if err := newTable.Close(); err != nil {
   733  		return err
   734  	}
   735  	files, err := ioutil.ReadDir(migrationPath)
   736  	if err != nil {
   737  		return err
   738  	}
   739  	// Move migrated files to ancients dir.
   740  	for _, f := range files {
   741  		// This will replace the old index file as a side-effect.
   742  		if err := os.Rename(filepath.Join(migrationPath, f.Name()), filepath.Join(ancientsPath, f.Name())); err != nil {
   743  			return err
   744  		}
   745  	}
   746  	// Delete by now empty dir.
   747  	if err := os.Remove(migrationPath); err != nil {
   748  		return err
   749  	}
   750  
   751  	return nil
   752  }