gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/workeraccountpersist.go

package renter

// TODO: Derive the account secret key using the wallet seed. Can use:
// `account specifier || wallet seed || host pubkey` I believe.
//
// If we derive the seeds deterministically, that may mean that we can
// regenerate accounts even we fail to load them from disk. When we make a new
// account with a host, we should always query that host for a balance even if
// we think this is a new account, some previous run on siad may have created
// the account for us.
//
// TODO: How long does the host keep an account open? Does it keep the account
// open for the entire period? If not, we should probably adjust that on the
// host side, otherwise renters that go offline for a while are going to lose
// their accounts because the hosts will expire them. Does the renter track the
// expiration date of the accounts? Will it know upload load that the account is
// missing from the host not because of malice but because they expired?

import (
	"bytes"
	"io"
	"math/big"
	"os"
	"path/filepath"
	"sync"

	"gitlab.com/NebulousLabs/encoding"
	"gitlab.com/NebulousLabs/errors"
	"gitlab.com/SkynetLabs/skyd/build"
	skydPersist "gitlab.com/SkynetLabs/skyd/persist"
	"gitlab.com/SkynetLabs/skyd/skymodules"
	"go.sia.tech/siad/crypto"
	"go.sia.tech/siad/modules"
	"go.sia.tech/siad/persist"
	"go.sia.tech/siad/types"
)

const (
	// accountSize is the fixed account size in bytes
	accountSize     = 1 << 10 // 1024 bytes
	accountSizeV150 = 1 << 8  // 256 bytes
	accountsOffset  = 1 << 12 // 4kib to sector align
)

var (
	// accountsFilename is the filename of the accounts persistence file
	accountsFilename = "accounts.dat"

	// accountsTmpFilename is the filename of the temporary account file created
	// when upgrading the account's persistence file.
	accountsTmpFilename = "accounts.tmp.dat"

	// Metadata
	metadataHeader      = types.NewSpecifier("Accounts\n")
	metadataVersion     = types.NewSpecifier("v1.6.2\n")
	metadataVersionV161 = types.NewSpecifier("v1.6.1\n")
	metadataSize        = 2*types.SpecifierLen + 1 // 1 byte for 'clean' flag

	// Metadata validation errors
	errWrongHeader  = errors.New("wrong header")
	errWrongVersion = errors.New("wrong version")

	// Persistence data validation errors
	errInvalidChecksum = errors.New("invalid checksum")
)

type (
	// accountManager tracks the set of accounts known to the renter.
	accountManager struct {
		accounts map[string]*account

		// Utils. The file is global to all accounts, each account looks at a
		// specific offset within the file.
		mu           sync.Mutex
		staticFile   modules.File
		staticRenter *Renter
	}

	// accountsMetadata is the metadata of the accounts persist file
	accountsMetadata struct {
		Header  types.Specifier
		Version types.Specifier
		Clean   bool
	}

	// accountPersistence is the account's persistence object which holds all
	// data that gets persisted for a single account.
	accountPersistence struct {
		AccountID modules.AccountID
		HostKey   types.SiaPublicKey
		SecretKey crypto.SecretKey

		// balance details, aside from the balance we keep track of the balance
		// drift, in both directions, that may occur when the renter's account
		// balance becomes out of sync with the host's version of the balance
		Balance              types.Currency
		BalanceDriftPositive types.Currency
		BalanceDriftNegative types.Currency

		// spending details
		SpendingDownloads         types.Currency
		SpendingRegistryReads     types.Currency
		SpendingRegistryWrites    types.Currency
		SpendingRepairDownloads   types.Currency
		SpendingRepairUploads     types.Currency
		SpendingSnapshotDownloads types.Currency
		SpendingSnapshotUploads   types.Currency
		SpendingSubscriptions     types.Currency
		SpendingUploads           types.Currency

		// The following fields were added in v1.5.11
		//
		// residue is the amount of money that was still in the ephemeral
		// account at the moment the contract renews and the FundAccountCost
		// goes back to zero, we need to keep track of the residue in order to
		// correctly report the spending details in the next period
		Residue types.Currency

		// The following fields were added in v1.6.1
		//
		// host balance stores a local version of the balance that resets on
		// every balance sync we perform with the host. These fields are
		// necessary to more accurately track the balance drift
		HostBalance         types.Currency
		HostBalanceNegative types.Currency

		// The following fields were added in v1.6.2
		//
		// the following spending details indicate the amount spent on
		// maintenance functions, namely updating the price table and syncing
		// the account balance, when they're paid for using the ephemeral
		// account
		SpendingAccountBalance   types.Currency
		SpendingUpdatePriceTable types.Currency
	}

	// accountPersistenceV150 is how the account persistence struct looked
	// before adding the spending details in v156
	accountPersistenceV150 struct {
		AccountID modules.AccountID
		Balance   types.Currency
		HostKey   types.SiaPublicKey
		SecretKey crypto.SecretKey
	}
)

// newAccountManager will initialize the account manager for the renter.
func (r *Renter) newAccountManager() error {
	if r.staticAccountManager != nil {
		return errors.New("account manager already exists")
	}

	r.staticAccountManager = &accountManager{
		accounts: make(map[string]*account),

		staticRenter: r,
	}

	return r.staticAccountManager.load()
}

// managedPersist will write the account to the given file at the account's
// offset, without syncing the file.
func (a *account) managedPersist() error {
	a.mu.Lock()
	defer a.mu.Unlock()
	return a.persist()
}

// persist will write the account to the given file at the account's offset,
// without syncing the file.
func (a *account) persist() error {
	accountData := accountPersistence{
		AccountID: a.staticID,
		HostKey:   a.staticHostKey,
		SecretKey: a.staticSecretKey,

		// balance details
		Balance:              a.minExpectedBalance(),
		BalanceDriftPositive: a.balanceDriftPositive,
		BalanceDriftNegative: a.balanceDriftNegative,

		// spending details
		SpendingDownloads:         a.spending.downloads,
		SpendingRegistryReads:     a.spending.registryReads,
		SpendingRegistryWrites:    a.spending.registryWrites,
		SpendingRepairDownloads:   a.spending.repairDownloads,
		SpendingRepairUploads:     a.spending.repairUploads,
		SpendingSnapshotDownloads: a.spending.snapshotDownloads,
		SpendingSnapshotUploads:   a.spending.snapshotUploads,
		SpendingSubscriptions:     a.spending.subscriptions,
		SpendingUploads:           a.spending.uploads,

		// residue
		Residue: a.residue,

		// host balance
		//
		// NOTE: we want to take into account pending withdrawals here, but we
		// do not want to incorporate the negative balance into the host
		// balance, as we persist that separately
		HostBalance:         minExpectedBalance(a.hostBalance, types.ZeroCurrency, a.pendingWithdrawals),
		HostBalanceNegative: a.hostBalanceNegative,

		// maintenance spending details
		//
		// NOTE: these fields are added to the bottom here to avoid writing too
		// much compat code
		SpendingAccountBalance:   a.spending.accountBalance,
		SpendingUpdatePriceTable: a.spending.updatePriceTable,
	}

	_, err := a.staticFile.WriteAt(accountData.bytes(), a.staticOffset)
	return errors.AddContext(err, "unable to write the account to disk")
}

// bytes is a helper method on the persistence object that outputs the bytes to
// put on disk, these include the checksum and the marshaled persistence object.
func (ap accountPersistence) bytes() []byte {
	accBytes := encoding.Marshal(ap)
	accBytesMaxSize := accountSize - crypto.HashSize // leave room for checksum
	if len(accBytes) > accBytesMaxSize {
		build.Critical("marshaled object is larger than expected size", len(accBytes))
		return nil
	}

	// Calculate checksum on padded account bytes. Upon load, the padding will
	// be ignored by the unmarshaling.
	accBytesPadded := make([]byte, accBytesMaxSize)
	copy(accBytesPadded, accBytes)
	checksum := crypto.HashBytes(accBytesPadded)

	// create final byte slice of account size
	b := make([]byte, accountSize)
	copy(b[:len(checksum)], checksum[:])
	copy(b[len(checksum):], accBytesPadded)
	return b
}

// loadBytes is a helper method that takes a byte slice, containing a checksum
// and the account bytes, and unmarshals them onto the persistence object if the
// checksum is valid.
func (ap *accountPersistence) loadBytes(b []byte) error {
	// extract checksum and verify it
	checksum := b[:crypto.HashSize]
	accBytes := b[crypto.HashSize:]
	accHash := crypto.HashBytes(accBytes)
	if !bytes.Equal(checksum, accHash[:]) {
		return errInvalidChecksum
	}

	// unmarshal the account bytes onto the persistence object
	return errors.AddContext(encoding.Unmarshal(accBytes, ap), "failed to unmarshal account bytes")
}

// EphemeralAccountSpending returns a breakdown of the costs of the account
// expenditures per spending category.
func (am *accountManager) EphemeralAccountSpending() []skymodules.EphemeralAccountSpending {
	am.mu.Lock()
	var accounts []*account
	for _, account := range am.accounts {
		accounts = append(accounts, account)
	}
	am.mu.Unlock()

	var eass []skymodules.EphemeralAccountSpending
	for _, account := range accounts {
		account.mu.Lock()
		b := account.balance
		dp := account.balanceDriftPositive
		dn := account.balanceDriftNegative
		s := account.spending
		r := account.residue
		account.mu.Unlock()

		var eas skymodules.EphemeralAccountSpending
		eas.HostKey = account.staticHostKey

		eas.AccountBalanceCost = s.accountBalance
		eas.DownloadsCost = s.downloads
		eas.RegistryReadsCost = s.registryReads
		eas.RegistryWritesCost = s.registryWrites
		eas.RepairDownloadsCost = s.repairDownloads
		eas.RepairUploadsCost = s.repairUploads
		eas.SnapshotDownloadsCost = s.snapshotDownloads
		eas.SnapshotUploadsCost = s.snapshotUploads
		eas.SubscriptionsCost = s.subscriptions
		eas.UpdatePriceTableCost = s.updatePriceTable
		eas.UploadsCost = s.uploads

		eas.Balance = b
		eas.Residue = r

		var drift *big.Int
		if dp.Cmp(dn) > 0 {
			drift = dp.Sub(dn).Big()
		} else {
			drift = dn.Sub(dp).Big()
			drift = drift.Neg(drift)
		}
		eas.BalanceDrift = *drift

		eass = append(eass, eas)
	}
	return eass
}

// managedOpenAccount returns an account for the given host. If it does not
// exist already one is created.
func (am *accountManager) managedOpenAccount(hostKey types.SiaPublicKey) (acc *account, err error) {
	// Check if we already have an account. Due to a race condition around
	// account creation, we need to check that the account was persisted to disk
	// before we can start using it, this happens with the 'staticReady' and
	// 'externActive' variables of the account. See the rest of this functions
	// implementation to understand how they are used in practice.
	am.mu.Lock()
	acc, exists := am.accounts[hostKey.String()]
	if exists {
		am.mu.Unlock()
		<-acc.staticReady
		if acc.externActive {
			return acc, nil
		}
		return nil, errors.New("account creation failed")
	}
	// Open a new account.
	offset := accountsOffset + len(am.accounts)*accountSize
	aid, sk := modules.NewAccountID()
	acc = &account{
		staticID:        aid,
		staticHostKey:   hostKey,
		staticSecretKey: sk,

		staticFile:   am.staticFile,
		staticOffset: int64(offset),

		staticAlerter:       am.staticRenter.staticAlerter,
		staticBalanceTarget: am.staticRenter.staticAccountBalanceTarget,
		staticLog:           am.staticRenter.staticLog,
		staticReady:         make(chan struct{}),
	}
	am.accounts[hostKey.String()] = acc
	am.mu.Unlock()
	// Defer a close on 'staticReady'. By default, 'externActive' is false, so
	// if there is an error, the account will be marked as unusable.
	defer close(acc.staticReady)

	// Defer a function to delete the account if the persistence fails. This is
	// technically a race condition, but the alternative is holding the lock on
	// the account mangager while doing an fsync, which is not ideal.
	defer func() {
		if err != nil {
			am.mu.Lock()
			delete(am.accounts, hostKey.String())
			am.mu.Unlock()
		}
	}()

	// Save the file. After the file gets written to disk, perform a sync
	// because we want to ensure that the secret key of the account can be
	// recovered before we start using the account.
	err = acc.managedPersist()
	if err != nil {
		return nil, errors.AddContext(err, "failed to persist account")
	}
	err = acc.staticFile.Sync()
	if err != nil {
		return nil, errors.AddContext(err, "failed to sync accounts file")
	}

	// Mark the account as usable so that anyone who tried to open the account
	// after this function ran will see that the account is persisted correctly.
	acc.mu.Lock()
	acc.externActive = true
	acc.mu.Unlock()
	return acc, nil
}

// managedSaveAndClose is called on shutdown and ensures the account data is
// properly persisted to disk
func (am *accountManager) managedSaveAndClose() error {
	am.mu.Lock()
	defer am.mu.Unlock()

	// Save the account data to disk.
	clean := true
	var persistErrs error
	for _, account := range am.accounts {
		err := account.managedPersist()
		if err != nil {
			clean = false
			persistErrs = errors.Compose(persistErrs, err)
			continue
		}
	}
	// If there was an error saving any of the accounts, the system is not clean
	// and we do not need to update the metadata for the file.
	if !clean {
		return errors.AddContext(persistErrs, "unable to persist all accounts cleanly upon shutdown")
	}

	// Sync the file before updating the header. We want to make sure that the
	// accounts have been put into a clean and finalized state before writing an
	// update to the metadata.
	err := am.staticFile.Sync()
	if err != nil {
		return errors.AddContext(err, "failed to sync accounts file")
	}

	// update the metadata and mark the file as clean
	if err = am.updateMetadata(accountsMetadata{
		Header:  metadataHeader,
		Version: metadataVersion,
		Clean:   true,
	}); err != nil {
		return errors.AddContext(err, "failed to update accounts file metadata")
	}

	// Close the account file.
	return am.staticFile.Close()
}

// checkMetadata will load the metadata from the account file and return whether
// or not the previous shutdown was clean. If the metadata does not match the
// expected metadata, an error will be returned.
//
// NOTE: If we change the version of the file, this is probably the function
// that should handle doing the persist upgrade. Inside of this function there
// would be a call to the upgrade function.
func (am *accountManager) checkMetadata() (bool, error) {
	// Read metadata.
	metadata, err := readAccountsMetadata(am.staticFile)
	if err != nil {
		return false, errors.AddContext(err, "failed to read metadata from accounts file")
	}

	// Validate the metadata.
	if metadata.Header != metadataHeader {
		return false, errors.AddContext(errWrongHeader, "failed to verify accounts metadata")
	}
	if metadata.Version != metadataVersion {
		return false, errors.AddContext(errWrongVersion, "failed to verify accounts metadata")
	}
	return metadata.Clean, nil
}

// handleInterruptedUpgrade ensures that an interrupted upgrade can be recovered
// from. It does so by checking for the existence of a tmp accounts file, if
// that file is present we want to handle it occordingly.
func (am *accountManager) handleInterruptedUpgrade() error {
	// convenience variables
	r := am.staticRenter
	tmpFilePath := filepath.Join(r.persistDir, accountsTmpFilename)

	// check whether the tmp file exists
	tmpFileExists, err := fileExists(tmpFilePath)
	if err != nil {
		return errors.AddContext(err, "error checking if tmp file exists")
	}

	// if the tmp file does not exist, we don't have to do anything
	if !tmpFileExists {
		return nil
	}

	// open the tmp file
	tmpFile, err := r.staticDeps.OpenFile(tmpFilePath, os.O_RDWR, defaultFilePerm)
	if err != nil {
		return errors.AddContext(err, "error opening tmp account file")
	}

	// read the metadata, there can only be two scenarios:
	// - the tmp file is clean, continue from that file
	// - the tmp file is dirty, remove it
	tmpFileMetadata, err := readAccountsMetadata(tmpFile)
	if err == nil && tmpFileMetadata.Clean {
		return am.upgradeCopyAccountsFromTmpFile(tmpFile)
	}

	return errors.Compose(tmpFile.Close(), r.staticDeps.RemoveFile(tmpFilePath))
}

// managedLoad will pull all of the accounts off of disk and load them into the
// account manager. This should complete before the accountManager is made
// available to other processes.
func (am *accountManager) load() error {
	// Open the accounts file.
	clean, err := am.openFile()
	if err != nil {
		return errors.AddContext(err, "failed to open accounts file")
	}

	// Read the raw account data and decode them into accounts. We start at an
	// offset of 'accountsOffset' because the metadata precedes the accounts
	// data.
	for offset := int64(accountsOffset); ; offset += accountSize {
		// read the account at offset
		acc, err := am.readAccountAt(offset)
		if errors.Contains(err, io.EOF) {
			break
		} else if err != nil {
			am.staticRenter.staticLog.Println("ERROR: could not load account", err)
			continue
		}

		// reset the account balances after an unclean shutdown
		if !clean {
			acc.balance = types.ZeroCurrency
		}
		am.accounts[acc.staticHostKey.String()] = acc
	}

	// Ensure that when the renter is shut down, the save and close function
	// runs.
	if am.staticRenter.staticDeps.Disrupt("InterruptAccountSaveOnShutdown") {
		// Dependency injection to simulate an unclean shutdown.
		return nil
	}
	err = am.staticRenter.tg.AfterStop(am.managedSaveAndClose)
	if err != nil {
		return errors.AddContext(err, "unable to schedule a save and close with the thread group")
	}
	return nil
}

// openFile will open the file of the account manager and set the account
// manager's file variable.
//
// openFile will return 'true' if the previous shutdown was clean, and 'false'
// if the previous shutdown was not clean.
func (am *accountManager) openFile() (bool, error) {
	r := am.staticRenter

	// Sanity check that the file isn't already opened.
	if am.staticFile != nil {
		r.staticLog.Critical("double open detected on account manager")
		return false, errors.New("accounts file already open")
	}

	// Open the accounts file
	accountsFile, err := am.openAccountsFile(accountsFilename)
	if err != nil {
		return false, errors.AddContext(err, "error opening account file")
	}
	am.staticFile = accountsFile

	// Read accounts metadata
	metadata, err := readAccountsMetadata(am.staticFile)
	if err != nil {
		return false, errors.AddContext(err, "error reading account metadata")
	}

	// Handle a potentially interrupted upgrade
	err = am.handleInterruptedUpgrade()
	if err != nil {
		return false, errors.AddContext(err, "error occurred while trying to recover from an intterupted upgrade")
	}

	// Check accounts metadata
	_, err = am.checkMetadata()
	if err != nil && !errors.Contains(err, errWrongVersion) {
		return false, errors.AddContext(err, "error reading account metadata")
	}

	// If the metadata contains a wrong version, run the upgrade code
	if errors.Contains(err, errWrongVersion) {
		err = am.upgradeFromV161ToV162()
		if err != nil && errors.Contains(err, errWrongVersion) {
			err = am.upgradeFromV156ToV162()
			if err != nil && errors.Contains(err, errWrongVersion) {
				err = am.upgradeFromV150ToV162()
			}
		}
		if err != nil {
			return false, errors.AddContext(err, "error upgrading accounts file")
		}

		// log the successful upgrade
		am.staticRenter.staticLog.Println("successfully upgraded accounts file to v161")
	}

	// Whether this is a new file or an existing file, we need to set the header
	// on the metadata. When opening an account, the header should represent an
	// unclean shutdown. This will be flipped to a header that represents a
	// clean shutdown upon closing.
	err = am.updateMetadata(accountsMetadata{
		Header:  metadataHeader,
		Version: metadataVersion,
		Clean:   false,
	})
	if err != nil {
		return false, errors.AddContext(err, "unable to update the account metadata")
	}

	// Sync the metadata to ensure the acounts will load as dirty before any
	// accounts are created.
	err = am.staticFile.Sync()
	if err != nil {
		return false, errors.AddContext(err, "failed to sync accounts file")
	}

	return metadata.Clean, nil
}

// openAccountsFile is a helper function that will open an accounts file with
// given filename. If the accounts file does not exist prior to calling this
// function, it will be created and provided with the metadata header.
func (am *accountManager) openAccountsFile(filename string) (modules.File, error) {
	r := am.staticRenter

	// check whether the file exists
	accountsFilepath := filepath.Join(r.persistDir, filename)
	accountsFileExists, err := fileExists(accountsFilepath)
	if err != nil {
		return nil, err
	}

	// open the file and create it if necessary
	accountsFile, err := r.staticDeps.OpenFile(accountsFilepath, os.O_RDWR|os.O_CREATE, defaultFilePerm)
	if err != nil {
		return nil, errors.AddContext(err, "error opening account file")
	}

	// make sure a newly created accounts file has the metadata header
	if !accountsFileExists {
		_, err = accountsFile.WriteAt(encoding.Marshal(accountsMetadata{
			Header:  metadataHeader,
			Version: metadataVersion,
			Clean:   false,
		}), 0)
		err = errors.Compose(err, accountsFile.Sync())
		if err != nil {
			return accountsFile, errors.AddContext(err, "error writing metadata to accounts file")
		}
	}

	return accountsFile, nil
}

// readAccountAt tries to read an account object from the account persist file
// at the given offset.
func (am *accountManager) readAccountAt(offset int64) (*account, error) {
	acc, err := readAccountFromFile(am.staticFile, offset)
	if err != nil {
		return nil, err
	}

	acc.staticAlerter = am.staticRenter.staticAlerter
	acc.staticBalanceTarget = am.staticRenter.staticAccountBalanceTarget
	acc.staticLog = am.staticRenter.staticLog

	close(acc.staticReady)
	return acc, nil
}

// upgradeFromV161ToV162 is compat code that upgrades the accounts file from
// v161 to v162. This version introduced two new maintenance spending fields.
func (am *accountManager) upgradeFromV161ToV162() error {
	// open the accounts file
	accountsFile, err := am.openAccountsFile(accountsFilename)
	if err != nil {
		return errors.AddContext(err, "error opening account file")
	}

	// read the metadata
	metadata, err := readAccountsMetadata(accountsFile)
	if err != nil {
		return errors.AddContext(err, "failed to read accounts metadata")
	}

	// error out if this is not v161
	if metadata.Version != metadataVersionV161 {
		return errWrongVersion
	}

	// that's all it takes, the upgrade code will overwrite the header with the
	// proper version and accounts will be loaded taking the original 'clean'
	// flag into account
	return nil
}

// upgradeFromV156ToV162 is compat code that upgrades the accounts file from
// v156 to v162. This version introduced a bug fix for the way we handle refunds
// and track drift, which require a reset of the account balances.
func (am *accountManager) upgradeFromV156ToV162() error {
	// convenience variables
	r := am.staticRenter

	// open the accounts file
	accountsFile, err := am.openAccountsFile(accountsFilename)
	if err != nil {
		return errors.AddContext(err, "error opening account file")
	}

	// read the metadata
	metadata, err := readAccountsMetadata(accountsFile)
	if err != nil {
		return errors.AddContext(err, "failed to read accounts metadata")
	}

	// error out if this is not v156
	if metadata.Version != persist.MetadataVersionv156 {
		return errWrongVersion
	}

	// open a tmp accounts file
	tmpFile, err := am.openAccountsFile(accountsTmpFilename)
	if err != nil {
		return errors.AddContext(err, "failed to open tmp accounts file")
	}

	// read the accounts from the accounts file, but link them to the tmp file,
	// when calling persist on the account it will write the account into the
	// tmp file
	accounts := compatV156ReadAccounts(r.staticLog, am.staticFile, tmpFile)
	for _, acc := range accounts {
		// the v156 -> v161 requires a balance and drift reset
		// the v161 -> v162 only entails a version bump
		acc.balance = types.ZeroCurrency
		acc.balanceDriftPositive = types.ZeroCurrency
		acc.balanceDriftNegative = types.ZeroCurrency
		if err := acc.managedPersist(); err != nil {
			r.staticLog.Println("failed to upgrade account from v156 to v161", err)
		}
	}

	// sync the tmp file
	err = tmpFile.Sync()
	if err != nil {
		return errors.AddContext(err, "failed to sync tmp file")
	}

	// update the header and mark it clean
	_, err = tmpFile.WriteAt(encoding.Marshal(accountsMetadata{
		Header:  metadataHeader,
		Version: metadataVersion,
		Clean:   true,
	}), 0)
	if err != nil {
		return errors.AddContext(err, "failed to write header to tmp file")
	}

	// sync the tmp file, this step is very important because if it completes
	// successfully, and the upgrade fails over this point, the tmp file will be
	// used to recover from an interrupted upgrade.
	err = tmpFile.Sync()
	if err != nil {
		return errors.AddContext(err, "failed to sync tmp file")
	}

	// copy the accounts from the tmp file to the accounts file, this is
	// extracted into a separate method as the recovery flow might have to pick
	// up from where we left off in case of failure during an initial attempt
	return am.upgradeCopyAccountsFromTmpFile(tmpFile)
}

// upgradeFromV150ToV162 is compat code that upgrades the accounts file from
// v150 to v162. The new accounts take up more space on disk, so we have to read
// all of them, assign them new offets and rewrite them to the accounts file.
func (am *accountManager) upgradeFromV150ToV162() error {
	// convenience variables
	r := am.staticRenter

	// open a tmp accounts file
	tmpFile, err := am.openAccountsFile(accountsTmpFilename)
	if err != nil {
		return errors.AddContext(err, "failed to open tmp accounts file")
	}

	// read the accounts from the accounts file, but link them to the tmp file,
	// when calling persist on the account it will write the account into the
	// tmp file
	accounts := compatV150ReadAccounts(r.staticLog, am.staticFile, tmpFile)
	for _, acc := range accounts {
		if err := acc.managedPersist(); err != nil {
			r.staticLog.Println("failed to upgrade account from v150 to v161", err)
		}
	}

	// sync the tmp file
	err = tmpFile.Sync()
	if err != nil {
		return errors.AddContext(err, "failed to sync tmp file")
	}

	// update the header and mark it clean
	_, err = tmpFile.WriteAt(encoding.Marshal(accountsMetadata{
		Header:  metadataHeader,
		Version: metadataVersion,
		Clean:   true,
	}), 0)
	if err != nil {
		return errors.AddContext(err, "failed to write header to tmp file")
	}

	// sync the tmp file, this step is very important because if it completes
	// successfully, and the upgrade fails over this point, the tmp file will be
	// used to recover from an interrupted upgrade.
	err = tmpFile.Sync()
	if err != nil {
		return errors.AddContext(err, "failed to sync tmp file")
	}

	// copy the accounts from the tmp file to the accounts file, this is
	// extracted into a separate method as the recovery flow might have to pick
	// up from where we left off in case of failure during an initial attempt
	return am.upgradeCopyAccountsFromTmpFile(tmpFile)
}

// upgradeCopyAccountsFromTmpFile will copy the contents of the tmp file into
// the accounts file. This is a separate method as this function is called
// during the happy flow, but it is also potentially the steps required when
// trying to recover from a failed initial update attempt.
func (am *accountManager) upgradeCopyAccountsFromTmpFile(tmpFile modules.File) (err error) {
	// convenience variables
	r := am.staticRenter
	tmpFilePath := filepath.Join(r.persistDir, accountsTmpFilename)

	// copy the tmp file to the accounts file
	_, err = io.Copy(am.staticFile, tmpFile)
	if err != nil {
		return errors.AddContext(err, "failed to copy the temporary accounts file to the actual accounts file location")
	}

	// sync the accounts file
	err = am.staticFile.Sync()
	if err != nil {
		return errors.AddContext(err, "failed to sync accounts file")
	}

	// seek to the beginning of the file
	_, err = am.staticFile.Seek(0, io.SeekStart)
	if err != nil {
		return errors.AddContext(err, "failed to seek to the beginning of the accounts file")
	}

	// delete the tmp file
	return errors.AddContext(errors.Compose(tmpFile.Close(), r.staticDeps.RemoveFile(tmpFilePath)), "failed to delete accounts file")
}

// updateMetadata writes the given metadata to the accounts file.
func (am *accountManager) updateMetadata(meta accountsMetadata) error {
	_, err := am.staticFile.WriteAt(encoding.Marshal(meta), 0)
	return err
}

// compatV150ReadAccounts is a helper function that reads the accounts from the
// accounts file assuming they are persisted using the v150 persistence object
// and parameters. Extracted to keep the compat code clean.
func compatV150ReadAccounts(log *skydPersist.Logger, accountsFile modules.File, tmpFile modules.File) []*account {
	// the offset needs to be the new accountsOffset
	newOffset := int64(accountsOffset)

	// collect all accounts from the current accounts file
	var accounts []*account
	for offset := int64(accountSizeV150); ; offset += accountSizeV150 {
		// read account bytes
		accountBytes := make([]byte, accountSizeV150)
		_, err := accountsFile.ReadAt(accountBytes, offset)
		if errors.Contains(err, io.EOF) {
			break
		} else if err != nil {
			log.Println("ERROR: could not read account data", err)
			continue
		}

		// load the account bytes onto the a persistence object
		var accountDataV150 accountPersistenceV150
		err = encoding.Unmarshal(accountBytes[crypto.HashSize:], &accountDataV150)
		if err != nil {
			log.Println("ERROR: could not load account bytes", err)
			continue
		}

		accounts = append(accounts, &account{
			staticID:        accountDataV150.AccountID,
			staticHostKey:   accountDataV150.HostKey,
			staticSecretKey: accountDataV150.SecretKey,

			balance: accountDataV150.Balance,

			staticOffset: newOffset,
			staticFile:   tmpFile,
		})
		newOffset += accountSize
	}

	return accounts
}

// compatV156ReadAccounts is a helper function that reads the accounts from the
// accounts file assuming they are persisted using the v156 persistence object
// and parameters. Extracted to keep the compat code clean.
func compatV156ReadAccounts(log *skydPersist.Logger, accountsFile modules.File, tmpFile modules.File) []*account {
	// collect all accounts from the current accounts file
	var accounts []*account
	for offset := int64(accountSize); ; offset += accountSize {
		// read the account from the file
		acc, err := readAccountFromFile(accountsFile, offset)
		if errors.Contains(err, io.EOF) {
			break
		} else if err != nil {
			log.Println("ERROR: could not read account data", err)
			continue
		}

		// add it to the accounts
		accounts = append(accounts, acc)
	}

	return accounts
}

// fileExists is a small helper function that checks whether a file at given
// path exists, it abstracts checking whether the error from the stat is an
// `IsNotExists` error or not.
func fileExists(path string) (bool, error) {
	_, statErr := os.Stat(path)
	if statErr == nil {
		return true, nil
	}
	if os.IsNotExist(statErr) {
		return false, nil
	}
	return false, errors.AddContext(statErr, "error calling stat on file")
}

// readAccountsMetadata is a small helper function that tries to read the
// metadata object from the given file.
func readAccountsMetadata(file modules.File) (*accountsMetadata, error) {
	// Seek to the beginning of the file
	_, err := file.Seek(0, io.SeekStart)
	if err != nil {
		return nil, errors.AddContext(err, "failed to seek to the beginning of the file")
	}

	// Read metadata.
	buffer := make([]byte, metadataSize)
	_, err = io.ReadFull(file, buffer)
	if err != nil {
		return nil, errors.AddContext(err, "failed to read metadata from file")
	}

	// Seek to the beginning of the file
	_, err = file.Seek(0, io.SeekStart)
	if err != nil {
		return nil, errors.AddContext(err, "failed to seek to the beginning of the file")
	}

	// Decode metadata
	var metadata accountsMetadata
	err = encoding.Unmarshal(buffer, &metadata)
	if err != nil {
		return nil, errors.AddContext(err, "failed to decode metadata")
	}

	return &metadata, nil
}

// readAccountFromFile is a helper function that reads an account from the given
// file at given offset.
func readAccountFromFile(file modules.File, offset int64) (*account, error) {
	// read account bytes
	accountBytes := make([]byte, accountSize)
	_, err := file.ReadAt(accountBytes, offset)
	if err != nil {
		return nil, errors.AddContext(err, "failed to read account bytes")
	}

	// load the account bytes onto the a persistence object
	var accountData accountPersistence
	err = accountData.loadBytes(accountBytes)
	if err != nil {
		return nil, errors.AddContext(err, "failed to load account bytes")
	}

	return &account{
		staticID:        accountData.AccountID,
		staticHostKey:   accountData.HostKey,
		staticSecretKey: accountData.SecretKey,

		// balance details
		balance:              accountData.Balance,
		balanceDriftPositive: accountData.BalanceDriftPositive,
		balanceDriftNegative: accountData.BalanceDriftNegative,

		// spending details
		spending: spendingDetails{
			downloads:         accountData.SpendingDownloads,
			registryReads:     accountData.SpendingRegistryReads,
			registryWrites:    accountData.SpendingRegistryWrites,
			repairDownloads:   accountData.SpendingRepairDownloads,
			repairUploads:     accountData.SpendingRepairUploads,
			snapshotDownloads: accountData.SpendingSnapshotDownloads,
			snapshotUploads:   accountData.SpendingSnapshotUploads,
			subscriptions:     accountData.SpendingSubscriptions,
			uploads:           accountData.SpendingUploads,
		},

		// residue
		residue: accountData.Residue,

		// host balance
		hostBalance:         accountData.HostBalance,
		hostBalanceNegative: accountData.HostBalanceNegative,

		staticReady:  make(chan struct{}),
		externActive: true,

		staticOffset: offset,
		staticFile:   file,
	}, nil
}